def write(self, data, hints): collection_name = hints.get('collection_name', None) prefix = hints.get('prefix', 'results_') if not collection_name: collection_name = mongo.get_results_collection_name(results_mongo_db, prefix=prefix) collection = mongo.getCollection(results_mongo_db, collection_name) counted = 0 for r in data: if r.keys(): oid = mongo._insert(results_mongo_db, collection_name, r) #todo: see if this is applicable, or set an option for it if "__sketch_id__" not in r: spec = { "_id" : oid } r['__sketch_id__'] = str(oid) mongo._update(results_mongo_db, collection_name, spec, r) counted += 1 if not counted: return { 'collection_name' : None, 'num_records': counted } return { 'collection_name' : collection_name, 'num_records': counted}
def importCall(request, collection, database=None): """ View used to import data. """ #this loads an instance of mapper from mappermanager import mappingManager, codedMappers #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = { 'parser' : [], 'mapper' : [] } out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB if request.POST: mapper = None mapperName = getMapper(request) #todo: decide to use name or id for referencing mapper in request if mapperName: if mapperName in codedMappers: mapperObject = codedMappers[mapperName] else: sketchMapper = SketchMapper.objects.get(name=mapperName) mapper = sketchMapper.mapper record_errors_number = 0 ok_records = [] MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS if 'data' in request.POST and 'parser' in request.POST: parser_name = request.POST['parser'].lower() data = request.POST['data'] try: #parsing phase parser = recordparser.parserFactory(parser_name, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data) continue #mapping phase if mapper is not None: try: newRecord = mappingManager.mapRecord(d, mapping, { '__mapper_name__' : mapperName }) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) #mapper is none, record is imported as it is else: ok_records.append(d) if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS: break #commit phase if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: #creating the collection model and set owner=user if collection does not exits #TODO: we could check again the number of allowed collections here, as in decorator try: collectionInstance = SketchCollection.objects.get(name=collection, database=database) except: collectionInstance = SketchCollection(owner=request.user, name=collection, database=database) collectionInstance.save() #finally inserting records for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)