def importCall(request, collection, database=None): """ View used to import data. """ #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = { 'parser' : [], 'mapper' : [] } out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() mongo.connect() #TODO: mapping should come from url, in form of id mapping = { '__key__' : 'id_str', # '__upperName__' : { 'transform' : 'upperCase', 'args' : ['name'] }, # '__fullName__' : { 'transform' : 'concatStrings', 'args' : ['name', 'surname'] }, # '__fullNameUpper__' : { 'transform' : 'concatStrings', # 'args' : [{ 'transform' : 'upperCase', 'args' : ['name'] }, { 'transform' : 'upperCase', 'args' : ['surname'] }] }, } if request.POST: record_errors_number = 0 ok_records = [] #TODO: PARAMETRIZE THIS, maybe in settings MAX_ERROR_RECORDS = 10000 if 'data' in request.POST and 'format' in request.POST: format = request.POST['format'].lower() data = request.POST['data'] try: parser = recordparser.parserFactory(format, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data) continue try: newRecord = mappingManager.mapRecord(d, mapping) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS: break if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None): """ View used to import data. """ #this loads an instance of mapper from mappermanager import mappingManager, codedMappers #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = { 'parser' : [], 'mapper' : [] } out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB if request.POST: mapper = None mapperName = getMapper(request) #todo: decide to use name or id for referencing mapper in request if mapperName: if mapperName in codedMappers: mapperObject = codedMappers[mapperName] else: sketchMapper = SketchMapper.objects.get(name=mapperName) mapper = sketchMapper.mapper record_errors_number = 0 ok_records = [] MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS if 'data' in request.POST and 'parser' in request.POST: parser_name = request.POST['parser'].lower() data = request.POST['data'] try: #parsing phase parser = recordparser.parserFactory(parser_name, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data) continue #mapping phase if mapper is not None: try: newRecord = mappingManager.mapRecord(d, mapping, { '__mapper_name__' : mapperName }) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) #mapper is none, record is imported as it is else: ok_records.append(d) if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS: break #commit phase if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: #creating the collection model and set owner=user if collection does not exits #TODO: we could check again the number of allowed collections here, as in decorator try: collectionInstance = SketchCollection.objects.get(name=collection, database=database) except: collectionInstance = SketchCollection(owner=request.user, name=collection, database=database) collectionInstance.save() #finally inserting records for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None): """ View used to import data. """ #this loads an instance of mapper from mappermanager import mappingManager, codedMappers #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = {'parser': [], 'mapper': []} out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() mongo.connect() if request.POST: mapper = None mapperName = getMapper(request) #todo: decide to use name or id for referencing mapper in request if mapperName: if mapperName in codedMappers: mapperObject = codedMappers[mapperName] else: sketchMapper = SketchMapper.objects.get(name=mapperName) mapper = sketchMapper.mapper record_errors_number = 0 ok_records = [] MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS if 'data' in request.POST and 'parser' in request.POST: parser_name = request.POST['parser'].lower() data = request.POST['data'] try: #parsing phase parser = recordparser.parserFactory(parser_name, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append( str(d.exception_message) + ":" + d.raw_data) continue #mapping phase if mapper is not None: try: newRecord = mappingManager.mapRecord( d, mapping, {'__mapper_name__': mapperName}) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) #mapper is none, record is imported as it is else: ok_records.append(d) if len(out['error_records']['mapper']) + len(out[ 'error_records']['parser']) > MAX_ERROR_RECORDS: break #commit phase if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: #creating the collection model and set owner=user if collection does not exits #TODO: we could check again the number of allowed collections here, as in decorator try: collectionInstance = SketchCollection.objects.get( name=collection, database=database) except: collectionInstance = SketchCollection( owner=request.user, name=collection, database=database) collectionInstance.save() #finally inserting records for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)