Exemplo n.º 1
0
 def write(self, data, hints):
     
     collection_name = hints.get('collection_name', None)
     prefix = hints.get('prefix', 'results_')
     if not collection_name:
         collection_name = mongo.get_results_collection_name(results_mongo_db, prefix=prefix)
     collection = mongo.getCollection(results_mongo_db, collection_name)
     counted = 0
     for r in data:
         if r.keys():
             oid =  mongo._insert(results_mongo_db, collection_name, r)
             #todo: see if this is applicable, or set an option for it
             if "__sketch_id__" not in r:
                 spec = { "_id" : oid }
                 r['__sketch_id__'] =  str(oid) 
                 mongo._update(results_mongo_db, collection_name, spec, r)
             counted += 1
             
     if not counted:
         return { 'collection_name' : None, 'num_records': counted }
     
     return { 'collection_name' : collection_name, 'num_records': counted}
Exemplo n.º 2
0
def importCall(request, collection, database=None):
    """
    View used to import data.
    """
    #this loads an instance of mapper    
    from mappermanager import mappingManager, codedMappers

    #TODO: separate data collection and processing and write a view that handles FILES
    
    out = createBaseResponseObject()
    
    out['error_records'] = { 'parser' : [], 'mapper' : [] }
    out['ok_records_number'] = 0
    
    database = database or settings.MONGO_SERVER_DEFAULT_DB
    
    if request.POST:
        
        mapper = None
        mapperName = getMapper(request)
        #todo: decide to use name or id for referencing mapper in request
        if mapperName:
            if mapperName in codedMappers:
                mapperObject = codedMappers[mapperName]
            else:
                sketchMapper = SketchMapper.objects.get(name=mapperName)
                mapper = sketchMapper.mapper
            
    
        record_errors_number = 0
        ok_records = []
        MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS
    
        if 'data' in request.POST and 'parser' in request.POST:
            parser_name = request.POST['parser'].lower()
            data = request.POST['data']
            
            try:
                #parsing phase
                parser = recordparser.parserFactory(parser_name, data)
                for d in parser.objects():
                    if d is recordparser.ParserError:
                        out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data)
                        continue
                    
                    #mapping phase
                    if mapper is not None:
                        try:
                            newRecord = mappingManager.mapRecord(d, mapping, { '__mapper_name__' : mapperName })
                            ok_records.append(newRecord)
                    
                        except:
                            out['error_records']['mapper'].append(d)
    
                    #mapper is none, record is imported as it is
                    else:
                        ok_records.append(d)
                    
                    if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS:
                        break
                        
    
                #commit phase
                if 'commit' in request.POST and request.POST['commit']:
                    try:
                        commit = int(request.POST['commit'])
                    except:
                        commit = 0
                        
                    if commit:
                        #creating the collection model and set owner=user if collection does not exits
                        #TODO: we could check again the number of allowed collections here, as in decorator
                        try:
                            collectionInstance = SketchCollection.objects.get(name=collection, database=database)
                        except:
                            collectionInstance = SketchCollection(owner=request.user, name=collection, database=database)
                            collectionInstance.save()
    
                        #finally inserting records
                        for record in ok_records:
                            mongo_id = mongo._insert(database, collection, record)
                            out['results'].append(mongo_id)
                            
            except Exception, e:
                out['errors'] = str(e)
                out['status'] = 0
                 
            out['ok_records_number'] = len(ok_records)