def importCall(request, collection, database=None):
"""
View used to import data.
"""
#TODO: separate data collection and processing and write a view that handles FILES
out = createBaseResponseObject()
out['error_records'] = { 'parser' : [], 'mapper' : [] }
out['ok_records_number'] = 0
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
mongo.connect()
#TODO: mapping should come from url, in form of id
mapping = { '__key__' : 'id_str',
# '__upperName__' : { 'transform' : 'upperCase', 'args' : ['name'] },
# '__fullName__' : { 'transform' : 'concatStrings', 'args' : ['name', 'surname'] },
# '__fullNameUpper__' : { 'transform' : 'concatStrings',
# 'args' : [{ 'transform' : 'upperCase', 'args' : ['name'] }, { 'transform' : 'upperCase', 'args' : ['surname'] }] },
}
if request.POST:
record_errors_number = 0
ok_records = []
#TODO: PARAMETRIZE THIS, maybe in settings
MAX_ERROR_RECORDS = 10000
if 'data' in request.POST and 'format' in request.POST:
format = request.POST['format'].lower()
data = request.POST['data']
try:
parser = recordparser.parserFactory(format, data)
for d in parser.objects():
if d is recordparser.ParserError:
out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data)
continue
try:
newRecord = mappingManager.mapRecord(d, mapping)
ok_records.append(newRecord)
except:
out['error_records']['mapper'].append(d)
if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS:
break
if 'commit' in request.POST and request.POST['commit']:
try:
commit = int(request.POST['commit'])
except:
commit = 0
if commit:
for record in ok_records:
mongo_id = mongo._insert(database, collection, record)
out['results'].append(mongo_id)
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None):
"""
View used to import data.
"""
#this loads an instance of mapper
from mappermanager import mappingManager, codedMappers
#TODO: separate data collection and processing and write a view that handles FILES
out = createBaseResponseObject()
out['error_records'] = { 'parser' : [], 'mapper' : [] }
out['ok_records_number'] = 0
database = database or settings.MONGO_SERVER_DEFAULT_DB
if request.POST:
mapper = None
mapperName = getMapper(request)
#todo: decide to use name or id for referencing mapper in request
if mapperName:
if mapperName in codedMappers:
mapperObject = codedMappers[mapperName]
else:
sketchMapper = SketchMapper.objects.get(name=mapperName)
mapper = sketchMapper.mapper
record_errors_number = 0
ok_records = []
MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS
if 'data' in request.POST and 'parser' in request.POST:
parser_name = request.POST['parser'].lower()
data = request.POST['data']
try:
#parsing phase
parser = recordparser.parserFactory(parser_name, data)
for d in parser.objects():
if d is recordparser.ParserError:
out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data)
continue
#mapping phase
if mapper is not None:
try:
newRecord = mappingManager.mapRecord(d, mapping, { '__mapper_name__' : mapperName })
ok_records.append(newRecord)
except:
out['error_records']['mapper'].append(d)
#mapper is none, record is imported as it is
else:
ok_records.append(d)
if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS:
break
#commit phase
if 'commit' in request.POST and request.POST['commit']:
try:
commit = int(request.POST['commit'])
except:
commit = 0
if commit:
#creating the collection model and set owner=user if collection does not exits
#TODO: we could check again the number of allowed collections here, as in decorator
try:
collectionInstance = SketchCollection.objects.get(name=collection, database=database)
except:
collectionInstance = SketchCollection(owner=request.user, name=collection, database=database)
collectionInstance.save()
#finally inserting records
for record in ok_records:
mongo_id = mongo._insert(database, collection, record)
out['results'].append(mongo_id)
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None):
"""
View used to import data.
"""
#this loads an instance of mapper
from mappermanager import mappingManager, codedMappers
#TODO: separate data collection and processing and write a view that handles FILES
out = createBaseResponseObject()
out['error_records'] = {'parser': [], 'mapper': []}
out['ok_records_number'] = 0
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
mongo.connect()
if request.POST:
mapper = None
mapperName = getMapper(request)
#todo: decide to use name or id for referencing mapper in request
if mapperName:
if mapperName in codedMappers:
mapperObject = codedMappers[mapperName]
else:
sketchMapper = SketchMapper.objects.get(name=mapperName)
mapper = sketchMapper.mapper
record_errors_number = 0
ok_records = []
MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS
if 'data' in request.POST and 'parser' in request.POST:
parser_name = request.POST['parser'].lower()
data = request.POST['data']
try:
#parsing phase
parser = recordparser.parserFactory(parser_name, data)
for d in parser.objects():
if d is recordparser.ParserError:
out['error_records']['parser'].append(
str(d.exception_message) + ":" + d.raw_data)
continue
#mapping phase
if mapper is not None:
try:
newRecord = mappingManager.mapRecord(
d, mapping, {'__mapper_name__': mapperName})
ok_records.append(newRecord)
except:
out['error_records']['mapper'].append(d)
#mapper is none, record is imported as it is
else:
ok_records.append(d)
if len(out['error_records']['mapper']) + len(out[
'error_records']['parser']) > MAX_ERROR_RECORDS:
break
#commit phase
if 'commit' in request.POST and request.POST['commit']:
try:
commit = int(request.POST['commit'])
except:
commit = 0
if commit:
#creating the collection model and set owner=user if collection does not exits
#TODO: we could check again the number of allowed collections here, as in decorator
try:
collectionInstance = SketchCollection.objects.get(
name=collection, database=database)
except:
collectionInstance = SketchCollection(
owner=request.user,
name=collection,
database=database)
collectionInstance.save()
#finally inserting records
for record in ok_records:
mongo_id = mongo._insert(database, collection,
record)
out['results'].append(mongo_id)
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
out['ok_records_number'] = len(ok_records)
