def query(request, collection, command, database=None):
"""
View to send commands to handler.
Deprecated.
"""
out = createBaseResponseObject()
database = database or settings.MONGO_SERVER_DEFAULT_DB
commandMethod = getattr(mongo, command, None)
try:
if not commandMethod or command not in mongo.available_commands:
raise Exception("Command %s not supported. Available commands are: %s" % (command, ", ".join(mongo.available_commands)))
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
if collection not in existing_collections:
raise Exception("Collection %s does not exist" % collection)
results = commandMethod(database, collection, request)
if results:
out['results'] = results
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def inner_decorator(request, collection, *args, **kwargs):
out = createBaseResponseObject()
database = kwargs.get(database, settings.MONGO_SERVER_DEFAULT_DB)
try:
#check user and collection
collectionInstance = SketchCollection.objects.get(
name=collection, database=database)
wa = collectionInstance.hasWriteAccess(request.user)
if wa:
return view(request,
collection,
database=database,
*args,
**kwargs)
except SketchCollection.DoesNotExist:
#TODO: we could limit the number of collections here
return view(request,
collection,
database=database,
*args,
**kwargs)
except Exception, e:
out['status'] = 0
out['errors'] = [str(e)]
return HttpResponse(json.dumps(out))
def parsers(request):
out = createBaseResponseObject()
try:
out['results'] = recordparser.ALLOWED_PARSERS.keys()
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def parsers(request):
out = createBaseResponseObject()
try:
out['results'] = recordparser.ALLOWED_PARSERS.keys()
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def processObjects(request, collection, database=None):
"""
this view applies processing to a set of records
"""
#TODO: get the records with a filter or with a list of ids
#TODO: handle a list of processing functions and argss to be passed in
#TODO: consider GET vs POST for calling this view. If no records are changed server side,
# GET method should be used
#TODO: consider writing output to a collection
#TODO: leverage map/reduce when possible
database = database or settings.MONGO_SERVER_DEFAULT_DB
from processingmanager import processingManager
if 'ids' in request.GET:
ids = request.GET['ids']
#TODO: get records by id
records = []
if 'query' in request.GET:
query = request.GET['query']
#todo: get records by query
records = []
if 'in_collection' in request.GET:
in_collection = request.GET['in_collection']
records = mongo.objectsFromCollection(database, in_collection)
write_collection = getWriteCollection(request)
processor = getProcessor(request)
if processor not in processingManager.getProcessors():
raise
#processing cycle
if write_collection:
for record in records:
pass
else:
for record in records:
pass
out = createBaseResponseObject()
database = database or settings.MONGO_SERVER_DEFAULT_DB
return HttpResponse(json.dumps(out, default=bson.json_util.default))
def getServerInfo():
try:
out = createBaseResponseObject()
mongo.connect()
existing_dbs = mongo.connection.database_names()
out['results'] = existing_dbs
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def formatters(request):
formatters = formattersManager.getFormatters()
out = createBaseResponseObject()
try:
out['results'] = formatters
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def objects(request, collection, database=None):
"""
Main view to send commands to handler
"""
out = createBaseResponseObject()
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
try:
mongo.connect()
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
if collection not in existing_collections:
raise Exception("Collection %s does not exist" % collection)
query_dict = getQueryDict(request)
offset = getOffset(request)
limit = getLimit(request)
formatter = getFormatter(request)
from formattersmanager import formattersManager
formatters = formattersManager.getFormatters()
if formatter and formatter not in formatters:
raise Exception("Formatter %s is not available" % str(formatter))
if formatter:
formatter_callback = formattersManager.getFormatter(formatter)
else:
formatter_callback = None
query_result = mongo.objects(database,
collection,
query_dict=query_dict,
offset=offset,
limit=limit,
formatter_callback=formatter_callback)
records = query_result['records']
has_more = query_result['has_more']
out['results'] = records
out['has_more'] = has_more
except Exception, e:
raise
out['errors'] = str(e)
out['status'] = 0
def mappers(request):
out = createBaseResponseObject()
try:
maps = SketchMapper.objects.all()
out['results'] = []
for map in maps:
out['results'].append(instanceDict(map))
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def transforms(request):
import mappermanager
transforms = mappermanager.mappingManager.getTransforms()
out = createBaseResponseObject()
try:
out['results'] = transforms
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def processors(request):
import processingmanager
processors = processingmanager.processingManager.getProcessors()
out = createBaseResponseObject()
try:
out['results'] = processors
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def transforms(request):
import mappermanager
transforms = mappermanager.mappingManager.getTransforms()
out = createBaseResponseObject()
try:
out['results'] = transforms
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def getServerInfo():
mongo = MongoWrapper()
try:
out = createBaseResponseObject()
mongo.connect()
existing_dbs = mongo.connection.database_names()
out['results'] = existing_dbs
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def inner_decorator(request,*args, **kwargs):
out = createBaseResponseObject()
try:
if request.user.is_authenticated():
return view(request, *args, **kwargs)
except Exception, e:
out['status'] = 0
out['errors'] = [str(e)]
return HttpResponse(json.dumps(out))
def processors(request):
import processingmanager
processors = processingmanager.processingManager.getProcessors()
out = createBaseResponseObject()
try:
out['results'] = processors
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def inner_decorator(request, *args, **kwargs):
try:
if request.user.is_authenticated():
return view(request, *args, **kwargs)
except:
pass
out = createBaseResponseObject()
out["status"] = 0
out["errors"] = ["You must be logged in to use this feature"]
return HttpResponse(json.dumps(out))
def inner_decorator(request, *args, **kwargs):
out = createBaseResponseObject()
try:
if request.user.is_authenticated():
return view(request, *args, **kwargs)
except Exception, e:
out['status'] = 0
out['errors'] = [str(e)]
return HttpResponse(json.dumps(out))
def formatters(request):
import formattersmanager
formatters = formattersmanager.formattersManager.getFormatters()
out = createBaseResponseObject()
try:
out['results'] = formatters
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def mappers(request):
out = createBaseResponseObject()
try:
maps = SketchMapper.objects.all()
out['results'] = []
for map in maps:
out['results'].append(instanceDict(map))
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def serverMeta(request):
mongo = MongoWrapper()
try:
out = createBaseResponseObject()
mongo.connect()
existing_dbs = mongo.connection.database_names()
out['results'] = existing_dbs
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def object(request, collection, oid, database=None):
"""
Single object view
"""
database = database or settings.MONGO_SERVER_DEFAULT_DB
out = createBaseResponseObject()
if request.DELETE:
mongo.dropObjectByOid(database, collection, oid)
out['results'].append(oid)
return HttpResponse(json.dumps(out, default=bson.json_util.default))
def processObjects(request, collection, database=None):
"""
this view applies processing to a set of records
"""
#TODO: get the records with a filter or with a list of ids
#TODO: handle a list of processing functions and argss to be passed in
#TODO: consider GET vs POST for calling this view. If no records are changed server side,
# GET method should be used
#TODO: consider writing output to a collection
#TODO: leverage map/reduce when possible
out = createBaseResponseObject()
return HttpResponse(json.dumps(out, default=bson.json_util.default))
def objects(request, collection, database=None):
"""
Query view, gets some objects
"""
out = createBaseResponseObject()
database = database or settings.MONGO_SERVER_DEFAULT_DB
try:
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
if collection not in existing_collections:
raise Exception("Collection %s does not exist" % collection)
query_dict = getQueryDict(request)
offset = getOffset(request)
limit = getLimit(request)
write_collection = getWriteCollection(request)
formatter = getFormatter(request)
formatters = formattersManager.getFormatters()
if formatter and formatter not in formatters:
raise Exception("Formatter %s is not available" % str(formatter))
if formatter:
formatter_callback = formattersManager.getFormatter(formatter)
else:
formatter_callback = None
drop_collection = request.GET.get('drop_collection', None)
if drop_collection and drop_collection.startswith('results'):
mongo.dropCollection(database, drop_collection)
query_result = mongo.objects(database, collection, query_dict=query_dict, offset=offset, limit=limit,
formatter_callback=formatter_callback, write_collection=write_collection)
out['results'] = query_result['records']
out['num_records'] = query_result['num_records']
out['has_more'] = query_result['has_more']
out['collection_out'] = query_result['collection_out']
except Exception, e:
raise
out['errors'] = str(e)
out['status'] = 0
def objects(request, collection, database=None):
"""
Main view to send commands to handler
"""
out = createBaseResponseObject()
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
try:
mongo.connect()
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
if collection not in existing_collections:
raise Exception("Collection %s does not exist" % collection)
query_dict = getQueryDict(request)
offset = getOffset(request)
limit = getLimit(request)
formatter = getFormatter(request)
from formattersmanager import formattersManager
formatters = formattersManager.getFormatters()
if formatter and formatter not in formatters:
raise Exception("Formatter %s is not available" % str(formatter))
if formatter:
formatter_callback = formattersManager.getFormatter(formatter)
else:
formatter_callback = None
query_result = mongo.objects(database, collection, query_dict=query_dict, offset=offset, limit=limit,
formatter_callback=formatter_callback)
records = query_result['records']
has_more = query_result['has_more']
out['results'] = records
out['has_more'] = has_more
except Exception, e:
raise
out['errors'] = str(e)
out['status'] = 0
def processObjects(request, collection, database=None):
"""
this view applies processing to a set of records
"""
#TODO: get the records with a filter or with a list of ids
#TODO: handle a list of processing functions and argss to be passed in
#TODO: consider GET vs POST for calling this view. If no records are changed server side,
# GET method should be used
#TODO: consider writing output to a collection
#TODO: leverage map/reduce when possible
out = createBaseResponseObject()
return HttpResponse(json.dumps(out, default=bson.json_util.default))
def inner_decorator(request, *args, **kwargs):
if "collection" not in kwargs:
return func(request, *args, **kwargs)
collection = kwargs["collection"]
try:
# TODO: check user and collection
return view(request, *args, **kwargs)
except:
pass
out = createBaseResponseObject()
out["status"] = 0
out["errors"] = ["You must own collection %s" % collection]
return HttpResponse(json.dumps(out))
def inner_decorator(request, collection, *args, **kwargs):
out = createBaseResponseObject()
database = kwargs.get(database, settings.MONGO_SERVER_DEFAULT_DB)
try:
#check user and collection
collectionInstance = SketchCollection.objects.get(name=collection, database=database)
wa = collectionInstance.hasReadAccess(request.user)
if wa:
return view(request, collection, database=database, *args, **kwargs)
except Exception, e:
out['status'] = 0
out['errors'] = [str(e)]
return HttpResponse(json.dumps(out))
def object(request, collection, oid, database=None):
database = database or settings.MONGO_SERVER_DEFAULT_DB
out = createBaseResponseObject()
mongo = MongoWrapper()
mongo.connect()
if request.DELETE:
mongo.dropObjectByOid(database, collection, oid)
out['results'].append(oid)
try:
mongo.connection.close()
except:
pass
return HttpResponse(json.dumps(out, default=bson.json_util.default))
def getDbInfo(database):
try:
out = createBaseResponseObject()
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
#filtering out "system.indexes" collection
existing_collections = [x for x in existing_collections if x != "system.indexes"]
out['results'] = existing_collections
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def object(request, collection, oid, database=None):
database = database or settings.MONGO_SERVER_DEFAULT_DB
out = createBaseResponseObject()
mongo = MongoWrapper()
mongo.connect()
if request.DELETE:
mongo.dropObjectByOid(database, collection, oid)
out['results'].append(oid)
try:
mongo.connection.close()
except:
pass
return HttpResponse(json.dumps(out, default=bson.json_util.default))
def dbMeta(request, database):
mongo = MongoWrapper()
try:
out = createBaseResponseObject()
mongo.connect()
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
out['results'] = existing_collections
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def getDbInfo(database):
mongo = MongoWrapper()
try:
out = createBaseResponseObject()
mongo.connect()
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
#filtering out "system.indexes" collection
existing_collections = [
x for x in existing_collections if x != "system.indexes"
]
out['results'] = existing_collections
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def operation(request):
out = createBaseResponseObject()
if request.POST:
try:
source_name = request.POST['source_name']
source_arguments_json = request.POST.get('source_arguments', '{}')
source_arguments = json.loads(source_arguments_json, object_hook=bson.json_util.object_hook)
map_operations_data_json = request.POST.get('map_operations_data', '[]');
map_operations_data =json.loads(map_operations_data_json, object_hook=bson.json_util.object_hook)
reduce_operations_data_json = request.POST.get('reduce_operations_data', '[]');
reduce_operations_data =json.loads(reduce_operations_data_json, object_hook=bson.json_util.object_hook)
m = sketch.operations.SketchOperation(source_name, source_arguments = source_arguments, map_operations_data=map_operations_data,
reduce_operations_data=reduce_operations_data)
try:
op_results = m.perform()
out['collection_out'] = op_results['collection_name']
out['num_records'] = op_results['num_records']
except Exception, e:
raise
out['errors'] = str(e)
out['status'] = 0
out['num_records'] = 0
except Exception, e:
raise
out['errors'] = str(e)
out['status'] = 0
out['num_records'] = 0
out['collection_name'] = None
def query(request, collection, command, database=None):
"""
Main view to send commands to handler
"""
out = createBaseResponseObject()
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
commandMethod = getattr(mongo, command, None)
try:
if not commandMethod or command not in mongo.available_commands:
raise Exception(
"Command %s not supported. Available commands are: %s" %
(command, ", ".join(mongo.available_commands)))
mongo.connect()
existing_dbs = mongo.connection.database_names()
if database not in existing_dbs:
raise Exception("Database %s does not exist" % database)
database_object = mongo.getDb(database)
existing_collections = database_object.collection_names()
if collection not in existing_collections:
raise Exception("Collection %s does not exist" % collection)
results = commandMethod(database, collection, request)
if results:
out['results'] = results
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
def importCall(request, collection, database=None):
"""
View used to import data.
"""
#this loads an instance of mapper
from mappermanager import mappingManager, codedMappers
#TODO: separate data collection and processing and write a view that handles FILES
out = createBaseResponseObject()
out['error_records'] = { 'parser' : [], 'mapper' : [] }
out['ok_records_number'] = 0
database = database or settings.MONGO_SERVER_DEFAULT_DB
if request.POST:
mapper = None
mapperName = getMapper(request)
#todo: decide to use name or id for referencing mapper in request
if mapperName:
if mapperName in codedMappers:
mapperObject = codedMappers[mapperName]
else:
sketchMapper = SketchMapper.objects.get(name=mapperName)
mapper = sketchMapper.mapper
record_errors_number = 0
ok_records = []
MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS
if 'data' in request.POST and 'parser' in request.POST:
parser_name = request.POST['parser'].lower()
data = request.POST['data']
try:
#parsing phase
parser = recordparser.parserFactory(parser_name, data)
for d in parser.objects():
if d is recordparser.ParserError:
out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data)
continue
#mapping phase
if mapper is not None:
try:
newRecord = mappingManager.mapRecord(d, mapping, { '__mapper_name__' : mapperName })
ok_records.append(newRecord)
except:
out['error_records']['mapper'].append(d)
#mapper is none, record is imported as it is
else:
ok_records.append(d)
if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS:
break
#commit phase
if 'commit' in request.POST and request.POST['commit']:
try:
commit = int(request.POST['commit'])
except:
commit = 0
if commit:
#creating the collection model and set owner=user if collection does not exits
#TODO: we could check again the number of allowed collections here, as in decorator
try:
collectionInstance = SketchCollection.objects.get(name=collection, database=database)
except:
collectionInstance = SketchCollection(owner=request.user, name=collection, database=database)
collectionInstance.save()
#finally inserting records
for record in ok_records:
mongo_id = mongo._insert(database, collection, record)
out['results'].append(mongo_id)
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None):
"""
View used to import data.
"""
#TODO: separate data collection and processing and write a view that handles FILES
out = createBaseResponseObject()
out['error_records'] = { 'parser' : [], 'mapper' : [] }
out['ok_records_number'] = 0
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
mongo.connect()
#TODO: mapping should come from url, in form of id
mapping = { '__key__' : 'id_str',
# '__upperName__' : { 'transform' : 'upperCase', 'args' : ['name'] },
# '__fullName__' : { 'transform' : 'concatStrings', 'args' : ['name', 'surname'] },
# '__fullNameUpper__' : { 'transform' : 'concatStrings',
# 'args' : [{ 'transform' : 'upperCase', 'args' : ['name'] }, { 'transform' : 'upperCase', 'args' : ['surname'] }] },
}
if request.POST:
record_errors_number = 0
ok_records = []
#TODO: PARAMETRIZE THIS, maybe in settings
MAX_ERROR_RECORDS = 10000
if 'data' in request.POST and 'format' in request.POST:
format = request.POST['format'].lower()
data = request.POST['data']
try:
parser = recordparser.parserFactory(format, data)
for d in parser.objects():
if d is recordparser.ParserError:
out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data)
continue
try:
newRecord = mappingManager.mapRecord(d, mapping)
ok_records.append(newRecord)
except:
out['error_records']['mapper'].append(d)
if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS:
break
if 'commit' in request.POST and request.POST['commit']:
try:
commit = int(request.POST['commit'])
except:
commit = 0
if commit:
for record in ok_records:
mongo_id = mongo._insert(database, collection, record)
out['results'].append(mongo_id)
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None):
"""
View used to import data.
"""
#this loads an instance of mapper
from mappermanager import mappingManager, codedMappers
#TODO: separate data collection and processing and write a view that handles FILES
out = createBaseResponseObject()
out['error_records'] = {'parser': [], 'mapper': []}
out['ok_records_number'] = 0
database = database or settings.MONGO_SERVER_DEFAULT_DB
mongo = MongoWrapper()
mongo.connect()
if request.POST:
mapper = None
mapperName = getMapper(request)
#todo: decide to use name or id for referencing mapper in request
if mapperName:
if mapperName in codedMappers:
mapperObject = codedMappers[mapperName]
else:
sketchMapper = SketchMapper.objects.get(name=mapperName)
mapper = sketchMapper.mapper
record_errors_number = 0
ok_records = []
MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS
if 'data' in request.POST and 'parser' in request.POST:
parser_name = request.POST['parser'].lower()
data = request.POST['data']
try:
#parsing phase
parser = recordparser.parserFactory(parser_name, data)
for d in parser.objects():
if d is recordparser.ParserError:
out['error_records']['parser'].append(
str(d.exception_message) + ":" + d.raw_data)
continue
#mapping phase
if mapper is not None:
try:
newRecord = mappingManager.mapRecord(
d, mapping, {'__mapper_name__': mapperName})
ok_records.append(newRecord)
except:
out['error_records']['mapper'].append(d)
#mapper is none, record is imported as it is
else:
ok_records.append(d)
if len(out['error_records']['mapper']) + len(out[
'error_records']['parser']) > MAX_ERROR_RECORDS:
break
#commit phase
if 'commit' in request.POST and request.POST['commit']:
try:
commit = int(request.POST['commit'])
except:
commit = 0
if commit:
#creating the collection model and set owner=user if collection does not exits
#TODO: we could check again the number of allowed collections here, as in decorator
try:
collectionInstance = SketchCollection.objects.get(
name=collection, database=database)
except:
collectionInstance = SketchCollection(
owner=request.user,
name=collection,
database=database)
collectionInstance.save()
#finally inserting records
for record in ok_records:
mongo_id = mongo._insert(database, collection,
record)
out['results'].append(mongo_id)
except Exception, e:
out['errors'] = str(e)
out['status'] = 0
out['ok_records_number'] = len(ok_records)