def query(request, collection, command, database=None): """ Main view to send commands to handler """ out = createBaseResponseObject() database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() commandMethod = getattr(mongo, command, None) try: if not commandMethod or command not in mongo.available_commands: raise Exception("Command %s not supported. Available commands are: %s" % (command, ", ".join(mongo.available_commands))) mongo.connect() existing_dbs = mongo.connection.database_names() if database not in existing_dbs: raise Exception("Database %s does not exist" % database) database_object = mongo.getDb(database) existing_collections = database_object.collection_names() if collection not in existing_collections: raise Exception("Collection %s does not exist" % collection) results = commandMethod(database, collection, request) if results: out['results'] = results except Exception, e: out['errors'] = str(e) out['status'] = 0
def getServerInfo(): mongo = MongoWrapper() try: out = createBaseResponseObject() mongo.connect() existing_dbs = mongo.connection.database_names() out['results'] = existing_dbs except Exception, e: out['errors'] = str(e) out['status'] = 0
def objects(request, collection, database=None): """ Main view to send commands to handler """ out = createBaseResponseObject() database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() try: mongo.connect() existing_dbs = mongo.connection.database_names() if database not in existing_dbs: raise Exception("Database %s does not exist" % database) database_object = mongo.getDb(database) existing_collections = database_object.collection_names() if collection not in existing_collections: raise Exception("Collection %s does not exist" % collection) query_dict = getQueryDict(request) offset = getOffset(request) limit = getLimit(request) formatter = getFormatter(request) from formattersmanager import formattersManager formatters = formattersManager.getFormatters() if formatter and formatter not in formatters: raise Exception("Formatter %s is not available" % str(formatter)) if formatter: formatter_callback = formattersManager.getFormatter(formatter) else: formatter_callback = None query_result = mongo.objects(database, collection, query_dict=query_dict, offset=offset, limit=limit, formatter_callback=formatter_callback) records = query_result['records'] has_more = query_result['has_more'] out['results'] = records out['has_more'] = has_more except Exception, e: raise out['errors'] = str(e) out['status'] = 0
def topscan(market): print("{}".format(market)) try: mongo = MongoWrapper.getInstance().getClient() bot = ScanBot({ "market": market, "candlesize": "1d" }, name='topscanner') res = bot.process({'scrape': True}) mongo.crypto.scanner.replace_one({ 'market': market, 'candlesize': '1d' }, res, upsert=True) bot = ScanBot({ "market": market, "candlesize": "1h" }, name='topscanner') res = bot.process({'scrape': True}) mongo.crypto.scanner.replace_one({ 'market': market, 'candlesize': '1h' }, res, upsert=True) bot = ScanBot({ "market": market, "candlesize": "15m" }, name='topscanner') res = bot.process({'scrape': True}) mongo.crypto.scanner.replace_one( { 'market': market, 'candlesize': '15m' }, res, upsert=True) bot = ScanBot({ "market": market, "candlesize": "5m" }, name='topscanner') res = bot.process({'scrape': False}) mongo.crypto.scanner.replace_one({ 'market': market, 'candlesize': '5m' }, res, upsert=True) except Exception as ex: print("Exception: {}".format(ex))
def object(request, collection, oid, database=None): database = database or settings.MONGO_SERVER_DEFAULT_DB out = createBaseResponseObject() mongo = MongoWrapper() mongo.connect() if request.DELETE: mongo.dropObjectByOid(database, collection, oid) out['results'].append(oid) try: mongo.connection.close() except: pass return HttpResponse(json.dumps(out, default=bson.json_util.default))
def dbMeta(request, database): mongo = MongoWrapper() try: out = createBaseResponseObject() mongo.connect() existing_dbs = mongo.connection.database_names() if database not in existing_dbs: raise Exception("Database %s does not exist" % database) database_object = mongo.getDb(database) existing_collections = database_object.collection_names() out['results'] = existing_collections except Exception, e: out['errors'] = str(e) out['status'] = 0
def getDbInfo(database): mongo = MongoWrapper() try: out = createBaseResponseObject() mongo.connect() existing_dbs = mongo.connection.database_names() if database not in existing_dbs: raise Exception("Database %s does not exist" % database) database_object = mongo.getDb(database) existing_collections = database_object.collection_names() #filtering out "system.indexes" collection existing_collections = [x for x in existing_collections if x != "system.indexes"] out['results'] = existing_collections except Exception, e: out['errors'] = str(e) out['status'] = 0
def getDbInfo(database): mongo = MongoWrapper() try: out = createBaseResponseObject() mongo.connect() existing_dbs = mongo.connection.database_names() if database not in existing_dbs: raise Exception("Database %s does not exist" % database) database_object = mongo.getDb(database) existing_collections = database_object.collection_names() #filtering out "system.indexes" collection existing_collections = [ x for x in existing_collections if x != "system.indexes" ] out['results'] = existing_collections except Exception, e: out['errors'] = str(e) out['status'] = 0
def query(request, collection, command, database=None): """ Main view to send commands to handler """ out = createBaseResponseObject() database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() commandMethod = getattr(mongo, command, None) try: if not commandMethod or command not in mongo.available_commands: raise Exception( "Command %s not supported. Available commands are: %s" % (command, ", ".join(mongo.available_commands))) mongo.connect() existing_dbs = mongo.connection.database_names() if database not in existing_dbs: raise Exception("Database %s does not exist" % database) database_object = mongo.getDb(database) existing_collections = database_object.collection_names() if collection not in existing_collections: raise Exception("Collection %s does not exist" % collection) results = commandMethod(database, collection, request) if results: out['results'] = results except Exception, e: out['errors'] = str(e) out['status'] = 0
def importCall(request, collection, database=None): """ View used to import data. """ #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = { 'parser' : [], 'mapper' : [] } out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() mongo.connect() #TODO: mapping should come from url, in form of id mapping = { '__key__' : 'id_str', # '__upperName__' : { 'transform' : 'upperCase', 'args' : ['name'] }, # '__fullName__' : { 'transform' : 'concatStrings', 'args' : ['name', 'surname'] }, # '__fullNameUpper__' : { 'transform' : 'concatStrings', # 'args' : [{ 'transform' : 'upperCase', 'args' : ['name'] }, { 'transform' : 'upperCase', 'args' : ['surname'] }] }, } if request.POST: record_errors_number = 0 ok_records = [] #TODO: PARAMETRIZE THIS, maybe in settings MAX_ERROR_RECORDS = 10000 if 'data' in request.POST and 'format' in request.POST: format = request.POST['format'].lower() data = request.POST['data'] try: parser = recordparser.parserFactory(format, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data) continue try: newRecord = mappingManager.mapRecord(d, mapping) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS: break if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)
def importCall(request, collection, database=None): """ View used to import data. """ #this loads an instance of mapper from mappermanager import mappingManager, codedMappers #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = { 'parser' : [], 'mapper' : [] } out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() mongo.connect() if request.POST: mapper = None mapperName = getMapper(request) #todo: decide to use name or id for referencing mapper in request if mapperName: if mapperName in codedMappers: mapperObject = codedMappers[mapperName] else: sketchMapper = SketchMapper.objects.get(name=mapperName) mapper = sketchMapper.mapper record_errors_number = 0 ok_records = [] MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS if 'data' in request.POST and 'parser' in request.POST: parser_name = request.POST['parser'].lower() data = request.POST['data'] try: #parsing phase parser = recordparser.parserFactory(parser_name, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append(str(d.exception_message) + ":" +d.raw_data) continue #mapping phase if mapper is not None: try: newRecord = mappingManager.mapRecord(d, mapping, { '__mapper_name__' : mapperName }) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) #mapper is none, record is imported as it is else: ok_records.append(d) if len(out['error_records']['mapper']) + len(out['error_records']['parser']) > MAX_ERROR_RECORDS: break #commit phase if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: #creating the collection model and set owner=user if collection does not exits #TODO: we could check again the number of allowed collections here, as in decorator try: collectionInstance = SketchCollection.objects.get(name=collection, database=database) except: collectionInstance = SketchCollection(owner=request.user, name=collection, database=database) collectionInstance.save() #finally inserting records for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)
def __init__(self, config={}): self.bittrex = Bittrex() self.mongo = MongoWrapper.getInstance().getClient() self.history = None self.pendingorders = None self.bal = None
def importCall(request, collection, database=None): """ View used to import data. """ #this loads an instance of mapper from mappermanager import mappingManager, codedMappers #TODO: separate data collection and processing and write a view that handles FILES out = createBaseResponseObject() out['error_records'] = {'parser': [], 'mapper': []} out['ok_records_number'] = 0 database = database or settings.MONGO_SERVER_DEFAULT_DB mongo = MongoWrapper() mongo.connect() if request.POST: mapper = None mapperName = getMapper(request) #todo: decide to use name or id for referencing mapper in request if mapperName: if mapperName in codedMappers: mapperObject = codedMappers[mapperName] else: sketchMapper = SketchMapper.objects.get(name=mapperName) mapper = sketchMapper.mapper record_errors_number = 0 ok_records = [] MAX_ERROR_RECORDS = settings.MAX_ERROR_RECORDS if 'data' in request.POST and 'parser' in request.POST: parser_name = request.POST['parser'].lower() data = request.POST['data'] try: #parsing phase parser = recordparser.parserFactory(parser_name, data) for d in parser.objects(): if d is recordparser.ParserError: out['error_records']['parser'].append( str(d.exception_message) + ":" + d.raw_data) continue #mapping phase if mapper is not None: try: newRecord = mappingManager.mapRecord( d, mapping, {'__mapper_name__': mapperName}) ok_records.append(newRecord) except: out['error_records']['mapper'].append(d) #mapper is none, record is imported as it is else: ok_records.append(d) if len(out['error_records']['mapper']) + len(out[ 'error_records']['parser']) > MAX_ERROR_RECORDS: break #commit phase if 'commit' in request.POST and request.POST['commit']: try: commit = int(request.POST['commit']) except: commit = 0 if commit: #creating the collection model and set owner=user if collection does not exits #TODO: we could check again the number of allowed collections here, as in decorator try: collectionInstance = SketchCollection.objects.get( name=collection, database=database) except: collectionInstance = SketchCollection( owner=request.user, name=collection, database=database) collectionInstance.save() #finally inserting records for record in ok_records: mongo_id = mongo._insert(database, collection, record) out['results'].append(mongo_id) except Exception, e: out['errors'] = str(e) out['status'] = 0 out['ok_records_number'] = len(ok_records)