def run(): loop = asyncio.get_event_loop() db = database.Database(host=config.cfg['database']['host'], user=config.cfg['database']['user'], password=config.cfg['database']['password'], database=config.cfg['database']['database'], loop=loop) try: pool = loop.run_until_complete(db.create_pool()) except Exception: log.exception('Could not set up PostgreSQL. Exiting.') return bot = AphidBot(command_prefix=config.cfg['bot']['prefix'], guild_id=config.cfg['bot']['guild'], loop=loop, pool=pool) log.info('Starting bot') try: bot.run(config.cfg['bot']['token'], reconnect=True) except Exception: log.exception('Encountered fatal exception') log.info('Closing event loop') loop.close()
def data_stats(request, user, scopes, users_to_return, user_roles, own_data): db = database.Database() roles_to_use = [] if own_data and 'researcher' in user_roles: roles_to_use = ['researcher'] if own_data and 'developer' in user_roles: roles_to_use = ['developer'] collections_to_use = request.GET.get('collections', '') if collections_to_use == '': collections_to_use = [x for x in COLLECTIONS] else: collections_to_use = collections_to_use.split(',') collections_to_use = [ x for x in set(collections_to_use).intersection(set(COLLECTIONS)) ] query = {'$query': {}} if not 'all' in users_to_return: query['$query']['_id'] = {'$in': users_to_return} results = {} for collection in collections_to_use: results[collection] = {} for x in db.getDocuments(query=query, collection=NAME + '_' + collection, roles=roles_to_use): results[collection][x['_id']] = int(x['value']['count']) return results
def run(): db = database.Database() mapper = Code(""" function() { var date = new Date(this.data.TIMESTAMP*1000); var month = ('0'+(date.getMonth()+1)).substr(-2); var day = ('0'+date.getDate()).substr(-2); var dateString = date.getFullYear() + '-' + month + '-' + day; var key = this.user + '_' + dateString; var dayCount = 0; if (date.getHours() > 1) dayCount = 1; var value = { user: this.user, date: dateString, count: 1, day_count: dayCount, timestamp_added: this.timestamp_added }; emit(key, value); }; """) reducer = Code(""" function(key, values) { var reducedObject = { user: key.split('_')[0], date: key.split('_')[1], count: 0, day_count: 0, timestamp_added: 0 }; values.forEach( function(value) { reducedObject.count += value.count; reducedObject.day_count += value.day_count; if (reducedObject.timestamp_added < value.timestamp_added) { reducedObject.timestamp_added = value.timestamp_added; } }); return reducedObject; }; """) for collection in COLLECTIONS: print "%s %s"%(NAME, collection) output_collection = NAME+'_'+collection min_v = findPreviousMax(db, output_collection) max_v = time.time()-5*60 print min_v, max_v if min_v > max_v: continue r = (db.getDatabase(collection))[collection].map_reduce(mapper, reducer, out={'reduce':output_collection}, query={'timestamp_added':{'$gt': min_v, '$lt': max_v}}) (db.getDatabase(output_collection))[output_collection].ensure_index('timestamp_added') return return True
def __init__(self, engine='mysql'): self.log = audit.getLogger(__name__) if engine == 'mysql': self.engine = mysql_wrapper.DBWrapper() self.filestorage_wrapper = filesystem_wrapper.FileSystemWrapper() elif engine == 'mongo': self.engine = database.Database() else: raise ValueError(engine + ' is not a valid engine type')
def populate(): pid = os.getpid() touchCreationLock() db = database.Database() if not start(pid): removeCreationLock() return run(db) end(pid)
def getDatabaseStatus(): try: db = database.Database() doc_id = db.insert({'test': 'test'}, 'test') response = db.getDatabase('test')['test'].remove({'_id': doc_id}) if response['ok'] == 1: return 'OK' else: return response except pymongo.errors.PyMongoError as e: return str(e)
def getDatabaseStats(): try: db = database.Database() return_value = {} return_value[db.default_database] = db.getDatabase( db.default_database).command('dbstats') for d in db.available_databases: return_value[db.available_databases[d]] = db.getDatabase( d).command('dbstats') return return_value except pymongo.errors.PyMongoError: return {'error': 'something went terribly wrong'}
def run(): db = database.Database() mapper = Code(""" function() { var key = this.user; var value = { count: 1, timestamp_added: this.timestamp_added }; emit(key, value); }; """) reducer = Code(""" function(key, values) { var reducedObject = { count: 0, timestamp_added: 0 }; values.forEach( function(value) { reducedObject.count += value.count; if (reducedObject.timestamp_added < value.timestamp_added) { reducedObject.timestamp_added = value.timestamp_added; } }); return reducedObject; }; """) for collection in COLLECTIONS: print collection output_collection = NAME + '_' + collection min_v = findPreviousMax(db, output_collection) max_v = time.time() - 5 * 60 print min_v, max_v if min_v > max_v: continue r = (db.getDatabase(collection))[collection].map_reduce( mapper, reducer, out={'reduce': output_collection}, query={'timestamp_added': { '$gt': min_v, '$lt': max_v }}) (db.getDatabase(output_collection) )[output_collection].ensure_index('timestamp_added') return return True
import json from web3 import Web3 from behave import * from utils.events import * import utils.database as storage db = storage.Database() # Database scenario @given(u'inputdata is') def step_impl(context): context.data = json.loads(context.text) @when(u'the insert("{collection}", inputdata) method is invoked') def step_impl(context, collection): context.response = db.insert(collection, context.data) @then(u'the new document is inserted') def step_impl(context): assert context.response is True # Calculation scenario @given(u'the current price of ETH is "{price}"$ on coinmarketcap') def step_impl(context, price): context.price = float(price)
def dataBuild(request, probe_settings, users_to_return, decrypted=False, own_data=False, roles=[]): _start_time = time.time() results = None query = None proc_req = None response = {} response['meta'] = {} try: if len(users_to_return) == 0: raise BadRequestException( 'error', 403, 'The current token does not allow to view data from any users') proc_req = processApiCall(request, probe_settings, users_to_return) query = buildQuery(users_to_return, proc_req) collection = probe_settings['collection'] if own_data and 'researcher' in roles: collection += '_researcher' db = database.Database() docs = db.getDocumentsCustom(query=query, collection=collection,\ fields = proc_req['fields']) ### hinting # if the users are specified, we use the hint with users if proc_req['users'] is not None: docs = docs.hint([('timestamp', proc_req['order']), ('facebook_id', 1), ('user', 1)]) # else, we use only id and timestamp else: docs = docs.hint([('timestamp', proc_req['order']), ('facebook_id', 1)]) #pagination (skipping) if proc_req['after'] is not None: docs = docs.skip(1) #apply limit docs.limit(proc_req['limit']) try: results = cursorToArray(docs, decrypted=decrypted, probe=probe_settings['collection']) except Exception as e: raise BadRequestException( 'error', 500, 'The request caused a DB malfunction: ' + str(e)) results_count = len(results) response['meta']['status'] = proc_req['status'] response['meta']['results_count'] = len(results) response['meta']['api_call'] = proc_req response['meta']['query'] = query response['results'] = results if len(results) > 0: response['meta']['paging'] = {} response['meta']['paging']['cursors'] = {} response['meta']['paging']['cursors']['after'] =OrderedDict([\ (proc_req['sortby'],getValueOfFullKey(results[-1], proc_req['sortby'])),\ ('facebook_id',results[-1]['facebook_id']), ('user',results[-1]['user'])]) if results_count == proc_req['limit']: if proc_req['after'] is not None: response['meta']['paging']['links'] =\ {'next':re.sub('&after=[^ &]+','&after=' + urlize_dict(response['meta']['paging']['cursors']['after']),request.build_absolute_uri())} else: response['meta']['paging']['links'] = \ {'next':request.build_absolute_uri() + '&after=' + urlize_dict(response['meta']['paging']['cursors']['after'])} except BadRequestException as e: response['meta']['status'] = e.value proc_req = {'format': 'json'} response['meta']['execution_time_seconds'] = time.time() - _start_time callback = request.REQUEST.get('callback', '') if len(callback) > 0: data = '%s(%s);' % (callback, json.dumps(response)) log.info(audit.message(request, response['meta'])) return HttpResponse(data, content_type="text/plain", status=response['meta']['status']['code']) if decrypted: pass if proc_req['format'] == 'pretty': log.info(audit.message(request, response['meta'])) return render_to_response('pretty_json.html', {'response': json.dumps(response, indent=2)}) elif proc_req['format'] == 'csv': output = '#' + json.dumps(response['meta'], indent=2).replace( '\n', '\n#') + '\n' if probe_settings['scope'] == 'connector_raw.locationfacebook': output2 = '' output += locationfacebook_to_csv(results, output2) else: output += array_to_csv(results, probe_settings['collection']) log.info(audit.message(request, response['meta'])) return HttpResponse(output, content_type="text/plain", status=response['meta']['status']['code']) else: log.info(audit.message(request, response['meta'])) return HttpResponse(json.dumps(response), content_type="application/json", status=response['meta']['status']['code']) return HttpResponse('hello decrypted')
def locationBuild(request, users_to_return, decrypted=False, own_data=False, roles=[]): _start_time = time.time() pretty = booleanize(request.REQUEST.get('pretty', False)) results = None query = None proc_req = None response = {} response['meta'] = {} try: if len(users_to_return) == 0: raise BadRequestException( 'error', 403, 'The current token does not allow to view data from any users') proc_req = processApiCall(request, users_to_return) query = buildQuery(users_to_return, proc_req) collection = 'edu_mit_media_funf_probe_builtin_LocationProbe' if own_data and 'researcher' in roles: collection += '_researcher' db = database.Database() docs = db.getDocumentsCustom(query=query, collection=collection,\ fields = proc_req['fields']) ### hinting # if the users are specified, we use the hint with users if proc_req['users'] is not None: docs = docs.hint([('data.TIMESTAMP', proc_req['order']), ('_id', 1), ('user', 1)]) # else, we use only id and timestamp else: docs = docs.hint([('data.TIMESTAMP', proc_req['order']), ('_id', 1)]) #pagination (skipping) if proc_req['after'] is not None: docs = docs.skip(1) #apply limit docs.limit(proc_req['limit']) try: results = cursorToArray(docs) except Exception as e: raise BadRequestException( 'error', 500, 'The request caused a DB malfunction: ' + str(e)) results_count = len(results) response['meta']['status'] = proc_req['status'] response['meta']['results_count'] = len(results) response['meta']['api_call'] = request.REQUEST response['meta']['query'] = query response['results'] = results if len(results) > 0: response['meta']['paging'] = {} response['meta']['paging']['cursors'] = {} response['meta']['paging']['cursors']['after'] =\ {proc_req['sortby']:getValueOfFullKey(results[-1], proc_req['sortby']),\ '_id':results[-1]['_id'], 'user':results[-1]['user']} if results_count == proc_req['limit']: if proc_req['after'] is not None: response['meta']['paging']['links'] =\ {'next':re.sub('&after=[^ &]+','&after=' + urlize_dict(response['meta']['paging']['cursors']['after']),request.build_absolute_uri())} else: response['meta']['paging']['links'] = \ {'next':request.build_absolute_uri() + '&after=' + urlize_dict(response['meta']['paging']['cursors']['after'])} except BadRequestException as e: response['meta']['status'] = e.value response['meta']['execution_time_seconds'] = time.time() - _start_time callback = request.REQUEST.get('callback', '') if len(callback) > 0: data = '%s(%s);' % (callback, json.dumps(response)) return HttpResponse(data, content_type="text/javascript", status=response['meta']['status']['code']) if decrypted: pass if pretty: return render_to_response('pretty_json.html', {'response': json.dumps(response, indent=2)}) else: return HttpResponse(json.dumps(response), content_type="application/json", status=response['meta']['status']['code']) return HttpResponse('hello decrypted')
ddate = check_date(row['created'].strip()) args.append( (ddate, row['value'].strip(), row['category'].strip(), row['description'].strip())) if len(args) > 0: logging.debug('sending data: {}'.format(args)) g.send(args) finally: g.close() finally: if fh: fh.close() if __name__ == '__main__': import sys import os logging.debug(sys.argv) _path, _ = os.path.split(sys.argv[0]) _path = os.path.abspath(_path) if len(sys.argv) > 1: db = database.Database(name=sys.argv[1], debug=True) db.connect() try: populate(filename=os.path.join(_path, 'fill_from_csv.template'), database=db) finally: db.close() else: raise SystemExit('provide a database path')
import discord, os, psycopg2 from utils import roblox from utils import database from discord.ext import commands description = '''Bloxify help''' bot = commands.Bot(command_prefix=['b25'], description=description, case_insensitive=True) bot.roblox = roblox.Roblox() bot.connection = database.Database() @bot.event async def on_ready(): print('ready') for file in os.listdir("./classes"): if file.endswith('.py'): bot.load_extension(f'classes.{file.replace(".py", "")}') bot.run('NDcwMjc3MzI2NTIyMDIzOTQ2.XYeumw.1qfVwHPDhRYZ-p4K3BvBF7NAV5c')
cli = sys.modules['flask.cli'] cli.show_server_banner = lambda *x: None app.config.from_object(Config) db = None @app.route('/') def query(): return render_template('query.html') @app.route('/pass', methods=['POST']) def passphrase(): device_id = request.form['id'] devices = [] for device in db.devices: if device['id'] == device_id: devices.append(device) print("received: " + device_id) return jsonify(devices) def main(): Thread(target=ssl_server.main, args=(db,)).start() app.run(host='0.0.0.0', debug=False) if __name__ == '__main__': db = database.Database() main()
def dataBuild(request, probe_settings, users_to_return, questions_to_return, decrypted = False, own_data = False, roles = []): _start_time = time.time() results = None query = None proc_req = None response = {} response['meta'] = {} try: if len(users_to_return) == 0: raise BadRequestException('error',403,'The current token does not allow to view data from any users') proc_req = processApiCall(request, probe_settings, users_to_return, questions_to_return) query = buildQuery(users_to_return,questions_to_return, proc_req) collection = probe_settings['collection'] if own_data and 'researcher' in roles: collection += '_researcher' db = database.Database() docs = db.getDocumentsCustom(query=query, collection=collection,\ fields = proc_req['fields']) ### hinting hint = [('form_version',1),('last_answered',proc_req['order']), ('variable_name',1), ('user',1)] docs.hint(hint) #pagination (skipping) if proc_req['after'] is not None: docs = docs.skip(1) #apply limit docs.limit(proc_req['limit']) try: results = cursorToArray(docs, decrypted = decrypted, probe=probe_settings['collection']) except Exception as e: raise BadRequestException('error',500,'The request caused a DB malfunction: ' + str(e) + '. Used hint: ' + str(hint)) results_count = len(results) response['meta']['status'] = proc_req['status'] response['meta']['results_count'] = len(results) response['meta']['api_call'] = proc_req response['meta']['query'] = query response['results'] = results if len(results) > 0: response['meta']['paging'] = {} response['meta']['paging']['cursors'] = {} response['meta']['paging']['cursors']['after'] =OrderedDict([\ ('form_version',results[-1]['form_version']),('last_answered', results[-1]['last_answered']),\ ('variable_name',results[-1]['variable_name']), ('user',results[-1]['user'])]) if results_count == proc_req['limit']: if proc_req['after'] is not None: response['meta']['paging']['links'] =\ {'next':re.sub('&after=[^ &]+','&after=' + urlize_dict(response['meta']['paging']['cursors']['after']),request.build_absolute_uri())} else: response['meta']['paging']['links'] = \ {'next':request.build_absolute_uri() + '&after=' + urlize_dict(response['meta']['paging']['cursors']['after'])} except BadRequestException as e: response['meta']['status'] = e.value proc_req = {'format':'json'} response['meta']['execution_time_seconds'] = time.time()-_start_time callback = request.REQUEST.get('callback','') if len(callback) > 0: data = '%s(%s);' % (callback, json.dumps(response)) log.info(audit.message(request, response['meta']['api_call'])) return HttpResponse(data, content_type="text/plain", status=response['meta']['status']['code']) if decrypted: pass #auditdb= audit.Audit() #doc_audit=response['meta'] #users_return=[] #users_results = cursorToArray(results, decrypted = decrypted, probe=probe_settings['collection']) #for data_users in users_results: # if data_users['user'] not in users_return: # users_return.append(data_users['user']) #doc_audit['users']=users_return #doc_audit=transform.transform(doc_audit) #auditdb.d(typ='prueba',tag='prueba2',doc=doc_audit,onlyfile=False) if proc_req['format'] == 'pretty': log.info(audit.message(request, response['meta']['api_call'])) return render_to_response('pretty_json.html', {'response': json.dumps(response, indent=2)}) elif proc_req['format'] == 'csv': output = '#' + json.dumps(response['meta'], indent=2).replace('\n','\n#') + '\n' output += array_to_csv(results,probe_settings['collection']) log.info(audit.message(request, response['meta']['api_call'])) return HttpResponse(output, content_type="text/plain", status=response['meta']['status']['code']) else: log.info(audit.message(request, response['meta']['api_call'])) return HttpResponse(json.dumps(response), content_type="application/json", status=response['meta']['status']['code']) return HttpResponse('hello decrypted')
return valid if __name__ == '__main__': r.set_loop_type('asyncio') with open('config.json') as f: config = json.load(f) bot = AutoShardedBot(command_prefix=get_prefix, help_attrs=dict(hidden=True)) bot.startup = datetime.now() bot.messages_seen = 0 bot.r = r bot.connection = bot.loop.run_until_complete(r.connect(db='parallax')) bot.db = database.Database(bot) bot.config = config for f in os.listdir('extensions'): if f.endswith('.py'): try: bot.load_extension(f'extensions.{f[:-3]}') except SyntaxError as exception: print(f'Failed to load {f}: {exception}') @bot.event async def on_message(message): bot.messages_seen += 1 if not bot.is_ready() or message.author.bot: return
def load_file(filename): #pdb.set_trace() log.log('Debug', 'Trying to populate db with ' + filename) mConnector = ConnectorFunf.objects.all()[0] db = database.Database() anonymizerObject = Anonymizer() documents_to_insert = defaultdict(list) proc_dir = os.path.join(mConnector.decrypted_path, 'processing') if not os.path.exists(proc_dir): os.makedirs(proc_dir) decrypted_filepath = os.path.join(mConnector.decrypted_path, filename) processing_filepath = os.path.join(proc_dir, filename) current_filepath = decrypted_filepath if os.path.exists( decrypted_filepath) and not os.path.exists(processing_filepath): try: # move to processing shutil.move(decrypted_filepath, proc_dir) current_filepath = processing_filepath # open connection to db file conn = sqlite3.connect(processing_filepath) cursor = conn.cursor() # get the meta data from db file meta = {} (meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac']) = \ cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac from file_info').fetchone() meta['device_id'] = anonymizerObject.anonymizeValue( 'device_id', meta['device_id']) #pdb.set_trace() # get the user associated with the token #meta['user'] = authorization_manager.getAuthorizationForToken(\ # 'connector_funf.submit_data', meta['token']).user meta['user'] = '******' for row in cursor.execute('select * from data'): doc = row_to_doc(row, meta['user'], anonymizerObject) if doc == None: continue documents_to_insert[doc['probe']].append( dict(doc.items() + meta.items())) cursor.close() #pdb.set_trace() for probe in documents_to_insert: db.insert(documents_to_insert[probe], probe) os.remove(current_filepath) except Exception as e: log.log('Error', str(e)) if not 'already exists' in str(e): top = traceback.extract_stack()[-1] fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\ + '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])])) else: pass return False