def run():
loop = asyncio.get_event_loop()
db = database.Database(host=config.cfg['database']['host'],
user=config.cfg['database']['user'],
password=config.cfg['database']['password'],
database=config.cfg['database']['database'],
loop=loop)
try:
pool = loop.run_until_complete(db.create_pool())
except Exception:
log.exception('Could not set up PostgreSQL. Exiting.')
return
bot = AphidBot(command_prefix=config.cfg['bot']['prefix'],
guild_id=config.cfg['bot']['guild'],
loop=loop,
pool=pool)
log.info('Starting bot')
try:
bot.run(config.cfg['bot']['token'], reconnect=True)
except Exception:
log.exception('Encountered fatal exception')
log.info('Closing event loop')
loop.close()
def data_stats(request, user, scopes, users_to_return, user_roles, own_data):
db = database.Database()
roles_to_use = []
if own_data and 'researcher' in user_roles: roles_to_use = ['researcher']
if own_data and 'developer' in user_roles: roles_to_use = ['developer']
collections_to_use = request.GET.get('collections', '')
if collections_to_use == '': collections_to_use = [x for x in COLLECTIONS]
else: collections_to_use = collections_to_use.split(',')
collections_to_use = [
x for x in set(collections_to_use).intersection(set(COLLECTIONS))
]
query = {'$query': {}}
if not 'all' in users_to_return:
query['$query']['_id'] = {'$in': users_to_return}
results = {}
for collection in collections_to_use:
results[collection] = {}
for x in db.getDocuments(query=query,
collection=NAME + '_' + collection,
roles=roles_to_use):
results[collection][x['_id']] = int(x['value']['count'])
return results
def run():
db = database.Database()
mapper = Code("""
function() {
var date = new Date(this.data.TIMESTAMP*1000);
var month = ('0'+(date.getMonth()+1)).substr(-2);
var day = ('0'+date.getDate()).substr(-2);
var dateString = date.getFullYear() + '-' + month + '-' + day;
var key = this.user + '_' + dateString;
var dayCount = 0;
if (date.getHours() > 1) dayCount = 1;
var value = {
user: this.user,
date: dateString,
count: 1,
day_count: dayCount,
timestamp_added: this.timestamp_added
};
emit(key, value);
};
""")
reducer = Code("""
function(key, values) {
var reducedObject = {
user: key.split('_')[0],
date: key.split('_')[1],
count: 0,
day_count: 0,
timestamp_added: 0
};
values.forEach( function(value) {
reducedObject.count += value.count;
reducedObject.day_count += value.day_count;
if (reducedObject.timestamp_added < value.timestamp_added) {
reducedObject.timestamp_added = value.timestamp_added;
}
});
return reducedObject;
};
""")
for collection in COLLECTIONS:
print "%s %s"%(NAME, collection)
output_collection = NAME+'_'+collection
min_v = findPreviousMax(db, output_collection)
max_v = time.time()-5*60
print min_v, max_v
if min_v > max_v: continue
r = (db.getDatabase(collection))[collection].map_reduce(mapper, reducer, out={'reduce':output_collection}, query={'timestamp_added':{'$gt': min_v, '$lt': max_v}})
(db.getDatabase(output_collection))[output_collection].ensure_index('timestamp_added')
return
return True
def __init__(self, engine='mysql'):
self.log = audit.getLogger(__name__)
if engine == 'mysql':
self.engine = mysql_wrapper.DBWrapper()
self.filestorage_wrapper = filesystem_wrapper.FileSystemWrapper()
elif engine == 'mongo':
self.engine = database.Database()
else:
raise ValueError(engine + ' is not a valid engine type')
def populate():
pid = os.getpid()
touchCreationLock()
db = database.Database()
if not start(pid):
removeCreationLock()
return
run(db)
end(pid)
def getDatabaseStatus():
try:
db = database.Database()
doc_id = db.insert({'test': 'test'}, 'test')
response = db.getDatabase('test')['test'].remove({'_id': doc_id})
if response['ok'] == 1:
return 'OK'
else:
return response
except pymongo.errors.PyMongoError as e:
return str(e)
def getDatabaseStats():
try:
db = database.Database()
return_value = {}
return_value[db.default_database] = db.getDatabase(
db.default_database).command('dbstats')
for d in db.available_databases:
return_value[db.available_databases[d]] = db.getDatabase(
d).command('dbstats')
return return_value
except pymongo.errors.PyMongoError:
return {'error': 'something went terribly wrong'}
def run():
db = database.Database()
mapper = Code("""
function() {
var key = this.user;
var value = {
count: 1,
timestamp_added: this.timestamp_added
};
emit(key, value);
};
""")
reducer = Code("""
function(key, values) {
var reducedObject = {
count: 0,
timestamp_added: 0
};
values.forEach( function(value) {
reducedObject.count += value.count;
if (reducedObject.timestamp_added < value.timestamp_added) {
reducedObject.timestamp_added = value.timestamp_added;
}
});
return reducedObject;
};
""")
for collection in COLLECTIONS:
print collection
output_collection = NAME + '_' + collection
min_v = findPreviousMax(db, output_collection)
max_v = time.time() - 5 * 60
print min_v, max_v
if min_v > max_v: continue
r = (db.getDatabase(collection))[collection].map_reduce(
mapper,
reducer,
out={'reduce': output_collection},
query={'timestamp_added': {
'$gt': min_v,
'$lt': max_v
}})
(db.getDatabase(output_collection)
)[output_collection].ensure_index('timestamp_added')
return
return True
import json
from web3 import Web3
from behave import *
from utils.events import *
import utils.database as storage
db = storage.Database()
# Database scenario
@given(u'inputdata is')
def step_impl(context):
context.data = json.loads(context.text)
@when(u'the insert("{collection}", inputdata) method is invoked')
def step_impl(context, collection):
context.response = db.insert(collection, context.data)
@then(u'the new document is inserted')
def step_impl(context):
assert context.response is True
# Calculation scenario
@given(u'the current price of ETH is "{price}"$ on coinmarketcap')
def step_impl(context, price):
context.price = float(price)
def dataBuild(request,
probe_settings,
users_to_return,
decrypted=False,
own_data=False,
roles=[]):
_start_time = time.time()
results = None
query = None
proc_req = None
response = {}
response['meta'] = {}
try:
if len(users_to_return) == 0:
raise BadRequestException(
'error', 403,
'The current token does not allow to view data from any users')
proc_req = processApiCall(request, probe_settings, users_to_return)
query = buildQuery(users_to_return, proc_req)
collection = probe_settings['collection']
if own_data and 'researcher' in roles: collection += '_researcher'
db = database.Database()
docs = db.getDocumentsCustom(query=query, collection=collection,\
fields = proc_req['fields'])
### hinting
# if the users are specified, we use the hint with users
if proc_req['users'] is not None:
docs = docs.hint([('timestamp', proc_req['order']),
('facebook_id', 1), ('user', 1)])
# else, we use only id and timestamp
else:
docs = docs.hint([('timestamp', proc_req['order']),
('facebook_id', 1)])
#pagination (skipping)
if proc_req['after'] is not None:
docs = docs.skip(1)
#apply limit
docs.limit(proc_req['limit'])
try:
results = cursorToArray(docs,
decrypted=decrypted,
probe=probe_settings['collection'])
except Exception as e:
raise BadRequestException(
'error', 500, 'The request caused a DB malfunction: ' + str(e))
results_count = len(results)
response['meta']['status'] = proc_req['status']
response['meta']['results_count'] = len(results)
response['meta']['api_call'] = proc_req
response['meta']['query'] = query
response['results'] = results
if len(results) > 0:
response['meta']['paging'] = {}
response['meta']['paging']['cursors'] = {}
response['meta']['paging']['cursors']['after'] =OrderedDict([\
(proc_req['sortby'],getValueOfFullKey(results[-1], proc_req['sortby'])),\
('facebook_id',results[-1]['facebook_id']),
('user',results[-1]['user'])])
if results_count == proc_req['limit']:
if proc_req['after'] is not None:
response['meta']['paging']['links'] =\
{'next':re.sub('&after=[^ &]+','&after=' + urlize_dict(response['meta']['paging']['cursors']['after']),request.build_absolute_uri())}
else:
response['meta']['paging']['links'] = \
{'next':request.build_absolute_uri() + '&after=' + urlize_dict(response['meta']['paging']['cursors']['after'])}
except BadRequestException as e:
response['meta']['status'] = e.value
proc_req = {'format': 'json'}
response['meta']['execution_time_seconds'] = time.time() - _start_time
callback = request.REQUEST.get('callback', '')
if len(callback) > 0:
data = '%s(%s);' % (callback, json.dumps(response))
log.info(audit.message(request, response['meta']))
return HttpResponse(data,
content_type="text/plain",
status=response['meta']['status']['code'])
if decrypted:
pass
if proc_req['format'] == 'pretty':
log.info(audit.message(request, response['meta']))
return render_to_response('pretty_json.html',
{'response': json.dumps(response, indent=2)})
elif proc_req['format'] == 'csv':
output = '#' + json.dumps(response['meta'], indent=2).replace(
'\n', '\n#') + '\n'
if probe_settings['scope'] == 'connector_raw.locationfacebook':
output2 = ''
output += locationfacebook_to_csv(results, output2)
else:
output += array_to_csv(results, probe_settings['collection'])
log.info(audit.message(request, response['meta']))
return HttpResponse(output,
content_type="text/plain",
status=response['meta']['status']['code'])
else:
log.info(audit.message(request, response['meta']))
return HttpResponse(json.dumps(response),
content_type="application/json",
status=response['meta']['status']['code'])
return HttpResponse('hello decrypted')
def locationBuild(request,
users_to_return,
decrypted=False,
own_data=False,
roles=[]):
_start_time = time.time()
pretty = booleanize(request.REQUEST.get('pretty', False))
results = None
query = None
proc_req = None
response = {}
response['meta'] = {}
try:
if len(users_to_return) == 0:
raise BadRequestException(
'error', 403,
'The current token does not allow to view data from any users')
proc_req = processApiCall(request, users_to_return)
query = buildQuery(users_to_return, proc_req)
collection = 'edu_mit_media_funf_probe_builtin_LocationProbe'
if own_data and 'researcher' in roles: collection += '_researcher'
db = database.Database()
docs = db.getDocumentsCustom(query=query, collection=collection,\
fields = proc_req['fields'])
### hinting
# if the users are specified, we use the hint with users
if proc_req['users'] is not None:
docs = docs.hint([('data.TIMESTAMP', proc_req['order']),
('_id', 1), ('user', 1)])
# else, we use only id and timestamp
else:
docs = docs.hint([('data.TIMESTAMP', proc_req['order']),
('_id', 1)])
#pagination (skipping)
if proc_req['after'] is not None:
docs = docs.skip(1)
#apply limit
docs.limit(proc_req['limit'])
try:
results = cursorToArray(docs)
except Exception as e:
raise BadRequestException(
'error', 500, 'The request caused a DB malfunction: ' + str(e))
results_count = len(results)
response['meta']['status'] = proc_req['status']
response['meta']['results_count'] = len(results)
response['meta']['api_call'] = request.REQUEST
response['meta']['query'] = query
response['results'] = results
if len(results) > 0:
response['meta']['paging'] = {}
response['meta']['paging']['cursors'] = {}
response['meta']['paging']['cursors']['after'] =\
{proc_req['sortby']:getValueOfFullKey(results[-1], proc_req['sortby']),\
'_id':results[-1]['_id'],
'user':results[-1]['user']}
if results_count == proc_req['limit']:
if proc_req['after'] is not None:
response['meta']['paging']['links'] =\
{'next':re.sub('&after=[^ &]+','&after=' + urlize_dict(response['meta']['paging']['cursors']['after']),request.build_absolute_uri())}
else:
response['meta']['paging']['links'] = \
{'next':request.build_absolute_uri() + '&after=' + urlize_dict(response['meta']['paging']['cursors']['after'])}
except BadRequestException as e:
response['meta']['status'] = e.value
response['meta']['execution_time_seconds'] = time.time() - _start_time
callback = request.REQUEST.get('callback', '')
if len(callback) > 0:
data = '%s(%s);' % (callback, json.dumps(response))
return HttpResponse(data,
content_type="text/javascript",
status=response['meta']['status']['code'])
if decrypted:
pass
if pretty:
return render_to_response('pretty_json.html',
{'response': json.dumps(response, indent=2)})
else:
return HttpResponse(json.dumps(response),
content_type="application/json",
status=response['meta']['status']['code'])
return HttpResponse('hello decrypted')
ddate = check_date(row['created'].strip())
args.append(
(ddate, row['value'].strip(), row['category'].strip(),
row['description'].strip()))
if len(args) > 0:
logging.debug('sending data: {}'.format(args))
g.send(args)
finally:
g.close()
finally:
if fh:
fh.close()
if __name__ == '__main__':
import sys
import os
logging.debug(sys.argv)
_path, _ = os.path.split(sys.argv[0])
_path = os.path.abspath(_path)
if len(sys.argv) > 1:
db = database.Database(name=sys.argv[1], debug=True)
db.connect()
try:
populate(filename=os.path.join(_path, 'fill_from_csv.template'),
database=db)
finally:
db.close()
else:
raise SystemExit('provide a database path')
import discord, os, psycopg2
from utils import roblox
from utils import database
from discord.ext import commands
description = '''Bloxify help'''
bot = commands.Bot(command_prefix=['b25'],
description=description,
case_insensitive=True)
bot.roblox = roblox.Roblox()
bot.connection = database.Database()
@bot.event
async def on_ready():
print('ready')
for file in os.listdir("./classes"):
if file.endswith('.py'):
bot.load_extension(f'classes.{file.replace(".py", "")}')
bot.run('NDcwMjc3MzI2NTIyMDIzOTQ2.XYeumw.1qfVwHPDhRYZ-p4K3BvBF7NAV5c')
cli = sys.modules['flask.cli']
cli.show_server_banner = lambda *x: None
app.config.from_object(Config)
db = None
@app.route('/')
def query():
return render_template('query.html')
@app.route('/pass', methods=['POST'])
def passphrase():
device_id = request.form['id']
devices = []
for device in db.devices:
if device['id'] == device_id:
devices.append(device)
print("received: " + device_id)
return jsonify(devices)
def main():
Thread(target=ssl_server.main, args=(db,)).start()
app.run(host='0.0.0.0', debug=False)
if __name__ == '__main__':
db = database.Database()
main()
def dataBuild(request, probe_settings, users_to_return, questions_to_return, decrypted = False, own_data = False, roles = []):
_start_time = time.time()
results = None
query = None
proc_req = None
response = {}
response['meta'] = {}
try:
if len(users_to_return) == 0:
raise BadRequestException('error',403,'The current token does not allow to view data from any users')
proc_req = processApiCall(request, probe_settings, users_to_return, questions_to_return)
query = buildQuery(users_to_return,questions_to_return, proc_req)
collection = probe_settings['collection']
if own_data and 'researcher' in roles: collection += '_researcher'
db = database.Database()
docs = db.getDocumentsCustom(query=query, collection=collection,\
fields = proc_req['fields'])
### hinting
hint = [('form_version',1),('last_answered',proc_req['order']), ('variable_name',1), ('user',1)]
docs.hint(hint)
#pagination (skipping)
if proc_req['after'] is not None:
docs = docs.skip(1)
#apply limit
docs.limit(proc_req['limit'])
try:
results = cursorToArray(docs, decrypted = decrypted, probe=probe_settings['collection'])
except Exception as e:
raise BadRequestException('error',500,'The request caused a DB malfunction: ' + str(e) + '. Used hint: ' + str(hint))
results_count = len(results)
response['meta']['status'] = proc_req['status']
response['meta']['results_count'] = len(results)
response['meta']['api_call'] = proc_req
response['meta']['query'] = query
response['results'] = results
if len(results) > 0:
response['meta']['paging'] = {}
response['meta']['paging']['cursors'] = {}
response['meta']['paging']['cursors']['after'] =OrderedDict([\
('form_version',results[-1]['form_version']),('last_answered', results[-1]['last_answered']),\
('variable_name',results[-1]['variable_name']),
('user',results[-1]['user'])])
if results_count == proc_req['limit']:
if proc_req['after'] is not None:
response['meta']['paging']['links'] =\
{'next':re.sub('&after=[^ &]+','&after=' + urlize_dict(response['meta']['paging']['cursors']['after']),request.build_absolute_uri())}
else:
response['meta']['paging']['links'] = \
{'next':request.build_absolute_uri() + '&after=' + urlize_dict(response['meta']['paging']['cursors']['after'])}
except BadRequestException as e:
response['meta']['status'] = e.value
proc_req = {'format':'json'}
response['meta']['execution_time_seconds'] = time.time()-_start_time
callback = request.REQUEST.get('callback','')
if len(callback) > 0:
data = '%s(%s);' % (callback, json.dumps(response))
log.info(audit.message(request, response['meta']['api_call']))
return HttpResponse(data, content_type="text/plain", status=response['meta']['status']['code'])
if decrypted:
pass
#auditdb= audit.Audit()
#doc_audit=response['meta']
#users_return=[]
#users_results = cursorToArray(results, decrypted = decrypted, probe=probe_settings['collection'])
#for data_users in users_results:
# if data_users['user'] not in users_return:
# users_return.append(data_users['user'])
#doc_audit['users']=users_return
#doc_audit=transform.transform(doc_audit)
#auditdb.d(typ='prueba',tag='prueba2',doc=doc_audit,onlyfile=False)
if proc_req['format'] == 'pretty':
log.info(audit.message(request, response['meta']['api_call']))
return render_to_response('pretty_json.html', {'response': json.dumps(response, indent=2)})
elif proc_req['format'] == 'csv':
output = '#' + json.dumps(response['meta'], indent=2).replace('\n','\n#') + '\n'
output += array_to_csv(results,probe_settings['collection'])
log.info(audit.message(request, response['meta']['api_call']))
return HttpResponse(output, content_type="text/plain", status=response['meta']['status']['code'])
else:
log.info(audit.message(request, response['meta']['api_call']))
return HttpResponse(json.dumps(response), content_type="application/json", status=response['meta']['status']['code'])
return HttpResponse('hello decrypted')
return valid
if __name__ == '__main__':
r.set_loop_type('asyncio')
with open('config.json') as f:
config = json.load(f)
bot = AutoShardedBot(command_prefix=get_prefix,
help_attrs=dict(hidden=True))
bot.startup = datetime.now()
bot.messages_seen = 0
bot.r = r
bot.connection = bot.loop.run_until_complete(r.connect(db='parallax'))
bot.db = database.Database(bot)
bot.config = config
for f in os.listdir('extensions'):
if f.endswith('.py'):
try:
bot.load_extension(f'extensions.{f[:-3]}')
except SyntaxError as exception:
print(f'Failed to load {f}: {exception}')
@bot.event
async def on_message(message):
bot.messages_seen += 1
if not bot.is_ready() or message.author.bot:
return
def load_file(filename):
#pdb.set_trace()
log.log('Debug', 'Trying to populate db with ' + filename)
mConnector = ConnectorFunf.objects.all()[0]
db = database.Database()
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
proc_dir = os.path.join(mConnector.decrypted_path, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(mConnector.decrypted_path, filename)
processing_filepath = os.path.join(proc_dir, filename)
current_filepath = decrypted_filepath
if os.path.exists(
decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac']) = \
cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac from file_info').fetchone()
meta['device_id'] = anonymizerObject.anonymizeValue(
'device_id', meta['device_id'])
#pdb.set_trace()
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
meta['user'] = '******'
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject)
if doc == None:
continue
documents_to_insert[doc['probe']].append(
dict(doc.items() + meta.items()))
cursor.close()
#pdb.set_trace()
for probe in documents_to_insert:
db.insert(documents_to_insert[probe], probe)
os.remove(current_filepath)
except Exception as e:
log.log('Error', str(e))
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return False
