def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
if request.method == 'GET':
template = {'token': request.GET['bearer_token']}
return HttpResponse("File not found", status=404)
pds = None
#scope = AccessRange.objects.get(key="funf_write")
authorization = PDSAuthorization("funf_write", audit_enabled=False)
if (not authorization.is_authorized(request)):
return HttpResponse("Unauthorized", status=401)
scope = 'funf_write'
token = request.GET['bearer_token']
datastore_owner_uuid = request.GET["datastore_owner__uuid"]
datastore_owner, ds_owner_created = Profile.objects.get_or_create(
uuid=datastore_owner_uuid)
print "Creating IDS for %s" % datastore_owner_uuid
#internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab",
"Social Health Tracker", token)
#collection = connection[datastore_owner.getDBName()]["funf"]
funf_password = "******"
key = decrypt.key_from_password(str(funf_password))
print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid
for filename, file in request.FILES.items():
try:
try:
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(
convert_string(row[1])))
#print json_insert$
# Insert into PDS$
pds_data = {}
pds_data['time'] = json_insert.get('timestamp')
pds_data['value'] = json_insert
pds_data['key'] = name
insert_pds(internalDataStore, token, pds_data)
inserted.append(convert_string(json_insert) + '\n')
result = {'success': True, 'rows_inserted': len(inserted)}
print "Inserted %s rows" % len(inserted)
except Exception as e:
print "Exception from funf_connector on pds:"
print "%s" % e
result = {'success': False, 'error_message': e.message}
finally:
response_dict = {"status": "success"}
return HttpResponse(json.dumps(result), content_type='application/json')
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
connection = None
token = request.GET['bearer_token']
if request.method == 'GET':
template = {'token': token}
return render_to_response('upload.html', template, RequestContext(request))
pds = None
scope = AccessRange.objects.get(key="funf_write")
authenticator = JSONAuthenticator(scope=scope)
try:
# Validate the request.
authenticator.validate(request)
except AuthenticationException as e:
# Return an error response.
print e
return authenticator.error_response(content="You didn't authenticate.")
profile = authenticator.user.get_profile()
funf_password = profile.funf_password
print "Registry set_funf_data for uuid: %s" % profile.uuid
for filename, file in request.FILES.items():
try:
file_path = upload_dir + file.name
try:
key = decrypt.key_from_password(str(funf_password))
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1])))
#print json_insert
# Insert into PDS
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=name
insert_pds(profile, token, pds_data)
#print "inserting row..."
#print pds_data
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'message':''.join(inserted)}
#remove_file(file_path)
except Exception as e:
print e.message
print traceback.format_exc()
result = {'success':False, 'error_message':e.message}
# It doesn't matter what we return at this point - the phones are just checking the response status
response_dict = {"status":"success"}
return HttpResponse(json.dumps(response_dict), content_type='application/json')
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
if request.method == 'GET':
template = {'token':request.GET['bearer_token']}
return HttpResponse("File not found", status=404)
pds = None
#scope = AccessRange.objects.get(key="funf_write")
authorization = PDSAuthorization("funf_write", audit_enabled=False)
if (not authorization.is_authorized(request)):
return HttpResponse("Unauthorized", status=401)
scope = 'funf_write'
token = request.GET['bearer_token']
datastore_owner_uuid = request.GET["datastore_owner__uuid"]
datastore_owner, ds_owner_created = Profile.objects.get_or_create(uuid = datastore_owner_uuid)
print "Creating IDS for %s" % datastore_owner_uuid
#internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token)
#collection = connection[datastore_owner.getDBName()]["funf"]
funf_password = "******"
key = decrypt.key_from_password(str(funf_password))
print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid
for filename, file in request.FILES.items():
try:
try:
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1])))
#print json_insert$
# Insert into PDS$
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=name
insert_pds(internalDataStore, token, pds_data)
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'rows_inserted': len(inserted)}
print "Inserted %s rows" % len(inserted)
except Exception as e:
print "Exception from funf_connector on pds:"
print "%s"%e
result = {'success':False, 'error_message':e.message}
finally:
response_dict = {"status":"success"}
return HttpResponse(json.dumps(result), content_type='application/json')
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(mConnector.decrypted_path, f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False
try:
# check if still exists, might have been moved in another thread
if os.path.exists(
upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
if decrypt_if_not_db_file(proc_filename, key, extension=None):
decryption_success = True
fail.safe_move(proc_filename, mConnector.decrypted_path)
log.log('Debug', 'Still here #1')
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
#log.log('Debug','Still here #2')
database_single_population.load_file(f)
return True
else:
return False
except Exception as e:
#find out when it happened
action = ''
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if not str(e).contains('already exists'):
fail.fail(
curr_filename, mConnector.decryption_failed_path,
'Exception thrown: ' + str(e) + '. While ' + action +
' file: ' + f)
log.log('error', 'README ^^^^^^^^^^^^^')
else:
log.log(
'error', 'Exception thrown: ' + str(e) + '. While ' +
action + ' file: ' + f)
except Exception as e1:
pass
return False
'''
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(myConnector['decrypted_path'], f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False;
try:
# check if still exists, might have been moved in another thread
if os.path.exists(upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
decryption_start = time.time();
if decrypt_if_not_db_file(proc_filename, key, extension=None):
log.debug({'dtime': time.time()-decryption_start})
decryption_success = True;
fail.safe_move(proc_filename, myConnector['decrypted_path'])
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
else:
fail.fail(curr_filename, myConnector['decryption_failed_path'], 'Could not decrypt file: ' + f)
return False
return True
else:
return False
except Exception as e:
#find out when it happened
action = '';
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if 'already exists' not in str(e):
fail.fail(curr_filename, myConnector['decryption_failed_path'], 'Exception thrown: ' + str(e) + '. While ' + action + ' file: ' + f)
else:
log.error({'error': str(e), 'action': str(action), 'file': str(f)})
except Exception as e1:
pass
return False;
'''
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(mConnector.decrypted_path, f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False;
try:
# check if still exists, might have been moved in another thread
if os.path.exists(upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
if decrypt_if_not_db_file(proc_filename, key, extension=None):
decryption_success = True;
fail.safe_move(proc_filename, mConnector.decrypted_path)
log.log('Debug','Still here #1')
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
#log.log('Debug','Still here #2')
database_single_population.load_file(f)
return True
else:
return False
except Exception as e:
#find out when it happened
action = '';
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if not str(e).contains('already exists'):
fail.fail(curr_filename, mConnector.decryption_failed_path, 'Exception thrown: ' + str(e) + '. While ' + action + ' file: ' + f)
log.log('error', 'README ^^^^^^^^^^^^^')
else:
log.log('error','Exception thrown: ' + str(e) + '. While ' + action + ' file: ' + f);
except Exception as e1:
pass
return False;
'''
def decrypt_all(files, key=None, password=None):
key = key or decrypt.key_from_password(password or decrypt.prompt_for_password())
print "Decrypting files"
failed_files = []
for file_name in files:
success = dbdecrypt.decrypt_if_not_db_file(file_name, key, options.extension)
if not success:
failed_files.append(file_name)
if failed_files:
print "\n\n\nWARNING: Some of the files failed to decrypt! \nType another password to attempt to decrypt these files, \nor leave blank to continue without decrypting the remaining files.\n"
another_password = decrypt.prompt_for_password()
if another_password:
decrypt_all(failed_files, password=another_password)
def decrypt_all(files, key=None, password=None):
key = key or decrypt.key_from_password(password
or decrypt.prompt_for_password())
print "Decrypting files"
failed_files = []
for file_name in files:
success = dbdecrypt.decrypt_if_not_db_file(file_name, key,
options.extension)
if not success:
failed_files.append(file_name)
if failed_files:
print "\n\n\nWARNING: Some of the files failed to decrypt! \nType another password to attempt to decrypt these files, \nor leave blank to continue without decrypting the remaining files.\n"
another_password = decrypt.prompt_for_password()
if another_password:
decrypt_all(failed_files, password=another_password)
def process_data():
# load decrypt password
with open(password_file, 'r') as file:
encryption_password = file.read().split("\n")[0]
encryption_key = decrypt.key_from_password(encryption_password)
# dbDecrypt all files in raw
db_files = glob.glob(os.path.join(raw_data_dir, '*.db'))
failed_files = []
for fl in db_files:
if not dbdecrypt.decrypt_if_not_db_file(fl, encryption_key):
failed_files.append(fl)
# merge all files in raw
decrypted_files = list(set(db_files) - set(failed_files))
dbmerge.merge(db_files=decrypted_files, out_file=processing_file, overwrite=True, attempt_salvage=True)
# replace current file
shutil.move(processing_file, merged_data_file)
def decrypt_directory(directory_to_decrypt=service_config.CONNECTORS['connector_funf']['config']['upload_path']): decrypted_directory_path = service_config.CONNECTORS['connector_funf']['config']['decrypted_path'] decryption_failed_path = service_config.CONNECTORS['connector_funf']['config']['decryption_failed_path'] backup_path = service_config.CONNECTORS['connector_funf']['config']['backup_path'] #TODO #raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db')] raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.orig')] filenames = [os.path.join(directory_to_decrypt, filename) for filename in raw_filenames] proc_dir = os.path.join(directory_to_decrypt, 'processing') failed_filenames = [] for f in filenames: try: shutil.move(f, proc_dir) except Exception as e: fail.fail(f, decryption_failed_path, 'Exception thrown: ' + str(e) + '. While decrypting file: ' + f) failed_filenames.append(os.path.basename(f)) raw_filenames = [e for e in raw_filenames if e not in failed_filenames] filenames = [os.path.join(proc_dir, filename) for filename in raw_filenames] for filename in filenames: try: # If successfull we move the decrypted file to the decrypted directory if decrypt_if_not_db_file(filename, key, extension=None): fail.safe_move(filename, decrypted_directory_path) # Move original db file to original directory if it exists -- if db file is already decrypted .orig will not exist #orig_filename = filename + '.orig' orig_filename = filename if os.path.exists(orig_filename): fail.safe_move(orig_filename, backup_path) # If decryption fails we move the file to the failed directory else: fail.fail(filename, decryption_failed_path, 'Could not decrypt file: ' + filename) except Exception as e: # If anything goes wrong we move the file to the failed directory fail.fail(filename, decryption_failed_path, 'Exception thrown: ' + str(e) + '. While decrypting file: ' + filename)
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
connection = None
for filename, file in request.FILES.items():
logging.debug(filename)
if request.method == 'GET':
template = {'token':request.GET['bearer_token']}
return render_to_response('upload.html',
template,
RequestContext(request))
pds = None
scope = AccessRange.objects.get(key="funf_write")
# authenticator = Authenticator(scope=scope)
authenticator = JSONAuthenticator(scope=scope)
try:
# Validate the request.
authenticator.validate(request)
except AuthenticationException as e:
# Return an error response.
print e
return authenticator.error_response(content="You didn't authenticate.")
profile = authenticator.user.get_profile()
funf_password = profile.funf_password
try:
scope = 'funf_write'
token = request.GET['bearer_token']
try:
key = decrypt.key_from_password(str(funf_password))
print key
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select value from data")
inserted = []
for row in cur:
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row)))
print json_insert
# Insert into PDS
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=json_insert.get('probe')
insert_pds(profile, token, pds_data)
print "inserting row..."
print pds_data
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'message':''.join(inserted)}
except Exception as e:
result = {'success':False, 'error_message':e.message}
finally:
response_dict = {"status":"success"}
return HttpResponse(json.dumps(response_dict), content_type='application/json')
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(myConnector['decrypted_path'], f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False
try:
# check if still exists, might have been moved in another thread
if os.path.exists(
upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
decryption_start = time.time()
if decrypt_if_not_db_file(proc_filename, key, extension=None):
log.debug({'dtime': time.time() - decryption_start})
decryption_success = True
fail.safe_move(proc_filename, myConnector['decrypted_path'])
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
else:
fail.fail(curr_filename, myConnector['decryption_failed_path'],
'Could not decrypt file: ' + f)
return False
return True
else:
return False
except Exception as e:
#find out when it happened
action = ''
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if 'already exists' not in str(e):
fail.fail(
curr_filename, myConnector['decryption_failed_path'],
'Exception thrown: ' + str(e) + '. While ' + action +
' file: ' + f)
else:
log.error({
'error': str(e),
'action': str(action),
'file': str(f)
})
except Exception as e1:
pass
return False
'''
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
connection = None
token = request.GET['bearer_token']
if request.method == 'GET':
template = {'token': token}
return render_to_response('upload.html', template,
RequestContext(request))
pds = None
scope = AccessRange.objects.get(key="funf_write")
authenticator = JSONAuthenticator(scope=scope)
try:
# Validate the request.
authenticator.validate(request)
except AuthenticationException as e:
# Return an error response.
print e
return authenticator.error_response(content="You didn't authenticate.")
profile = authenticator.user.get_profile()
funf_password = profile.funf_password
print "Registry set_funf_data for uuid: %s" % profile.uuid
for filename, file in request.FILES.items():
try:
file_path = upload_dir + file.name
try:
key = decrypt.key_from_password(str(funf_password))
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(
convert_string(row[1])))
#print json_insert
# Insert into PDS
pds_data = {}
pds_data['time'] = json_insert.get('timestamp')
pds_data['value'] = json_insert
pds_data['key'] = name
insert_pds(profile, token, pds_data)
#print "inserting row..."
#print pds_data
inserted.append(convert_string(json_insert) + '\n')
result = {'success': True, 'message': ''.join(inserted)}
#remove_file(file_path)
except Exception as e:
print e.message
print traceback.format_exc()
result = {'success': False, 'error_message': e.message}
# It doesn't matter what we return at this point - the phones are just checking the response status
response_dict = {"status": "success"}
return HttpResponse(json.dumps(response_dict),
content_type='application/json')
