def data(request): '''decrypt funf database files, and upload them to your PDS''' result = {} if request.method == 'GET': template = {'token': request.GET['bearer_token']} return HttpResponse("File not found", status=404) pds = None #scope = AccessRange.objects.get(key="funf_write") authorization = PDSAuthorization("funf_write", audit_enabled=False) if (not authorization.is_authorized(request)): return HttpResponse("Unauthorized", status=401) scope = 'funf_write' token = request.GET['bearer_token'] datastore_owner_uuid = request.GET["datastore_owner__uuid"] datastore_owner, ds_owner_created = Profile.objects.get_or_create( uuid=datastore_owner_uuid) print "Creating IDS for %s" % datastore_owner_uuid #internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token) internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token) #collection = connection[datastore_owner.getDBName()]["funf"] funf_password = "******" key = decrypt.key_from_password(str(funf_password)) print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid for filename, file in request.FILES.items(): try: try: file_path = upload_dir + file.name write_file(str(file_path), file) except Exception as ex: print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR" dbdecrypt.decrypt_if_not_db_file(file_path, key) con = sqlite3.connect(file_path) cur = con.cursor() cur.execute("select name, value from data") inserted = [] for row in cur: name = convert_string(row[0]) json_insert = clean_keys(json.JSONDecoder().decode( convert_string(row[1]))) #print json_insert$ # Insert into PDS$ pds_data = {} pds_data['time'] = json_insert.get('timestamp') pds_data['value'] = json_insert pds_data['key'] = name insert_pds(internalDataStore, token, pds_data) inserted.append(convert_string(json_insert) + '\n') result = {'success': True, 'rows_inserted': len(inserted)} print "Inserted %s rows" % len(inserted) except Exception as e: print "Exception from funf_connector on pds:" print "%s" % e result = {'success': False, 'error_message': e.message} finally: response_dict = {"status": "success"} return HttpResponse(json.dumps(result), content_type='application/json')
def data(request): '''decrypt funf database files, and upload them to your PDS''' result = {} connection = None token = request.GET['bearer_token'] if request.method == 'GET': template = {'token': token} return render_to_response('upload.html', template, RequestContext(request)) pds = None scope = AccessRange.objects.get(key="funf_write") authenticator = JSONAuthenticator(scope=scope) try: # Validate the request. authenticator.validate(request) except AuthenticationException as e: # Return an error response. print e return authenticator.error_response(content="You didn't authenticate.") profile = authenticator.user.get_profile() funf_password = profile.funf_password print "Registry set_funf_data for uuid: %s" % profile.uuid for filename, file in request.FILES.items(): try: file_path = upload_dir + file.name try: key = decrypt.key_from_password(str(funf_password)) write_file(str(file_path), file) except Exception as ex: print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR" dbdecrypt.decrypt_if_not_db_file(file_path, key) con = sqlite3.connect(file_path) cur = con.cursor() cur.execute("select name, value from data") inserted = [] for row in cur: name = convert_string(row[0]) json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1]))) #print json_insert # Insert into PDS pds_data= {} pds_data['time']=json_insert.get('timestamp') pds_data['value']=json_insert pds_data['key']=name insert_pds(profile, token, pds_data) #print "inserting row..." #print pds_data inserted.append(convert_string(json_insert)+'\n') result = {'success': True, 'message':''.join(inserted)} #remove_file(file_path) except Exception as e: print e.message print traceback.format_exc() result = {'success':False, 'error_message':e.message} # It doesn't matter what we return at this point - the phones are just checking the response status response_dict = {"status":"success"} return HttpResponse(json.dumps(response_dict), content_type='application/json')
def data(request): '''decrypt funf database files, and upload them to your PDS''' result = {} if request.method == 'GET': template = {'token':request.GET['bearer_token']} return HttpResponse("File not found", status=404) pds = None #scope = AccessRange.objects.get(key="funf_write") authorization = PDSAuthorization("funf_write", audit_enabled=False) if (not authorization.is_authorized(request)): return HttpResponse("Unauthorized", status=401) scope = 'funf_write' token = request.GET['bearer_token'] datastore_owner_uuid = request.GET["datastore_owner__uuid"] datastore_owner, ds_owner_created = Profile.objects.get_or_create(uuid = datastore_owner_uuid) print "Creating IDS for %s" % datastore_owner_uuid #internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token) internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token) #collection = connection[datastore_owner.getDBName()]["funf"] funf_password = "******" key = decrypt.key_from_password(str(funf_password)) print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid for filename, file in request.FILES.items(): try: try: file_path = upload_dir + file.name write_file(str(file_path), file) except Exception as ex: print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR" dbdecrypt.decrypt_if_not_db_file(file_path, key) con = sqlite3.connect(file_path) cur = con.cursor() cur.execute("select name, value from data") inserted = [] for row in cur: name = convert_string(row[0]) json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1]))) #print json_insert$ # Insert into PDS$ pds_data= {} pds_data['time']=json_insert.get('timestamp') pds_data['value']=json_insert pds_data['key']=name insert_pds(internalDataStore, token, pds_data) inserted.append(convert_string(json_insert)+'\n') result = {'success': True, 'rows_inserted': len(inserted)} print "Inserted %s rows" % len(inserted) except Exception as e: print "Exception from funf_connector on pds:" print "%s"%e result = {'success':False, 'error_message':e.message} finally: response_dict = {"status":"success"} return HttpResponse(json.dumps(result), content_type='application/json')
def decrypt_all(files, key=None, password=None): key = key or decrypt.key_from_password(password or decrypt.prompt_for_password()) print "Decrypting files" failed_files = [] for file_name in files: success = dbdecrypt.decrypt_if_not_db_file(file_name, key, options.extension) if not success: failed_files.append(file_name) if failed_files: print "\n\n\nWARNING: Some of the files failed to decrypt! \nType another password to attempt to decrypt these files, \nor leave blank to continue without decrypting the remaining files.\n" another_password = decrypt.prompt_for_password() if another_password: decrypt_all(failed_files, password=another_password)
def process_data(): # load decrypt password with open(password_file, 'r') as file: encryption_password = file.read().split("\n")[0] encryption_key = decrypt.key_from_password(encryption_password) # dbDecrypt all files in raw db_files = glob.glob(os.path.join(raw_data_dir, '*.db')) failed_files = [] for fl in db_files: if not dbdecrypt.decrypt_if_not_db_file(fl, encryption_key): failed_files.append(fl) # merge all files in raw decrypted_files = list(set(db_files) - set(failed_files)) dbmerge.merge(db_files=decrypted_files, out_file=processing_file, overwrite=True, attempt_salvage=True) # replace current file shutil.move(processing_file, merged_data_file)
decrypt.decrypt([file_name], key, extension) if is_funf_database(file_name): print "Success!" return True else: print "FAILED!!!!" print "File is either encrypted with another method, another key, or is not a valid sqlite3 db file." print "Keeping original file." shutil.move(decrypt.backup_file(file_name, extension), file_name) return False if __name__ == '__main__': usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]" description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. " parser = OptionParser(usage="%s\n\n%s" % (usage, description)) parser.add_option("-i", "--inplace", dest="extension", default=None, help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension,) parser.add_option("-k", "--key", dest="key", default=None, help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",) parser.add_option("-p", "--pass", dest="password", default=None, help="The password used to decrypt the files.",) (options, args) = parser.parse_args() if options.key: key = options.key elif options.password: key = decrypt.key_from_password(options.password) else: key = decrypt.key_from_password(decrypt.prompt_for_password()) for file_name in args: decrypt_if_not_db_file(file_name, key, options.extension)
def data(request): '''decrypt funf database files, and upload them to your PDS''' result = {} connection = None for filename, file in request.FILES.items(): logging.debug(filename) if request.method == 'GET': template = {'token':request.GET['bearer_token']} return render_to_response('upload.html', template, RequestContext(request)) pds = None scope = AccessRange.objects.get(key="funf_write") # authenticator = Authenticator(scope=scope) authenticator = JSONAuthenticator(scope=scope) try: # Validate the request. authenticator.validate(request) except AuthenticationException as e: # Return an error response. print e return authenticator.error_response(content="You didn't authenticate.") profile = authenticator.user.get_profile() funf_password = profile.funf_password try: scope = 'funf_write' token = request.GET['bearer_token'] try: key = decrypt.key_from_password(str(funf_password)) print key file_path = upload_dir + file.name write_file(str(file_path), file) except Exception as ex: print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR" dbdecrypt.decrypt_if_not_db_file(file_path, key) con = sqlite3.connect(file_path) cur = con.cursor() cur.execute("select value from data") inserted = [] for row in cur: json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row))) print json_insert # Insert into PDS pds_data= {} pds_data['time']=json_insert.get('timestamp') pds_data['value']=json_insert pds_data['key']=json_insert.get('probe') insert_pds(profile, token, pds_data) print "inserting row..." print pds_data inserted.append(convert_string(json_insert)+'\n') result = {'success': True, 'message':''.join(inserted)} except Exception as e: result = {'success':False, 'error_message':e.message} finally: response_dict = {"status":"success"} return HttpResponse(json.dumps(response_dict), content_type='application/json')
from dbdecrypt import decrypt_if_not_db_file from decrypt import key_from_password from utils import SECURE_settings from utils import fail from django.conf import settings from connectors.connector_funf.models import ConnectorFunf import connectors.connector_funf.database_single_population as database_single_population import connectors.connectors_config import time import pdb from sensible_audit import audit log = audit.getLogger(__name__) myConnector = connectors.connectors_config.CONNECTORS['ConnectorFunf']['config'] key = key_from_password(SECURE_settings.CONNECTORS['connector_funf']['db_pass']); def decrypt(): decrypt_directory() def decrypt_directory(directory_to_decrypt=myConnector['upload_path']): ''' NEW VERSION: moves a file, decrypts it, moves on to the next file ''' raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db')] failed_filenames = [] already_moved_counter = 0;
if __name__ == "__main__": usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]" description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. " parser = OptionParser(usage="%s\n\n%s" % (usage, description)) parser.add_option( "-i", "--inplace", dest="extension", default=None, help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension, ) parser.add_option( "-k", "--key", dest="key", default=None, help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.", ) (options, args) = parser.parse_args() key = options.key if options.key else decrypt.key_from_password(decrypt.prompt_for_password()) try: for file_name in args: decrypt_if_not_db_file(file_name, key, options.extension) except Exception as e: import sys sys.exit("ERROR: " + str(e))
def decrypt_if_not_db_file(file_name, key, extension=None): if is_funf_database(file_name): print "Already decrypted: '%s'" % file_name return True else: print ("Attempting to decrypt: '%s'..." % file_name), decrypt.decrypt([file_name], key, extension) if is_funf_database(file_name): print "Success!" os.remove(file_name+'.orig') return True else: print "FAILED!!!!" os.remove(file_name) os.remove(file_name+'.orig') return False if __name__ == '__main__': usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]" description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. " parser = OptionParser(usage="%s\n\n%s" % (usage, description)) parser.add_option("-i", "--inplace", dest="extension", default=None, help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension,) parser.add_option("-k", "--key", dest="key", default=None, help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",) (options, args) = parser.parse_args() key = decrypt.key_from_password(options.key) for file_name in args: decrypt_if_not_db_file(file_name, key, options.extension)
if is_funf_database(file_name): print "Already decrypted: '%s'" % file_name return True else: print ("Attempting to decrypt: '%s'..." % file_name), decrypt.decrypt([file_name], key, extension) if is_funf_database(file_name): print "Success!" return True else: print "FAILED!!!!" print "File is either encrypted with another method, another key, or is not a valid sqlite3 db file." print "Keeping original file." shutil.move(decrypt.backup_file(file_name, extension), file_name) return False if __name__ == '__main__': usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]" description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. " parser = OptionParser(usage="%s\n\n%s" % (usage, description)) parser.add_option("-i", "--inplace", dest="extension", default=None, help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension,) parser.add_option("-k", "--key", dest="key", default=None, help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",) (options, args) = parser.parse_args() #key = options.key if options.key else decrypt.key_from_password(decrypt.prompt_for_password()) key = decrypt.key_from_password("changeme"); for file_name in args: decrypt_if_not_db_file(file_name, key, options.extension)
import os import shutil import fnmatch from dbdecrypt import decrypt_if_not_db_file from decrypt import key_from_password from utils import SECURE_settings from utils import log, fail from django.conf import settings from connectors.connector_funf.models import ConnectorFunf import connectors.connector_funf.database_single_population as database_single_population import pdb mConnector = ConnectorFunf.objects.all()[0] key = key_from_password( SECURE_settings.CONNECTORS['connector_funf']['db_pass']) def decrypt(): decrypt_directory() def decrypt_directory(directory_to_decrypt=mConnector.upload_path): ''' NEW VERSION: moves a file, decrypts it, moves on to the next file ''' raw_filenames = [ filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db') ]
print "Keeping original file." shutil.move(decrypt.backup_file(file_name, extension), file_name) return False if __name__ == '__main__': usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]" description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. " parser = OptionParser(usage="%s\n\n%s" % (usage, description)) parser.add_option( "-i", "--inplace", dest="extension", default=None, help= "The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension, ) parser.add_option( "-k", "--key", dest="key", default=None, help= "The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.", ) (options, args) = parser.parse_args() key = options.key if options.key else decrypt.key_from_password( decrypt.prompt_for_password()) for file_name in args: decrypt_if_not_db_file(file_name, key, options.extension)
import sys import os import shutil import fnmatch from dbdecrypt import decrypt_if_not_db_file from decrypt import key_from_password from utils import service_config from utils import SECURE_service_config from utils import log, fail key = key_from_password(SECURE_service_config.CONNECTORS['connector_funf']['db_pass']) def decrypt(): decrypt_directory() def decrypt_directory(directory_to_decrypt=service_config.CONNECTORS['connector_funf']['config']['upload_path']): decrypted_directory_path = service_config.CONNECTORS['connector_funf']['config']['decrypted_path'] decryption_failed_path = service_config.CONNECTORS['connector_funf']['config']['decryption_failed_path'] backup_path = service_config.CONNECTORS['connector_funf']['config']['backup_path'] #TODO #raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db')] raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.orig')] filenames = [os.path.join(directory_to_decrypt, filename) for filename in raw_filenames] proc_dir = os.path.join(directory_to_decrypt, 'processing') failed_filenames = [] for f in filenames: try: shutil.move(f, proc_dir) except Exception as e: fail.fail(f, decryption_failed_path, 'Exception thrown: ' + str(e) + '. While decrypting file: ' + f)
def data(request): '''decrypt funf database files, and upload them to your PDS''' result = {} connection = None token = request.GET['bearer_token'] if request.method == 'GET': template = {'token': token} return render_to_response('upload.html', template, RequestContext(request)) pds = None scope = AccessRange.objects.get(key="funf_write") authenticator = JSONAuthenticator(scope=scope) try: # Validate the request. authenticator.validate(request) except AuthenticationException as e: # Return an error response. print e return authenticator.error_response(content="You didn't authenticate.") profile = authenticator.user.get_profile() funf_password = profile.funf_password print "Registry set_funf_data for uuid: %s" % profile.uuid for filename, file in request.FILES.items(): try: file_path = upload_dir + file.name try: key = decrypt.key_from_password(str(funf_password)) write_file(str(file_path), file) except Exception as ex: print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR" dbdecrypt.decrypt_if_not_db_file(file_path, key) con = sqlite3.connect(file_path) cur = con.cursor() cur.execute("select name, value from data") inserted = [] for row in cur: name = convert_string(row[0]) json_insert = clean_keys(json.JSONDecoder().decode( convert_string(row[1]))) #print json_insert # Insert into PDS pds_data = {} pds_data['time'] = json_insert.get('timestamp') pds_data['value'] = json_insert pds_data['key'] = name insert_pds(profile, token, pds_data) #print "inserting row..." #print pds_data inserted.append(convert_string(json_insert) + '\n') result = {'success': True, 'message': ''.join(inserted)} #remove_file(file_path) except Exception as e: print e.message print traceback.format_exc() result = {'success': False, 'error_message': e.message} # It doesn't matter what we return at this point - the phones are just checking the response status response_dict = {"status": "success"} return HttpResponse(json.dumps(response_dict), content_type='application/json')