def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
if request.method == 'GET':
template = {'token': request.GET['bearer_token']}
return HttpResponse("File not found", status=404)
pds = None
#scope = AccessRange.objects.get(key="funf_write")
authorization = PDSAuthorization("funf_write", audit_enabled=False)
if (not authorization.is_authorized(request)):
return HttpResponse("Unauthorized", status=401)
scope = 'funf_write'
token = request.GET['bearer_token']
datastore_owner_uuid = request.GET["datastore_owner__uuid"]
datastore_owner, ds_owner_created = Profile.objects.get_or_create(
uuid=datastore_owner_uuid)
print "Creating IDS for %s" % datastore_owner_uuid
#internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab",
"Social Health Tracker", token)
#collection = connection[datastore_owner.getDBName()]["funf"]
funf_password = "******"
key = decrypt.key_from_password(str(funf_password))
print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid
for filename, file in request.FILES.items():
try:
try:
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(
convert_string(row[1])))
#print json_insert$
# Insert into PDS$
pds_data = {}
pds_data['time'] = json_insert.get('timestamp')
pds_data['value'] = json_insert
pds_data['key'] = name
insert_pds(internalDataStore, token, pds_data)
inserted.append(convert_string(json_insert) + '\n')
result = {'success': True, 'rows_inserted': len(inserted)}
print "Inserted %s rows" % len(inserted)
except Exception as e:
print "Exception from funf_connector on pds:"
print "%s" % e
result = {'success': False, 'error_message': e.message}
finally:
response_dict = {"status": "success"}
return HttpResponse(json.dumps(result), content_type='application/json')
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
connection = None
token = request.GET['bearer_token']
if request.method == 'GET':
template = {'token': token}
return render_to_response('upload.html', template, RequestContext(request))
pds = None
scope = AccessRange.objects.get(key="funf_write")
authenticator = JSONAuthenticator(scope=scope)
try:
# Validate the request.
authenticator.validate(request)
except AuthenticationException as e:
# Return an error response.
print e
return authenticator.error_response(content="You didn't authenticate.")
profile = authenticator.user.get_profile()
funf_password = profile.funf_password
print "Registry set_funf_data for uuid: %s" % profile.uuid
for filename, file in request.FILES.items():
try:
file_path = upload_dir + file.name
try:
key = decrypt.key_from_password(str(funf_password))
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1])))
#print json_insert
# Insert into PDS
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=name
insert_pds(profile, token, pds_data)
#print "inserting row..."
#print pds_data
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'message':''.join(inserted)}
#remove_file(file_path)
except Exception as e:
print e.message
print traceback.format_exc()
result = {'success':False, 'error_message':e.message}
# It doesn't matter what we return at this point - the phones are just checking the response status
response_dict = {"status":"success"}
return HttpResponse(json.dumps(response_dict), content_type='application/json')
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
if request.method == 'GET':
template = {'token':request.GET['bearer_token']}
return HttpResponse("File not found", status=404)
pds = None
#scope = AccessRange.objects.get(key="funf_write")
authorization = PDSAuthorization("funf_write", audit_enabled=False)
if (not authorization.is_authorized(request)):
return HttpResponse("Unauthorized", status=401)
scope = 'funf_write'
token = request.GET['bearer_token']
datastore_owner_uuid = request.GET["datastore_owner__uuid"]
datastore_owner, ds_owner_created = Profile.objects.get_or_create(uuid = datastore_owner_uuid)
print "Creating IDS for %s" % datastore_owner_uuid
#internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", "Activity", token)
internalDataStore = getInternalDataStore(datastore_owner, "Living Lab", "Social Health Tracker", token)
#collection = connection[datastore_owner.getDBName()]["funf"]
funf_password = "******"
key = decrypt.key_from_password(str(funf_password))
print "PDS: set_funf_data on uuid: %s" % datastore_owner_uuid
for filename, file in request.FILES.items():
try:
try:
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row[1])))
#print json_insert$
# Insert into PDS$
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=name
insert_pds(internalDataStore, token, pds_data)
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'rows_inserted': len(inserted)}
print "Inserted %s rows" % len(inserted)
except Exception as e:
print "Exception from funf_connector on pds:"
print "%s"%e
result = {'success':False, 'error_message':e.message}
finally:
response_dict = {"status":"success"}
return HttpResponse(json.dumps(result), content_type='application/json')
def decrypt_all(files, key=None, password=None):
key = key or decrypt.key_from_password(password or decrypt.prompt_for_password())
print "Decrypting files"
failed_files = []
for file_name in files:
success = dbdecrypt.decrypt_if_not_db_file(file_name, key, options.extension)
if not success:
failed_files.append(file_name)
if failed_files:
print "\n\n\nWARNING: Some of the files failed to decrypt! \nType another password to attempt to decrypt these files, \nor leave blank to continue without decrypting the remaining files.\n"
another_password = decrypt.prompt_for_password()
if another_password:
decrypt_all(failed_files, password=another_password)
def decrypt_all(files, key=None, password=None):
key = key or decrypt.key_from_password(password
or decrypt.prompt_for_password())
print "Decrypting files"
failed_files = []
for file_name in files:
success = dbdecrypt.decrypt_if_not_db_file(file_name, key,
options.extension)
if not success:
failed_files.append(file_name)
if failed_files:
print "\n\n\nWARNING: Some of the files failed to decrypt! \nType another password to attempt to decrypt these files, \nor leave blank to continue without decrypting the remaining files.\n"
another_password = decrypt.prompt_for_password()
if another_password:
decrypt_all(failed_files, password=another_password)
def process_data():
# load decrypt password
with open(password_file, 'r') as file:
encryption_password = file.read().split("\n")[0]
encryption_key = decrypt.key_from_password(encryption_password)
# dbDecrypt all files in raw
db_files = glob.glob(os.path.join(raw_data_dir, '*.db'))
failed_files = []
for fl in db_files:
if not dbdecrypt.decrypt_if_not_db_file(fl, encryption_key):
failed_files.append(fl)
# merge all files in raw
decrypted_files = list(set(db_files) - set(failed_files))
dbmerge.merge(db_files=decrypted_files, out_file=processing_file, overwrite=True, attempt_salvage=True)
# replace current file
shutil.move(processing_file, merged_data_file)
decrypt.decrypt([file_name], key, extension)
if is_funf_database(file_name):
print "Success!"
return True
else:
print "FAILED!!!!"
print "File is either encrypted with another method, another key, or is not a valid sqlite3 db file."
print "Keeping original file."
shutil.move(decrypt.backup_file(file_name, extension), file_name)
return False
if __name__ == '__main__':
usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]"
description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. "
parser = OptionParser(usage="%s\n\n%s" % (usage, description))
parser.add_option("-i", "--inplace", dest="extension", default=None,
help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension,)
parser.add_option("-k", "--key", dest="key", default=None,
help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",)
parser.add_option("-p", "--pass", dest="password", default=None,
help="The password used to decrypt the files.",)
(options, args) = parser.parse_args()
if options.key:
key = options.key
elif options.password:
key = decrypt.key_from_password(options.password)
else:
key = decrypt.key_from_password(decrypt.prompt_for_password())
for file_name in args:
decrypt_if_not_db_file(file_name, key, options.extension)
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
connection = None
for filename, file in request.FILES.items():
logging.debug(filename)
if request.method == 'GET':
template = {'token':request.GET['bearer_token']}
return render_to_response('upload.html',
template,
RequestContext(request))
pds = None
scope = AccessRange.objects.get(key="funf_write")
# authenticator = Authenticator(scope=scope)
authenticator = JSONAuthenticator(scope=scope)
try:
# Validate the request.
authenticator.validate(request)
except AuthenticationException as e:
# Return an error response.
print e
return authenticator.error_response(content="You didn't authenticate.")
profile = authenticator.user.get_profile()
funf_password = profile.funf_password
try:
scope = 'funf_write'
token = request.GET['bearer_token']
try:
key = decrypt.key_from_password(str(funf_password))
print key
file_path = upload_dir + file.name
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to "+file_path+". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select value from data")
inserted = []
for row in cur:
json_insert = clean_keys(json.JSONDecoder().decode(convert_string(row)))
print json_insert
# Insert into PDS
pds_data= {}
pds_data['time']=json_insert.get('timestamp')
pds_data['value']=json_insert
pds_data['key']=json_insert.get('probe')
insert_pds(profile, token, pds_data)
print "inserting row..."
print pds_data
inserted.append(convert_string(json_insert)+'\n')
result = {'success': True, 'message':''.join(inserted)}
except Exception as e:
result = {'success':False, 'error_message':e.message}
finally:
response_dict = {"status":"success"}
return HttpResponse(json.dumps(response_dict), content_type='application/json')
from dbdecrypt import decrypt_if_not_db_file from decrypt import key_from_password from utils import SECURE_settings from utils import fail from django.conf import settings from connectors.connector_funf.models import ConnectorFunf import connectors.connector_funf.database_single_population as database_single_population import connectors.connectors_config import time import pdb from sensible_audit import audit log = audit.getLogger(__name__) myConnector = connectors.connectors_config.CONNECTORS['ConnectorFunf']['config'] key = key_from_password(SECURE_settings.CONNECTORS['connector_funf']['db_pass']); def decrypt(): decrypt_directory() def decrypt_directory(directory_to_decrypt=myConnector['upload_path']): ''' NEW VERSION: moves a file, decrypts it, moves on to the next file ''' raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db')] failed_filenames = [] already_moved_counter = 0;
if __name__ == "__main__":
usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]"
description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. "
parser = OptionParser(usage="%s\n\n%s" % (usage, description))
parser.add_option(
"-i",
"--inplace",
dest="extension",
default=None,
help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'."
% decrypt.default_extension,
)
parser.add_option(
"-k",
"--key",
dest="key",
default=None,
help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",
)
(options, args) = parser.parse_args()
key = options.key if options.key else decrypt.key_from_password(decrypt.prompt_for_password())
try:
for file_name in args:
decrypt_if_not_db_file(file_name, key, options.extension)
except Exception as e:
import sys
sys.exit("ERROR: " + str(e))
def decrypt_if_not_db_file(file_name, key, extension=None):
if is_funf_database(file_name):
print "Already decrypted: '%s'" % file_name
return True
else:
print ("Attempting to decrypt: '%s'..." % file_name),
decrypt.decrypt([file_name], key, extension)
if is_funf_database(file_name):
print "Success!"
os.remove(file_name+'.orig')
return True
else:
print "FAILED!!!!"
os.remove(file_name)
os.remove(file_name+'.orig')
return False
if __name__ == '__main__':
usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]"
description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. "
parser = OptionParser(usage="%s\n\n%s" % (usage, description))
parser.add_option("-i", "--inplace", dest="extension", default=None,
help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension,)
parser.add_option("-k", "--key", dest="key", default=None,
help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",)
(options, args) = parser.parse_args()
key = decrypt.key_from_password(options.key)
for file_name in args:
decrypt_if_not_db_file(file_name, key, options.extension)
if is_funf_database(file_name):
print "Already decrypted: '%s'" % file_name
return True
else:
print ("Attempting to decrypt: '%s'..." % file_name),
decrypt.decrypt([file_name], key, extension)
if is_funf_database(file_name):
print "Success!"
return True
else:
print "FAILED!!!!"
print "File is either encrypted with another method, another key, or is not a valid sqlite3 db file."
print "Keeping original file."
shutil.move(decrypt.backup_file(file_name, extension), file_name)
return False
if __name__ == '__main__':
usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]"
description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. "
parser = OptionParser(usage="%s\n\n%s" % (usage, description))
parser.add_option("-i", "--inplace", dest="extension", default=None,
help="The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'." % decrypt.default_extension,)
parser.add_option("-k", "--key", dest="key", default=None,
help="The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",)
(options, args) = parser.parse_args()
#key = options.key if options.key else decrypt.key_from_password(decrypt.prompt_for_password())
key = decrypt.key_from_password("changeme");
for file_name in args:
decrypt_if_not_db_file(file_name, key, options.extension)
import os
import shutil
import fnmatch
from dbdecrypt import decrypt_if_not_db_file
from decrypt import key_from_password
from utils import SECURE_settings
from utils import log, fail
from django.conf import settings
from connectors.connector_funf.models import ConnectorFunf
import connectors.connector_funf.database_single_population as database_single_population
import pdb
mConnector = ConnectorFunf.objects.all()[0]
key = key_from_password(
SECURE_settings.CONNECTORS['connector_funf']['db_pass'])
def decrypt():
decrypt_directory()
def decrypt_directory(directory_to_decrypt=mConnector.upload_path):
'''
NEW VERSION: moves a file, decrypts it, moves on to the next file
'''
raw_filenames = [
filename for filename in os.listdir(directory_to_decrypt)
if fnmatch.fnmatch(filename, '*.db')
]
print "Keeping original file."
shutil.move(decrypt.backup_file(file_name, extension), file_name)
return False
if __name__ == '__main__':
usage = "%prog [options] [sqlite_file1.db [sqlite_file2.db...]]"
description = "Safely decrypt Sqlite3 db files. Checks to see if the file can be opened by Sqlite. If so, the file is left alone, otherwise the file is decrypted. Uses the decrypt script, so it always keeps a backup of the original encrypted files. "
parser = OptionParser(usage="%s\n\n%s" % (usage, description))
parser.add_option(
"-i",
"--inplace",
dest="extension",
default=None,
help=
"The extension to rename the original file to. Will not overwrite file if it already exists. Defaults to '%s'."
% decrypt.default_extension,
)
parser.add_option(
"-k",
"--key",
dest="key",
default=None,
help=
"The DES key used to decrypt the files. Uses the default hard coded one if one is not supplied.",
)
(options, args) = parser.parse_args()
key = options.key if options.key else decrypt.key_from_password(
decrypt.prompt_for_password())
for file_name in args:
decrypt_if_not_db_file(file_name, key, options.extension)
import sys import os import shutil import fnmatch from dbdecrypt import decrypt_if_not_db_file from decrypt import key_from_password from utils import service_config from utils import SECURE_service_config from utils import log, fail key = key_from_password(SECURE_service_config.CONNECTORS['connector_funf']['db_pass']) def decrypt(): decrypt_directory() def decrypt_directory(directory_to_decrypt=service_config.CONNECTORS['connector_funf']['config']['upload_path']): decrypted_directory_path = service_config.CONNECTORS['connector_funf']['config']['decrypted_path'] decryption_failed_path = service_config.CONNECTORS['connector_funf']['config']['decryption_failed_path'] backup_path = service_config.CONNECTORS['connector_funf']['config']['backup_path'] #TODO #raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db')] raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.orig')] filenames = [os.path.join(directory_to_decrypt, filename) for filename in raw_filenames] proc_dir = os.path.join(directory_to_decrypt, 'processing') failed_filenames = [] for f in filenames: try: shutil.move(f, proc_dir) except Exception as e: fail.fail(f, decryption_failed_path, 'Exception thrown: ' + str(e) + '. While decrypting file: ' + f)
def data(request):
'''decrypt funf database files, and upload them to your PDS'''
result = {}
connection = None
token = request.GET['bearer_token']
if request.method == 'GET':
template = {'token': token}
return render_to_response('upload.html', template,
RequestContext(request))
pds = None
scope = AccessRange.objects.get(key="funf_write")
authenticator = JSONAuthenticator(scope=scope)
try:
# Validate the request.
authenticator.validate(request)
except AuthenticationException as e:
# Return an error response.
print e
return authenticator.error_response(content="You didn't authenticate.")
profile = authenticator.user.get_profile()
funf_password = profile.funf_password
print "Registry set_funf_data for uuid: %s" % profile.uuid
for filename, file in request.FILES.items():
try:
file_path = upload_dir + file.name
try:
key = decrypt.key_from_password(str(funf_password))
write_file(str(file_path), file)
except Exception as ex:
print "failed to write file to " + file_path + ". Please make sure you have write permission to the directory set in settings.SERVER_UPLOAD_DIR"
dbdecrypt.decrypt_if_not_db_file(file_path, key)
con = sqlite3.connect(file_path)
cur = con.cursor()
cur.execute("select name, value from data")
inserted = []
for row in cur:
name = convert_string(row[0])
json_insert = clean_keys(json.JSONDecoder().decode(
convert_string(row[1])))
#print json_insert
# Insert into PDS
pds_data = {}
pds_data['time'] = json_insert.get('timestamp')
pds_data['value'] = json_insert
pds_data['key'] = name
insert_pds(profile, token, pds_data)
#print "inserting row..."
#print pds_data
inserted.append(convert_string(json_insert) + '\n')
result = {'success': True, 'message': ''.join(inserted)}
#remove_file(file_path)
except Exception as e:
print e.message
print traceback.format_exc()
result = {'success': False, 'error_message': e.message}
# It doesn't matter what we return at this point - the phones are just checking the response status
response_dict = {"status": "success"}
return HttpResponse(json.dumps(response_dict),
content_type='application/json')
