def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(mConnector.decrypted_path, f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False
try:
# check if still exists, might have been moved in another thread
if os.path.exists(
upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
if decrypt_if_not_db_file(proc_filename, key, extension=None):
decryption_success = True
fail.safe_move(proc_filename, mConnector.decrypted_path)
log.log('Debug', 'Still here #1')
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
#log.log('Debug','Still here #2')
database_single_population.load_file(f)
return True
else:
return False
except Exception as e:
#find out when it happened
action = ''
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if not str(e).contains('already exists'):
fail.fail(
curr_filename, mConnector.decryption_failed_path,
'Exception thrown: ' + str(e) + '. While ' + action +
' file: ' + f)
log.log('error', 'README ^^^^^^^^^^^^^')
else:
log.log(
'error', 'Exception thrown: ' + str(e) + '. While ' +
action + ' file: ' + f)
except Exception as e1:
pass
return False
'''
def load_file(filename):
#pdb.set_trace()
log.log('Debug', 'Trying to populate db with ' + filename);
mConnector = ConnectorFunf.objects.all()[0]
db = database.Database()
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
proc_dir = os.path.join(mConnector.decrypted_path, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(mConnector.decrypted_path, filename)
processing_filepath = os.path.join(proc_dir,filename)
current_filepath = decrypted_filepath
if os.path.exists(decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac']) = \
cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac from file_info').fetchone()
meta['device_id'] = anonymizerObject.anonymizeValue('device_id',meta['device_id'])
#pdb.set_trace()
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
meta['user'] = '******'
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject )
if doc == None:
continue
documents_to_insert[doc['probe']].append(dict(doc.items() + meta.items()))
cursor.close();
#pdb.set_trace()
for probe in documents_to_insert:
db.insert(documents_to_insert[probe], probe)
os.remove(current_filepath);
except Exception as e:
log.log('Error', str(e));
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return False
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(myConnector['decrypted_path'], f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False;
try:
# check if still exists, might have been moved in another thread
if os.path.exists(upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
decryption_start = time.time();
if decrypt_if_not_db_file(proc_filename, key, extension=None):
log.debug({'dtime': time.time()-decryption_start})
decryption_success = True;
fail.safe_move(proc_filename, myConnector['decrypted_path'])
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
else:
fail.fail(curr_filename, myConnector['decryption_failed_path'], 'Could not decrypt file: ' + f)
return False
return True
else:
return False
except Exception as e:
#find out when it happened
action = '';
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if 'already exists' not in str(e):
fail.fail(curr_filename, myConnector['decryption_failed_path'], 'Exception thrown: ' + str(e) + '. While ' + action + ' file: ' + f)
else:
log.error({'error': str(e), 'action': str(action), 'file': str(f)})
except Exception as e1:
pass
return False;
'''
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(mConnector.decrypted_path, f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False;
try:
# check if still exists, might have been moved in another thread
if os.path.exists(upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
if decrypt_if_not_db_file(proc_filename, key, extension=None):
decryption_success = True;
fail.safe_move(proc_filename, mConnector.decrypted_path)
log.log('Debug','Still here #1')
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
#log.log('Debug','Still here #2')
database_single_population.load_file(f)
return True
else:
return False
except Exception as e:
#find out when it happened
action = '';
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if not str(e).contains('already exists'):
fail.fail(curr_filename, mConnector.decryption_failed_path, 'Exception thrown: ' + str(e) + '. While ' + action + ' file: ' + f)
log.log('error', 'README ^^^^^^^^^^^^^')
else:
log.log('error','Exception thrown: ' + str(e) + '. While ' + action + ' file: ' + f);
except Exception as e1:
pass
return False;
'''
def decrypt_directory(directory_to_decrypt=service_config.CONNECTORS['connector_funf']['config']['upload_path']): decrypted_directory_path = service_config.CONNECTORS['connector_funf']['config']['decrypted_path'] decryption_failed_path = service_config.CONNECTORS['connector_funf']['config']['decryption_failed_path'] backup_path = service_config.CONNECTORS['connector_funf']['config']['backup_path'] #TODO #raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.db')] raw_filenames = [filename for filename in os.listdir(directory_to_decrypt) if fnmatch.fnmatch(filename, '*.orig')] filenames = [os.path.join(directory_to_decrypt, filename) for filename in raw_filenames] proc_dir = os.path.join(directory_to_decrypt, 'processing') failed_filenames = [] for f in filenames: try: shutil.move(f, proc_dir) except Exception as e: fail.fail(f, decryption_failed_path, 'Exception thrown: ' + str(e) + '. While decrypting file: ' + f) failed_filenames.append(os.path.basename(f)) raw_filenames = [e for e in raw_filenames if e not in failed_filenames] filenames = [os.path.join(proc_dir, filename) for filename in raw_filenames] for filename in filenames: try: # If successfull we move the decrypted file to the decrypted directory if decrypt_if_not_db_file(filename, key, extension=None): fail.safe_move(filename, decrypted_directory_path) # Move original db file to original directory if it exists -- if db file is already decrypted .orig will not exist #orig_filename = filename + '.orig' orig_filename = filename if os.path.exists(orig_filename): fail.safe_move(orig_filename, backup_path) # If decryption fails we move the file to the failed directory else: fail.fail(filename, decryption_failed_path, 'Could not decrypt file: ' + filename) except Exception as e: # If anything goes wrong we move the file to the failed directory fail.fail(filename, decryption_failed_path, 'Exception thrown: ' + str(e) + '. While decrypting file: ' + filename)
def run(db):
print 'running'
authorizationManager = AuthorizationManager()
decrypted_path = settings.CONNECTORS['connector_funf']['config']['decrypted_path']
load_failed_path = settings.CONNECTORS['connector_funf']['config']['load_failed_path']
#TODO
raw_filenames = [filename for filename in os.listdir(decrypted_path) if fnmatch.fnmatch(filename, '*.orig')]
anonymizerObject = Anonymizer()
raw_filenames = raw_filenames[:settings.CONNECTORS['connector_funf']['config']['max_population_files']]
filenames = [os.path.join(decrypted_path, filename) for filename in raw_filenames]
print raw_filenames
proc_dir = os.path.join(decrypted_path, 'processing')
failed_filenames = []
for f in filenames:
try:
shutil.move(f, proc_dir)
except Exception as e:
failed_filenames.append(os.path.basename(f))
raw_filenames = [e for e in raw_filenames if e not in failed_filenames]
filenames = [os.path.join(proc_dir, filename) for filename in raw_filenames]
cleanFailedFilenames(failed_filenames)
cursor = None
documents_to_insert = defaultdict(list)
filenames_to_remove = []
nof_files = len(filenames)
file_count = 0
for filename in filenames:
file_count += 1
if not os.path.exists(filename):
continue
log.log('Debug', 'Populating to DB, file(%d/%d): %s' % (file_count, nof_files, filename))
try:
conn = sqlite3.connect(filename)
cursor = conn.cursor()
except Exception as e:
fail.fail(filename, load_failed_path, 'Exception thrown:' + str(e) + '. While trying to open sqlite file: ' + filename)
continue
try:
device = cursor.execute('select device from file_info').fetchone()[0]
uuid = cursor.execute('select uuid from file_info').fetchone()[0]
device_id = ''
try:
device_id = anonymizerObject.anonymizeValue('device_id',str(cursor.execute('select device_id from file_info').fetchone()[0]))
# device_id = str(cursor.execute('select device_id from file_info').fetchone()[0])
except Exception as e:
fail.fail(filename, load_failed_path, 'Exception thrown: ' + str(e) + '. While trying to extract device_id from file: ' + filename)
continue
#TODO: replace device_id with token
try:
# user = anonymizerObject.anonymizeValue('user', authorizationManager.getAuthorizationForToken('connector_funf', 'all_probes', device_id)['user'])
user = authorizationManager.getAuthorizationForToken('connector_funf', 'all_probes', device_id)['user']
except KeyError: user = None
if not user:
log.log('Debug', 'User does not exist for device id: ' + str(device_id))
fail.fail(filename, load_failed_path, 'No user found in database. Device id: ' + str(device_id))
continue
for row in cursor.execute('select * from data'):
name = row[1]
timestamp = row[2]
#TODO: separate this sanitization
data_raw = row[3].replace('android.bluetooth.device.extra.DEVICE','android_bluetooth_device_extra_DEVICE')
data_raw = data_raw.replace('android.bluetooth.device.extra.NAME', 'android_bluetooth_device_extra_NAME')
data_raw = data_raw.replace('android.bluetooth.device.extra.CLASS', 'android_bluetooth_device_extra_CLASS')
data_raw = data_raw.replace('android.bluetooth.device.extra.RSSI', 'android_bluetooth_device_extra_RSSI')
data = json.loads(data_raw)
doc = {}
doc['_id'] = hashlib.sha1(json.dumps(data)).hexdigest()+'_'+user+'_'+str(int(timestamp))
doc['uuid'] = uuid
doc['device'] = device
doc['device_id'] = device_id
doc['user'] = user
doc['probe'] = data['PROBE'].replace('.','_')
doc['data'] = anonymizerObject.anonymizeDocument(data, doc['probe'])
doc['name'] = name
doc['timestamp'] = float(timestamp)
doc['timestamp_added'] = time.time()
documents_to_insert[doc['probe']].append(doc)
except Exception as e:
fail.fail(filename, load_failed_path, 'Exception thrown: ' + str(e) + '. While extracting data from file: ' + filename)
# traceback.print_exc(file=sys.stdout)
continue
cursor.close()
log.log('Debug', 'Adding file to be populated')
filenames_to_remove.append(filename)
#TODO: make sure that the duplicates logic works
for probe in documents_to_insert:
try:
db.insert(documents_to_insert[probe], probe)
except Exception as e:
# print 'problem!!!' + probe + ' '
# traceback.print_exc(file=sys.stdout)
pass
for filename in filenames_to_remove:
print "removing ",filename
os.remove(filename)
def load_file(filename):
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
#inserted_counter = 0
proc_dir = os.path.join(myConnector['decrypted_path'], 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(myConnector['decrypted_path'], filename)
processing_filepath = os.path.join(proc_dir,filename)
current_filepath = decrypted_filepath
load_failed_path = myConnector['load_failed_path']
roles = []
if os.path.exists(decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
moving_start = time.time()
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
reading_start = time.time()
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac'], meta['timestamp']) = cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac, created from file_info').fetchone()
meta['user'] = None
try:
(user, token) = get_user_name(meta['sensible_token'], meta['device_id'], meta['timestamp'])
meta['user'] = user.username
meta['sensible_token'] = token
roles = [x.role for x in UserRole.objects.get(user=user).roles.all()]
except: pass
if meta['user'] == None:
if not os.path.exists(myConnector['decrypted_not_authorized']):
os.makedirs(myConnector['decrypted_not_authorized'])
shutil.move(current_filepath, myConnector['decrypted_not_authorized'])
return (0,0)
meta['device_id'] = anonymizerObject.anonymizeValue('device_id',meta['device_id'])
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject )
if doc == None:
continue
documents_to_insert[doc['probe']].append(dict(doc.items() + meta.items()))
cursor.close();
# for probe in documents_to_insert:
# inserted_counter += len(documents_to_insert[probe])
# try:
# db.insert(documents_to_insert[probe], probe, roles)
# except Exception as e:
# pass
os.remove(current_filepath);
except Exception as e:
log.error({'message': 'population_error', 'error':str(e)})
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return (0,0)
# return inserted_counter
return (documents_to_insert, roles)
def load_file(filename):
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
#inserted_counter = 0
proc_dir = os.path.join(myConnector['decrypted_path'], 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(myConnector['decrypted_path'], filename)
processing_filepath = os.path.join(proc_dir, filename)
current_filepath = decrypted_filepath
load_failed_path = myConnector['load_failed_path']
roles = []
if os.path.exists(
decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
moving_start = time.time()
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
reading_start = time.time()
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(
meta['device'], meta['uuid'], meta['device_id'],
meta['sensible_token'], meta['device_bt_mac'],
meta['timestamp']
) = cursor.execute(
'select device, uuid, device_id, sensible_token, device_bt_mac, created from file_info'
).fetchone()
meta['user'] = None
try:
(user, token) = get_user_name(meta['sensible_token'],
meta['device_id'],
meta['timestamp'])
meta['user'] = user.username
meta['sensible_token'] = token
roles = [
x.role
for x in UserRole.objects.get(user=user).roles.all()
]
except:
pass
if meta['user'] == None:
if not os.path.exists(myConnector['decrypted_not_authorized']):
os.makedirs(myConnector['decrypted_not_authorized'])
shutil.move(current_filepath,
myConnector['decrypted_not_authorized'])
return (0, 0)
meta['device_id'] = anonymizerObject.anonymizeValue(
'device_id', meta['device_id'])
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject)
if doc == None:
continue
documents_to_insert[doc['probe']].append(
dict(doc.items() + meta.items()))
cursor.close()
# for probe in documents_to_insert:
# inserted_counter += len(documents_to_insert[probe])
# try:
# db.insert(documents_to_insert[probe], probe, roles)
# except Exception as e:
# pass
os.remove(current_filepath)
except Exception as e:
log.error({'message': 'population_error', 'error': str(e)})
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return (0, 0)
# return inserted_counter
return (documents_to_insert, roles)
def run(db):
print 'running'
authorizationManager = AuthorizationManager()
decrypted_path = settings.CONNECTORS['connector_funf']['config'][
'decrypted_path']
load_failed_path = settings.CONNECTORS['connector_funf']['config'][
'load_failed_path']
#TODO
raw_filenames = [
filename for filename in os.listdir(decrypted_path)
if fnmatch.fnmatch(filename, '*.orig')
]
anonymizerObject = Anonymizer()
raw_filenames = raw_filenames[:settings.CONNECTORS['connector_funf']
['config']['max_population_files']]
filenames = [
os.path.join(decrypted_path, filename) for filename in raw_filenames
]
print raw_filenames
proc_dir = os.path.join(decrypted_path, 'processing')
failed_filenames = []
for f in filenames:
try:
shutil.move(f, proc_dir)
except Exception as e:
failed_filenames.append(os.path.basename(f))
raw_filenames = [e for e in raw_filenames if e not in failed_filenames]
filenames = [
os.path.join(proc_dir, filename) for filename in raw_filenames
]
cleanFailedFilenames(failed_filenames)
cursor = None
documents_to_insert = defaultdict(list)
filenames_to_remove = []
nof_files = len(filenames)
file_count = 0
for filename in filenames:
file_count += 1
if not os.path.exists(filename):
continue
log.log(
'Debug', 'Populating to DB, file(%d/%d): %s' %
(file_count, nof_files, filename))
try:
conn = sqlite3.connect(filename)
cursor = conn.cursor()
except Exception as e:
fail.fail(
filename, load_failed_path, 'Exception thrown:' + str(e) +
'. While trying to open sqlite file: ' + filename)
continue
try:
device = cursor.execute(
'select device from file_info').fetchone()[0]
uuid = cursor.execute('select uuid from file_info').fetchone()[0]
device_id = ''
try:
device_id = anonymizerObject.anonymizeValue(
'device_id',
str(
cursor.execute(
'select device_id from file_info').fetchone()[0]))
# device_id = str(cursor.execute('select device_id from file_info').fetchone()[0])
except Exception as e:
fail.fail(
filename, load_failed_path, 'Exception thrown: ' + str(e) +
'. While trying to extract device_id from file: ' +
filename)
continue
#TODO: replace device_id with token
try:
# user = anonymizerObject.anonymizeValue('user', authorizationManager.getAuthorizationForToken('connector_funf', 'all_probes', device_id)['user'])
user = authorizationManager.getAuthorizationForToken(
'connector_funf', 'all_probes', device_id)['user']
except KeyError:
user = None
if not user:
log.log('Debug',
'User does not exist for device id: ' + str(device_id))
fail.fail(
filename, load_failed_path,
'No user found in database. Device id: ' + str(device_id))
continue
for row in cursor.execute('select * from data'):
name = row[1]
timestamp = row[2]
#TODO: separate this sanitization
data_raw = row[3].replace(
'android.bluetooth.device.extra.DEVICE',
'android_bluetooth_device_extra_DEVICE')
data_raw = data_raw.replace(
'android.bluetooth.device.extra.NAME',
'android_bluetooth_device_extra_NAME')
data_raw = data_raw.replace(
'android.bluetooth.device.extra.CLASS',
'android_bluetooth_device_extra_CLASS')
data_raw = data_raw.replace(
'android.bluetooth.device.extra.RSSI',
'android_bluetooth_device_extra_RSSI')
data = json.loads(data_raw)
doc = {}
doc['_id'] = hashlib.sha1(
json.dumps(data)).hexdigest() + '_' + user + '_' + str(
int(timestamp))
doc['uuid'] = uuid
doc['device'] = device
doc['device_id'] = device_id
doc['user'] = user
doc['probe'] = data['PROBE'].replace('.', '_')
doc['data'] = anonymizerObject.anonymizeDocument(
data, doc['probe'])
doc['name'] = name
doc['timestamp'] = float(timestamp)
doc['timestamp_added'] = time.time()
documents_to_insert[doc['probe']].append(doc)
except Exception as e:
fail.fail(
filename, load_failed_path, 'Exception thrown: ' + str(e) +
'. While extracting data from file: ' + filename)
# traceback.print_exc(file=sys.stdout)
continue
cursor.close()
log.log('Debug', 'Adding file to be populated')
filenames_to_remove.append(filename)
#TODO: make sure that the duplicates logic works
for probe in documents_to_insert:
try:
db.insert(documents_to_insert[probe], probe)
except Exception as e:
# print 'problem!!!' + probe + ' '
# traceback.print_exc(file=sys.stdout)
pass
for filename in filenames_to_remove:
print "removing ", filename
os.remove(filename)
def decrypt_file(directory_to_decrypt, f):
#pdb.set_trace()
proc_dir = os.path.join(directory_to_decrypt, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
upload_filename = os.path.join(directory_to_decrypt, f)
proc_filename = os.path.join(proc_dir, f)
decrypted_filename = os.path.join(myConnector['decrypted_path'], f)
curr_filename = upload_filename #for keeping track of the file's current location
decryption_success = False
try:
# check if still exists, might have been moved in another thread
if os.path.exists(
upload_filename) and not os.path.exists(proc_filename):
# move it to processing
shutil.move(upload_filename, proc_dir)
curr_filename = proc_filename
# decrypt
decryption_start = time.time()
if decrypt_if_not_db_file(proc_filename, key, extension=None):
log.debug({'dtime': time.time() - decryption_start})
decryption_success = True
fail.safe_move(proc_filename, myConnector['decrypted_path'])
curr_filename = decrypted_filename
orig_filename = proc_filename + '.orig'
if os.path.exists(orig_filename):
os.remove(orig_filename)
else:
fail.fail(curr_filename, myConnector['decryption_failed_path'],
'Could not decrypt file: ' + f)
return False
return True
else:
return False
except Exception as e:
#find out when it happened
action = ''
if curr_filename == upload_filename:
action = 'moving to /processing'
elif curr_filename == proc_filename and decryption_success == False:
action = 'decrypting'
elif curr_filename == proc_filename and decryption_success == True:
action = 'moving to /decrypted'
elif curr_filename == decrypted_filename:
action = 'removing the .orig file of'
try:
if 'already exists' not in str(e):
fail.fail(
curr_filename, myConnector['decryption_failed_path'],
'Exception thrown: ' + str(e) + '. While ' + action +
' file: ' + f)
else:
log.error({
'error': str(e),
'action': str(action),
'file': str(f)
})
except Exception as e1:
pass
return False
'''
def load_file(filename):
#pdb.set_trace()
log.log('Debug', 'Trying to populate db with ' + filename)
mConnector = ConnectorFunf.objects.all()[0]
db = database.Database()
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
proc_dir = os.path.join(mConnector.decrypted_path, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(mConnector.decrypted_path, filename)
processing_filepath = os.path.join(proc_dir, filename)
current_filepath = decrypted_filepath
if os.path.exists(
decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac']) = \
cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac from file_info').fetchone()
meta['device_id'] = anonymizerObject.anonymizeValue(
'device_id', meta['device_id'])
#pdb.set_trace()
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
meta['user'] = '******'
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject)
if doc == None:
continue
documents_to_insert[doc['probe']].append(
dict(doc.items() + meta.items()))
cursor.close()
#pdb.set_trace()
for probe in documents_to_insert:
db.insert(documents_to_insert[probe], probe)
os.remove(current_filepath)
except Exception as e:
log.log('Error', str(e))
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return False
