def cursorToArray(cursor, decrypted=False, probe=''):
array = [doc for doc in cursor]
if decrypted and 'BluetoothProbe' in probe:
anonymizer = Anonymizer()
return anonymizer.deanonymizeDocument(array, probe)
else:
return array
def cursorToArray(cursor, decrypted=False, probe="", is_researcher=False, map_to_users=False):
array = [doc for doc in cursor]
if "BluetoothProbe" in probe:
if decrypted:
anonymizer = Anonymizer()
return anonymizer.deanonymizeDocument(array, probe)
elif is_researcher and map_to_users:
deviceInventory = device_inventory.DeviceInventory()
for doc in array:
for device in doc["data"]["DEVICES"]:
try:
user_temp = deviceInventory.mapBtToUser(
device["android_bluetooth_device_extra_DEVICE"]["mAddress"],
doc["data"]["TIMESTAMP"],
use_mac_if_empty=False,
)
if user_temp is not None:
device["user"] = user_temp
else:
device["user"] = ""
except KeyError:
device["user"] = ""
return array
else:
return array
else:
return array
def cursorToArray(cursor, decrypted = False, probe = '', is_researcher=False, map_to_users=False): array = [] for row in cursor: if 'timestamp' in row: row['timestamp'] = int(time.mktime(row['timestamp'].timetuple())) if 'timestamp_added' in row: row['timestamp_added'] = int(time.mktime(row['timestamp_added'].timetuple())) array.append(row) if 'ExperienceSamplingProbe' in probe: for doc in array: doc['answer'] = json.loads(base64.b64decode(doc['answer'])) if 'EpidemicProbe' in probe: for doc in array: doc['data'] = json.loads(base64.b64decode(doc['data'])) if 'BluetoothProbe' not in probe: return array if decrypted: anonymizer = Anonymizer() return anonymizer.deanonymizeDocument(array, probe) if is_researcher and map_to_users: deviceInventory = device_inventory.DeviceInventory() for doc in array: try: user_temp = deviceInventory.mapBtToUser(doc['bt_mac'], doc['timestamp'], use_mac_if_empty=False) if user_temp is not None: doc['scanned_user'] = user_temp else: doc['scanned_user'] = '' except KeyError: doc['scanned_user'] = '' return array
def cursorToArray(cursor, decrypted = False, probe = ''): array = [doc for doc in cursor] if decrypted and 'BluetoothProbe' in probe: anonymizer = Anonymizer() return anonymizer.deanonymizeDocument(array, probe) else: return array
def load_file(filename):
#pdb.set_trace()
log.log('Debug', 'Trying to populate db with ' + filename);
mConnector = ConnectorFunf.objects.all()[0]
db = database.Database()
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
proc_dir = os.path.join(mConnector.decrypted_path, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(mConnector.decrypted_path, filename)
processing_filepath = os.path.join(proc_dir,filename)
current_filepath = decrypted_filepath
if os.path.exists(decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac']) = \
cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac from file_info').fetchone()
meta['device_id'] = anonymizerObject.anonymizeValue('device_id',meta['device_id'])
#pdb.set_trace()
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
meta['user'] = '******'
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject )
if doc == None:
continue
documents_to_insert[doc['probe']].append(dict(doc.items() + meta.items()))
cursor.close();
#pdb.set_trace()
for probe in documents_to_insert:
db.insert(documents_to_insert[probe], probe)
os.remove(current_filepath);
except Exception as e:
log.log('Error', str(e));
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return False
def get_friends_connections(request, user, scopes, users_to_return, user_roles, own_data):
collection = 'main'
if 'researcher' in user_roles: collection = 'researcher'
elif 'developer' in user_roles: collection = 'developer'
if len(users_to_return) > 1 or 'all' in users_to_return: return {'error':'This functionality is not yet implemented'}
conn, cursor = get_cursor()
friends, connections = get_ego_network(cursor, collection, users_to_return[0])
anon = Anonymizer()
return anon.deanonymizeDocument({'friends':list(friends), 'connections':list(connections)}, 'dk_dtu_compute_facebook_friends')
def cursorToArray(cursor, decrypted = False, probe = '', is_researcher=False, map_to_users=False): array = [doc for doc in cursor] if 'BluetoothProbe' in probe: if decrypted: anonymizer = Anonymizer() return anonymizer.deanonymizeDocument(array, probe) elif is_researcher and map_to_users: deviceInventory = device_inventory.DeviceInventory() for doc in array: for device in doc['data']['DEVICES']: try: user_temp = deviceInventory.mapBtToUser(device['android_bluetooth_device_extra_DEVICE']['mAddress'], doc['data']['TIMESTAMP'], use_mac_if_empty=False) if user_temp is not None: device['user'] = user_temp else: device['user'] = '' except KeyError: device['user'] = '' return array else: return array else: return array
def run(db):
print 'running'
authorizationManager = AuthorizationManager()
decrypted_path = settings.CONNECTORS['connector_funf']['config']['decrypted_path']
load_failed_path = settings.CONNECTORS['connector_funf']['config']['load_failed_path']
#TODO
raw_filenames = [filename for filename in os.listdir(decrypted_path) if fnmatch.fnmatch(filename, '*.orig')]
anonymizerObject = Anonymizer()
raw_filenames = raw_filenames[:settings.CONNECTORS['connector_funf']['config']['max_population_files']]
filenames = [os.path.join(decrypted_path, filename) for filename in raw_filenames]
print raw_filenames
proc_dir = os.path.join(decrypted_path, 'processing')
failed_filenames = []
for f in filenames:
try:
shutil.move(f, proc_dir)
except Exception as e:
failed_filenames.append(os.path.basename(f))
raw_filenames = [e for e in raw_filenames if e not in failed_filenames]
filenames = [os.path.join(proc_dir, filename) for filename in raw_filenames]
cleanFailedFilenames(failed_filenames)
cursor = None
documents_to_insert = defaultdict(list)
filenames_to_remove = []
nof_files = len(filenames)
file_count = 0
for filename in filenames:
file_count += 1
if not os.path.exists(filename):
continue
log.log('Debug', 'Populating to DB, file(%d/%d): %s' % (file_count, nof_files, filename))
try:
conn = sqlite3.connect(filename)
cursor = conn.cursor()
except Exception as e:
fail.fail(filename, load_failed_path, 'Exception thrown:' + str(e) + '. While trying to open sqlite file: ' + filename)
continue
try:
device = cursor.execute('select device from file_info').fetchone()[0]
uuid = cursor.execute('select uuid from file_info').fetchone()[0]
device_id = ''
try:
device_id = anonymizerObject.anonymizeValue('device_id',str(cursor.execute('select device_id from file_info').fetchone()[0]))
# device_id = str(cursor.execute('select device_id from file_info').fetchone()[0])
except Exception as e:
fail.fail(filename, load_failed_path, 'Exception thrown: ' + str(e) + '. While trying to extract device_id from file: ' + filename)
continue
#TODO: replace device_id with token
try:
# user = anonymizerObject.anonymizeValue('user', authorizationManager.getAuthorizationForToken('connector_funf', 'all_probes', device_id)['user'])
user = authorizationManager.getAuthorizationForToken('connector_funf', 'all_probes', device_id)['user']
except KeyError: user = None
if not user:
log.log('Debug', 'User does not exist for device id: ' + str(device_id))
fail.fail(filename, load_failed_path, 'No user found in database. Device id: ' + str(device_id))
continue
for row in cursor.execute('select * from data'):
name = row[1]
timestamp = row[2]
#TODO: separate this sanitization
data_raw = row[3].replace('android.bluetooth.device.extra.DEVICE','android_bluetooth_device_extra_DEVICE')
data_raw = data_raw.replace('android.bluetooth.device.extra.NAME', 'android_bluetooth_device_extra_NAME')
data_raw = data_raw.replace('android.bluetooth.device.extra.CLASS', 'android_bluetooth_device_extra_CLASS')
data_raw = data_raw.replace('android.bluetooth.device.extra.RSSI', 'android_bluetooth_device_extra_RSSI')
data = json.loads(data_raw)
doc = {}
doc['_id'] = hashlib.sha1(json.dumps(data)).hexdigest()+'_'+user+'_'+str(int(timestamp))
doc['uuid'] = uuid
doc['device'] = device
doc['device_id'] = device_id
doc['user'] = user
doc['probe'] = data['PROBE'].replace('.','_')
doc['data'] = anonymizerObject.anonymizeDocument(data, doc['probe'])
doc['name'] = name
doc['timestamp'] = float(timestamp)
doc['timestamp_added'] = time.time()
documents_to_insert[doc['probe']].append(doc)
except Exception as e:
fail.fail(filename, load_failed_path, 'Exception thrown: ' + str(e) + '. While extracting data from file: ' + filename)
# traceback.print_exc(file=sys.stdout)
continue
cursor.close()
log.log('Debug', 'Adding file to be populated')
filenames_to_remove.append(filename)
#TODO: make sure that the duplicates logic works
for probe in documents_to_insert:
try:
db.insert(documents_to_insert[probe], probe)
except Exception as e:
# print 'problem!!!' + probe + ' '
# traceback.print_exc(file=sys.stdout)
pass
for filename in filenames_to_remove:
print "removing ",filename
os.remove(filename)
def load_file(filename):
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
#inserted_counter = 0
proc_dir = os.path.join(myConnector['decrypted_path'], 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(myConnector['decrypted_path'], filename)
processing_filepath = os.path.join(proc_dir,filename)
current_filepath = decrypted_filepath
load_failed_path = myConnector['load_failed_path']
roles = []
if os.path.exists(decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
moving_start = time.time()
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
reading_start = time.time()
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac'], meta['timestamp']) = cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac, created from file_info').fetchone()
meta['user'] = None
try:
(user, token) = get_user_name(meta['sensible_token'], meta['device_id'], meta['timestamp'])
meta['user'] = user.username
meta['sensible_token'] = token
roles = [x.role for x in UserRole.objects.get(user=user).roles.all()]
except: pass
if meta['user'] == None:
if not os.path.exists(myConnector['decrypted_not_authorized']):
os.makedirs(myConnector['decrypted_not_authorized'])
shutil.move(current_filepath, myConnector['decrypted_not_authorized'])
return (0,0)
meta['device_id'] = anonymizerObject.anonymizeValue('device_id',meta['device_id'])
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject )
if doc == None:
continue
documents_to_insert[doc['probe']].append(dict(doc.items() + meta.items()))
cursor.close();
# for probe in documents_to_insert:
# inserted_counter += len(documents_to_insert[probe])
# try:
# db.insert(documents_to_insert[probe], probe, roles)
# except Exception as e:
# pass
os.remove(current_filepath);
except Exception as e:
log.error({'message': 'population_error', 'error':str(e)})
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return (0,0)
# return inserted_counter
return (documents_to_insert, roles)
try:
writer = OpenDataWriter(config)
is_postgres_alive = writer._db_manager.is_alive()
except Exception:
logger.log_heartbeat('Failed to initialize PostgreSQL connection.', 'opendata-anonymizer', 'FAILED')
logger.log_error('postgresql_connection_failed',
'Failed initializing postgresql database connector. ERROR: {0}'.format(
traceback.format_exc().replace('\n', '')
))
raise
try:
logger.log_info('anonymization_process_started', 'Started anonymizing logs.')
logger.log_heartbeat('Started anonymizing logs.', 'opendata-anonymizer', 'SUCCEEDED')
anonymizer_instance = Anonymizer(reader, writer, config, logger_manager=logger)
if anonymization_limit:
records = anonymizer_instance.anonymize_with_limit(anonymization_limit)
else:
records = anonymizer_instance.anonymize()
except:
logger.log_error('anonymization_process_failed',
'Failed to anonymize. ERROR: {0}'.format(
traceback.format_exc().replace('\n', '')
))
logger.log_heartbeat('Error occurred during log anonymization', 'opendata-anonymizer', 'FAILED')
raise
logger.log_heartbeat('Finished anonymization session', 'opendata-anonymizer', 'SUCCEEDED')
except:
def load_file(filename):
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
#inserted_counter = 0
proc_dir = os.path.join(myConnector['decrypted_path'], 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(myConnector['decrypted_path'], filename)
processing_filepath = os.path.join(proc_dir, filename)
current_filepath = decrypted_filepath
load_failed_path = myConnector['load_failed_path']
roles = []
if os.path.exists(
decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
moving_start = time.time()
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
reading_start = time.time()
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(
meta['device'], meta['uuid'], meta['device_id'],
meta['sensible_token'], meta['device_bt_mac'],
meta['timestamp']
) = cursor.execute(
'select device, uuid, device_id, sensible_token, device_bt_mac, created from file_info'
).fetchone()
meta['user'] = None
try:
(user, token) = get_user_name(meta['sensible_token'],
meta['device_id'],
meta['timestamp'])
meta['user'] = user.username
meta['sensible_token'] = token
roles = [
x.role
for x in UserRole.objects.get(user=user).roles.all()
]
except:
pass
if meta['user'] == None:
if not os.path.exists(myConnector['decrypted_not_authorized']):
os.makedirs(myConnector['decrypted_not_authorized'])
shutil.move(current_filepath,
myConnector['decrypted_not_authorized'])
return (0, 0)
meta['device_id'] = anonymizerObject.anonymizeValue(
'device_id', meta['device_id'])
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject)
if doc == None:
continue
documents_to_insert[doc['probe']].append(
dict(doc.items() + meta.items()))
cursor.close()
# for probe in documents_to_insert:
# inserted_counter += len(documents_to_insert[probe])
# try:
# db.insert(documents_to_insert[probe], probe, roles)
# except Exception as e:
# pass
os.remove(current_filepath)
except Exception as e:
log.error({'message': 'population_error', 'error': str(e)})
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return (0, 0)
# return inserted_counter
return (documents_to_insert, roles)
def run(db):
print 'running'
authorizationManager = AuthorizationManager()
decrypted_path = settings.CONNECTORS['connector_funf']['config'][
'decrypted_path']
load_failed_path = settings.CONNECTORS['connector_funf']['config'][
'load_failed_path']
#TODO
raw_filenames = [
filename for filename in os.listdir(decrypted_path)
if fnmatch.fnmatch(filename, '*.orig')
]
anonymizerObject = Anonymizer()
raw_filenames = raw_filenames[:settings.CONNECTORS['connector_funf']
['config']['max_population_files']]
filenames = [
os.path.join(decrypted_path, filename) for filename in raw_filenames
]
print raw_filenames
proc_dir = os.path.join(decrypted_path, 'processing')
failed_filenames = []
for f in filenames:
try:
shutil.move(f, proc_dir)
except Exception as e:
failed_filenames.append(os.path.basename(f))
raw_filenames = [e for e in raw_filenames if e not in failed_filenames]
filenames = [
os.path.join(proc_dir, filename) for filename in raw_filenames
]
cleanFailedFilenames(failed_filenames)
cursor = None
documents_to_insert = defaultdict(list)
filenames_to_remove = []
nof_files = len(filenames)
file_count = 0
for filename in filenames:
file_count += 1
if not os.path.exists(filename):
continue
log.log(
'Debug', 'Populating to DB, file(%d/%d): %s' %
(file_count, nof_files, filename))
try:
conn = sqlite3.connect(filename)
cursor = conn.cursor()
except Exception as e:
fail.fail(
filename, load_failed_path, 'Exception thrown:' + str(e) +
'. While trying to open sqlite file: ' + filename)
continue
try:
device = cursor.execute(
'select device from file_info').fetchone()[0]
uuid = cursor.execute('select uuid from file_info').fetchone()[0]
device_id = ''
try:
device_id = anonymizerObject.anonymizeValue(
'device_id',
str(
cursor.execute(
'select device_id from file_info').fetchone()[0]))
# device_id = str(cursor.execute('select device_id from file_info').fetchone()[0])
except Exception as e:
fail.fail(
filename, load_failed_path, 'Exception thrown: ' + str(e) +
'. While trying to extract device_id from file: ' +
filename)
continue
#TODO: replace device_id with token
try:
# user = anonymizerObject.anonymizeValue('user', authorizationManager.getAuthorizationForToken('connector_funf', 'all_probes', device_id)['user'])
user = authorizationManager.getAuthorizationForToken(
'connector_funf', 'all_probes', device_id)['user']
except KeyError:
user = None
if not user:
log.log('Debug',
'User does not exist for device id: ' + str(device_id))
fail.fail(
filename, load_failed_path,
'No user found in database. Device id: ' + str(device_id))
continue
for row in cursor.execute('select * from data'):
name = row[1]
timestamp = row[2]
#TODO: separate this sanitization
data_raw = row[3].replace(
'android.bluetooth.device.extra.DEVICE',
'android_bluetooth_device_extra_DEVICE')
data_raw = data_raw.replace(
'android.bluetooth.device.extra.NAME',
'android_bluetooth_device_extra_NAME')
data_raw = data_raw.replace(
'android.bluetooth.device.extra.CLASS',
'android_bluetooth_device_extra_CLASS')
data_raw = data_raw.replace(
'android.bluetooth.device.extra.RSSI',
'android_bluetooth_device_extra_RSSI')
data = json.loads(data_raw)
doc = {}
doc['_id'] = hashlib.sha1(
json.dumps(data)).hexdigest() + '_' + user + '_' + str(
int(timestamp))
doc['uuid'] = uuid
doc['device'] = device
doc['device_id'] = device_id
doc['user'] = user
doc['probe'] = data['PROBE'].replace('.', '_')
doc['data'] = anonymizerObject.anonymizeDocument(
data, doc['probe'])
doc['name'] = name
doc['timestamp'] = float(timestamp)
doc['timestamp_added'] = time.time()
documents_to_insert[doc['probe']].append(doc)
except Exception as e:
fail.fail(
filename, load_failed_path, 'Exception thrown: ' + str(e) +
'. While extracting data from file: ' + filename)
# traceback.print_exc(file=sys.stdout)
continue
cursor.close()
log.log('Debug', 'Adding file to be populated')
filenames_to_remove.append(filename)
#TODO: make sure that the duplicates logic works
for probe in documents_to_insert:
try:
db.insert(documents_to_insert[probe], probe)
except Exception as e:
# print 'problem!!!' + probe + ' '
# traceback.print_exc(file=sys.stdout)
pass
for filename in filenames_to_remove:
print "removing ", filename
os.remove(filename)
def load_file(filename):
#pdb.set_trace()
log.log('Debug', 'Trying to populate db with ' + filename)
mConnector = ConnectorFunf.objects.all()[0]
db = database.Database()
anonymizerObject = Anonymizer()
documents_to_insert = defaultdict(list)
proc_dir = os.path.join(mConnector.decrypted_path, 'processing')
if not os.path.exists(proc_dir):
os.makedirs(proc_dir)
decrypted_filepath = os.path.join(mConnector.decrypted_path, filename)
processing_filepath = os.path.join(proc_dir, filename)
current_filepath = decrypted_filepath
if os.path.exists(
decrypted_filepath) and not os.path.exists(processing_filepath):
try:
# move to processing
shutil.move(decrypted_filepath, proc_dir)
current_filepath = processing_filepath
# open connection to db file
conn = sqlite3.connect(processing_filepath)
cursor = conn.cursor()
# get the meta data from db file
meta = {}
(meta['device'], meta['uuid'], meta['device_id'], meta['sensible_token'], meta['device_bt_mac']) = \
cursor.execute('select device, uuid, device_id, sensible_token, device_bt_mac from file_info').fetchone()
meta['device_id'] = anonymizerObject.anonymizeValue(
'device_id', meta['device_id'])
#pdb.set_trace()
# get the user associated with the token
#meta['user'] = authorization_manager.getAuthorizationForToken(\
# 'connector_funf.submit_data', meta['token']).user
meta['user'] = '******'
for row in cursor.execute('select * from data'):
doc = row_to_doc(row, meta['user'], anonymizerObject)
if doc == None:
continue
documents_to_insert[doc['probe']].append(
dict(doc.items() + meta.items()))
cursor.close()
#pdb.set_trace()
for probe in documents_to_insert:
db.insert(documents_to_insert[probe], probe)
os.remove(current_filepath)
except Exception as e:
log.log('Error', str(e))
if not 'already exists' in str(e):
top = traceback.extract_stack()[-1]
fail.fail(current_filepath, load_failed_path, 'Exception with file: ' + filename\
+ '\n' + ', '.join([type(e).__name__, os.path.basename(top[0]), str(top[1])]))
else:
pass
return False
