def process_data():
# load decrypt password
with open(password_file, 'r') as file:
encryption_password = file.read().split("\n")[0]
encryption_key = decrypt.key_from_password(encryption_password)
# dbDecrypt all files in raw
db_files = glob.glob(os.path.join(raw_data_dir, '*.db'))
failed_files = []
for fl in db_files:
if not dbdecrypt.decrypt_if_not_db_file(fl, encryption_key):
failed_files.append(fl)
# merge all files in raw
decrypted_files = list(set(db_files) - set(failed_files))
dbmerge.merge(db_files=decrypted_files, out_file=processing_file, overwrite=True, attempt_salvage=True)
# replace current file
shutil.move(processing_file, merged_data_file)
# Default to all zip files in script directory
zip_files = args
if len(args) == 0:
zip_files = glob.glob( os.path.join(path, '*.zip') )
if len(zip_files) == 0:
print "Missing zip file..."
else:
os.chdir(path)
subfolder = os.path.join(path, _data_folder)
unzip_to_folder(zip_files, subfolder)
db_files = glob.glob( os.path.join(subfolder, '*.db') )
print "Funf data files are encrypted. Please enter the password you entered on the device to decrypt them."
decrypt_all(db_files, key=options.key)
dbmerge.merge(db_files, merged_data_path, overwrite=True)
viz_gen.main()
shutil.rmtree(subfolder)
try:
shutil.rmtree('kmz')
except:
print "No location data"
files_to_remove = ['bw.png', 'classic.png', 'most_visited.kml']
for file in files_to_remove:
try:
os.remove(file)
except:
# Default to all zip files in script directory
zip_files = args
if len(args) == 0:
zip_files = glob.glob(os.path.join(path, '*.zip'))
if len(zip_files) == 0:
print "Missing zip file..."
else:
os.chdir(path)
subfolder = os.path.join(path, _data_folder)
unzip_to_folder(zip_files, subfolder)
db_files = glob.glob(os.path.join(subfolder, '*.db'))
print "Funf data files are encrypted. Please enter the password you entered on the device to decrypt them."
decrypt_all(db_files, key=options.key)
dbmerge.merge(db_files, merged_data_path, overwrite=True)
viz_gen.main()
shutil.rmtree(subfolder)
try:
shutil.rmtree('kmz')
except:
print "No location data"
files_to_remove = ['bw.png', 'classic.png', 'most_visited.kml']
for file in files_to_remove:
try:
os.remove(file)
except:
pass
def merge_files(files):
"""
Decrypt a list of files, after confirming with the user.
"""
print("merging "+str(files));
merge(files)
sleep_df['sleep_duration'] = sleep_df.sleep_duration.astype(np.timedelta64)
sleep_df.index = pd.to_datetime(sleep_df.index)
return sleep_df
def run_analysis(screen_probe_csv_file, output_file):
screen_probe = pd.read_csv(screen_probe_csv_file, usecols=['timestamp', 'screenOn'],
parse_dates=['timestamp'], date_parser=lambda x: datetime.datetime.fromtimestamp(float(x)).replace(tzinfo=None))
screen_probe = screen_probe.set_index(screen_probe.timestamp)
screen_probe = screen_probe.sort_index()
screen_probe.to_csv(output_file)
if __name__ == '__main__':
parser = OptionParser("usage: %prog arg1 arg2")
(options, args) = parser.parse_args()
try:
os.chdir("/var/uploads/test_data")
# Merges all of the files in test data, overwriting the old merged.db file
dbmerge.merge(None, "/var/uploads/processed/merged.db", True)
print "Merge completed"
# Convert merged.db file to a csv file that can be processed and placed in the processed folder
for file in glob.glob("/var/uploads/processed/*.db"):
db2csv.convert(file, "/var/uploads/processed/done")
print "db2csv completed"
print args[0]
print args[1]
run_analysis(args[0], args[1])
except Exception as e:
sys.exit("ERROR: " + str(e))
import os,sys
from dbmerge import merge
# merges database already decrypted
path = "T:"
print os.listdir(path)
mergeList = [f for f in os.listdir(path) if f.find('.db') > 0 and f.find('.orig') == 0]
print mergeList
merge(mergeList)
