def __init__(self, conf):
self.config = conf
#set logging configuration
logging.basicConfig(format='%(levelname)s:%(asctime)s - %(message)s')
self.log = logging.getLogger()
self.log.setLevel(logging.getLevelName(self.config.log_level))
self.dbx = DropboxHandler(self.config.access_token, self.config.dropbox_timeout, self.config.dropbox_chunck)
def pushEquippedFileSystem(self, cloudService):
# initialize chosen session
if cloudService == 'drive':
cloudHandler = GoogleDriveHandler()
elif cloudService == 'dropbox':
# get access token if exists
accessToken = self.getSetting('dbAccessToken')
if accessToken is None:
print("> Initializing dropbox for first time")
cloudHandler = DropboxHandler(accessToken)
# update saved access code
self.setSetting('dbAccessToken', cloudHandler.access_token)
else:
print("> Error: Unrecognized cloud service")
return
# get UUIDs for equipped system
equippedSystem = self.getEquippedSystem()
if equippedSystem is None:
return
info = self.getSystemInfo(equippedSystem)
if info is None:
print("> Error: No info found for system '{}'".format(
equippedSystem))
filesInfo = info['files'] if 'files' in info else []
fileUUIDs = {}
for fileName, fInfo in filesInfo.items():
if 'uuid' in fInfo:
fileUUIDs[fileName] = fInfo['uuid']
if len(fileUUIDs) == 0:
print(
"> No encrypted files to push! Encrypt the equipped filesystem using the 'encrypt' command."
)
return
# push and timestamp (seconds) each file being pushed
for fileName, uuid in fileUUIDs.items():
# push file
if cloudService == 'drive':
res = cloudHandler.upsert_file(uuid, "crypt/")
else:
res = cloudHandler.upsert_file(uuid, "crypt/{}".format(uuid),
uuid)
if res is not None:
print("> Upload successful.")
else:
print("> Upload failed.")
return
filesInfo[fileName]['pushed'] = round(time.time())
# update the database
systems = self.db.table('systems')
System = Query()
systems.update({'files': filesInfo}, System.name == equippedSystem)
def main(argv):
config_path = ''
import_data = False
process_data = False
category = ''
try:
opts, args = getopt.getopt(argv, 'hi:c:p:',
['config=', 'import=', 'process='])
except getopt.GetoptError:
info()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
info()
sys.exit()
elif opt in ('-c', '--config'):
config_path = arg
elif opt in ('-i', '--import'):
import_data = bool(arg)
elif opt in ('-p', '--process'):
process_data = bool(arg)
try:
config = Config(config_path)
#set logging
logging.basicConfig(
format=
'%(levelname)s:%(asctime)s - %(pathname)s:%(lineno)d: %(message)s')
log = logging.getLogger()
log.setLevel(logging.getLevelName(config.log_level))
except Exception as ex:
print('There has been an error while initializing configuration.\n%s' %
(ex))
sys.exit(1)
try:
dbx = DropboxHandler(config.access_token, config.dropbox_timeout,
config.dropbox_chunck)
dbx.set_log(log)
except Exception as ex:
log.error(
'There has been an error while initializing dropbox handler.\n%s' %
(ex))
sys.exit(1)
try:
#import data from Google trends
if (import_data):
#download tickers file
if (not path.exists(config.tickers_folder)):
dbx.download_file(config.tickers_path, config.tickers_folder)
#download gtrends data
gt = GTrends(config.encoding, config.tz,
config.gtrends_timeout_connect,
config.gtrends_timeout_read, config.retries,
config.backoff_factor, config.geo, dbx)
gt.set_log(log)
download_all = gt.import_data(
config.tickers_folder, config.year_from, config.year_until,
config.categories, config.data_folder_monthly,
config.data_folder_daily, config.data_folder_monthly_dropbox,
config.data_folder_daily_dropbox)
print('download_all=%s' % str(download_all))
except Exception as ex:
log.error('There has been an error while importing data.\n%s' % (ex))
sys.exit(1)
try:
#process data and upload to dropbox
if (process_data):
p = Processor(config.prefix, config.output_size_mb, dbx)
p.set_log(log)
for category in config.categories:
category_type = category.split(':')
category_name = category_type[0]
category_type = category_type[1]
if (category_type == 'monthly'):
p.TL_data(config.data_folder_monthly_dropbox,
config.dropbox_folder_upload_monthly,
config.tmp_folder_monthly,
config.result_folder_monthly, 'monthly.csv',
category_name)
else:
p.TL_data(config.data_folder_daily_dropbox,
config.dropbox_folder_upload_daily,
config.tmp_folder_daily,
config.result_folder_daily, 'daily.csv',
category_name)
except Exception as ex:
log.error('There has been an error while processing data.\n%s' % (ex))
sys.exit(1)
sys.exit(0)
def pullEquippedFileSystem(self, cloudService):
# initialize chosen session
if cloudService == 'drive':
cloudHandler = GoogleDriveHandler()
elif cloudService == 'dropbox':
# get access token if exists
accessToken = self.getSetting('dbAccessToken')
if accessToken is None:
print("> Initializing dropbox for first time")
cloudHandler = DropboxHandler(accessToken)
# update saved access code
self.setSetting('dbAccessToken', cloudHandler.access_token)
else:
print("> Error: Unrecognized cloud service")
return
# get UUIDs for equipped system
equippedSystem = self.getEquippedSystem()
if equippedSystem is None:
return
info = self.getSystemInfo(equippedSystem)
if info is None:
print("> Error: No info found for system '{}'".format(
equippedSystem))
filesInfo = info['files'] if 'files' in info else []
fileUUIDs = {}
for fileName, fInfo in filesInfo.items():
if 'uuid' in fInfo:
fileUUIDs[fileName] = fInfo['uuid']
if len(fileUUIDs) == 0:
print(
"> No encrypted files to pull! Encrypt the equipped filesystem using the 'encrypt' command and push it using 'push'"
)
return
# pull and timestamp (seconds) each file being pushed
for fileName, uuid in fileUUIDs.items():
# ensure last encryption was before it was pushed
if 'encrypted' not in filesInfo[fileName]:
print(
"> File system not yet encrypted! Encrypt it using 'encrypt' and push using 'push'."
)
return
if (filesInfo[fileName]['encrypted'] >=
filesInfo[fileName]['pushed']):
cfrm = input(
"> Looks like file '{}' was re-encrypted locally after it was pushed. Resync by pushing now?"
.format(fileName))
if str.lower(cfrm) != "yes":
print("> Operation aborted.")
return
self.pushEquippedFileSystem(cloudService)
return
# create crypt if doesn't exist
if not os.path.exists('crypt'):
os.mkdir('crypt')
# pull file
if cloudService == 'drive':
res = cloudHandler.download_file(uuid)
elif cloudService == 'dropbox':
res = cloudHandler.download_file("crypt/{}".format(uuid), uuid)
else:
print("> Error: Unrecognized cloud service")
return
if res is not None:
print("> Download successful.")
else:
print(" Download failed.")
return
# timestamp with pulled time
filesInfo[fileName]['pulled'] = round(time.time())
print(
"> Successfully '{}' ({}) from {}. Decrypt the filesystem to import it from the crypt."
.format(uuid, fileName, cloudService))
# update the database
systems = self.db.table('systems')
System = Query()
systems.update({'files': filesInfo}, System.name == equippedSystem)
