Beispiel #1
0
 def __init__(self, conf):
     self.config = conf
     #set logging configuration
     logging.basicConfig(format='%(levelname)s:%(asctime)s - %(message)s')
     self.log = logging.getLogger()
     self.log.setLevel(logging.getLevelName(self.config.log_level))        
     self.dbx = DropboxHandler(self.config.access_token, self.config.dropbox_timeout, self.config.dropbox_chunck)
    def pushEquippedFileSystem(self, cloudService):

        # initialize chosen session
        if cloudService == 'drive':
            cloudHandler = GoogleDriveHandler()

        elif cloudService == 'dropbox':
            # get access token if exists
            accessToken = self.getSetting('dbAccessToken')
            if accessToken is None:
                print("> Initializing dropbox for first time")
            cloudHandler = DropboxHandler(accessToken)

            # update saved access code
            self.setSetting('dbAccessToken', cloudHandler.access_token)
        else:
            print("> Error: Unrecognized cloud service")
            return

        # get UUIDs for equipped system
        equippedSystem = self.getEquippedSystem()
        if equippedSystem is None:
            return
        info = self.getSystemInfo(equippedSystem)
        if info is None:
            print("> Error: No info found for system '{}'".format(
                equippedSystem))
        filesInfo = info['files'] if 'files' in info else []
        fileUUIDs = {}
        for fileName, fInfo in filesInfo.items():
            if 'uuid' in fInfo:
                fileUUIDs[fileName] = fInfo['uuid']

        if len(fileUUIDs) == 0:
            print(
                "> No encrypted files to push! Encrypt the equipped filesystem using the 'encrypt' command."
            )
            return

        # push and timestamp (seconds) each file being pushed
        for fileName, uuid in fileUUIDs.items():

            # push file
            if cloudService == 'drive':
                res = cloudHandler.upsert_file(uuid, "crypt/")
            else:
                res = cloudHandler.upsert_file(uuid, "crypt/{}".format(uuid),
                                               uuid)
            if res is not None:
                print("> Upload successful.")
            else:
                print("> Upload failed.")
                return

            filesInfo[fileName]['pushed'] = round(time.time())

        # update the database
        systems = self.db.table('systems')
        System = Query()
        systems.update({'files': filesInfo}, System.name == equippedSystem)
Beispiel #3
0
def main(argv):
    config_path = ''
    import_data = False
    process_data = False
    category = ''
    try:
        opts, args = getopt.getopt(argv, 'hi:c:p:',
                                   ['config=', 'import=', 'process='])
    except getopt.GetoptError:
        info()
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            info()
            sys.exit()
        elif opt in ('-c', '--config'):
            config_path = arg
        elif opt in ('-i', '--import'):
            import_data = bool(arg)
        elif opt in ('-p', '--process'):
            process_data = bool(arg)

    try:
        config = Config(config_path)

        #set logging
        logging.basicConfig(
            format=
            '%(levelname)s:%(asctime)s - %(pathname)s:%(lineno)d: %(message)s')
        log = logging.getLogger()
        log.setLevel(logging.getLevelName(config.log_level))
    except Exception as ex:
        print('There has been an error while initializing configuration.\n%s' %
              (ex))
        sys.exit(1)

    try:
        dbx = DropboxHandler(config.access_token, config.dropbox_timeout,
                             config.dropbox_chunck)
        dbx.set_log(log)
    except Exception as ex:
        log.error(
            'There has been an error while initializing dropbox handler.\n%s' %
            (ex))
        sys.exit(1)
    try:
        #import data from Google trends
        if (import_data):
            #download tickers file
            if (not path.exists(config.tickers_folder)):
                dbx.download_file(config.tickers_path, config.tickers_folder)

            #download gtrends data
            gt = GTrends(config.encoding, config.tz,
                         config.gtrends_timeout_connect,
                         config.gtrends_timeout_read, config.retries,
                         config.backoff_factor, config.geo, dbx)
            gt.set_log(log)
            download_all = gt.import_data(
                config.tickers_folder, config.year_from, config.year_until,
                config.categories, config.data_folder_monthly,
                config.data_folder_daily, config.data_folder_monthly_dropbox,
                config.data_folder_daily_dropbox)
            print('download_all=%s' % str(download_all))
    except Exception as ex:
        log.error('There has been an error while importing data.\n%s' % (ex))
        sys.exit(1)

    try:
        #process data and upload to dropbox
        if (process_data):
            p = Processor(config.prefix, config.output_size_mb, dbx)
            p.set_log(log)
            for category in config.categories:
                category_type = category.split(':')
                category_name = category_type[0]
                category_type = category_type[1]
                if (category_type == 'monthly'):
                    p.TL_data(config.data_folder_monthly_dropbox,
                              config.dropbox_folder_upload_monthly,
                              config.tmp_folder_monthly,
                              config.result_folder_monthly, 'monthly.csv',
                              category_name)
                else:
                    p.TL_data(config.data_folder_daily_dropbox,
                              config.dropbox_folder_upload_daily,
                              config.tmp_folder_daily,
                              config.result_folder_daily, 'daily.csv',
                              category_name)

    except Exception as ex:
        log.error('There has been an error while processing data.\n%s' % (ex))
        sys.exit(1)

    sys.exit(0)
    def pullEquippedFileSystem(self, cloudService):
        # initialize chosen session
        if cloudService == 'drive':
            cloudHandler = GoogleDriveHandler()

        elif cloudService == 'dropbox':
            # get access token if exists
            accessToken = self.getSetting('dbAccessToken')
            if accessToken is None:
                print("> Initializing dropbox for first time")
            cloudHandler = DropboxHandler(accessToken)

            # update saved access code
            self.setSetting('dbAccessToken', cloudHandler.access_token)

        else:
            print("> Error: Unrecognized cloud service")
            return

        # get UUIDs for equipped system
        equippedSystem = self.getEquippedSystem()
        if equippedSystem is None:
            return
        info = self.getSystemInfo(equippedSystem)
        if info is None:
            print("> Error: No info found for system '{}'".format(
                equippedSystem))
        filesInfo = info['files'] if 'files' in info else []
        fileUUIDs = {}
        for fileName, fInfo in filesInfo.items():
            if 'uuid' in fInfo:
                fileUUIDs[fileName] = fInfo['uuid']

        if len(fileUUIDs) == 0:
            print(
                "> No encrypted files to pull! Encrypt the equipped filesystem using the 'encrypt' command and push it using 'push'"
            )
            return

        # pull and timestamp (seconds) each file being pushed
        for fileName, uuid in fileUUIDs.items():

            # ensure last encryption was before it was pushed
            if 'encrypted' not in filesInfo[fileName]:
                print(
                    "> File system not yet encrypted! Encrypt it using 'encrypt' and push using 'push'."
                )
                return
            if (filesInfo[fileName]['encrypted'] >=
                    filesInfo[fileName]['pushed']):
                cfrm = input(
                    "> Looks like file '{}' was re-encrypted locally after it was pushed. Resync by pushing now?"
                    .format(fileName))
                if str.lower(cfrm) != "yes":
                    print("> Operation aborted.")
                    return
                self.pushEquippedFileSystem(cloudService)
                return

            # create crypt if doesn't exist
            if not os.path.exists('crypt'):
                os.mkdir('crypt')

            # pull file
            if cloudService == 'drive':
                res = cloudHandler.download_file(uuid)
            elif cloudService == 'dropbox':
                res = cloudHandler.download_file("crypt/{}".format(uuid), uuid)
            else:
                print("> Error: Unrecognized cloud service")
                return

            if res is not None:
                print("> Download successful.")
            else:
                print(" Download failed.")
                return

            # timestamp with pulled time
            filesInfo[fileName]['pulled'] = round(time.time())
            print(
                "> Successfully '{}' ({}) from {}. Decrypt the filesystem to import it from the crypt."
                .format(uuid, fileName, cloudService))

        # update the database
        systems = self.db.table('systems')
        System = Query()
        systems.update({'files': filesInfo}, System.name == equippedSystem)