def post_download_maintenance(database_options, download_list, period=None, verbose=False): """ Perform tasks that require all data to be downloaded first, such as the source cross validator function. :param database_options: Dictionary of the postgres database options :param download_list: List of dictionaries, with each dictionary containing all of the relevant variables for the specific source :param period: Optional integer indicating the prior number of days whose values should be cross validated. If None is provided, then the entire set of values will be validated. :param verbose: Boolean of whether debugging prints should occur. """ intervals = {} for source in download_list: if source['interval'] == 'daily': intervals['daily_prices'] = True elif source['interval'] == 'minute': print('The cross validator for minute prices is disabled because ' 'there is currently only one source (Google Finance). Thus, ' 'it does not make sense to run it. When you add a second ' 'source you can re-enable it by un-commenting out the line ' 'below this message in post_download_maintenance within ' 'pySecMaster.py') # intervals['minute_prices'] = True else: raise SystemError('No interval was provided for %s in ' 'data_download in pySecMaster.py' % source['interval']) for key, value in intervals.items(): # The key is the table name to process table = key if verbose: print('Starting cross validator for %s' % table) tsids_df = query_all_active_tsids( database=database_options['database'], user=database_options['user'], password=database_options['password'], host=database_options['host'], port=database_options['port'], table=table, period=period) tsid_list = tsids_df['tsid'].values CrossValidate(database=database_options['database'], user=database_options['user'], password=database_options['password'], host=database_options['host'], port=database_options['port'], table=table, tsid_list=tsid_list, period=period, verbose=verbose)
def post_download_maintenance(database_options, download_list, period=None, verbose=False): """ Perform tasks that require all data to be downloaded first, such as the source cross validator function. :param database_options: Dictionary of the postgres database options :param download_list: List of dictionaries, with each dictionary containing all of the relevant variables for the specific source :param period: Optional integer indicating the prior number of days whose values should be cross validated. If None is provided, then the entire set of values will be validated. :param verbose: Boolean of whether debugging prints should occur. """ intervals = {} for source in download_list: if source['interval'] == 'daily': intervals['daily_prices'] = True elif source['interval'] == 'minute': print('The cross validator for minute prices is disabled because ' 'there is currently only one source (Google Finance). Thus, ' 'it does not make sense to run it. When you add a second ' 'source you can re-enable it by un-commenting out the line ' 'below this message in post_download_maintenance within ' 'pySecMaster.py') # intervals['minute_prices'] = True else: raise SystemError('No interval was provided for %s in ' 'data_download in pySecMaster.py' % source['interval']) for key, value in intervals.items(): # The key is the table name to process table = key if verbose: print('Starting cross validator for %s' % table) tsids_df = query_all_active_tsids( database=database_options['database'], user=database_options['user'], password=database_options['password'], host=database_options['host'], port=database_options['port'], table=table, period=period) tsid_list = tsids_df['tsid'].values CrossValidate( database=database_options['database'], user=database_options['user'], password=database_options['password'], host=database_options['host'], port=database_options['port'], table=table, tsid_list=tsid_list, period=period, verbose=verbose)
if self.verbose: print('%s data cross-validation took %0.2f seconds to complete.' % (tsid, time.time() - tsid_start)) if __name__ == '__main__': from utilities.user_dir import user_dir userdir = user_dir() test_table = 'daily_prices' test_tsids_df = query_all_active_tsids( database=userdir['postgresql']['pysecmaster_db'], user=userdir['postgresql']['pysecmaster_user'], password=userdir['postgresql']['pysecmaster_password'], host=userdir['postgresql']['pysecmaster_host'], port=userdir['postgresql']['pysecmaster_port'], table=test_table) test_tsid_list = test_tsids_df['tsid'].values CrossValidate( database=userdir['postgresql']['pysecmaster_db'], user=userdir['postgresql']['pysecmaster_user'], password=userdir['postgresql']['pysecmaster_password'], host=userdir['postgresql']['pysecmaster_host'], port=userdir['postgresql']['pysecmaster_port'], table=test_table, tsid_list=test_tsid_list, verbose=True)
if self.verbose: print('%s data cross-validation took %0.2f seconds to complete.' % (tsid, time.time() - tsid_start)) if __name__ == '__main__': from utilities.user_dir import user_dir userdir = user_dir() test_table = 'daily_prices' test_tsids_df = query_all_active_tsids( database=userdir['postgresql']['pysecmaster_db'], user=userdir['postgresql']['pysecmaster_user'], password=userdir['postgresql']['pysecmaster_password'], host=userdir['postgresql']['pysecmaster_host'], port=userdir['postgresql']['pysecmaster_port'], table=test_table) test_tsid_list = test_tsids_df['tsid'].values CrossValidate(database=userdir['postgresql']['pysecmaster_db'], user=userdir['postgresql']['pysecmaster_user'], password=userdir['postgresql']['pysecmaster_password'], host=userdir['postgresql']['pysecmaster_host'], port=userdir['postgresql']['pysecmaster_port'], table=test_table, tsid_list=test_tsid_list, verbose=True)