def post_download_maintenance(database_options,
download_list,
period=None,
verbose=False):
""" Perform tasks that require all data to be downloaded first, such as the
source cross validator function.
:param database_options: Dictionary of the postgres database options
:param download_list: List of dictionaries, with each dictionary containing
all of the relevant variables for the specific source
:param period: Optional integer indicating the prior number of days whose
values should be cross validated. If None is provided, then the
entire set of values will be validated.
:param verbose: Boolean of whether debugging prints should occur.
"""
intervals = {}
for source in download_list:
if source['interval'] == 'daily':
intervals['daily_prices'] = True
elif source['interval'] == 'minute':
print('The cross validator for minute prices is disabled because '
'there is currently only one source (Google Finance). Thus, '
'it does not make sense to run it. When you add a second '
'source you can re-enable it by un-commenting out the line '
'below this message in post_download_maintenance within '
'pySecMaster.py')
# intervals['minute_prices'] = True
else:
raise SystemError('No interval was provided for %s in '
'data_download in pySecMaster.py' %
source['interval'])
for key, value in intervals.items():
# The key is the table name to process
table = key
if verbose:
print('Starting cross validator for %s' % table)
tsids_df = query_all_active_tsids(
database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
table=table,
period=period)
tsid_list = tsids_df['tsid'].values
CrossValidate(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
table=table,
tsid_list=tsid_list,
period=period,
verbose=verbose)
def post_download_maintenance(database_options, download_list, period=None,
verbose=False):
""" Perform tasks that require all data to be downloaded first, such as the
source cross validator function.
:param database_options: Dictionary of the postgres database options
:param download_list: List of dictionaries, with each dictionary containing
all of the relevant variables for the specific source
:param period: Optional integer indicating the prior number of days whose
values should be cross validated. If None is provided, then the
entire set of values will be validated.
:param verbose: Boolean of whether debugging prints should occur.
"""
intervals = {}
for source in download_list:
if source['interval'] == 'daily':
intervals['daily_prices'] = True
elif source['interval'] == 'minute':
print('The cross validator for minute prices is disabled because '
'there is currently only one source (Google Finance). Thus, '
'it does not make sense to run it. When you add a second '
'source you can re-enable it by un-commenting out the line '
'below this message in post_download_maintenance within '
'pySecMaster.py')
# intervals['minute_prices'] = True
else:
raise SystemError('No interval was provided for %s in '
'data_download in pySecMaster.py' %
source['interval'])
for key, value in intervals.items():
# The key is the table name to process
table = key
if verbose:
print('Starting cross validator for %s' % table)
tsids_df = query_all_active_tsids(
database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
table=table,
period=period)
tsid_list = tsids_df['tsid'].values
CrossValidate(
database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
table=table, tsid_list=tsid_list, period=period, verbose=verbose)
if self.verbose:
print('%s data cross-validation took %0.2f seconds to complete.' %
(tsid, time.time() - tsid_start))
if __name__ == '__main__':
from utilities.user_dir import user_dir
userdir = user_dir()
test_table = 'daily_prices'
test_tsids_df = query_all_active_tsids(
database=userdir['postgresql']['pysecmaster_db'],
user=userdir['postgresql']['pysecmaster_user'],
password=userdir['postgresql']['pysecmaster_password'],
host=userdir['postgresql']['pysecmaster_host'],
port=userdir['postgresql']['pysecmaster_port'],
table=test_table)
test_tsid_list = test_tsids_df['tsid'].values
CrossValidate(
database=userdir['postgresql']['pysecmaster_db'],
user=userdir['postgresql']['pysecmaster_user'],
password=userdir['postgresql']['pysecmaster_password'],
host=userdir['postgresql']['pysecmaster_host'],
port=userdir['postgresql']['pysecmaster_port'],
table=test_table,
tsid_list=test_tsid_list, verbose=True)
if self.verbose:
print('%s data cross-validation took %0.2f seconds to complete.' %
(tsid, time.time() - tsid_start))
if __name__ == '__main__':
from utilities.user_dir import user_dir
userdir = user_dir()
test_table = 'daily_prices'
test_tsids_df = query_all_active_tsids(
database=userdir['postgresql']['pysecmaster_db'],
user=userdir['postgresql']['pysecmaster_user'],
password=userdir['postgresql']['pysecmaster_password'],
host=userdir['postgresql']['pysecmaster_host'],
port=userdir['postgresql']['pysecmaster_port'],
table=test_table)
test_tsid_list = test_tsids_df['tsid'].values
CrossValidate(database=userdir['postgresql']['pysecmaster_db'],
user=userdir['postgresql']['pysecmaster_user'],
password=userdir['postgresql']['pysecmaster_password'],
host=userdir['postgresql']['pysecmaster_host'],
port=userdir['postgresql']['pysecmaster_port'],
table=test_table,
tsid_list=test_tsid_list,
verbose=True)
