def pysecmaster(self, settings_dict):
"""
Calls the functions that operate the pySecMaster. Emits signals back to
the main gui for further processing, using the dataReady process.
:param settings_dict: Dictionary of all parameters to be passed back
to the pySecMaster.py functions.
"""
self.dataReady.emit('Building the pySecMaster in the %s database '
'located at host %s\n' %
(settings_dict['database_options']['database'],
settings_dict['database_options']['host']))
maintenance(
database_options=settings_dict['database_options'],
quandl_key=settings_dict['quandl_key'],
quandl_ticker_source=settings_dict['quandl_ticker_source'],
database_list=settings_dict['quandl_db_list'],
threads=settings_dict['threads'],
quandl_update_range=settings_dict['quandl_update_range'],
csidata_update_range=settings_dict['google_fin_update_range'],
symbology_sources=settings_dict['symbology_sources'])
data_download(database_options=settings_dict['database_options'],
quandl_key=settings_dict['quandl_key'],
download_list=settings_dict['download_list'],
threads=settings_dict['threads'],
verbose=True)
self.dataReady.emit('Finished running the pySecMaster process\n')
self.finished.emit()
def pysecmaster(self, settings_dict):
"""
Calls the functions that operate the pySecMaster. Emits signals back to
the main gui for further processing, using the dataReady process.
:param settings_dict: Dictionary of all parameters to be passed back
to the pySecMaster.py functions.
"""
self.dataReady.emit('Building the pySecMaster in the %s database '
'located at host %s\n' %
(settings_dict['database_options']['database'],
settings_dict['database_options']['host']))
maintenance(database_options=settings_dict['database_options'],
quandl_key=settings_dict['quandl_key'],
quandl_ticker_source=settings_dict['quandl_ticker_source'],
database_list=settings_dict['quandl_db_list'],
threads=settings_dict['threads'],
quandl_update_range=settings_dict['quandl_update_range'],
csidata_update_range=settings_dict['google_fin_update_range'],
symbology_sources=settings_dict['symbology_sources'])
data_download(database_options=settings_dict['database_options'],
quandl_key=settings_dict['quandl_key'],
download_list=settings_dict['download_list'],
threads=settings_dict['threads'],
verbose=True)
self.dataReady.emit('Finished running the pySecMaster process\n')
self.finished.emit()
def main(verbose=False):
old_db_location = 'C:/Users/Josh/Desktop/pySecMaster_m old.db'
new_db_location = 'C:/Users/Josh/Desktop/pySecMaster_m.db'
table = 'minute_prices' # daily_prices, minute_prices
# Create a new database where the old prices will be copied to
symbology_sources = ['csi_data', 'tsid', 'quandl_wiki', 'quandl_goog',
'seeking_alpha', 'yahoo']
os.chdir('..') # Need to move up a folder in order to access load_tables
maintenance(database_link=new_db_location, quandl_ticker_source='csidata',
database_list=['WIKI'], threads=8, quandl_key='',
quandl_update_range=30, csidata_update_range=5,
symbology_sources=symbology_sources)
# Retrieve a list of all the tickers from the existing database table
qcodes_df = query_existing_qcodes(db_location=old_db_location,
table=table, verbose=True)
for index, row in qcodes_df.iterrows():
ticker = row['q_code']
copy_start = time.time()
# Retrieve all price data for this ticker
raw_price_df = query_qcode_data(db_location=old_db_location,
table=table, qcode=ticker,
verbose=False)
# Change the q_code column to a tsid column
clean_price_df = convert_qcode_to_tsid(db_location=new_db_location,
price_df=raw_price_df,
table=table, qcode=ticker)
tsid = clean_price_df.loc[0, 'tsid']
# If there is no tsid, don't attempt to insert the data to the database
if tsid:
# Add the data to the database if there is not existing data
insert_df_to_db(db_location=new_db_location,
price_df=clean_price_df, table=table, verbose=True)
if verbose:
print('Moving the %s from %s to %s took %0.2f seconds' %
(table, ticker, tsid, time.time() - copy_start))
def main(verbose=False):
old_db_location = 'C:/Users/Josh/Desktop/pySecMaster_m old.db'
new_db_location = 'C:/Users/Josh/Desktop/pySecMaster_m.db'
table = 'minute_prices' # daily_prices, minute_prices
# Create a new database where the old prices will be copied to
symbology_sources = ['csi_data', 'tsid', 'quandl_wiki', 'quandl_goog',
'seeking_alpha', 'yahoo']
os.chdir('..') # Need to move up a folder in order to access load_tables
maintenance(database_link=new_db_location, quandl_ticker_source='csidata',
database_list=['WIKI'], threads=8, quandl_key='',
quandl_update_range=30, csidata_update_range=5,
symbology_sources=symbology_sources)
# Retrieve a list of all the tickers from the existing database table
qcodes_df = query_existing_qcodes(db_location=old_db_location,
table=table, verbose=True)
for index, row in qcodes_df.iterrows():
ticker = row['q_code']
copy_start = time.time()
# Retrieve all price data for this ticker
raw_price_df = query_qcode_data(db_location=old_db_location,
table=table, qcode=ticker,
verbose=False)
# Change the q_code column to a tsid column
clean_price_df = convert_qcode_to_tsid(db_location=new_db_location,
price_df=raw_price_df,
table=table, qcode=ticker)
tsid = clean_price_df.loc[0, 'tsid']
# If there is no tsid, don't attempt to insert the data to the database
if tsid:
# Add the data to the database if there is not existing data
insert_df_to_db(db_location=new_db_location,
price_df=clean_price_df, table=table, verbose=True)
if verbose:
print('Moving the %s from %s to %s took %0.2f seconds' %
(table, ticker, tsid, time.time() - copy_start))
def main(verbose=False):
old_db_location = 'C:/Users/joshs/Programming/Databases/pySecMaster/' \
'pySecMaster_m old.db'
new_db_location = 'C:/Users/joshs/Programming/Databases/pySecMaster/' \
'pySecMaster_m new.db'
table = 'minute_prices'
# Create a new pySecMaster minute database
symbology_sources = ['csi_data', 'tsid', 'quandl_wiki', 'quandl_goog',
'seeking_alpha', 'yahoo']
os.chdir('..') # Need to move up a folder in order to access load_tables
maintenance(database_link=new_db_location, quandl_ticker_source='csidata',
database_list=['WIKI'], threads=8, quandl_key='',
quandl_update_range=30, csidata_update_range=5,
symbology_sources=symbology_sources)
# Retrieve a list of all existing tsid's
current_tsid_df = query_existing_tsids(db_location=old_db_location,
table=table, verbose=verbose)
# Cycle through each tsid
for index, row in current_tsid_df.iterrows():
tsid = row['tsid']
tsid_start = time.time()
# Query the existing tsid's price times
existing_price_df = query_tsid_data(db_location=old_db_location,
table=table, tsid=tsid,
verbose=verbose)
# Change any incorrect times to best guess times (98% confident)
updated_price_df = verify_minute_time(price_df=existing_price_df,
tsid=tsid)
# Update the database times with the corrected times
insert_df_to_db(db_location=new_db_location, table=table,
price_df=updated_price_df, verbose=verbose)
if verbose:
print('Verifying the %s times for %s took %0.2f seconds' %
(table, tsid, time.time() - tsid_start))
