def pysecmaster(self, settings_dict): """ Calls the functions that operate the pySecMaster. Emits signals back to the main gui for further processing, using the dataReady process. :param settings_dict: Dictionary of all parameters to be passed back to the pySecMaster.py functions. """ self.dataReady.emit('Building the pySecMaster in the %s database ' 'located at host %s\n' % (settings_dict['database_options']['database'], settings_dict['database_options']['host'])) maintenance( database_options=settings_dict['database_options'], quandl_key=settings_dict['quandl_key'], quandl_ticker_source=settings_dict['quandl_ticker_source'], database_list=settings_dict['quandl_db_list'], threads=settings_dict['threads'], quandl_update_range=settings_dict['quandl_update_range'], csidata_update_range=settings_dict['google_fin_update_range'], symbology_sources=settings_dict['symbology_sources']) data_download(database_options=settings_dict['database_options'], quandl_key=settings_dict['quandl_key'], download_list=settings_dict['download_list'], threads=settings_dict['threads'], verbose=True) self.dataReady.emit('Finished running the pySecMaster process\n') self.finished.emit()
def pysecmaster(self, settings_dict): """ Calls the functions that operate the pySecMaster. Emits signals back to the main gui for further processing, using the dataReady process. :param settings_dict: Dictionary of all parameters to be passed back to the pySecMaster.py functions. """ self.dataReady.emit('Building the pySecMaster in the %s database ' 'located at host %s\n' % (settings_dict['database_options']['database'], settings_dict['database_options']['host'])) maintenance(database_options=settings_dict['database_options'], quandl_key=settings_dict['quandl_key'], quandl_ticker_source=settings_dict['quandl_ticker_source'], database_list=settings_dict['quandl_db_list'], threads=settings_dict['threads'], quandl_update_range=settings_dict['quandl_update_range'], csidata_update_range=settings_dict['google_fin_update_range'], symbology_sources=settings_dict['symbology_sources']) data_download(database_options=settings_dict['database_options'], quandl_key=settings_dict['quandl_key'], download_list=settings_dict['download_list'], threads=settings_dict['threads'], verbose=True) self.dataReady.emit('Finished running the pySecMaster process\n') self.finished.emit()
def main(verbose=False): old_db_location = 'C:/Users/Josh/Desktop/pySecMaster_m old.db' new_db_location = 'C:/Users/Josh/Desktop/pySecMaster_m.db' table = 'minute_prices' # daily_prices, minute_prices # Create a new database where the old prices will be copied to symbology_sources = ['csi_data', 'tsid', 'quandl_wiki', 'quandl_goog', 'seeking_alpha', 'yahoo'] os.chdir('..') # Need to move up a folder in order to access load_tables maintenance(database_link=new_db_location, quandl_ticker_source='csidata', database_list=['WIKI'], threads=8, quandl_key='', quandl_update_range=30, csidata_update_range=5, symbology_sources=symbology_sources) # Retrieve a list of all the tickers from the existing database table qcodes_df = query_existing_qcodes(db_location=old_db_location, table=table, verbose=True) for index, row in qcodes_df.iterrows(): ticker = row['q_code'] copy_start = time.time() # Retrieve all price data for this ticker raw_price_df = query_qcode_data(db_location=old_db_location, table=table, qcode=ticker, verbose=False) # Change the q_code column to a tsid column clean_price_df = convert_qcode_to_tsid(db_location=new_db_location, price_df=raw_price_df, table=table, qcode=ticker) tsid = clean_price_df.loc[0, 'tsid'] # If there is no tsid, don't attempt to insert the data to the database if tsid: # Add the data to the database if there is not existing data insert_df_to_db(db_location=new_db_location, price_df=clean_price_df, table=table, verbose=True) if verbose: print('Moving the %s from %s to %s took %0.2f seconds' % (table, ticker, tsid, time.time() - copy_start))
def main(verbose=False): old_db_location = 'C:/Users/joshs/Programming/Databases/pySecMaster/' \ 'pySecMaster_m old.db' new_db_location = 'C:/Users/joshs/Programming/Databases/pySecMaster/' \ 'pySecMaster_m new.db' table = 'minute_prices' # Create a new pySecMaster minute database symbology_sources = ['csi_data', 'tsid', 'quandl_wiki', 'quandl_goog', 'seeking_alpha', 'yahoo'] os.chdir('..') # Need to move up a folder in order to access load_tables maintenance(database_link=new_db_location, quandl_ticker_source='csidata', database_list=['WIKI'], threads=8, quandl_key='', quandl_update_range=30, csidata_update_range=5, symbology_sources=symbology_sources) # Retrieve a list of all existing tsid's current_tsid_df = query_existing_tsids(db_location=old_db_location, table=table, verbose=verbose) # Cycle through each tsid for index, row in current_tsid_df.iterrows(): tsid = row['tsid'] tsid_start = time.time() # Query the existing tsid's price times existing_price_df = query_tsid_data(db_location=old_db_location, table=table, tsid=tsid, verbose=verbose) # Change any incorrect times to best guess times (98% confident) updated_price_df = verify_minute_time(price_df=existing_price_df, tsid=tsid) # Update the database times with the corrected times insert_df_to_db(db_location=new_db_location, table=table, price_df=updated_price_df, verbose=verbose) if verbose: print('Verifying the %s times for %s took %0.2f seconds' % (table, tsid, time.time() - tsid_start))