def build_symbology(database_options):
tables_to_load = ["data_vendor", "exchanges"]
csidata_type = "stock" # stock, commodity
csidata_update_range = 7
# Don't change these unless you know what you are doing
# http://www.csidata.com/factsheets.php?type=stock&format=html
csidata_url = "http://www.csidata.com/factsheets.php?"
symbology_sources = ["csi_data", "tsid", "quandl_wiki", "quandl_goog", "seeking_alpha", "yahoo"]
LoadTables(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
tables_to_load=tables_to_load,
)
CSIDataExtractor(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
db_url=csidata_url,
data_type=csidata_type,
redownload_time=csidata_update_range,
)
create_symbology(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
source_list=symbology_sources,
)
def build_symbology(database_options):
tables_to_load = ['data_vendor', 'exchanges']
csidata_type = 'stock' # stock, commodity
csidata_update_range = 7
# Don't change these unless you know what you are doing
# http://www.csidata.com/factsheets.php?type=stock&format=html
csidata_url = 'http://www.csidata.com/factsheets.php?'
symbology_sources = [
'csi_data', 'tsid', 'quandl_wiki', 'quandl_eod', 'quandl_goog',
'seeking_alpha', 'yahoo'
]
LoadTables(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
tables_to_load=tables_to_load)
CSIDataExtractor(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
db_url=csidata_url,
data_type=csidata_type,
redownload_time=csidata_update_range)
create_symbology(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
source_list=symbology_sources)
def maintenance(database_options, quandl_key, quandl_ticker_source,
database_list, threads, quandl_update_range,
csidata_update_range, symbology_sources):
"""
:param database_options: Dictionary of the postgres database options
:param quandl_key: Optional string of the Quandl API key
:param quandl_ticker_source: String of which source the Quandl data should
use when determining which codes to download (csidata, quandl)
:param database_list: List of strings indicating which Quandl databases
should have their codes downloaded (WIKI, GOOG, YAHOO)
:param threads: Integer of the threads to run when downloading Quandl codes
:param quandl_update_range: Integer of the number of days before the
Quandl codes should be updated
:param csidata_update_range: Integer of the number of days before the CSI
Data factsheet should be updated
:param symbology_sources: List of strings of which symbology sources
should be created (csi_data, tsid, quandl_wiki)
"""
print('Starting Security Master table maintenance function. This can take '
'some time to finish if large databases are used. If this fails, '
'rerun it after a few minutes.')
# Create the SQL tables if they don't already exist
create_database(admin_user=database_options['admin_user'],
admin_password=database_options['admin_password'],
database=database_options['database'],
user=database_options['user'])
main_tables(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'])
data_tables(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'])
events_tables(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'])
LoadTables(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
tables_to_load=tables_to_load,
load_tables=userdir['load_tables'])
# Always extract CSI values, as they are used for the symbology table
CSIDataExtractor(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
db_url=csidata_url,
data_type=csidata_type,
redownload_time=csidata_update_range)
if quandl_ticker_source == 'quandl':
QuandlCodeExtract(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
quandl_token=quandl_key,
database_list=database_list,
database_url=database_url,
update_range=quandl_update_range,
threads=threads)
create_symbology(database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
source_list=symbology_sources)
NASDAQSectorIndustryExtractor(
database=database_options['database'],
user=database_options['user'],
password=database_options['password'],
host=database_options['host'],
port=database_options['port'],
db_url=nasdaq_sector_industry_url,
exchange_list=nasdaq_sector_industry_extractor_exchanges,
redownload_time=nasdaq_sector_industry_redownload_time)
def maintenance(
database_options,
quandl_key,
quandl_ticker_source,
database_list,
threads,
quandl_update_range,
csidata_update_range,
symbology_sources,
):
"""
:param database_options: Dictionary of the postgres database options
:param quandl_key: Optional string of the Quandl API key
:param quandl_ticker_source: String of which source the Quandl data should
use when determining which codes to download (csidata, quandl)
:param database_list: List of strings indicating which Quandl databases
should have their codes downloaded (WIKI, GOOG, YAHOO)
:param threads: Integer of the threads to run when downloading Quandl codes
:param quandl_update_range: Integer of the number of days before the
Quandl codes should be updated
:param csidata_update_range: Integer of the number of days before the CSI
Data factsheet should be updated
:param symbology_sources: List of strings of which symbology sources
should be created (csi_data, tsid, quandl_wiki)
"""
print(
"Starting Security Master table maintenance function. This can take "
"some time to finish if large databases are used. If this fails, "
"rerun it after a few minutes."
)
# Create the SQL tables if they don't already exist
create_database(
admin_user=database_options["admin_user"],
admin_password=database_options["admin_password"],
database=database_options["database"],
user=database_options["user"],
)
main_tables(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
)
data_tables(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
)
events_tables(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
)
LoadTables(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
tables_to_load=tables_to_load,
load_tables=userdir["load_tables"],
)
# Always extract CSI values, as they are used for the symbology table
CSIDataExtractor(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
db_url=csidata_url,
data_type=csidata_type,
redownload_time=csidata_update_range,
)
if quandl_ticker_source == "quandl":
QuandlCodeExtract(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
quandl_token=quandl_key,
database_list=database_list,
database_url=database_url,
update_range=quandl_update_range,
threads=threads,
)
create_symbology(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
source_list=symbology_sources,
)
NASDAQSectorIndustryExtractor(
database=database_options["database"],
user=database_options["user"],
password=database_options["password"],
host=database_options["host"],
port=database_options["port"],
db_url=nasdaq_sector_industry_url,
exchange_list=nasdaq_sector_industry_extractor_exchanges,
redownload_time=nasdaq_sector_industry_redownload_time,
)