def main(args):
'''
************************** Step 1: get arguments from argparse variables **************************
'''
logger = init_root_logger('logger.log', 'INFO')
pga = db_info.get_db_info(args.config_name, args.db_config_csv_path)
DB_USER_NAME = pga.username
if DB_USER_NAME is not None and len(DB_USER_NAME) < 1:
DB_USER_NAME = None
DB_PASSWORD = pga.password
dburl = pga.dburl
databasename = pga.databasename
'''
************************** Step 2: build an instance of BuildDb **************************
'''
bdb = build_db.BuildDb(None,
dburl=dburl,
databasename=databasename,
schema_name=args.schema_name,
username=DB_USER_NAME,
password=DB_PASSWORD)
'''
************************** Step 3: make a new table, if testorlive==live **************************
'''
if str(args.testorlive).lower() == 'live':
print(f'creating schema {bdb.schema_name}')
bdb.pga.exec_sql_raw(f"create schema IF NOT EXISTS {bdb.schema_name};")
print(f'creating futures table')
bdb.make_underlying_table()
print(f'creating options table')
bdb.make_options_table()
'''
************************* END ****************************
'''
logger.info("finished")
parser.add_argument('--db_config_csv_path',type=str,
help='path to the csv file that holds config_name,dburl,databasename,username,password info for the postgres db that you will update (default is ./postgres_info.csv',
default="./postgres_info.csv")
parser.add_argument('--config_name',type=str,
help='value of the config_name column in the db config csv file (default is local',
default="local")
parser.add_argument('--strike_divisor_json_path',type=str,
help='if specified, a path to a json file that contains divisors for each commodity in contract_list',
default = './divisor_dict.json')
parser.add_argument('--write_to_postgres',type=str,
help='if True the data will be written to postgres. Otherwise, a psql COPY command will be printed to the console. Default=False',
default="False")
args = parser.parse_args()
logger = build_db.init_root_logger(args.log_file_path, args.logging_level)
pga = db_info.get_db_info(args.config_name, args.db_config_csv_path)
WRITE_TO_POSTGRES = str(args.write_to_postgres).lower()=='true'
# create a builder just so that you can get it's logger and it's pga instance
logger.info(f"fetching commod lists for options and futures")
opt_contract_list = get_commod_list_from_max_settle_date(pga,tablename=OPTTAB)
fut_contract_list = get_commod_list_from_max_settle_date(pga,tablename=FUTTAB)
# get contract list of commod symbols (like CL and ES) for options and futures
# get first day to start upload
df_all_options = None
df_all_futures = None
yyyymmdds_to_fetch = get_dates_to_fetch(pga,OPTTAB)
for yyyymmdd in tqdm_notebook(yyyymmdds_to_fetch):
# importlib.reload(db_info)
# In[2]:
import warnings
warnings.filterwarnings('ignore')
# ### important global variables
# In[3]:
DEBUG_IT = False
opttab = 'sec_schema.options_table'
futtab = 'sec_schema.underlying_table'
pga = db_info.get_db_info()
SYMBOL_TO_RESEARCH = 'ES'
STRIKE_DIVISORS = {}
# df_expiry_dates_additions = pd.read_csv('df_expiry_dates_additions.csv')
df_expiry_dates_additions = pd.read_csv('live_option_expirations.csv')
# ### methods to build options
# In[4]:
def _get_contract_number_from_symbol(symbol):
c = symbol[0:2]
if c in ['CL', 'ES', 'NG']:
return 2
MONTH_CODES = 'FGHJKMNQUVXZ'
DICT_MONTH_NUMS = {MONTH_CODES[i]: i + 1 for i in range(len(MONTH_CODES))}
# ### important global variables
# In[3]:
DEBUG_IT = False
opttab = 'sec_schema.options_table'
futtab = 'sec_schema.underlying_table'
config_name = None
argvs = sys.argv
if len(argvs) > 1:
config_name = argvs[1]
pga = db_info.get_db_info(config_name=config_name)
SYMBOL_TO_RESEARCH = 'ES'
STRIKE_DIVISORS = {}
# df_expiry_dates_additions = pd.read_csv('df_expiry_dates_additions.csv')
df_expiry_dates_additions = pd.read_csv('live_option_expirations.csv')
# ### methods to build options
# In[4]:
def _get_contract_number_from_symbol(symbol):
c = symbol[0:2]
if c in ['CL', 'CB', 'ES', 'GE', 'NG']:
return 2
'--strike_divisor_json_path',
type=str,
help=
'if specified, a path to a json file that contains divisors for each commodity in contract_list',
default='./divisor_dict.json')
parser.add_argument(
'--write_to_postgres',
type=str,
help=
'if True the data will be written to postgres. Otherwise, a psql COPY command will be printed to the console. Default=False',
default="False")
args = parser.parse_args()
# create logger and pga instance
logger = build_db.init_root_logger('logfile.log', 'INFO')
pga = db_info.get_db_info(args.config_name, './postgres_info.csv')
# see if we are updating db
write_to_postgres = str(args.write_to_postgres).lower() == 'true'
logger.info(f"fetching commod lists for options and futures")
# get commodities to be processed
commods = args.contract_list.split(',')
# get dates to be processed for options, grouped by commod
print(f"geting dates to process for {OPTTAB} symols: {commods}")
dict_options_yyyymmdds_per_commod = get_dates_to_fetch(
pga, OPTTAB, commods)
options_yyyymmdds_to_fetch = []
for yyyymmdds in dict_options_yyyymmdds_per_commod.values():
options_yyyymmdds_to_fetch.extend(yyyymmdds)
options_yyyymmdds_to_fetch = list(set(options_yyyymmdds_to_fetch))