def read_and_write_quotes(snapshot_date):
mkt_conf = config.read_markets()
active_mkt_conf = config.active_markets(mkt_conf, snapshot_date)
retrieved_markets = {}
for mkt_name, mkt_conf in active_mkt_conf.items():
print('{}: retrieving {}. . . '.format(snapshot_date, mkt_name),
end='')
mkt = contract.Market(mkt_name)
# Market snapshot may have been retrieved already
if mkt_name not in retrieved_markets:
quotes_dfs = px_hist.read_quote_frames(mkt_conf)
retrieved_markets.update(quotes_dfs)
quotes_df = retrieved_markets[mkt_name]
with open_store(mode='a') as hdf_store:
key = quote_key(market=mkt)
if key in hdf_store:
prev_df = hdf_store[key]
dedupe_idx = quotes_df.index.difference(prev_df.index)
iter_df = quotes_df.reindex(dedupe_idx)
else:
iter_df = quotes_df
hdf_store.put(key=key, value=iter_df, format='t', append=True)
print('{}: completed {}'.format(pd.Timestamp.now(), mkt_name))
def __init__(self, market_name, contract_name):
self.contract_name = contract_name
self.market = Market(market_name)
a = config.find_asset(config.read_markets(), market_name,
contract_name)
self.asset_id = int(a['id'])
self.asset_to_market_id = int(a['order'])
def _active_markets(self, ts, exp_num_active_markets, exp_num_fed_bundles):
mkt_conf = config.read_markets()
active_mkt_conf = config.active_markets(mkt_conf, ts)
fed = 'FedPolicyB'
self.assertEqual(len(active_mkt_conf), exp_num_active_markets)
self.assertTrue(fed in active_mkt_conf)
num_active_bundles = len(active_mkt_conf[fed][config.BUNDLE])
self.assertEqual(num_active_bundles, exp_num_fed_bundles)
def __init__(self, username=None, password=None):
self._username = username
self._password = password
self._logger = None
# Start session
self._session = requests.Session()
# Lookup tables
self._market_asset_dict = read_markets()
self._asset_market_dict = _asset_market_dict(self._market_asset_dict)
self._order_market_dict = _order_market_dict(self._market_asset_dict)
def testFindBundle(self):
mkt_conf = config.read_markets()
for mkt_name, mkt_dict in mkt_conf.items():
bundles = mkt_dict[config.BUNDLE]
if config.BUNDLE_ID in bundles: # bundles is expected bundle
bundle = config.find_bundle(mkt_conf, mkt_name, '')
self.assertEqual(bundles, bundle)
else:
for bundle_exp, exp_bundle in bundles.items():
bundle = config.find_bundle(mkt_conf, mkt_name, bundle_exp)
self.assertEqual(exp_bundle, bundle)
def retrieve_and_store_daily_data():
"""
For each market in the markets config file, retrieve the daily snapshots
from the IEM site. Previous daily snapshot data for that market is
overwritten in the process
:return: None
"""
for mkt_name in config.read_markets().keys():
mkt = contract.Market(mkt_name)
px_hist_df = read_daily_market_data(mkt_name)
with open_store(mode='a') as hdf_store:
key = history_key(market=mkt)
hdf_store.put(key=key, value=px_hist_df, format='t')
with open('conf/db.json', 'r') as fp:
db_conf = json.load(fp)
# Engine connect to MySQL
sa_url = 'sqlalchemy.url'
mysql_url, dbname = db_conf[sa_url].rsplit(sep='/', maxsplit=1)
mysql_engine = sa.engine_from_config({sa_url: mysql_url})
create_fmt = 'CREATE DATABASE IF NOT EXISTS {dbname}'
mysql_engine.execute(create_fmt.format(dbname=dbname))
# Engine connecting to MySQL database
engine = sa.engine_from_config(db_conf)
metadata = sa.MetaData()
mkt_conf = config.read_markets()
create_all(metadata, mkt_conf)
metadata.create_all(engine)
insert_config_data(engine)
# Query market table. Does metadata drop and create the table?
db_mkt_df = pd.read_sql_table(config.MARKETS, engine, index_col=config.ID)
db_bundle_df = pd.read_sql_table(config.BUNDLES, engine, index_col=config.ID)
db_asset_df = pd.read_sql_table(config.ASSETS, engine, index_col=config.ID)
# TODO: Populate data
markets = [contract.Market(mkt_nm) for mkt_nm in mkt_conf.keys()]
with recorder.open_store('data/hist_iem.hdf') as hdf_store:
for market in markets:
mkt_hist_key = recorder.history_key(market)
def testMarketConstructor(self):
conf = config.read_markets()
for mkt_name in conf.keys():
m = Market(mkt_name)
self.assertEqual(m.name, mkt_name)
self.assertEqual(m.id, conf[mkt_name]['id'])
cumsum_df = cumsum_frame(bundle_history_srs)
# First trade index
fst_trade_idx = (cumsum_df.sum(axis=1) > 0).idxmax()
nonzero_cum_units_df = cumsum_df.loc[fst_trade_idx:]
plt.figure(get_new_fignum())
plt.plot(nonzero_cum_units_df)
plt.show()
if __name__ == '__main__':
mkt_name = 'FedPolicyB'
mkt = contract.Market(mkt_name)
with recorder.open_store(mode='r') as hdf_store:
px_hist_df = hdf_store[recorder.history_key(market=mkt)]
mkts_json = config.read_markets()
mkt_conf = mkts_json[mkt_name]
# Clean dataframe
df = px_hist_df.copy().reset_index()
# Expiry
df[EXPIRY] = df[iem.CONTRACT].apply(expiry)
# Expiry date
expiry_date_json = mkt_conf[config.BUNDLE]
kwargs = {'expiry_date_json': expiry_date_json}
df[config.EXPIRY_DATE] = df[EXPIRY].apply(expiry_date_series, **kwargs)
# Days to expiration
df[DAYS_TO_EXPIRY_DATE] = df[config.EXPIRY_DATE] - df[iem.DATE]
# Bundle frame
bundle = '1116'
def __init__(self, market_name, market_fp=None):
self.name = market_name
self.id = config.read_markets(market_fp)[market_name]['id']
def __init__(self, market_name, expiration):
self.market = Market(market_name)
self.expiration = expiration
b = config.find_bundle(config.read_markets(), market_name, expiration)
self.id = int(b[config.BUNDLE_ID])