def all_tickers_for_bundle(fields,
bundle_name,
raw_path=os.path.join(BASE, RAW_FLDR)):
tickers = get_ticker_sid_dict_from_bundle(bundle_name)
#populate_raw_data({"WMT":3173, "HD":2912, "DOGGY":69, "CSCO":2809}, fields, raw_path)
populate_raw_data(tickers, fields, raw_path)
return len(tickers)
def create_sid_table_from_file(filepath):
"""reads the raw file, maps tickers -> SIDS,
then maps sector strings to integers, and saves
to the file: SID_FILE"""
register(
BUNDLE_NAME,
int,
)
df = pd.read_csv(filepath, index_col="ticker")
assert df.shape[0] > 10001 # there should be more than 10k tickers
df = df[df.exchange != 'None']
df = df[df.exchange != 'INDEX']
coded_sectors_for_ticker = df["sector"].map(SECTOR_CODING)
ae_d = get_ticker_sid_dict_from_bundle(BUNDLE_NAME)
N = max(ae_d.values()) + 1
# create empty 1-D array to hold data where index = SID
sectors = np.full(N, -1, np.dtype('int64'))
# iterate over Assets in the bundle, and fill in sectors
for ticker, sid in ae_d.items():
sectors[sid] = coded_sectors_for_ticker.get(ticker, -1)
print(sectors)
# finally save the file to disk
np.save(ZIPLINE_DATA_DIR + SID_FILE, sectors)
def all_tickers_for_bundle(input_fields,
output_fields,
bundle_name,
raw_path=os.path.join(BASE, RAW_FLDR)):
tickers = get_ticker_sid_dict_from_bundle(bundle_name)
populate_raw_data(tickers, input_fields, output_fields, raw_path)
return len(tickers)
def all_tickers_for_bundle(fields,
dims,
bundle_name,
raw_path=os.path.join(BASE, RAW_FLDR)):
tickers = get_ticker_sid_dict_from_bundle(bundle_name)
#populate_raw_data(tickers, fields, dims, raw_path)
populate_raw_data_from_database_SP500mem(tickers, fields, dims, raw_path,
ACTION_CODING, Init_DATE)
def test_reads_file(self):
bundle_name = 'sep'
tickers = get_ticker_sid_dict_from_bundle(bundle_name)
self.assertTrue(len(tickers) > 0) # test the bundle is not empty
BASE = os.path.dirname(os.path.realpath(__file__))
RAW_FLDR = "raw" # folder to store the raw text file
fields = ['netinc', 'equity', 'bvps', 'sps', 'fcfps', 'price'] # basic QV
dims = ['ARQ', 'ARQ', 'ARQ', 'ARQ', 'ARQ', 'ARQ']
populate_raw_data_from_dump(tickers, fields, dims, raw_path=os.path.join(BASE, RAW_FLDR))
self.assertEquals(True, True)
def create_sid_table_from_file(filepath):
"""reads the raw file, maps tickers -> SIDS,
then maps sector strings to integers, and saves
to the file: SID_FILE"""
df = pd.read_csv(filepath, index_col="Symbol")
df = df.drop_duplicates()
coded_sectors_for_ticker = df["Sector"].map(SECTOR_CODING)
ae_d = get_ticker_sid_dict_from_bundle('quantopian-quandl')
N = max(ae_d.values()) + 1
# create empty 1-D array to hold data where index = SID
sectors = np.full(N, -1, np.dtype('int64'))
# iterate over Assets in the bundle, and fill in sectors
for ticker, sid in ae_d.items():
sectors[sid] = coded_sectors_for_ticker.get(ticker, -1)
np.save(BASE_PATH + SID_FILE, sectors)
def create_static_table_from_file(filepath):
"""Stores static items to a persisted np array.
The following static fields are currently persisted.
-Sector
-exchange
-category
"""
register(
BUNDLE_NAME,
int,
)
df = pd.read_csv(filepath, index_col="ticker")
assert df.shape[0] > 10001 # there should be more than 10k tickers
df = df[df.exchange != 'None']
df = df[df.exchange != 'INDEX']
df = df[df.table == 'SEP']
coded_sectors_for_ticker = df['sector'].map(SECTOR_CODING)
coded_exchange_for_ticker = df['exchange'].map(EXCHANGE_CODING)
coded_category_for_ticker = df['category'].map(CATEGORY_CODING)
ae_d = get_ticker_sid_dict_from_bundle(BUNDLE_NAME)
N = max(ae_d.values()) + 1
# create 2-D array to hold data where index = SID
sectors = np.full((3, N), -1, np.dtype('int64'))
# sectors = np.full(N, -1, np.dtype('int64'))
# iterate over Assets in the bundle, and fill in static fields
for ticker, sid in ae_d.items():
print(ticker, sid, coded_sectors_for_ticker.get(ticker, -1))
sectors[0, sid] = coded_sectors_for_ticker.get(ticker, -1)
sectors[1, sid] = coded_exchange_for_ticker.get(ticker, -1)
sectors[2, sid] = coded_category_for_ticker.get(ticker, -1)
print(sectors)
print(sectors[:, -10:])
# finally save the file to disk
np.save(ZIPLINE_DATA_DIR + STATIC_FILE, sectors)
def create_sid_table_from_file_ipo(ticker_df):
"""reads the raw file, maps tickers -> SIDS,
then maps sector strings to integers, and saves
to the file: SID_FILE"""
df = ticker_df
df = df.drop_duplicates()
# coded_sectors_for_ticker = df["Sector"].map(SECTOR_CODING)
ae_d = get_ticker_sid_dict_from_bundle('quandl')
N = max(ae_d.values()) + 1
# create empty 1-D array to hold data where index = SID
ipoyears = np.full(N, -1, np.dtype('int64'))
# iterate over Assets in the bundle, and fill in sectors
for ticker, sid in ae_d.items():
try:
ipoyears[sid] = int(df.loc[ticker].firstpricedate[:4])
except Exception as e:
print(e)
np.save(os.path.join(BASE_PATH , SID_FILE_IPO), ipoyears)
def num_tkrs_in_bundle(bundle_name):
return len(get_ticker_sid_dict_from_bundle(bundle_name))
def all_tickers_for_bundle(fields,
dims,
bundle_name,
raw_path=os.path.join(BASE, RAW_FLDR)):
tickers = get_ticker_sid_dict_from_bundle(bundle_name)
populate_raw_data(tickers, fields, dims, raw_path)
def __init__(self, *args, **kwargs):
super(TransactionData, self).__init__(*args, **kwargs)
self.N = len(get_ticker_sid_dict_from_bundle(
"quandl")) + 1 # max(sid)+1 get this from the bundle
self.data_path = os.path.join(str(Path.home()), "SF2.npy")
def __init__(self, *args, **kwargs):
super(Fundamentals, self).__init__(*args, **kwargs)
self.N = len(get_ticker_sid_dict_from_bundle(
"sep")) + 1 # max(sid)+1 get this from the bundle
self.data_path = zipline_root() + '/data/SF1.npy'
def all_tickers_for_bundle(fields):
tickers = get_ticker_sid_dict_from_bundle('quantopian-quandl')
populate_raw_data(tickers, fields)
def __init__(self, *args, **kwargs):
super(sp500member, self).__init__(*args, **kwargs)
self.N = len(get_ticker_sid_dict_from_bundle(
"sep")) + 1 # max(sid)+1 get this from the bundle
self.data_path = ZIPLINE_DATA_DIR + 'SP500mem.npy'
def create_static_table_from_database(ZIPLINE_DATA_DIR, STATIC_FILE,
BUNDLE_NAME, SECTOR_CODING,
EXCHANGE_CODING, CATEGORY_CODING):
"""Stores static items to a persisted np array.
The following static fields are currently persisted.
-Sector
-exchange
-category
-code ->siccode
"""
register(
BUNDLE_NAME,
int,
)
query = """SELECT ticker, code, sector, exchange_id, category FROM security WHERE ttable = 'SF1' """
df = pd.read_sql_query(query, engine)
# add the exchange based on the exchange and exchange_id relation
# get the exchange and exchange_id relation
name_ex_id = get_name_exchange_id()
my_EXCHANGE_CODING = name_ex_id.set_index('id')['name'].to_dict()
# add the exchange based on the exchange and exchange_id relation
df['exchange'] = df['exchange_id'].map(my_EXCHANGE_CODING)
df['sectors_'] = df['sector'].map(SECTOR_CODING)
df['exchange_'] = df['exchange'].map(EXCHANGE_CODING)
df['category_'] = df['category'].map(CATEGORY_CODING)
df['code_'] = df['code'].astype(
int) # just multiply the siccode by 10 and get integer
df = df.fillna(-1)
ae_d = get_ticker_sid_dict_from_bundle(BUNDLE_NAME)
N = max(ae_d.values()) + 1
# create 2-D array to hold data where index = SID
sectors = np.full((4, N), -1, np.dtype('int64'))
# sectors = np.full(N, -1, np.dtype('int64'))
# iterate over Assets in the bundle, and fill in static fields
for ticker, sid in tqdm(ae_d.items(), total=len(ae_d)):
#for ticker, sid in ae_d.items():
#sector_coded = coded_sectors_for_ticker.get(ticker)
if not df[df['ticker'] == ticker].empty:
#sector_ = df.sectors_[df['ticker']==ticker].iloc[0]
#exchange_ = df.exchange_[df['ticker']==ticker].iloc[0]
#category_ = df.category_[df['ticker']==ticker].iloc[0]
#code_ = df.code_[df['ticker']==ticker].iloc[0]
#print(ticker, sid, sector_coded, exchange_, category_, code_ ,'<-end')
sectors[0, sid] = df.sectors_[df['ticker'] == ticker].iloc[0]
sectors[1, sid] = df.exchange_[df['ticker'] == ticker].iloc[0]
sectors[2, sid] = df.category_[df['ticker'] == ticker].iloc[0]
sectors[3, sid] = df.code_[df['ticker'] == ticker].iloc[0]
else:
sectors[0, sid] = -1
sectors[1, sid] = -1
sectors[2, sid] = -1
sectors[3, sid] = -1
#print('ticker missing but filled with -1, everthing under control keep cool= ',ticker)
print(sectors)
print(sectors[:, -10:])
# finally save the file to disk
np.save(ZIPLINE_DATA_DIR + STATIC_FILE, sectors)
print("this worked master")
def __init__(self, *args, **kwargs):
super(Fundamentals, self).__init__(*args, **kwargs)
self.N = len(get_ticker_sid_dict_from_bundle(
"sep")) + 1 # max(sid)+1 get this from the bundle
self.data_path = '/Users/peterharrington/.zipline/data/' + 'SF1.npy'
def __init__(self, *args, **kwargs):
super(Fundamentals, self).__init__(*args, **kwargs)
self.N = len(get_ticker_sid_dict_from_bundle("quantopian-quandl")) + 1 # max(sid)+1 get this from the bundle
self.data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "SF1.npy")
