def all_tickers_for_bundle(fields, bundle_name, raw_path=os.path.join(BASE, RAW_FLDR)): tickers = get_ticker_sid_dict_from_bundle(bundle_name) #populate_raw_data({"WMT":3173, "HD":2912, "DOGGY":69, "CSCO":2809}, fields, raw_path) populate_raw_data(tickers, fields, raw_path) return len(tickers)
def create_sid_table_from_file(filepath): """reads the raw file, maps tickers -> SIDS, then maps sector strings to integers, and saves to the file: SID_FILE""" register( BUNDLE_NAME, int, ) df = pd.read_csv(filepath, index_col="ticker") assert df.shape[0] > 10001 # there should be more than 10k tickers df = df[df.exchange != 'None'] df = df[df.exchange != 'INDEX'] coded_sectors_for_ticker = df["sector"].map(SECTOR_CODING) ae_d = get_ticker_sid_dict_from_bundle(BUNDLE_NAME) N = max(ae_d.values()) + 1 # create empty 1-D array to hold data where index = SID sectors = np.full(N, -1, np.dtype('int64')) # iterate over Assets in the bundle, and fill in sectors for ticker, sid in ae_d.items(): sectors[sid] = coded_sectors_for_ticker.get(ticker, -1) print(sectors) # finally save the file to disk np.save(ZIPLINE_DATA_DIR + SID_FILE, sectors)
def all_tickers_for_bundle(input_fields, output_fields, bundle_name, raw_path=os.path.join(BASE, RAW_FLDR)): tickers = get_ticker_sid_dict_from_bundle(bundle_name) populate_raw_data(tickers, input_fields, output_fields, raw_path) return len(tickers)
def all_tickers_for_bundle(fields, dims, bundle_name, raw_path=os.path.join(BASE, RAW_FLDR)): tickers = get_ticker_sid_dict_from_bundle(bundle_name) #populate_raw_data(tickers, fields, dims, raw_path) populate_raw_data_from_database_SP500mem(tickers, fields, dims, raw_path, ACTION_CODING, Init_DATE)
def test_reads_file(self): bundle_name = 'sep' tickers = get_ticker_sid_dict_from_bundle(bundle_name) self.assertTrue(len(tickers) > 0) # test the bundle is not empty BASE = os.path.dirname(os.path.realpath(__file__)) RAW_FLDR = "raw" # folder to store the raw text file fields = ['netinc', 'equity', 'bvps', 'sps', 'fcfps', 'price'] # basic QV dims = ['ARQ', 'ARQ', 'ARQ', 'ARQ', 'ARQ', 'ARQ'] populate_raw_data_from_dump(tickers, fields, dims, raw_path=os.path.join(BASE, RAW_FLDR)) self.assertEquals(True, True)
def create_sid_table_from_file(filepath): """reads the raw file, maps tickers -> SIDS, then maps sector strings to integers, and saves to the file: SID_FILE""" df = pd.read_csv(filepath, index_col="Symbol") df = df.drop_duplicates() coded_sectors_for_ticker = df["Sector"].map(SECTOR_CODING) ae_d = get_ticker_sid_dict_from_bundle('quantopian-quandl') N = max(ae_d.values()) + 1 # create empty 1-D array to hold data where index = SID sectors = np.full(N, -1, np.dtype('int64')) # iterate over Assets in the bundle, and fill in sectors for ticker, sid in ae_d.items(): sectors[sid] = coded_sectors_for_ticker.get(ticker, -1) np.save(BASE_PATH + SID_FILE, sectors)
def create_static_table_from_file(filepath): """Stores static items to a persisted np array. The following static fields are currently persisted. -Sector -exchange -category """ register( BUNDLE_NAME, int, ) df = pd.read_csv(filepath, index_col="ticker") assert df.shape[0] > 10001 # there should be more than 10k tickers df = df[df.exchange != 'None'] df = df[df.exchange != 'INDEX'] df = df[df.table == 'SEP'] coded_sectors_for_ticker = df['sector'].map(SECTOR_CODING) coded_exchange_for_ticker = df['exchange'].map(EXCHANGE_CODING) coded_category_for_ticker = df['category'].map(CATEGORY_CODING) ae_d = get_ticker_sid_dict_from_bundle(BUNDLE_NAME) N = max(ae_d.values()) + 1 # create 2-D array to hold data where index = SID sectors = np.full((3, N), -1, np.dtype('int64')) # sectors = np.full(N, -1, np.dtype('int64')) # iterate over Assets in the bundle, and fill in static fields for ticker, sid in ae_d.items(): print(ticker, sid, coded_sectors_for_ticker.get(ticker, -1)) sectors[0, sid] = coded_sectors_for_ticker.get(ticker, -1) sectors[1, sid] = coded_exchange_for_ticker.get(ticker, -1) sectors[2, sid] = coded_category_for_ticker.get(ticker, -1) print(sectors) print(sectors[:, -10:]) # finally save the file to disk np.save(ZIPLINE_DATA_DIR + STATIC_FILE, sectors)
def create_sid_table_from_file_ipo(ticker_df): """reads the raw file, maps tickers -> SIDS, then maps sector strings to integers, and saves to the file: SID_FILE""" df = ticker_df df = df.drop_duplicates() # coded_sectors_for_ticker = df["Sector"].map(SECTOR_CODING) ae_d = get_ticker_sid_dict_from_bundle('quandl') N = max(ae_d.values()) + 1 # create empty 1-D array to hold data where index = SID ipoyears = np.full(N, -1, np.dtype('int64')) # iterate over Assets in the bundle, and fill in sectors for ticker, sid in ae_d.items(): try: ipoyears[sid] = int(df.loc[ticker].firstpricedate[:4]) except Exception as e: print(e) np.save(os.path.join(BASE_PATH , SID_FILE_IPO), ipoyears)
def num_tkrs_in_bundle(bundle_name): return len(get_ticker_sid_dict_from_bundle(bundle_name))
def all_tickers_for_bundle(fields, dims, bundle_name, raw_path=os.path.join(BASE, RAW_FLDR)): tickers = get_ticker_sid_dict_from_bundle(bundle_name) populate_raw_data(tickers, fields, dims, raw_path)
def __init__(self, *args, **kwargs): super(TransactionData, self).__init__(*args, **kwargs) self.N = len(get_ticker_sid_dict_from_bundle( "quandl")) + 1 # max(sid)+1 get this from the bundle self.data_path = os.path.join(str(Path.home()), "SF2.npy")
def __init__(self, *args, **kwargs): super(Fundamentals, self).__init__(*args, **kwargs) self.N = len(get_ticker_sid_dict_from_bundle( "sep")) + 1 # max(sid)+1 get this from the bundle self.data_path = zipline_root() + '/data/SF1.npy'
def all_tickers_for_bundle(fields): tickers = get_ticker_sid_dict_from_bundle('quantopian-quandl') populate_raw_data(tickers, fields)
def __init__(self, *args, **kwargs): super(sp500member, self).__init__(*args, **kwargs) self.N = len(get_ticker_sid_dict_from_bundle( "sep")) + 1 # max(sid)+1 get this from the bundle self.data_path = ZIPLINE_DATA_DIR + 'SP500mem.npy'
def create_static_table_from_database(ZIPLINE_DATA_DIR, STATIC_FILE, BUNDLE_NAME, SECTOR_CODING, EXCHANGE_CODING, CATEGORY_CODING): """Stores static items to a persisted np array. The following static fields are currently persisted. -Sector -exchange -category -code ->siccode """ register( BUNDLE_NAME, int, ) query = """SELECT ticker, code, sector, exchange_id, category FROM security WHERE ttable = 'SF1' """ df = pd.read_sql_query(query, engine) # add the exchange based on the exchange and exchange_id relation # get the exchange and exchange_id relation name_ex_id = get_name_exchange_id() my_EXCHANGE_CODING = name_ex_id.set_index('id')['name'].to_dict() # add the exchange based on the exchange and exchange_id relation df['exchange'] = df['exchange_id'].map(my_EXCHANGE_CODING) df['sectors_'] = df['sector'].map(SECTOR_CODING) df['exchange_'] = df['exchange'].map(EXCHANGE_CODING) df['category_'] = df['category'].map(CATEGORY_CODING) df['code_'] = df['code'].astype( int) # just multiply the siccode by 10 and get integer df = df.fillna(-1) ae_d = get_ticker_sid_dict_from_bundle(BUNDLE_NAME) N = max(ae_d.values()) + 1 # create 2-D array to hold data where index = SID sectors = np.full((4, N), -1, np.dtype('int64')) # sectors = np.full(N, -1, np.dtype('int64')) # iterate over Assets in the bundle, and fill in static fields for ticker, sid in tqdm(ae_d.items(), total=len(ae_d)): #for ticker, sid in ae_d.items(): #sector_coded = coded_sectors_for_ticker.get(ticker) if not df[df['ticker'] == ticker].empty: #sector_ = df.sectors_[df['ticker']==ticker].iloc[0] #exchange_ = df.exchange_[df['ticker']==ticker].iloc[0] #category_ = df.category_[df['ticker']==ticker].iloc[0] #code_ = df.code_[df['ticker']==ticker].iloc[0] #print(ticker, sid, sector_coded, exchange_, category_, code_ ,'<-end') sectors[0, sid] = df.sectors_[df['ticker'] == ticker].iloc[0] sectors[1, sid] = df.exchange_[df['ticker'] == ticker].iloc[0] sectors[2, sid] = df.category_[df['ticker'] == ticker].iloc[0] sectors[3, sid] = df.code_[df['ticker'] == ticker].iloc[0] else: sectors[0, sid] = -1 sectors[1, sid] = -1 sectors[2, sid] = -1 sectors[3, sid] = -1 #print('ticker missing but filled with -1, everthing under control keep cool= ',ticker) print(sectors) print(sectors[:, -10:]) # finally save the file to disk np.save(ZIPLINE_DATA_DIR + STATIC_FILE, sectors) print("this worked master")
def __init__(self, *args, **kwargs): super(Fundamentals, self).__init__(*args, **kwargs) self.N = len(get_ticker_sid_dict_from_bundle( "sep")) + 1 # max(sid)+1 get this from the bundle self.data_path = '/Users/peterharrington/.zipline/data/' + 'SF1.npy'
def __init__(self, *args, **kwargs): super(Fundamentals, self).__init__(*args, **kwargs) self.N = len(get_ticker_sid_dict_from_bundle("quantopian-quandl")) + 1 # max(sid)+1 get this from the bundle self.data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "SF1.npy")