"--ignore_mod_time",
dest="ignore_mod_time",
type=int,
default=0)
parser.add_option("-l", "--process_lag", dest="lag", type=float)
(options, args) = parser.parse_args()
assert options.ignore_mod_time in (0, 1, 2)
if options.debug:
util.set_debug()
else:
util.set_log_file("all", True)
if options.db == "pri":
newdb.init_db()
database = newdb.get_db()
elif options.db == "sec":
newdb.init_db(os.environ["SEC_DB_CONFIG_FILE"])
database = newdb.get_db()
else:
util.error("Valid database choices are [pri|sec]")
sys.exit(1)
# Check for previously running instance
if not database.getProcessedFilesLock():
util.warning("Not processing, previous instance running")
sys.exit(1)
#XXX may want to precache seen files for speed in loading
try:
def main():
global database
parser = OptionParser()
parser.add_option('-d',
'--date',
dest='date',
help='Date to get data distribution')
parser.add_option('-g',
'--groups',
dest='groups',
help='groups.ports file')
parser.add_option('-t',
'--tickers_file',
dest='tickers_file',
help='tickers file')
parser.add_option('-k',
'--keeptogether',
dest='keeptogether',
help='list of stocks to keep on the same server')
parser.add_option(
'-a',
'--addtoall',
dest='addtoall',
help='add symbols to all universes (SHOULD NOT BE TRADED)"',
default="SPY")
parser.add_option('-s',
'--secmaster',
dest='secmaster',
help='Security master file for list of valid symbols')
parser.add_option('-m',
'--overflow_mult',
dest='overflow_mult',
help='Multiply the overflow bucket\'s volume',
default=1.0)
opt, insts = parser.parse_args()
random.shuffle(insts)
if opt.date is None or opt.groups is None or opt.tickers_file is None or opt.secmaster is None:
util.error("All options must be set:")
exit(2)
if len(insts) < 1:
util.error("Must specify at least one instance.")
newdb.init_db()
database = newdb.get_db()
secmaster = get_symlist(opt.secmaster)
secid2tickers = get_secid2tickers(opt.tickers_file)
for secid, ticker in secid2tickers.items():
if ticker not in secmaster: del secid2tickers[secid]
universe = set(secid2tickers.values())
massive = get_massive(opt.groups)
dist = get_dist(opt.date, secid2tickers, massive, float(opt.overflow_mult))
nodist = universe - set(dist.keys())
util.info("%d symbols without data distribution: %s" %
(len(nodist), " ".join(nodist)))
dist.update(map(lambda k: (k, 0.0), universe - set(dist.keys())))
if opt.keeptogether is not None:
keep = get_symlist(opt.keeptogether)
else:
keep = set()
if opt.addtoall is not None:
all = set(opt.addtoall.split(","))
else:
all = set()
assign = distribute(dist, universe, massive, insts, keep, all)
for (vol, symset, instname) in assign:
util.info("Instance %s sees %4.2f%% volume, trades %d symbols" %
(instname, vol * 100, len(symset)))
symfile = open(instname, 'w')
symlist = list(symset)
symlist.sort()
symfile.writelines(map(lambda s: s + "\n", symlist))
symfile.close()
info["end"])
#check row quality
if died is not None and died <= born:
continue
if info["attributeValue"] == "":
continue
util.debug("Getting/Inserting Security {}.{}, {}".format(
info["coid"], info["issueid"], born))
secid = __getOrCreateSecid(info["coid"], info["issueid"],
info["country"], info["currency"], born)
if info["attributeName"] == "EXCHG":
__insertExchangeHistorical(secid, info["attributeName"],
info["attributeValue"], source, born,
died, 1)
else:
__insertXrefHistorical(secid, info["attributeName"],
info["attributeValue"], source, born, died)
#revert attribute autocreate
database.setAttributeAutoCreate(autocreate)
if __name__ == "__main__":
#util.DEBUG=True
newdb.init_db()
database = newdb.get_db()
database.start_transaction()
def __init__(self):
newdb.init_db()
database = newdb.get_db()
rows = database.execute(
"SELECT value FROM {} WHERE xref_type=%(type)s AND source=%(source)s AND born<=%(now)s AND (died>%(now)s OR died is NULL)"
.format(database.XREF_TABLE), {
"type": database.getXrefType("TIC"),
"now": util.now(),
"source": database.getSourceType("compustat_idhist")
}).fetchall()
tickers = [
row['value'] for row in rows
if re.match("[0-9].+", row["value"]) == None
]
util.info("Retrieving info on %d tickers" % len(tickers))
database.close()
fields = [
"symbol",
"name",
"exchange",
"error_flag",
"market_cap",
"avg_daily_volume",
"ex_dividend_date",
"dividend_pay_date",
"dividend_share_ratio",
"dividend_yield",
#"ebitda",
"earnings_share_ratio",
"eps_est_cur_year",
"eps_est_next_qtr",
"eps_est_next_year",
"pe_ratio",
"peg_ratio",
"price_book_ratio",
"price_eps_est_cur_year_ratio",
"price_eps_est_next_year_ratio",
"price_sales_ratio",
"short_ratio",
]
# Grab data
data = ystockquote.get_symbols(tickers, fields)
# Save data to temp dir
tempdir = tempfile.mkdtemp(dir=os.environ['TMP_DIR'])
f = open("%s/yahoo.csv" % tempdir, "w")
writer = csv.DictWriter(f, fields)
rows = [dict(zip(fields, fields))]
rows.extend(data.values())
writer.writerows(rows)
f.close()
# Zip file
result = os.system(
"zip -j %s/yahoo-%s.csv.zip %s/yahoo.csv 1>/dev/null" %
(tempdir, datetime.datetime.now().strftime("%Y%m%d%H%M"), tempdir))
if (result != 0):
shutil.rmtree(tempdir)
raise DataSourceError("Could not zip file")
os.remove("%s/yahoo.csv" % tempdir)
self._remote_dir = tempdir
