def diffIPOBar():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
bar = session.query(PostIPOPrice.company_id).distinct()
bsym =[b[0] for b in bar]
miss = []
for ipo in ipos:
if ipo.HistoricalIPO.company_id not in bsym:
miss.append(ipo.Company.symbol)
return miss
def open_price_dist():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
data = []
max_ratio = 0
max_symbol = ""
for ipo in ipos:
if ipo.HistoricalIPO.first_opening_price == 0:
continue
if ipo.HistoricalIPO.scoop_rating == 'N/A':
continue
if ipo.HistoricalIPO.price > 1:
ratio = ipo.HistoricalIPO.first_opening_price/ipo.HistoricalIPO.price
data.append({"symbol": ipo.Company.symbol, "price_rate": ratio, "scoop_rate": ipo.HistoricalIPO.scoop_rating})
if ratio >max_ratio:
max_ratio = ratio
max_symbol = ipo.Company.symbol
if ratio < 0.8:
print ipo.Company.symbol, ipo.Company.id, ratio
print max_symbol, max_ratio
span=numpy.arange(1,4,0.2)
fig = plt.Figure()
fig, axes = plt.subplots()
#fig.add_subplot(111)
#axes = fig.gca()
display_distribution(axes, data, "price_rate", 0.2)
plt.show()
def open_price_dist():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
data = []
max_ratio = 0
max_symbol = ""
for ipo in ipos:
if ipo.HistoricalIPO.first_opening_price == 0:
continue
if ipo.HistoricalIPO.scoop_rating == 'N/A':
continue
if ipo.HistoricalIPO.price > 1:
ratio = ipo.HistoricalIPO.first_opening_price / ipo.HistoricalIPO.price
data.append({
"symbol": ipo.Company.symbol,
"price_rate": ratio,
"scoop_rate": ipo.HistoricalIPO.scoop_rating
})
if ratio > max_ratio:
max_ratio = ratio
max_symbol = ipo.Company.symbol
if ratio < 0.8:
print ipo.Company.symbol, ipo.Company.id, ratio
print max_symbol, max_ratio
span = numpy.arange(1, 4, 0.2)
fig = plt.Figure()
fig, axes = plt.subplots()
#fig.add_subplot(111)
#axes = fig.gca()
display_distribution(axes, data, "price_rate", 0.2)
plt.show()
def check_ipo_data_validity():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
if not os.path.exists(tick_gz_path):
hi = HistoricalIPO.query.filter(
HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity = 1
session.commit()
else:
o_at = o_yh = v_at = v_yh = 0
pips_at = PostIPOPriceAT.query.filter(
PostIPOPriceAT.company_id == ipo.HistoricalIPO.company_id
).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
pips_yh = PostIPOPriceYahoo.query.filter(
PostIPOPriceYahoo.company_id == ipo.HistoricalIPO.company_id
).filter(
PostIPOPriceYahoo.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips_at) > 0:
o_at = pips_at[0].open
v_at = pips_at[0].volume
if len(pips_yh) > 0:
o_yh = pips_yh[0].open
v_yh = pips_yh[0].volume
open_vol = ipo.HistoricalIPO.open_vol
if v_at < v_yh / 1.2:
print 'incomplete tick--', symbol, 'at:', o_at, v_at, 'yh:', o_yh, v_yh, 'open_vol:', open_vol
hi = HistoricalIPO.query.filter(
HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity = 2
session.commit()
continue
if ipo.HistoricalIPO.open_vol < 5000:
#only if one of at or yh data is not present
if min(v_at, v_yh) == 0 or float(
max(v_at, v_yh) /
min(v_at, v_yh)) > 1.2 or abs(o_at - o_yh) < 0.02:
print 'suspicious volume--', symbol, 'at:', o_at, v_at, 'yh:', o_yh, v_yh, 'open_vol:', open_vol
hi = HistoricalIPO.query.filter(
HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity = 3
session.commit()
continue
#if float(max(v_at, v_yh))/min(v_at, v_yh) > 1.5 and float(max(v_at, v_yh)/min(v_at, v_yh))< 2.0:
#if float(max(v_at, v_yh))/min(v_at, v_yh) < 1.2 : #vol match, does not matter
if abs(o_at - o_yh) > 0.02:
hi = HistoricalIPO.query.filter(
HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity = 4
session.commit()
print 'mismatch open--', symbol, 'at:', o_at, v_at, 'yh:', o_yh, v_yh, 'open_vol:', open_vol
continue # open price match
def populate_post_ipo_yahoo(days=30):
try:
PostIPOPriceYahoo.__table__.create(bind=savant.db.create_engine())
except:
savant.db.session.rollback()
#ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(HistoricalIPO.first_day_volume != None).filter(Company.symbol == 'TRTLU').all()
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(
HistoricalIPO.first_day_volume != None).all()
for ipo in ipos:
sym = ipo.Company.symbol
print sym
stock = yahoo.Share(sym)
ipo_date = ipo.HistoricalIPO.ipo_date
sdate = ipo_date.strftime('%Y-%m-%d')
edate = (ipo_date + datetime.timedelta(days)).strftime('%Y-%m-%d')
result = stock.get_historical(sdate, edate)
#the result contains: adj_close and close. THe adj_close should have considered divident or split. we use close here.
if len(result) == 0:
print "cannot download historical data from Yahoo Finance for", sym
continue
try:
if sdate != result[-1]["Date"]:
print "historical data on ipo date for", sym, "is not found"
continue
except:
print "result for ", sym, 'does not contain Date:', result[-1]
continue
if len(result) < 12:
print sym, 'contains only', len(result), 'daily bar!'
continue
for res in result:
try:
rec = {
"open": float(res["Open"]),
"close": float(res["Close"]),
"high": float(res["High"]),
"low": float(res["Low"]),
"volume": int(res["Volume"]),
"date": datetime.datetime.strptime(res["Date"], '%Y-%m-%d')
}
except:
print "invalide result for", sym, res
break
post_ipo_price = PostIPOPriceYahoo(**rec)
#print post_ipo_price
#post_ipo_price.datetime = price.name
# post_ipo_price.date = price.name.split(' ')[0]
#post_ipo_price.date = res["Date"]
post_ipo_price.company_id = ipo.Company.id
savant.db.session.add(post_ipo_price)
try:
savant.db.session.commit()
except:
savant.db.session.rollback()
print "cannot save ", sym
def checkIPOStartTime(strtime):
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
fttimestr = ipo.HistoricalIPO.first_trade_time
if fttimestr=="" or fttimestr=="N/A":
#print "no start time", ipo.Company.symbol, fttimestr
continue
fttime = time.strptime(fttimestr, '%H:%M:%S.%f')
if fttime< time.strptime(strtime, '%H:%M:%S.%f'):
print ipo.Company.symbol, fttimestr
def get_all_ipo_pv():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(
HistoricalIPO.validity == 0).filter(Company.symbol == 'AM').all()
print ipos
#outfile = open(settings.DATA_HOME+"/ipo_pv.csv", 'w')
outfile = open(settings.DATA_HOME + "/ipo_pv_am.csv", 'w')
for ipo in ipos:
# print ipo.Company.symbol, str(ipo.HistoricalIPO.ipo_date).replace("-", "")
sym = ipo.Company.symbol
ipo_date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
scoop_rate = ipo.HistoricalIPO.scoop_rating
ipo_price = ipo.HistoricalIPO.price
ipo_open_price = ipo.HistoricalIPO.first_opening_price
shares = ipo.HistoricalIPO.shares
outstanding = ipo.HistoricalIPO.outstanding
try:
sector = ipo.Company.sector.name.replace(',', ';')
except AttributeError:
sector = ""
try:
industry = ipo.Company.industry.name.replace(',', ':')
except AttributeError:
industry = ""
print sym, ipo_date
bar_gz_path = settings.DATA_HOME + '/data/' + ipo_date + '/' + sym + "_second_bar.csv.gz"
if not os.path.exists(bar_gz_path):
tick_gz_path = settings.DATA_HOME + "/data/" + ipo_date + "/" + sym + "_markethours.csv.gz"
if not os.path.exists(tick_gz_path):
print "no tick data found"
continue
processors.Tick2SecondBarConverter(sym, ipo_date)
print "generated second bar for", sym
res = get_peak_valley(sym, ipo_date,
ipo.HistoricalIPO.first_trade_time, g_range_list,
g_price_type)
if res == None:
print "cannot get peak and valley data for", sym
continue
outfile.write(sym + ", " + ipo_date + ', ' + str(scoop_rate) + ', ' +
sector + ', ' + industry + ', ' + str(shares) + ', ' +
str(outstanding) + ', ' + str(ipo_price) + ', ' +
str(ipo.HistoricalIPO.open_vol) + ', ' +
str(ipo.HistoricalIPO.first_opening_price) + ', ' +
str(ipo.HistoricalIPO.revenue) + ', ' +
str(ipo.HistoricalIPO.net_income) + ', ' +
flat_result(res) + '\n')
outfile.close()
def IPO_Vol_consitency():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
if ipo.HistoricalIPO.first_opening_price == None:
continue
pips = PostIPOPrice.query.filter(PostIPOPrice.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPrice.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips) < 1:
print 'cannot find post ipo price for', ipo.Company.symbol
continue
rate = ipo.HistoricalIPO.first_day_volume*1.0/pips[0].volume
if rate < 0.9 or rate > 1.1:
print ipo.Company.symbol, rate
def check_missing_post_ipo():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
if os.path.exists(tick_gz_path):
#pips = PostIPOPriceAT.query.filter(PostIPOPriceAT.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
pips = PostIPOPriceYahoo.query.filter(PostIPOPriceYahoo.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceYahoo.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips) < 1:
# print 'no bar from yahoo', ipo.Company.symbol
pip_at = PostIPOPriceYahoo.query.filter(PostIPOPriceAT.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
if len(pip_at) < 1:
print 'no bar data', symbol
def populate_post_ipo_yahoo(days=30):
try:
PostIPOPriceYahoo.__table__.create(bind = savant.db.create_engine())
except:
savant.db.session.rollback()
#ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(HistoricalIPO.first_day_volume != None).filter(Company.symbol == 'TRTLU').all()
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(HistoricalIPO.first_day_volume != None).all()
for ipo in ipos:
sym = ipo.Company.symbol
print sym
stock = yahoo.Share(sym)
ipo_date = ipo.HistoricalIPO.ipo_date
sdate = ipo_date.strftime('%Y-%m-%d')
edate = (ipo_date+ datetime.timedelta(days)).strftime('%Y-%m-%d')
result = stock.get_historical(sdate, edate)
#the result contains: adj_close and close. THe adj_close should have considered divident or split. we use close here.
if len(result) == 0:
print "cannot download historical data from Yahoo Finance for", sym
continue
try:
if sdate != result[-1]["Date"]:
print "historical data on ipo date for", sym, "is not found"
continue
except:
print "result for ", sym, 'does not contain Date:', result[-1]
continue
if len(result) < 12:
print sym, 'contains only', len(result), 'daily bar!'
continue
for res in result:
try:
rec = {"open": float(res["Open"]), "close": float(res["Close"]), "high": float(res["High"]), "low": float(res["Low"]), "volume": int(res["Volume"]), "date": datetime.datetime.strptime(res["Date"], '%Y-%m-%d')}
except:
print "invalide result for", sym, res
break
post_ipo_price = PostIPOPriceYahoo(**rec)
#print post_ipo_price
#post_ipo_price.datetime = price.name
# post_ipo_price.date = price.name.split(' ')[0]
#post_ipo_price.date = res["Date"]
post_ipo_price.company_id = ipo.Company.id
savant.db.session.add(post_ipo_price)
try:
savant.db.session.commit()
except:
savant.db.session.rollback()
print "cannot save ", sym
def get_all_ipo_pv():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(HistoricalIPO.validity == 0).filter(Company.symbol == 'AM').all()
print ipos
#outfile = open(settings.DATA_HOME+"/ipo_pv.csv", 'w')
outfile = open(settings.DATA_HOME+"/ipo_pv_am.csv", 'w')
for ipo in ipos:
# print ipo.Company.symbol, str(ipo.HistoricalIPO.ipo_date).replace("-", "")
sym = ipo.Company.symbol
ipo_date = str(ipo.HistoricalIPO.ipo_date).replace('-','')
scoop_rate = ipo.HistoricalIPO.scoop_rating
ipo_price = ipo.HistoricalIPO.price
ipo_open_price = ipo.HistoricalIPO.first_opening_price
shares = ipo.HistoricalIPO.shares
outstanding = ipo.HistoricalIPO.outstanding
try:
sector = ipo.Company.sector.name.replace(',', ';')
except AttributeError:
sector = ""
try:
industry = ipo.Company.industry.name.replace(',', ':')
except AttributeError:
industry = ""
print sym, ipo_date
bar_gz_path = settings.DATA_HOME+ '/data/' + ipo_date + '/' +sym+"_second_bar.csv.gz"
if not os.path.exists(bar_gz_path):
tick_gz_path = settings.DATA_HOME+"/data/"+ipo_date+"/"+sym+"_markethours.csv.gz"
if not os.path.exists(tick_gz_path):
print "no tick data found"
continue
processors.Tick2SecondBarConverter(sym, ipo_date)
print "generated second bar for", sym
res = get_peak_valley(sym, ipo_date, ipo.HistoricalIPO.first_trade_time, g_range_list, g_price_type)
if res == None:
print "cannot get peak and valley data for", sym
continue
outfile.write(sym + ", " + ipo_date + ', ' + str(scoop_rate) + ', ' +
sector + ', ' + industry + ', ' +
str(shares) + ', ' + str(outstanding) + ', ' +
str(ipo_price) + ', ' + str(ipo.HistoricalIPO.open_vol) + ', ' +
str(ipo.HistoricalIPO.first_opening_price) +', ' + str(ipo.HistoricalIPO.revenue) + ', ' +
str(ipo.HistoricalIPO.net_income) + ', ' +
flat_result(res) + '\n')
outfile.close()
def update_scoop_rate():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
if ipo.HistoricalIPO.scoop_rating != 0:
continue
sym = ipo.Company.symbol
rate = rate_finder(sym)
if rate == None:
continue
if rate == 'N/A' or rate == 'N/C':
rate = 0
hi = HistoricalIPO.query.filter(HistoricalIPO.company_id == ipo.Company.id).first()
if hi == None:
continue #should not happen
hi.scoop_rating = rate
session.commit()
def check_ipo_data_validity():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
if not os.path.exists(tick_gz_path):
hi = HistoricalIPO.query.filter(HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity= 1
session.commit()
else:
o_at = o_yh = v_at = v_yh = 0
pips_at = PostIPOPriceAT.query.filter(PostIPOPriceAT.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
pips_yh = PostIPOPriceYahoo.query.filter(PostIPOPriceYahoo.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceYahoo.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips_at) > 0:
o_at = pips_at[0].open
v_at = pips_at[0].volume
if len(pips_yh) > 0:
o_yh = pips_yh[0].open
v_yh = pips_yh[0].volume
open_vol = ipo.HistoricalIPO.open_vol
if v_at < v_yh/1.2:
print 'incomplete tick--', symbol, 'at:', o_at, v_at, 'yh:', o_yh, v_yh, 'open_vol:', open_vol
hi = HistoricalIPO.query.filter(HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity= 2
session.commit()
continue
if ipo.HistoricalIPO.open_vol < 5000:
#only if one of at or yh data is not present
if min(v_at, v_yh) == 0 or float(max(v_at, v_yh)/min(v_at, v_yh)) > 1.2 or abs(o_at - o_yh) < 0.02:
print 'suspicious volume--', symbol, 'at:', o_at, v_at, 'yh:', o_yh, v_yh, 'open_vol:', open_vol
hi = HistoricalIPO.query.filter(HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity= 3
session.commit()
continue
#if float(max(v_at, v_yh))/min(v_at, v_yh) > 1.5 and float(max(v_at, v_yh)/min(v_at, v_yh))< 2.0:
#if float(max(v_at, v_yh))/min(v_at, v_yh) < 1.2 : #vol match, does not matter
if abs(o_at - o_yh) > 0.02:
hi = HistoricalIPO.query.filter(HistoricalIPO.company_id == ipo.Company.id).first()
if hi is not None:
hi.validity= 4
session.commit()
print 'mismatch open--', symbol, 'at:', o_at, v_at, 'yh:', o_yh, v_yh, 'open_vol:', open_vol
continue # open price match
def find_split_stocks():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
if os.path.exists(tick_gz_path):
pips_at = PostIPOPriceAT.query.filter(PostIPOPriceAT.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
pips_yh = PostIPOPriceYahoo.query.filter(PostIPOPriceYahoo.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceYahoo.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips_at) < 1 or len(pips_yh) < 1:
continue
o_at = pips_at[0].open
o_yh = pips_yh[0].open
ratio = (2.0*max(o_at, o_yh))/min(o_at, o_yh)
#after split the price might be x or x.5 times of the other
if abs(o_at - o_yh) > 1 and ratio%1 < 0.01 and ratio < 500:
print symbol, o_at, o_yh
def compareIPOVol(times):
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
if ipo.HistoricalIPO.first_opening_price == 0.0:
continue
pips = PostIPOPrice.query.filter(PostIPOPrice.company_id ==ipo.HistoricalIPO.company_id).all()
if len(pips) < 2:
continue
ipovol = pips[0].volume
totalvol = 0
for pip in pips:
totalvol += pip.volume
totalvol -= ipovol
avevol = totalvol/(len(pips)-1)
if(ipovol < times * avevol):
print ipo.Company.symbol, ipovol, avevol
def scoop_openratio_assoc():
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
data = []
for ipo in ipos:
if ipo.HistoricalIPO.first_opening_price == 0:
continue
if ipo.HistoricalIPO.scoop_rating == 'N/A':
continue
if ipo.HistoricalIPO.price > 1:
ratio = ipo.HistoricalIPO.first_opening_price/ipo.HistoricalIPO.price
data.append({"symbol": ipo.Company.symbol, "price_rate": ratio, "scoop_rate": ipo.HistoricalIPO.scoop_rating})
fig = plt.Figure()
fig, axes = plt.subplots()
#fig.add_subplot(111)
#axes = fig.gca()
# display_distribution(axes, data, "price_rate", 0.2)
display_association(axes, data, "price_rate", "scoop_rate")
plt.show()
def check_missing_post_ipo():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
if os.path.exists(tick_gz_path):
#pips = PostIPOPriceAT.query.filter(PostIPOPriceAT.company_id ==ipo.HistoricalIPO.company_id).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
pips = PostIPOPriceYahoo.query.filter(
PostIPOPriceYahoo.company_id == ipo.HistoricalIPO.company_id
).filter(
PostIPOPriceYahoo.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips) < 1:
# print 'no bar from yahoo', ipo.Company.symbol
pip_at = PostIPOPriceYahoo.query.filter(
PostIPOPriceAT.company_id == ipo.HistoricalIPO.company_id
).filter(
PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
if len(pip_at) < 1:
print 'no bar data', symbol
def update_ipo_tick_info():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(
HistoricalIPO.open_vol == None).all()
for ipo in ipos:
sym = ipo.Company.symbol
ipo_date = ipo.HistoricalIPO.ipo_date.strftime('%Y%m%d')
ipo_data_dir = os.path.join(tickdata_dir, ipo_date)
ipo_data_path = os.path.join(ipo_data_dir,
"%s_markethours.csv.gz" % sym)
if os.path.exists(ipo_data_dir) and os.path.exists(ipo_data_path):
#handle exceptions. WLH has trades prior IPO and it does not have market open signal.
# if sym == 'WLH':
# open_vol = 1188834
# elif sym == 'FCAU':
# open_vol = 242453
# else:
print sym
ticks = data_processor.get_ticks_by_date(sym, ipo_date, ipo_date)
analyzer = TickDataAnalyzer(ticks)
open_vol = analyzer.get_open_vol()
hi = HistoricalIPO.query.filter(
HistoricalIPO.company_id == ipo.Company.id).first()
if hi == None:
continue #should not happen
hi.open_vol = open_vol
hi.first_opening_price = analyzer.get_opening_price()
hi.first_closing_price = analyzer.get_closing_price()
hi.first_trade_time = analyzer.get_first_trade_time()
hi.first_day_high = analyzer.get_high_price()
hi.first_day_low = analyzer.get_low_price()
hi.first_day_high_percent_change = analyzer.get_high_percent_change(
)
hi.first_day_low_percent_change = analyzer.get_low_percent_change()
hi.first_day_volume = analyzer.get_volume()
#print open_vol
session.commit()
def find_split_stocks():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
if os.path.exists(tick_gz_path):
pips_at = PostIPOPriceAT.query.filter(
PostIPOPriceAT.company_id == ipo.HistoricalIPO.company_id
).filter(PostIPOPriceAT.date == ipo.HistoricalIPO.ipo_date).all()
pips_yh = PostIPOPriceYahoo.query.filter(
PostIPOPriceYahoo.company_id == ipo.HistoricalIPO.company_id
).filter(
PostIPOPriceYahoo.date == ipo.HistoricalIPO.ipo_date).all()
if len(pips_at) < 1 or len(pips_yh) < 1:
continue
o_at = pips_at[0].open
o_yh = pips_yh[0].open
ratio = (2.0 * max(o_at, o_yh)) / min(o_at, o_yh)
#after split the price might be x or x.5 times of the other
if abs(o_at - o_yh) > 1 and ratio % 1 < 0.01 and ratio < 500:
print symbol, o_at, o_yh
def scoop_openratio_assoc():
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
data = []
for ipo in ipos:
if ipo.HistoricalIPO.first_opening_price == 0:
continue
if ipo.HistoricalIPO.scoop_rating == 'N/A':
continue
if ipo.HistoricalIPO.price > 1:
ratio = ipo.HistoricalIPO.first_opening_price / ipo.HistoricalIPO.price
data.append({
"symbol": ipo.Company.symbol,
"price_rate": ratio,
"scoop_rate": ipo.HistoricalIPO.scoop_rating
})
fig = plt.Figure()
fig, axes = plt.subplots()
#fig.add_subplot(111)
#axes = fig.gca()
# display_distribution(axes, data, "price_rate", 0.2)
display_association(axes, data, "price_rate", "scoop_rate")
plt.show()
from savant.db.models import *
from savant.db import session
from savant.config import settings
from savant.ticker.processors import *
#Tick2SecondBarConverter('WLRHU', "20140606")
#exit(0)
ipos = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
bar_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_second_bar.csv.gz'
if os.path.exists(tick_gz_path) and not os.path.exists(bar_gz_path):
print ipo.Company.symbol, str(ipo.HistoricalIPO.ipo_date).replace(
"-", "")
Tick2SecondBarConverter(
ipo.Company.symbol,
str(ipo.HistoricalIPO.ipo_date).replace('-', ''))
from savant.db.models import *
from savant.db import session
from savant.config import settings
from savant.ticker.processors import *
#Tick2SecondBarConverter('WLRHU', "20140606")
#exit(0)
ipos = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).all()
for ipo in ipos:
symbol = ipo.Company.symbol
date = str(ipo.HistoricalIPO.ipo_date).replace('-', '')
tick_gz_path = settings.DATA_HOME + '/data/' + date + '/' + symbol + '_markethours.csv.gz'
bar_gz_path = settings.DATA_HOME+ '/data/' + date + '/' +symbol + '_second_bar.csv.gz'
if os.path.exists(tick_gz_path) and not os.path.exists(bar_gz_path):
print ipo.Company.symbol, str(ipo.HistoricalIPO.ipo_date).replace("-", "")
Tick2SecondBarConverter(ipo.Company.symbol, str(ipo.HistoricalIPO.ipo_date).replace('-', ''))
def __get_pages(change, hr_change, min_close, min_vol):
global _pages, _syms, _dates
_pages = []
hr_change_perc = int(100 * hr_change)
f1 = filter_pattern(change, min_close, min_vol)
order = 0
ath = ATHttpConnection()
for i in f1:
his = session.query(Company, HistoricalIPO).filter(
Company.id == HistoricalIPO.company_id).filter(
Company.symbol == i[0]).all()
if len(his) > 0 and his[
0].HistoricalIPO.ipo_date >= i[1] - datetime.timedelta(25):
#print "this is in IPO period", i
continue
ave_vol = get_average_vol(i[0], get_previous_dates(i[1], 10))
if ave_vol is None:
#print "cannot find average vol", i
continue
if i[4] < 3 * ave_vol:
#print "vol not exceeding 3 times of everage volume", i
continue
dates = get_previous_dates(i[1], 15)
dates.reverse()
dates += get_following_dates(i[1], 15)
#print i[0],i[1], len(dates)
ds = Daily.query.filter(Daily.symbol == i[0]).filter(
and_(Daily.date >= dates[0], Daily.date <= dates[-1])).all()
#further filter out high volatile pre-step stocks:
if False:
#if (i[2]-ds[0].open) > 0.05 * i[2]:
# continue
qualify = True
for d in ds:
if d.date >= i[1]:
break
if abs(d.close - d.open) > 0.03 * d.open:
qualify = False
break
if not qualify:
continue
#filter by hourly change
hbar = ath.getHourlyBar(i[0], i[1].strftime("%Y%m%d"))
if hbar is None:
print "no hour bar", i[0], i[1]
breakout = False
hr_change_list = []
for ind, data in hbar.iterrows():
rate = int(100 * (data["close"] - data["open"]) / data["open"])
hr_change_list.append(rate)
if rate > hr_change_perc:
breakout = True
if not breakout:
continue
if hr_change_list[0] > hr_change_perc:
continue
print i[0], i[1], hr_change_list
quotes = []
for d in ds:
quotes.append(
[date2num(d.date), d.open, d.high, d.low, d.close, d.volume])
_pages.append(quotes)
_syms.append(i[0])
_dates.append(i[1])
order += 1
return order
def __get_pages(change, hr_change, min_close, min_vol):
global _pages, _syms, _dates
_pages = []
hr_change_perc = int(100*hr_change)
f1 = filter_pattern(change, min_close, min_vol)
order = 0
ath = ATHttpConnection()
for i in f1:
his = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(Company.symbol == i[0]).all()
if len(his)>0 and his[0].HistoricalIPO.ipo_date >= i[1] - datetime.timedelta(25):
#print "this is in IPO period", i
continue
ave_vol = get_average_vol(i[0], get_previous_dates(i[1], 10))
if ave_vol is None:
#print "cannot find average vol", i
continue
if i[4] < 3 * ave_vol:
#print "vol not exceeding 3 times of everage volume", i
continue
dates = get_previous_dates(i[1], 15)
dates.reverse()
dates += get_following_dates(i[1], 15)
#print i[0],i[1], len(dates)
ds = Daily.query.filter(Daily.symbol == i[0]).filter(and_(Daily.date >= dates[0], Daily.date <= dates[-1])).all()
#further filter out high volatile pre-step stocks:
if False:
#if (i[2]-ds[0].open) > 0.05 * i[2]:
# continue
qualify = True
for d in ds:
if d.date >= i[1]:
break
if abs(d.close-d.open) > 0.03 * d.open:
qualify = False
break
if not qualify:
continue
#filter by hourly change
hbar = ath.getHourlyBar(i[0], i[1].strftime("%Y%m%d"))
if hbar is None:
print "no hour bar", i[0], i[1]
breakout = False
hr_change_list = []
for ind, data in hbar.iterrows():
rate = int(100*(data["close"]-data["open"])/data["open"])
hr_change_list.append(rate)
if rate > hr_change_perc:
breakout = True
if not breakout:
continue
if hr_change_list[0]>hr_change_perc:
continue
print i[0], i[1], hr_change_list
quotes=[]
for d in ds:
quotes.append([date2num(d.date), d.open, d.high, d.low, d.close, d.volume])
_pages.append(quotes)
_syms.append(i[0])
_dates.append(i[1])
order += 1
return order
def patch_ipo_date():
wrong_ipo = { 'HHC': '20101105',
'CHKR': '20111111',
'NLNK': '20111111',
# 'WLH': '20130516',
# 'RTRX': '20140110',
# 'RXDX': '20140314',
'VGGL': '',
# 'FCAU': '20141212',
'BLMT': '20111005',
'XELB': ''}
res = session.query(Company, HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(Company.symbol.in_(wrong_ipo.keys())).all()
tickdata_dir = settings.DOWNLOAD_DIR
ticker = TickDataProcessor()
for r in res:
symbol = r.Company.symbol
id = r.Company.id
#remove them from tick data
datestr = r.HistoricalIPO.ipo_date.strftime('%Y%m%d')
try:
paths = ticker.get_ticks_paths_by_date(r.Company.symbol, datestr)
for path in paths:
if path != "":
os.remove(path)
except OSError:
print "cannot find the file", path
if wrong_ipo[symbol] == "":
# remove the data and remove the symbol from ipo related tables as this is not an actual IPO, might be SPO
HistoricalIPO.query.filter(HistoricalIPO.company_id == id).delete(synchronize_session='fetch')
session.commit()
IPOInfoUrl.query.filter(IPOInfoUrl.symbol==symbol).delete(synchronize_session='fetch')
session.commit()
PostIPOPrice.query.filter(PostIPOPrice.company_id==id).delete(synchronize_session='fetch')
session.commit()
else:
hi = HistoricalIPO.query.filter(HistoricalIPO.company_id == id).first()
hi.ipo_date=datetime.strptime(wrong_ipo[symbol], '%Y%m%d').date()
session.commit()
#fe tch data
ipo_data_dir = os.path.join(tickdata_dir, wrong_ipo[symbol])
ipo_data_path = os.path.join(ipo_data_dir, "%s_markethours.csv.gz" % symbol)
if os.path.exists(ipo_data_dir) and os.path.exists(ipo_data_path):
print "IPO data found"
else:
request = {"command": "get", "symbol": symbol, "date": wrong_ipo[symbol], "gettrade": "true", "getquote": "true"}
print request
print cjson.encode(request)
fetcher_caller = fetcher.FetcherCaller()
fetcher_caller.set_request(cjson.encode(request))
try:
response = fetcher_caller.send_request()
fetcher_caller.close()
except:
print "Unable to send fetch request"
continue
count_down = 60
fetched = False
while count_down > 0:
if os.path.exists(ipo_data_path):
print "IPO data fetched:", symbol
fetched = True
time.sleep(5)
break
time.sleep(1)
count_down -= 1
if not fetched:
print "Unable to download data for", symbol
def patch_ipo_date():
wrong_ipo = {
'HHC': '20101105',
'CHKR': '20111111',
'NLNK': '20111111',
# 'WLH': '20130516',
# 'RTRX': '20140110',
# 'RXDX': '20140314',
'VGGL': '',
# 'FCAU': '20141212',
'BLMT': '20111005',
'XELB': ''
}
res = session.query(
Company,
HistoricalIPO).filter(Company.id == HistoricalIPO.company_id).filter(
Company.symbol.in_(wrong_ipo.keys())).all()
tickdata_dir = settings.DOWNLOAD_DIR
ticker = TickDataProcessor()
for r in res:
symbol = r.Company.symbol
id = r.Company.id
#remove them from tick data
datestr = r.HistoricalIPO.ipo_date.strftime('%Y%m%d')
try:
paths = ticker.get_ticks_paths_by_date(r.Company.symbol, datestr)
for path in paths:
if path != "":
os.remove(path)
except OSError:
print "cannot find the file", path
if wrong_ipo[symbol] == "":
# remove the data and remove the symbol from ipo related tables as this is not an actual IPO, might be SPO
HistoricalIPO.query.filter(HistoricalIPO.company_id == id).delete(
synchronize_session='fetch')
session.commit()
IPOInfoUrl.query.filter(IPOInfoUrl.symbol == symbol).delete(
synchronize_session='fetch')
session.commit()
PostIPOPrice.query.filter(PostIPOPrice.company_id == id).delete(
synchronize_session='fetch')
session.commit()
else:
hi = HistoricalIPO.query.filter(
HistoricalIPO.company_id == id).first()
hi.ipo_date = datetime.strptime(wrong_ipo[symbol], '%Y%m%d').date()
session.commit()
#fe tch data
ipo_data_dir = os.path.join(tickdata_dir, wrong_ipo[symbol])
ipo_data_path = os.path.join(ipo_data_dir,
"%s_markethours.csv.gz" % symbol)
if os.path.exists(ipo_data_dir) and os.path.exists(ipo_data_path):
print "IPO data found"
else:
request = {
"command": "get",
"symbol": symbol,
"date": wrong_ipo[symbol],
"gettrade": "true",
"getquote": "true"
}
print request
print cjson.encode(request)
fetcher_caller = fetcher.FetcherCaller()
fetcher_caller.set_request(cjson.encode(request))
try:
response = fetcher_caller.send_request()
fetcher_caller.close()
except:
print "Unable to send fetch request"
continue
count_down = 60
fetched = False
while count_down > 0:
if os.path.exists(ipo_data_path):
print "IPO data fetched:", symbol
fetched = True
time.sleep(5)
break
time.sleep(1)
count_down -= 1
if not fetched:
print "Unable to download data for", symbol