def get(self):
yahoo = Yahoo()
yakujyos = db.GqlQuery("SELECT * FROM Yakujyo where open in ('"+STATUS_ORDER+"', '"+STATUS_OPEN+"')")
stocks = yahoo.search_stock(yakujyos)
mystocks = []
profit = 0
for idx,yakujyo in enumerate(yakujyos):
stock = {}
stock['ordered'] = None
stock['name'] = stocks.get(yakujyo.stock_code).get('name')
stock['code'] = stocks.get(yakujyo.stock_code).get('code')
stock['price'] = stocks.get(yakujyo.stock_code).get('price')
stock['sabun'] = stocks.get(yakujyo.stock_code).get('sabun')
stock['zenjitsuhi'] = stocks.get(yakujyo.stock_code).get('zenjitsuhi')
stock['dekidaka'] = stocks.get(yakujyo.stock_code).get('dekidaka')
stock['ordertype'] = yakujyo.yakujyo_type
stock['quantity'] = yakujyo.quantity
stock['idx'] = idx
if yakujyo.open == STATUS_ORDER:
stock['ordered'] = 1
if re.search(u'売', yakujyo.yakujyo_type):
stock['profit'] = yakujyo.quantity *( yakujyo.price - int(stocks.get(yakujyo.stock_code).get('price')))
elif re.search(u'買', yakujyo.yakujyo_type):
stock['profit'] = yakujyo.quantity *( int(stocks.get(yakujyo.stock_code).get('price')) - yakujyo.price )
profit += stock['profit']
mystocks.append(stock)
#logging.info(stocks)
template_values = {'stocks': mystocks,'profit': profit}
path = os.path.join(os.path.dirname(__file__), 'template/liststock.html.tmpl')
self.response.out.write(template.render(path, template_values))
return
def test_url_02(self):
url = "http://lyrics.gyao.yahoo.co.jp/ly/Y160641/"
obj = Lyric(url)
obj.parse()
self.assertEqual(obj.title, u"home")
self.assertEqual(obj.artist, u"クリス・ハート")
self.assertEqual(obj.lyricist, u"多胡邦夫")
self.assertEqual(obj.composer, u"多胡邦夫")
self.assertEqual(len(obj.lyric), 376)
def test_url_one_more(self):
url = "http://lyrics.gyao.yahoo.co.jp/ly/Y004402/"
obj = Lyric(url)
obj.parse()
self.assertEqual(obj.title, u"帰って来たヨッパライ")
self.assertEqual(obj.artist, u"ザ・フォーク・クルセダーズ")
self.assertEqual(obj.lyricist, u"ザ･フォーク･パロディ･ギャング")
self.assertEqual(obj.composer, u"加藤和彦")
self.assertEqual(len(obj.lyric), 536)
def get_wacc(self, stock):
r_e = self.r_f + stock.beta * (self.r_m - self.r_f)
lt_debt = self.get_info(stock.ticker, 'balance-sheet', [
'Long-Term Debt excl. Capitalized Leases',
'LT Debt excl. Capitalized Leases',
'LT Debt excl. Capital Lease Obligations'
])
if lt_debt is None:
self.warn('Skipping {0}...'.format(stock.ticker))
return None
else:
self.info('Good with {0}'.format(stock.ticker))
st_debt = self.get_info(stock.ticker, 'balance-sheet',
['Short Term Debt'])
debt = lt_debt + st_debt
interest_expense = self.get_info(stock.ticker, '', [
' Interest Expense', ' Total Interest Expense',
'Total Interest Expense', 'Interest Expense',
'Interest Expense, Net of Interest Capitalized'
])
r_d = 0
if debt != 0:
r_d = interest_expense / debt
equity = stock.market_cap
if equity < 1:
equity = Yahoo().get_market_cap(stock)
if equity:
stock.update(market_cap=equity)
else:
return None
value = equity + debt
income_tax_3 = self.get_info(stock.ticker, '',
['Income Tax', 'Income Taxes'], 3)
pretax_income_3 = self.get_info(stock.ticker, '',
[' Pretax Income', 'Pretax Income'], 3)
t_c = sum(income_tax_3) / sum(pretax_income_3)
wacc = (equity * r_e) / value + (debt * r_d * (1 - t_c)) / value
return wacc
from __future__ import print_function
import os, json, sys
from google import Google
from duckduckgo import Duckduckgo
from bing import Bing
from yahoo import Yahoo
scrapers = {
'g': Google(),
'b': Bing(),
'y': Yahoo(),
'd': Duckduckgo(),
}
def read_in():
lines = sys.stdin.readlines()
return json.loads(lines[0])
def small_test():
assert type(scrapers.google.results_search('fossasia')) is list
def feedgen(query, engine):
urls = scrapers[engine].results_search(query)
result = urls
print(result)
print(len(result))
return result
class Option_Scraper:
stock_list=[]
google=None
yahoo=None
def __init__(self):
self.google=Google()
self.yahoo=Yahoo()
def run(self):
self.load_stock_list()
if os.path.exists("./option_data")==False:
os.mkdir("./option_data")
print("Scrapping on " + time.strftime("%c") + "...")
#Scrapes twice because some stocks fail the first time
for temp in range(0, 1):
for stock in self.stock_list:
self.scrape_yahoo(stock)
#scrapes google if you want, but yahoo has more accurate data
# self.scrape_google(stock)
#scrapes Google Finance for option data
def scrape_google(self, stock_symbol):
expiration_dates=self.google.get_expiration_dates(stock_symbol)
for x in range(0, len(expiration_dates)):
date=[]
date.append(expiration_dates[x]['month'])
date.append(expiration_dates[x]['day'])
date.append(expiration_dates[x]['year'])
#expiration date stuff
exp_year=expiration_dates[x]['year']
exp_month=expiration_dates[x]['month']
exp_day=expiration_dates[x]['day']
cur_date=self.get_current_date()
cur_year=cur_date['year']
cur_month=cur_date['month']
cur_day=cur_date['day']
#get option data if hasn't already
path="./option_data/"+str(exp_month)+"-"+str(exp_day)+"-"+str(exp_year)+"/"+str(cur_month)+"-"+str(cur_day)+"-"+str(cur_year)+"/"+stock_symbol+"_calls.csv"
if os.path.isfile(path)==False:
option_data=self.google.get_option_data(stock_symbol, date)
self.save_option_data(stock_symbol, expiration_dates[x], option_data['call'], "calls")
self.save_option_data(stock_symbol, expiration_dates[x], option_data['put'], "puts")
#Scrapes Yahoo finance for option data
def scrape_yahoo(self, stock_symbol):
expiration_dates=self.yahoo.get_expiration_dates(stock_symbol)
#if no option dates
if len(expiration_dates)==0:
return
for x in range(0, len(expiration_dates['dates'])):
exp_year=expiration_dates['dates'][x]['year']
exp_month=expiration_dates['dates'][x]['month']
exp_day=expiration_dates['dates'][x]['day']
cur_date=self.get_current_date()
cur_year=cur_date['year']
cur_month=cur_date['month']
cur_day=cur_date['day']
#get option data if hasn't already
path="./option_data/"+str(exp_month)+"-"+str(exp_day)+"-"+str(exp_year)+"/"+str(cur_month)+"-"+str(cur_day)+"-"+str(cur_year)+"/"+stock_symbol+"_calls.csv"
if os.path.isfile(path)==False:
option_data=self.yahoo.get_option_data(stock_symbol, expiration_dates['date_ids'][x])
if option_data['call']!="":
self.save_option_data(stock_symbol, expiration_dates['dates'][x], option_data['call'], "calls")
self.save_option_data(stock_symbol, expiration_dates['dates'][x], option_data['put'], "puts")
#saves option_data to CSV in ./option_data/"+expiration_date+"/"+cur_date+".csv"
#option_type = "calls" or "puts"
def save_option_data(self, stock_symbol, expiration_date, option_data, option_type):
exp_year=expiration_date['year']
exp_month=expiration_date['month']
exp_day=expiration_date['day']
cur_date=self.get_current_date()
cur_year=cur_date['year']
cur_month=cur_date['month']
cur_day=cur_date['day']
#creates expiration folder if it doesn't exist
path="./option_data/"+str(exp_month)+"-"+str(exp_day)+"-"+str(exp_year)+"/"
if os.path.isdir(path)==False:
os.mkdir(path)
#creates current date folder if it doesn't exist
path+=str(cur_month)+"-"+str(cur_day)+"-"+str(cur_year)+"/"
if os.path.isdir(path)==False:
os.mkdir(path)
path+=stock_symbol+"_"+str(option_type)+".csv"
#converts option_data to csv table data stuff
new_data=[]
for x in range(0, len(option_data)):
if len(option_data[x])!=0:
try:
temp_data=[]
temp_data.append(option_data[x]['strike'])
temp_data.append(option_data[x]['price'])
temp_data.append(option_data[x]['bid'])
temp_data.append(option_data[x]['ask'])
temp_data.append(option_data[x]['volume'])
temp_data.append(option_data[x]['open_int'])
temp_data.append(option_data[x]['IV'])
except Exception as error:
print("Error "+str(stock_symbol)+"...")
temp_data=[]
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
new_data.append(temp_data)
with open( path , 'w', newline='') as file:
contents = csv.writer(file)
contents.writerows(new_data)
#returns stock list retrieved from txt file
def load_stock_list(self):
stocks=open("./stock_list.txt")
for symbol in stocks:
symbol=symbol.strip()
symbol=symbol.upper()
self.stock_list.append(symbol)
#returns current date in list format 2013-12-15 17:45:35.177000
#[0]=year
#[1]=month
#[2]=day
def get_current_date(self):
curDate=str(datetime.datetime.utcnow() + datetime.timedelta(hours=-8))
date=curDate.split(' ')
date=date[0]
date=date.split('-')
to_return={}
to_return['year']=date[0]
to_return['month']=date[1]
to_return['day']=date[2]
return to_return
def __init__(self): self.google=Google() self.yahoo=Yahoo()
def yahoo2df(event, context):
yahoo = Yahoo()
"""
STEP 1 : キーワード Ebay 検索
"""
page_number = 1
add_options = {
"page": page_number
}
query = event["query"]
response = yahoo.search(query=query, add_options=add_options)
pages = int(yahoo.get_total_pages(response))
items = yahoo.get_items(response)
df = yahoo.make_dataframe(items)
# Get All Pages.
if pages > 1:
for i in range(1, pages):
add_options = {
"page": i + 1
}
response = yahoo.search(query=query, add_options=add_options)
items = yahoo.get_items(response)
df = df.append(yahoo.make_dataframe(items))
"""
STEP 2 : 英翻訳、shortTitlte
"""
def get_en_title(title):
google = Google()
return google.translate(text=title, source="ja", target="en")
df["en_Title"] = df.Title.apply(get_en_title)
def get_en_short_title(en_title):
ptn = r"[^a-zA-Z0-9\s]"
title = re.sub(ptn, "", en_title)
doc = nlp(title)
# 名詞、固有名詞、数字以外を除去したカラムを作成
words = []
for token in doc:
if token.pos_ in ("PROPN", "NOUN", "NUM",):
words.append(token.text)
return " ".join(words)
df["en_short_Title"] = df.en_Title.apply(get_en_short_title)
def get_ja_short_title(title):
# ptn = r"[^a-zA-Z0-9\s]"
# title = re.sub(ptn, "", title)
doc = nlp_ginza(title)
# 名詞、固有名詞、数字以外を除去したカラムを作成
words = []
for token in doc:
print(token.pos_)
if token.pos_ in ("PROPN", "NOUN", "NUM",):
words.append(token.text)
return " ".join(words)
df["short_Title"] = df.Title.apply(get_ja_short_title)
# CSV 出力
df.to_csv("./data/yahoo_%s_%s.csv" % (query, TODAY,))
return df
# TODO implement
return {
'statusCode': 200,
'body': json.dumps('Hello from Lambda!')
}
from yahoo import Yahoo
from yandex import Yandex
from youtube import Youtube
scrapers = {
'ask': Ask(),
'baidu': Baidu(),
'bing': Bing(),
'dailymotion': Dailymotion(),
'duckduckgo': Duckduckgo(),
'exalead': Exalead(),
'google': Google(),
'mojeek': Mojeek(),
'parsijoo': Parsijoo(),
'quora': Quora(),
'yahoo': Yahoo(),
'yandex': Yandex(),
'youtube': Youtube()
}
def small_test():
assert isinstance(scrapers['google'].search('fossasia'), list)
def feedgen(query, engine, count=10):
engine = engine.lower()
# provide temporary backwards compatibility for old names
old_names = {
'ubaidu': 'baidu',
'vdailymotion': 'dailymotion',
class Option_Scraper:
stock_list=[]
google=None
yahoo=None
def __init__(self):
self.google=Google()
self.yahoo=Yahoo()
def run(self):
self.load_stock_list()
if os.path.exists("./option_data")==False:
os.mkdir("./option_data")
#Scrapes twice because some stocks fail the first time
for temp in range(0, 2):
for stock in self.stock_list[x]
self.scrape_yahoo(stock)
#scrapes google if you want, but yahoo has more accurate data
# self.scrape_google(stock)
print("\n\nScraping for a 2nd time\n")
#scrapes Google Finance for option data
def scrape_google(self, stock_symbol):
print("Scraping Google for "+str(stock_symbol)+"...")
expiration_dates=self.google.get_expiration_dates(stock_symbol)
for x in range(0, len(expiration_dates)):
date=[]
date.append(expiration_dates[x]['month'])
date.append(expiration_dates[x]['day'])
date.append(expiration_dates[x]['year'])
#expiration date stuff
exp_year=expiration_dates[x]['year']
exp_month=expiration_dates[x]['month']
exp_day=expiration_dates[x]['day']
cur_date=self.get_current_date()
cur_year=cur_date['year']
cur_month=cur_date['month']
cur_day=cur_date['day']
#get option data if hasn't already
path="./option_data/"+str(exp_month)+"-"+str(exp_day)+"-"+str(exp_year)+"/"+str(cur_month)+"-"+str(cur_day)+"-"+str(cur_year)+"/"+stock_symbol+"_calls.csv"
if os.path.isfile(path)==False:
option_data=self.google.get_option_data(stock_symbol, date)
self.save_option_data(stock_symbol, expiration_dates[x], option_data['call'], "calls")
self.save_option_data(stock_symbol, expiration_dates[x], option_data['put'], "puts")
#Scrapes Yahoo finance for option data
def scrape_yahoo(self, stock_symbol):
print("Scraping Yahoo for "+str(stock_symbol)+"...")
expiration_dates=self.yahoo.get_expiration_dates(stock_symbol)
#if no option dates
if len(expiration_dates)==0:
return
for x in range(0, len(expiration_dates['dates'])):
exp_year=expiration_dates['dates'][x]['year']
exp_month=expiration_dates['dates'][x]['month']
exp_day=expiration_dates['dates'][x]['day']
cur_date=self.get_current_date()
cur_year=cur_date['year']
cur_month=cur_date['month']
cur_day=cur_date['day']
#get option data if hasn't already
path="./option_data/"+str(exp_month)+"-"+str(exp_day)+"-"+str(exp_year)+"/"+str(cur_month)+"-"+str(cur_day)+"-"+str(cur_year)+"/"+stock_symbol+"_calls.csv"
if os.path.isfile(path)==False:
option_data=self.yahoo.get_option_data(stock_symbol, expiration_dates['date_ids'][x])
if option_data['call']!="":
self.save_option_data(stock_symbol, expiration_dates['dates'][x], option_data['call'], "calls")
self.save_option_data(stock_symbol, expiration_dates['dates'][x], option_data['put'], "puts")
#saves option_data to CSV in ./option_data/"+expiration_date+"/"+cur_date+".csv"
#option_type = "calls" or "puts"
def save_option_data(self, stock_symbol, expiration_date, option_data, option_type):
exp_year=expiration_date['year']
exp_month=expiration_date['month']
exp_day=expiration_date['day']
cur_date=self.get_current_date()
cur_year=cur_date['year']
cur_month=cur_date['month']
cur_day=cur_date['day']
#creates expiration folder if it doesn't exist
path="./option_data/"+str(exp_month)+"-"+str(exp_day)+"-"+str(exp_year)+"/"
if os.path.isdir(path)==False:
os.mkdir(path)
#creates current date folder if it doesn't exist
path+=str(cur_month)+"-"+str(cur_day)+"-"+str(cur_year)+"/"
if os.path.isdir(path)==False:
os.mkdir(path)
path+=stock_symbol+"_"+str(option_type)+".csv"
#converts option_data to csv table data stuff
new_data=[]
for x in range(0, len(option_data)):
if len(option_data[x])!=0:
try:
temp_data=[]
temp_data.append(option_data[x]['strike'])
temp_data.append(option_data[x]['price'])
temp_data.append(option_data[x]['bid'])
temp_data.append(option_data[x]['ask'])
temp_data.append(option_data[x]['volume'])
temp_data.append(option_data[x]['open_int'])
temp_data.append(option_data[x]['IV'])
except Exception as error:
print("Error "+str(stock_symbol)+"...")
temp_data=[]
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
temp_data.append(0)
new_data.append(temp_data)
with open( path , 'w', newline='') as file:
contents = csv.writer(file)
contents.writerows(new_data)
#returns stock list retrieved from txt file
def load_stock_list(self):
stocks=open("./stock_list.txt")
for symbol in stocks:
symbol=symbol.strip()
symbol=symbol.upper()
self.stock_list.append(symbol)
#returns current date in list format 2013-12-15 17:45:35.177000
#[0]=year
#[1]=month
#[2]=day
def get_current_date(self):
curDate=str(datetime.datetime.utcnow() + datetime.timedelta(hours=-8))
date=curDate.split(' ')
date=date[0]
date=date.split('-')
to_return={}
to_return['year']=date[0]
to_return['month']=date[1]
to_return['day']=date[2]
return to_return
def __init__(self): self.google=Google() self.yahoo=Yahoo()
class MarketMakerBot(SputnikSession):
external_markets = {}
yahoo = Yahoo()
bitstamp = BitStamp()
kraken = Kraken()
def startAutomationAfterMarkets(self):
self.get_external_market = task.LoopingCall(self.getExternalMarket)
self.get_external_market.start(float(self.factory.rate) * 6)
self.monitor_orders = task.LoopingCall(self.monitorOrders)
self.monitor_orders.start(float(self.factory.rate) * 1)
return True
# See if we have any orders on a given side
def cancelOrders(self, ticker, side):
for id, order in self.orders.items():
if order['is_cancelled'] or order['quantity_left'] <= 0:
continue
if order['side'] == side and order['contract'] == ticker:
self.cancelOrder(id)
def checkOrders(self, side):
for id, order in self.orders.items():
if order['is_cancelled'] or order['quantity_left'] <= 0:
continue
if order['side'] == side:
return True
return False
@inlineCallbacks
def getExternalMarket(self):
try:
bitstamp_book = yield self.bitstamp.getOrderBook('BTC/USD')
btcusd_bid = bitstamp_book['bids'][0]['price']
btcusd_ask = bitstamp_book['asks'][0]['price']
except Exception as e:
# Unable to get markets, just exit
print "unable to get external market data from bitstamp: %s" % e
raise e
for ticker, market in self.markets.iteritems():
new_ask = None
new_bid = None
if ticker not in self.factory.ignore_contracts:
if market['contract_type'] == "cash_pair":
if ticker == "BTC/USD":
new_bid = btcusd_bid
new_ask = btcusd_ask
else:
currency = market['denominated_contract_ticker']
try:
# Get Yahoo quote
yahoo_book = yield self.yahoo.getOrderBook(
'USD/%s' % currency)
bid = yahoo_book['bids'][0]['price']
ask = yahoo_book['asks'][0]['price']
except Exception as e:
# Unable to get markets, just exit
print "unable to get external market data from Yahoo: %s" % e
continue
new_bid = btcusd_bid * bid
new_ask = btcusd_ask * ask
elif market['contract_type'] == "futures":
got_spot = False
if ticker.startswith("USDBTC"):
# Ignore BTC and USD interest rates
new_bid_spot = 10000 / btcusd_ask
new_ask_spot = 10000 / btcusd_bid
# Assume 10bps USD risk-free rate
base_rate = 0.0010
got_spot = True
elif ticker.startswith("LTCBTC"):
kraken_book = yield self.kraken.getOrderBook('BTC/LTC')
btcltc_bid = kraken_book['bids'][0]['price']
btcltc_ask = kraken_book['asks'][0]['price']
new_bid_spot = 10000 / btcltc_ask
new_ask_spot = 10000 / btcltc_bid
# Assume 10% LTC risk-free rate
base_rate = 0.10
got_spot = True
if got_spot:
from datetime import datetime
import util
timedelta_to_expiry = util.timestamp_to_dt(
market['expiration']) - datetime.utcnow()
time_to_expiry = timedelta_to_expiry.total_seconds(
) / (365.25 * 24 * 60 * 60)
# Assume 5% BTC risk-free rate
btc_rate = 0.0500
import math
forward_factor = Decimal(
math.exp((base_rate - btc_rate) * time_to_expiry))
new_bid = new_bid_spot * forward_factor
new_ask = new_ask_spot * forward_factor
if new_ask is not None and new_bid is not None:
logging.info("%s: %f/%f" % (ticker, new_bid, new_ask))
# Make sure that the marketwe are making isn't crossed
if new_bid > new_ask:
tmp = new_bid
new_bid = new_ask
new_ask = tmp
# If it's matched or crossed make a spread just because
if self.price_to_wire(ticker,
new_bid) >= self.price_to_wire(
ticker, new_ask):
new_bid = min(new_bid, new_ask) - self.price_from_wire(
ticker, self.markets[ticker]['tick_size'])
new_ask = max(new_bid, new_ask) + self.price_from_wire(
ticker, self.markets[ticker]['tick_size'])
if ticker in self.external_markets:
if new_bid != self.external_markets[ticker]['bid']:
self.external_markets[ticker]['bid'] = new_bid
self.replaceBidAsk(ticker, new_bid, 'BUY')
if new_ask != self.external_markets[ticker]['ask']:
self.external_markets[ticker]['ask'] = new_ask
self.replaceBidAsk(ticker, new_ask, 'SELL')
else:
self.external_markets[ticker] = {
'bid': new_bid,
'ask': new_ask
}
self.replaceBidAsk(ticker, new_ask, 'SELL')
self.replaceBidAsk(ticker, new_bid, 'BUY')
def replaceBidAsk(self, ticker, new_ba, side):
self.cancelOrders(ticker, side)
if self.markets[ticker]['contract_type'] == "futures":
quantity = 10
else:
quantity = 2.5
self.placeOrder(ticker, quantity, new_ba, side)
def monitorOrders(self):
for ticker, market in self.external_markets.iteritems():
# Make sure we have orders open for both bid and ask
for side in ['BUY', 'SELL']:
total_qty = 0
for id, order in self.orders.items():
if order['side'] == side and order[
'is_cancelled'] is False and order[
'contract'] == ticker:
total_qty += order['quantity_left']
if self.markets[ticker]['contract_type'] == "futures":
qty_to_add = Decimal(10)
else:
qty_to_add = Decimal(2.5)
if qty_to_add > total_qty:
if side == 'BUY':
price = market['bid']
else:
price = market['ask']
self.placeOrder(ticker, qty_to_add - total_qty, price,
side)
