def checks(qdate,symbol,quotes):
#Check if vols are populated
if quotes.empty:
print(f"{qdate} {symbol} - Nothing returned on quotes, are rates populated?")
return 0
else:
while True:
vols = query.vols(["Time","Tenor","Strike"],["Tenor","Strike"],symbol,qdate,qtime=None,distinct=True)
if vols.empty == False: break
cout.warn(f"{qdate} {symbol} - missing vols from the selected date & symbol, unable to calibrate, waiting for fetch to populate")
sleep(60)
#Check if table exist and params are populated
tables = query.get("Params","select count(*) from information_schema.tables where table_name = 'main'").iloc[0,0]
if tables == 1:
q = f"select distinct Symbol,Date from dbo.main where Symbol = '{symbol}' and Date = convert(datetime,'{qdate}',103)"
params = query.get("Params",q)
if params.empty == False and replace == False:
cout.info(f"{qdate} {symbol} - params already populated, moving on to next")
return 0
elif params.empty == False and replace == True:
cout.warn(f"{qdate} {symbol} - params already populated, cleaning up")
q = f"delete from dbo.main where Symbol = '{symbol}' and Date = convert(datetime,'{qdate}',103)"
query.execute("Params",q)
else:
cout.info(f"{qdate} {symbol} - params are empty for the selected date & symbol")
def request(keyword, size, is_suggest):
URL = 'http://113.198.137.239:9200/tags/words/_search'
headers = {'Accept': 'text/plain', 'Content-type': 'application/json'}
http_proxy = "http://113.198.137.239:8080"
https_proxy = "https://113.198.137.239:8080"
proxyDict = {"http": http_proxy, "https": https_proxy}
result = requests.post(URL,
data=json.dumps(
query.get(size=size,
keyword=keyword,
is_suggest=is_suggest)),
headers=headers,
proxies=proxyDict)
if not is_suggest:
hits = json.loads(result.text)['hits']['hits']
if len(hits) > 0:
return [h['_source']['keyword'] for h in hits]
else:
hits = json.loads(result.text)['suggest']['autocomp'][0]
if len(hits) > 0:
return [h['text'] for h in hits['options']]
return []
def show_adv(id):
advertisement = query.get(id)
if 'error' in advertisement:
page_not_found('Wrong id')
images = query.get_images(id)
return render_template('advertisement.html',
advertisement=advertisement,
images=images)
def edit_adv(id):
images = query.get_images(id)
advertisement = query.get(id)
if advertisement:
if advertisement[1] != session['id']:
return page_not_found
return render_template('edit.html',
images=images,
advertisement=advertisement,
advertisement_id=id)
def checks(qdate, symbol, quotes, weekday):
#Check if quotes are populated
if quotes.empty:
print(
f"{qdate} {symbol} - Nothing returned on quotes, are rates populated?"
)
return 0
else:
while True:
remainder = query.opt_remainder(3, weekday, symbol, "USD", qdate,
14)
if remainder.empty: break
left = len(remainder)
cout.warn(
f"{qdate} {symbol} - {left} quotes missing, waiting for fetch to populate"
)
sleep(60)
#Check if table exist and vols are populated
tables = query.get(
"Vols",
f"select count(*) from information_schema.tables where table_name = '{symbol}'"
).iloc[0, 0]
if tables == 1:
q = f"select distinct Time,Tenor,Strike from dbo.[{symbol}] where Date = convert(datetime,'{qdate}',103) order by Tenor,Strike"
vols = query.get("Vols", q)
quotes = quotes[["Time", "Tenor", "Strike"]].drop_duplicates()
if len(vols) == len(quotes):
cout.info(
f"{qdate} {symbol} - vols already populated, moving on to next"
)
return 0
elif len(vols) > 0:
cout.warn(
f"{qdate} {symbol} - vols are partially populated, cleaning up"
)
q = f"delete from dbo.[{symbol}] where Date = convert(datetime,'{qdate}',103)"
query.execute("Vols", q)
else:
cout.info(
f"{qdate} {symbol} - vols are empty for the selected date & symbol"
)
def checkDatabase():
exists = False
if os.path.exists(DB_NAME):
exists = True
if not exists:
print "\nDatabase doesn't exists"
print "Creating database..."
print "Creating Table 'Server'..."
sql = query.get('ct_server')
executeQuery(sql, 'I', False)
print "Creating Table 'RFCs'..."
sql = query.get('ct_rfcs')
executeQuery(sql, 'I', False)
exists = True
print "Creating Table 'Signals'..."
sql = query.get('ct_signals')
executeQuery(sql, 'I', False)
return exists
def executeQuery(query_name, q = 'S', val=True, pars=()):
r = None
if val:
checkDatabase()
sql = query.get(query_name) % pars
con = sqlite3.connect(DB_NAME)
cur = con.cursor()
cur.execute(sql)
if q == 'S':
r = cur.fetchall()
if q == 'I':
r = con.commit()
cur.close()
con.close()
return r
def API_query():
'''
: text => edges
$
curl "http://127.0.0.1:5000/query" -H "Content-Type:text/json" -d '{"query": "_ -> + ACh"}'
'''
data = request.get_json(force=True)
q = data['query']
csrft = request.cookies.get('csrft')
if not csrft:
return
with db.as_collection(csrft):
results = list(edge.format() for edge in query.get(q))
response = {'results': results}
response = flask.jsonify(**response)
return response
def client():
ack_port = 16382
print('--欢迎使用文件分享系统--')
print('输入get 文件名来向网络中的设备搜索文件')
print('或输入quit退出')
print('注意：当该程序开启时，会向其它peer分享share文件夹中的文件')
while True:
command = input('>> ')
if command == 'quit':
#退出程序
send_quit()
return
else:
commands = command.split(' ')
if commands[0] == 'get':
if len(commands) == 1:
print('请在get之后输入要搜索的文件名')
continue
filename = commands[1]
query(filename)
print('正在搜索{}'.format(filename))
ack_res = listen_ack(ack_port)
if ack_res == None:
# 未搜索到
print('未搜索到相应文件')
continue
filepath = ack_res[0]
address = ack_res[1]
filesize = int(ack_res[2])
#print(filepath)
#print(address)
print('开始向{}下载{}'.format(address[0], filename))
res = get(filename, filepath, address, filesize)
if res:
print('下载完毕')
else:
print('下载过程出现问题，请稍后再试')
else:
print('指令{}无效'.format(commands[0]))
continue
def request(params):
"""
url : index - related / type : words
params : input - query string (keyword)
sort - key of sort [ genyear : generated year of keyword, co_occur : co-occurance count of related keyword]
order : order of sort [ 'asc', 'desc' ]
"""
URL = 'https://fd2b16254da343159c56f09ad393c420.us-west-1.aws.found.io:9243/related/words/_search'
headers = {'Accept': 'text/plain', 'Content-type': 'application/json'}
# if params['input'] is not list:
# params['input'] = [ params['input'] ]
# sort = 'relatedwords.co_occur' if co_occur else 'relatedwords.genyear'
r = requests.get(URL,
data=json.dumps(query.get(params)),
headers=headers,
auth=('elastic', 'Ftkn0jSUxwI867OzNmPiAVeu'))
hits = json.loads(r.text)['hits']['hits']
if len(hits) > 0:
return [h['_source']['relatedwords'] for h in hits]
return []
def rates(currency, tenor):
if currency == "EUR":
name = "EURIBOR" if tenor != "ON" else "EONIA"
else:
name = "LIBOR"
url_dict = links(currency)
url = url_dict[tenor]
res = requests.get(url).text
soup = BeautifulSoup(res, "html.parser")
tables = soup.findAll("table")[6].findAll("tr")[14:26]
df = pd.DataFrame(columns=["Date", "Type", "Tenor", "Rate", "Currency"])
for caption in tables:
caption_date = caption.text.strip().rsplit('0.', 1)[0].replace("-", "")
date = pd.to_datetime(caption_date)
rate = round(float(caption.text.strip()[-8:][:6]) / 100, 5)
row = [date, name, tenor, rate, currency]
print(row)
df.loc[len(df)] = row
df = pd.concat([df, query.get("Static", "select * from dbo.rates")],
axis=0)
df = df.drop_duplicates(["Date", "Tenor"])
df = parse_dates(df, ["Date"], "Date", "%d/%m/%Y")
query.post(df, "Static", "rates", "replace")
print("Posted")
import source_hist
import query
import utils
cout = utils.log(__file__, __name__)
curr = "USD"
window = "1 D"
qdate = "20/10/2020"
q = "select distinct Symbol from dbo.chains where Symbol != 'FP'"
symbols = query.get("Static", q)
symbols = symbols["Symbol"].sort_values().to_list()
try:
for symbol in symbols:
req = source_hist.QuotesReq(qdate, symbol, curr, window)
stk = query.stk_remainder(symbol, qdate)
while stk.empty == True:
cout.info(f"{symbol} - requesting spot quotes")
req.equities()
stk = query.stk_remainder(symbol, qdate)
else:
cout.info(f"{symbol} - stk table already populated")
opt = query.opt_remainder(3, symbol, curr, qdate)
while opt.empty == False:
cout.info(
f"{symbol} - requesting {len(opt.index)} contract quotes")
for i in opt.index:
cout.info(f"{qdate} {symbol} - calibrating vols for slices")
df = pd.DataFrame()
for qtime in query.trading_times(qdate,symbol):
dfx,eps,n = calibrator.surface(symbol,qdate,qtime,errors=True,n=True)
dfx["N"] = n #Sample size
dfx["E"] = eps[-1]/n #Mean error
df = pd.concat([df,dfx],axis=0)
query.post(df,"Params","main","append")
elapsed = round((time.time()-start_time)/60,3)
cout.info(f"{qdate} {symbol} - elapsed {elapsed} mins")
#---------------------------------------------------------------------------------------------------------
symbols = query.get("Static","select distinct Symbol from dbo.chains")["Symbol"].sort_values().to_list()
replace = False
qdates = ["16/11/2020"]
# start_date = input("Select start trade date in dd/mm/yyyy format: ")
# end_date = input("Select end trade date in dd/mm/yyyy format: ")
# qdates = [x.strftime('%d/%m/%Y') for x in utils.drange(start_date,end_date)]
try:
for qdate in qdates: main(qdate,symbols)
cout.terminate()
except:
cout.error("Error")
#Sort BIIB for 06/11/2020
def _quickGet(self,query,objectId):
try:
return query.get(objectId=objectId)
except:
return None
def surface(symbol,
qdate,
qtime=None,
errors=False,
post=True,
loc=0,
scale=None):
q = f"select * from dbo.{symbol} where Date = convert(datetime,'{qdate}',103) order by Time,Tenor,Strike"
df = query.get("Vols", q)
if qtime is not None: df = df[df["Time"] == qtime]
st = utils.pack(df)
del df
#----------------------------------------------------------------------------------
f = lambda LST: pricer.norm_weights(LST, loc, scale)
wgt = utils.apply(f, 1, st, ["Time", "Tenor"], ["LogStrike"])
sv = pricer.SSVI(errors)
f = lambda LST, VAR, TNR, VOL: sv.calibrate(LST, VAR, TNR, VOL, wgt)
if errors == True:
prm = utils.apply(f,
6,
st, ["Time"],
["LogStrike", "TotAtmfVar", "Tenor", "SmtVol"],
diff=True,
fill=False)
errors = prm[-1]
prm = np.asarray(prm[:-1]).T
else:
prm = utils.apply(f,
5,
st, ["Time"],
["LogStrike", "TotAtmfVar", "Tenor", "SmtVol"],
fill=False)
#----------------------------------------------------------------------------------
reduced = [
"Time", "Tenor", "SpotMid", "Forward", "CumDivDays", "Div", "ImpBor",
"Rate", "CallH0", "CallH1", "CallKh", "PutH0", "PutH1", "PutKh",
"TotAtmfVar"
]
st = utils.unique_nosort(st[reduced])
prm = utils.restack(prm, st["TotAtmfVar"], tile=False)
rho = sv.correlation(st["TotAtmfVar"], prm[:, 0], prm[:, 1], prm[:, 2])
phi = sv.power_law(st["TotAtmfVar"], prm[:, 3], prm[:, 4])
atm, skw, krt = sv.raw2jw(st["Tenor"] / base, st["TotAtmfVar"], phi, rho)
arr = unstruct(st)
arr = np.column_stack([arr, atm, skw, krt, phi, rho])
df = utils.unpack(arr, qdate, False, symbol)
if post == True:
query.post(df, "Params", "main", "replace")
return df
# symbol = "JPM"
# qdate = "16/10/2020"
# qtime = 2040
# errors = True
# df = surface(symbol,qdate,qtime=None,errors=False,post=False)
def getURL(filename):
solr = connection.get_connection()
queryURL = r'filename:(.)*' + filename
output = query.get(solr, constant.SOLR_CORE, queryURL,1 )
return output.result.dict['response']['docs'][0]['id']
def trading_times(symbol):
tms = query.get("Vols",f"select distinct Time from dbo.{symbol} where Date = convert(datetime,'25/09/2020',103)")["Time"].tolist()
return [str(x)[:2]+":"+str(x)[2:]+" UTC" for x in list(sorted(tms))]
def symbol_universe():
return query.get("Static","select distinct Symbol from dbo.chains where Symbol != 'FP' order by Symbol")["Symbol"].tolist()