def pyq_predict(n):
q('.debug.ST:.z.P')
print('generating knn prediction using kdb+ and python')
qpreds=q("{knn[pyedist2;training.features;training.labels;5;x#'testing.features]}",n)
print('done, time taken:')
print(q('.z.P-.debug.ST'))
return qpreds
def run(q_prompt=False):
if len(sys.argv) > 1:
try:
q(r'\l %s' % sys.argv[1])
except kerr:
pass
else:
del sys.argv[1]
if q_prompt:
q()
ptp.run()
def q(line, cell=None, _ns=None):
"""Run q code.
Options:
-l (dir|script) - pre-load database or script
-h host:port - execute on the given host
-o var - send output to a variable named var.
-i var1,..,varN - input variables
"""
if _ns is None:
_ns = vars(sys.modules['__main__'])
input = output = None
preload = []
try:
if cell is None:
r = pyq.q(line)
else:
h = pyq.q('0i')
if line:
for opt, value in getopt(line.split(), "h:l:o:i:")[0]:
if opt == '-l':
preload.append(value)
elif opt == '-h':
h = pyq.K(str(':' + value))
elif opt == '-o':
output = str(value) # (see #673)
elif opt == '-i':
input = str(value).split(',')
r = None
for script in preload:
h(pyq.kp(r"\l " + script))
if input is not None:
for chunk in logical_lines(cell):
func = "{[%s]%s}" % (';'.join(input), chunk)
args = tuple(_ns[i] for i in input)
r = h((pyq.kp(func),) + args)
else:
for chunk in logical_lines(cell):
r = h(pyq.kp(chunk))
except pyq.kerr as e:
print("'%s" % e)
else:
if output is not None:
if output.startswith('q.'):
pyq.q('@[`.;;]', output[2:], r)
else:
_ns[output] = r
else:
if r != Q_NONE:
return r
def test_inside_ci():
# Test designed for Gitlab CI
if 'GITLAB_CI' not in os.environ:
pytest.skip("This test designed for Gitlab CI only.")
qver, qmin = os.environ.get('KDB_VER', '0.0').split('.')
assert q(".z.K") == int(qver) + 0.1 * int(qmin)
def test_k0a(q, shape):
size = numpy.prod(shape)
x = 1.0 + q.til(size)
z = q('#', list(shape), x)
a = numpy.empty(shape)
_n.k0a(a, z)
while z._t == 0:
z = z.raze
assert z == a.flatten()
def test_inside_ci():
# Test designed for Gitlab CI
if 'GITLAB_CI' not in os.environ:
pytest.skip("This test designed for Gitlab CI only.")
qmin = int(os.environ.get('QMIN', '0'))
qver = int(os.environ.get('QVER', '0'))
assert q(".z.K") == qver + 0.1 * qmin
def py_predict(n):
q('.debug.ST:.z.P') #start a timer in q
features_train=np.array(q('flip training.features'))
features_test=np.array(q('{x#flip testing.features}',n))
labels_train=np.array(q('training.labels'))
labels_test=np.array(q('{x#testing.labels}',n))
classifier=build_classifier(features_train,labels_train)
print('generating prediction using sklearn nearest neighbors')
preds=classifier.predict(features_test) #TODO, respect N
print('done time taken:')
print(q('.z.P-.debug.ST'))
return preds
def run(q_prompt=False):
lines, columns = console_size()
q(r'\c %d %d' % (lines, columns))
if len(sys.argv) > 1:
try:
q(r'\l %s' % sys.argv[1])
except kerr as e:
print(e)
raise SystemExit(1)
else:
del sys.argv[1]
if q_prompt:
q()
ptp.run()
import os
from tempfile import mkstemp
import pyq
from io import StringIO
from getopt import getopt
import sys
STD_STREAM = [sys.stdin, sys.stdout, sys.stderr]
try:
string_types = (str, unicode)
except NameError:
string_types = (str,)
Q_NONE = pyq.q('::')
def logical_lines(lines):
if isinstance(lines, string_types):
lines = StringIO(lines)
buf = []
for line in lines:
if buf and not line.startswith(' '):
chunk = ''.join(buf).strip()
if chunk:
yield chunk
buf[:] = []
buf.append(line)
chunk = ''.join(buf).strip()
def test_pypath():
assert q('::', py.path.local('/abc/def')) == q("`:/abc/def")
def connect(self):
q.connectTp = q(
'{hopen hsym x}'
) #Takes string as input, eg q.connectTp("localhost:5000") < the q interpreter will see this as a symbol
self.handle = q.connectTp(self.tp)
from pyq import q, K
q.set(
':alpha/HDB/',
q(
'.Q.en', ':alpha',
q('([]Name:`$(); Ask:`float$(); Bid:`float$(); Last:`float$(); Volume:`float$(); Time:`timestamp$())'
)))
# ------- ------------------ -------------------------------
# 15Aug17 Joel Oliveira Original version
from pyq import q
import math
import pandas as pd
import time
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn import preprocessing, cross_validation
import datetime
import os
# Make sure python and Kdb are running on the same process
print "Python process: ", os.getpid()
print "Kdb process: ", q('.z.i,.z.f')
# Load data from the file
q('trade:("DFFFFFFFFFFFF"; enlist ",")0:`:aapl.csv')
#q.trade.show()
# Create a new column HL_PCT
q('trade:update Hl_Pct:((Adj_High - Adj_Low) % Adj_Close) * 100.0 from trade')
# Create a new column PCT_CHANGE
q('trade:update Pct_Change:((Adj_Close - Adj_Open) % Adj_Open) * 100.0 from trade'
)
# Remove non-used columns
q('trade: select Date, Adj_Close, Hl_Pct, Pct_Change, Adj_Volume from trade')
## usage pyq csv_drops/csv_loader.py -dir csv_drops/trade.csv
import pandas as pd
import numpy as np
from pyq import q
import sys
dir = sys.argv[
1 + sys.argv.index('--dir')] ## read csv directory from cmd line args
dt = pd.read_csv(dir) #use of pandas read_csv
q.dt = [np.array(dt[x]) for x in dt.columns] #convert dt columns into arrays
q.dt = q.dt.flip # flip to the correct format for upsert
h = q.hopen('::5010') # hopen to tickerplant
upsert2 = q('{[x;y].at.d:(x;y); y[;0] : "N"$string y[;0]; x upsert y}'
) # create function to update symbol timespan column
h((upsert2, 'trade', (q.dt)))
def q(line, cell=None, _ns=None):
"""Run q code.
Options:
-l (dir|script) - pre-load database or script
-h host:port - execute on the given host
-o var - send output to a variable named var.
-i var1,..,varN - input variables
-1/-2 - redirect stdout/stderr
"""
if cell is None:
return pyq.q(line)
if _ns is None:
_ns = vars(sys.modules['__main__'])
input = output = None
preload = []
outs = {}
try:
h = pyq.q('0i')
if line:
for opt, value in getopt(line.split(), "h:l:o:i:12")[0]:
if opt == '-l':
preload.append(value)
elif opt == '-h':
h = pyq.K(str(':' + value))
elif opt == '-o':
output = str(value) # (see #673)
elif opt == '-i':
input = str(value).split(',')
elif opt in ('-1', '-2'):
outs[int(opt[1])] = None
if outs:
if int(h) != 0:
raise ValueError("Cannot redirect remote std stream")
for fd in outs:
tmpfd, tmpfile = mkstemp()
try:
pyq.q(r'\%d %s' % (fd, tmpfile))
finally:
os.unlink(tmpfile)
os.close(tmpfd)
r = None
for script in preload:
h(pyq.kp(r"\l " + script))
if input is not None:
for chunk in logical_lines(cell):
func = "{[%s]%s}" % (';'.join(input), chunk)
args = tuple(_ns[i] for i in input)
if r != Q_NONE:
r.show()
r = h((pyq.kp(func), ) + args)
if outs:
_forward_outputs(outs)
else:
for chunk in logical_lines(cell):
if r != Q_NONE:
r.show()
r = h(pyq.kp(chunk))
if outs:
_forward_outputs(outs)
except pyq.kerr as e:
print("'%s" % e)
else:
if output is not None:
if output.startswith('q.'):
pyq.q('@[`.;;:;]', output[2:], r)
else:
_ns[output] = r
else:
if r != Q_NONE:
return r
def test_array(q):
x = q('([]d:"d"$0 1;t:"t"$0 1)')
a = _n.array(x)
assert x.d == a['d']
def test_k2a_date(q, unit, x, y):
dtype = 'M8[%s]' % unit
a = numpy.empty([1], dtype)
x = q(x).enlist
_n.k2a(a, x)
assert numpy.array_equiv(a, numpy.array([y], dtype))
def q_types(q):
return q("{{key x$()}'[x t]!`short$t:where not null x}.Q.t")
def test_issue_615(q):
x = q('0#`')
a = _n.array(x)
assert a.dtype == numpy.dtype('O')
def q(line, cell=None, _ns=None):
"""Run q code.
Options:
-l (dir|script) - pre-load database or script
-h host:port - execute on the given host
-o var - send output to a variable named var.
-i var1,..,varN - input variables
-1/-2 - redirect stdout/stderr
"""
if cell is None:
return pyq.q(line)
if _ns is None:
_ns = vars(sys.modules['__main__'])
input = output = None
preload = []
outs = {}
try:
h = pyq.q('0i')
if line:
for opt, value in getopt(line.split(), "h:l:o:i:12")[0]:
if opt == '-l':
preload.append(value)
elif opt == '-h':
h = pyq.K(str(':' + value))
elif opt == '-o':
output = str(value) # (see #673)
elif opt == '-i':
input = str(value).split(',')
elif opt in ('-1', '-2'):
outs[int(opt[1])] = None
if outs:
if int(h) != 0:
raise ValueError("Cannot redirect remote std stream")
for fd in outs:
tmpfd, tmpfile = mkstemp()
try:
pyq.q(r'\%d %s' % (fd, tmpfile))
finally:
os.unlink(tmpfile)
os.close(tmpfd)
r = None
for script in preload:
h(pyq.kp(r"\l " + script))
if input is not None:
for chunk in logical_lines(cell):
func = "{[%s]%s}" % (';'.join(input), chunk)
args = tuple(_ns[i] for i in input)
if r != Q_NONE:
r.show()
r = h((pyq.kp(func),) + args)
if outs:
forward_outputs(outs)
else:
for chunk in logical_lines(cell):
if r != Q_NONE:
r.show()
r = h(pyq.kp(chunk))
if outs:
forward_outputs(outs)
except pyq.kerr as e:
print("'%s" % e)
else:
if output is not None:
if output.startswith('q.'):
pyq.q('@[`.;;:;]', output[2:], r)
else:
_ns[output] = r
else:
if r != Q_NONE:
return r
import io
import os
from tempfile import mkstemp
import pyq
from getopt import getopt
import sys
StringIO = io.StringIO # LGTM.com does not like double import
STD_STREAM = [sys.stdin, sys.stdout, sys.stderr]
try:
string_types = (str, unicode)
except NameError:
string_types = (str, )
Q_NONE = pyq.q('::')
def logical_lines(lines):
"""Merge lines into chunks according to q rules"""
if isinstance(lines, string_types):
lines = StringIO(lines)
buf = []
for line in lines:
if buf and not line.startswith(' '):
chunk = ''.join(buf).strip()
if chunk:
yield chunk
buf[:] = []
buf.append(line)
from pyq import q
from datetime import datetime
import argparse
import csv
#Defining kdb/q helper functions
q.connect = q('{hopen hsym x}')
q.typeMap = q(
'`time`sym`price`size`bid`ask`asize`bsize`max_price`min_price`volume!("NSFIFFIIFFI")'
)
q.tradeTbl = q('([]time:"n"$(); sym:`symbol$();price:`float$();size:`int$())')
q.quoteTbl = q(
'([]time:"n"$(); sym:`symbol$();bid:`float$();ask:`float$(); bsize:`int$();asize:`int$())'
)
q.aggTbl = q(
'([]time:"n"$(); sym:`symbol$() ; max_price:`float$() ; min_price: `float$(); volume:`int$())'
)
q.header = q('{`$"," vs raze system raze "head -1 ",string x }')
q.createTbl = q('{[datatypes;header]flip datatypes$header!()}')
q.row_data = q('{[datatypes;row] (raze string datatypes)$(string row) }')
q.send_update = q('{[handle;tbl;data] handle(`.u.upd;tbl;data)}')
def parseArgs():
print("Parsing command line arguments")
parser = argparse.ArgumentParser()
parser.add_argument(
"-csv",
help=
"The filepath to the csv you want to publish to the kdb TP instance ",
type=str)
from bittrex.bittrex import Bittrex, API_V2_0
from datetime import datetime
import time, dictKDB
from pyq import q, K
get_bittrex = Bittrex(None, None)
starttime = time.time()
q.load(':alpha/HDB/')
while True:
market_result = get_bittrex.get_market_summaries()['result']
for res in market_result:
market_name = res['MarketName']
ask = float(res['Ask'])
bid = float(res['Bid'])
last = float(res['Last'])
volume = float(res['Volume'])
dt = res['TimeStamp']
if market_name in [
'USDT-BTC', 'USDT-ETH', 'USDT-LTC', 'USDT-XRP', 'USDT-NEO',
'USDT-BCC', 'USDT-ZEC', 'USDT-XMR', 'USDT-DASH'
]:
ts = datetime.strptime(dt, '%Y-%m-%dT%H:%M:%S.%f')
data = [market_name[5:], ask, bid, last, volume, ts]
data[0] = q('`:alpha/sym?', data[0])
q.insert(':alpha/HDB/', data)
q.get(':alpha/HDB/').show()
time.sleep(30.0)
# def __del__(self):
# self.my_client.close_connection()
if __name__ == "__main__":
# df = pd.read_csv("~/github/kdbLearn/companylist.csv")
# c=pyq.q.conn(host="127.0.0.1",port=12345)
# print(c('1+1'))
# print(dir(q))
# pyq.q.symbols.show()
# pyq.q("\l /home/acer/github/kdbLearn/companylist")
T0 = time.time()
pyq.q(
"symbols:(\"SSFSISS\";enlist\",\")0:`:/home/acer/github/kdbLearn/companylist.csv"
)
pyq.q(
"quotes:(\"DSEEE\";enlist\",\")0:`:/home/acer/github/kdbLearn/quotes.csv"
)
# pyq.q("`sym`date xkey `quotes")
# pyq.q("`Symbol xkey `symbols")
pyq.q("getallsymbols:{`Symbol xasc select Symbol from symbols}")
quotes = pyq.q("showquotes:{`show quotes}")
quotes.show()
x = pyq.q.getallsymbols()
# x.show()
from pyq import q
from sklearn.neighbors import KNeighborsClassifier
import numpy as np
#load the training data in q
print('loading data')
q('system"l knn.q"')
print('ready')
def build_classifier(X,y):
neighbors=KNeighborsClassifier(n_neighbors=5,algorithm='brute',metric='euclidean')
neighbors.fit(X,y)
return neighbors
def py_predict(n):
q('.debug.ST:.z.P') #start a timer in q
features_train=np.array(q('flip training.features'))
features_test=np.array(q('{x#flip testing.features}',n))
labels_train=np.array(q('training.labels'))
labels_test=np.array(q('{x#testing.labels}',n))
classifier=build_classifier(features_train,labels_train)
print('generating prediction using sklearn nearest neighbors')
preds=classifier.predict(features_test) #TODO, respect N
print('done time taken:')
print(q('.z.P-.debug.ST'))
return preds
def q_predict(n):
q('.debug.ST:.z.P')
print('generating knn prediction using kdb+')
qpreds=q("{knn[edist2;training.features;training.labels;5;x#'testing.features]}",n)
print('done, time taken:')
print(q('.z.P-.debug.ST'))
return qpreds
def test_symbol_enum_mix(q):
x = q('(`sym?`a`b;`x`y)')
a = _n.array(x)
assert a.tolist() == [['a', 'b'], ['x', 'y']]
import time
import pandas as pd
import pyq
from datetime import date
from polygon import WebSocketClient, STOCKS_CLUSTER
import numpy as np
if __name__ == "__main__":
df = pd.read_csv("~/github/kdbLearn/companylist.csv")
# c=pyq.q.conn(host="127.0.0.1",port=12345)
# print(c('1+1'))
# print(dir(q))
print("=============")
# pyq.q.symbols.show()
pyq.q("\l /home/acer/github/kdbLearn/companylist")
pyq.q("getallsymbols:{select Symbol from symbols}")
# pyq.q("getallsymbols:{select Symbol from symbols where Symbol=`MSFT}")
x = pyq.q.getallsymbols()
# x.show()
for name in x:
# name.show()
# print(name, type(name))
# print(name.Symbol, type(name.Symbol))
sym = str(name.Symbol)
print(sym, type(sym))
# print("type(x)", type(x))
# print(list(x))
def test_pypath():
p = py.path.local('/abc/def')
assert q('::', p) == ':' + p.strpath
