def f1(time, rdd):
lt = rdd.collect()
if not lt:
return
print '**** f1'
print lt
print '**** end f1'
f = open(
'/home/larry/l1304/workspace/finopt/data/mds_files/std/std-20151008.txt',
'a') # % datetime.datetime.now().strftime('%Y%m%d%H%M'), 'a')
msg = ''.join('%s,%s,%s,%s,%s\n' %
(s[0], s[1][0][0].strftime('%Y-%m-%d %H:%M:%S.%f'),
s[1][0][1], s[1][0][2], s[1][1]) for s in lt)
f.write(msg)
d = Q.value
# return rdd tuple (-,((-,-),-)): name = 0--, time 100, sd 101, mean 102, vol 11-
for s in lt:
if s[0].find('HSI-20151029-0') > 0 and (s[1][0][1] > 4.5
or s[1][1] > 100000):
msg = 'Unusal trading activity: %s (SD=%0.2f, mean px=%d, vol=%d) at %s\n'\
% (s[0], \
s[1][0][1], s[1][0][2],\
s[1][1],\
s[1][0][0].strftime('%m-%d %H:%M:%S'))
q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'],
d['port'], d['db'])
q.put(msg)
def process(time, rdd):
#print (time, rdd)
lt = (rdd.collect())
#print '\n'.join ('%d %s'% (l[0], ''.join(('%f'% e) for e in l[1])) for l in list)
if len(lt) == 2:
a = list(lt[0][1])
b = list(lt[1][1])
#print a, b
corr = 0.0
if len(a) > 1 and len(b) > 1:
if len(a) > len(b):
corr = numpy.corrcoef(a[:len(b)], b)
else:
corr = numpy.corrcoef(b[:len(a)], a)
print "%s corr---> %f" % (time.strftime('%Y%m%d %H:%M:%S'),
corr.tolist()[0][1])
d = Q.value
q = RedisQueue(d['qname'], d['namespace'], d['host'], d['port'],
d['db'])
corr = corr.tolist()[0][1]
if not numpy.isnan(corr):
print 'insert into redis'
q.put((time, corr))
def f1(time, rdd):
lt = rdd.collect()
print '**** f1'
print lt
print '**** end f1'
f = open('/home/larry/l1304/workspace/finopt/data/mds_files/std/std-%s.txt' % datetime.datetime.now().strftime('%Y%m%d%H%M'), 'a')
f.write(''.join('%s,%s,%s\n'%(s[0].strftime('%Y-%m-%d %H:%M:%S.%f'),s[1],s[2]) for s in lt))
d = Q.value
if float(lt[0][1]) > 4.6:
msg = 'Stock SD alert triggered: '.join('%s,%0.2f,%02.f\n'%(s[0].strftime('%Y-%m-%d %H:%M:%S.%f'),s[1],s[2]) for s in lt)
print msg
q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'], d['port'], d['db'])
q.put(msg)
def detect_trend(x1, y1, num_points_ahead, ric):
z4 = polyfit(x1, y1, 3)
p4 = poly1d(z4) # construct the polynomial
#print y1
z5 = polyfit(x1, y1, 4)
p5 = poly1d(z5)
extrap_y_max_limit = len(x1) * 2 # 360 days
x2 = linspace(
0, extrap_y_max_limit,
50) # 0, 160, 100 means 0 - 160 with 100 data points in between
pylab.switch_backend(
'agg'
) # switch to agg backend that support writing in non-main threads
pylab.plot(x1, y1, 'o', x2, p4(x2), '-g', x2, p5(x2), '-b')
pylab.legend(['%s to fit' % ric, '4th degree poly', '5th degree poly'])
#pylab.axis([0,160,0,10])
#
pylab.axis([
0, len(x1) * 1.1,
min(y1) * 0.997,
max(y1) * 1.002
]) # first pair tells the x axis boundary, 2nd pair y axis boundary
# compute the slopes of each set of data points
# sr - slope real contains the slope computed from real data points from d to d+5 days
# s4 - slope extrapolated by applying 4th degree polynomial
y_arraysize = len(y1)
# s4, intercept, r_value, p_value, std_err = stats.linregress(range(0,num_points_ahead), [p4(i) for i in range(y_arraysize,y_arraysize + num_points_ahead )])
# s5, intercept, r_value, p_value, std_err = stats.linregress(range(0,num_points_ahead), [p5(i) for i in range(y_arraysize,y_arraysize + num_points_ahead )])
s4, intercept, r_value, p_value, std_err = stats.linregress(x1, y1)
s5, intercept, r_value, p_value, std_err = stats.linregress(x1, y1)
rc = (1.0 if s4 > 0.0 else 0.0, 1.0 if s5 > 0.0 else 0.0)
print s4, s5, rc, y_arraysize
#pylab.show()
pylab.savefig('../data/extrapolation/%s-%s.png' % (ric, time))
d = Q.value
q = RedisQueue(d['qname'], d['namespace'], d['host'], d['port'],
d['db'])
q.put((time, y1))
# # clear memory
pylab.close()
return rc
def simple(time, rdd):
lt = (rdd.collect())
if lt:
first = lt[0][1][0]
last_pos = len(lt) - 1
last = lt[last_pos][1][0]
change = (last - first) / last
msg = '%0.6f, %0.2f, %0.2f, %0.2f' % (change, first, last, last_pos)
print 'process............. %d {%s}' % (NumProcessed.value, msg)
if abs(change) > Threshold.value:
d = Q.value
q = RedisQueue(d['alert_bot_q'][1], d['alert_bot_q'][0], d['host'],
d['port'], d['db'])
q.put(msg)
NumProcessed.add(1)