def count_steps(data):
print 'count_steps'
plot_data(data)
mag = vector_magnitude(data)
plot_mag(mag)
average = moving_average(data, 100)
plot_mag(average)
num_steps = 0
'''
This function counts the number of steps in data and returns the number of steps
'''
i = 0
found = False
stepArray = []
for x in average:
if (x >= 4 and x <= 4.03):
if found == False:
num_steps = num_steps + 1
stepArray.append(i)
found = True
else:
found = False
i = i + 1
plot_steps(average, stepArray)
return num_steps
def get_stat():
fields = ['loss', 'acc']
stat = {}
for phase in data._PHASES:
if phase == data._TRAIN:
iteration = sum([len(file) for file in files[data._TRAIN]]) / _BATCH_SIZE
elif phase == data._VAL:
iteration = sum([len(file) for file in files[data._VAL]]) / _BATCH_SIZE
raw_averages = {field: (net[field], util.moving_average(net[field], iteration)) for field in fields}
display = {}
display.update({'%s_raw' % field: raw_averages[field][0] for field in fields})
display.update({'%s_avg' % field: raw_averages[field][1] for field in fields})
summaries = []
summaries += [tf.scalar_summary('%s_%s_raw' % (data._NAME[phase], field), raw_averages[field][0]) for field in fields]
summaries += [tf.scalar_summary('%s_%s_avg' % (data._NAME[phase], field), raw_averages[field][1]) for field in fields]
summary = tf.merge_summary(summaries)
stat[phase] = dict(
iteration=iteration,
display=display,
summary=summary)
return stat
def graphs_to_image(*args, title="", moving_average=True):
fig = plt.figure(figsize=(8, 6))
ax1 = fig.add_subplot(111)
ax1.set_title(title)
ax1.legend(loc='upper left')
colors = ["r", "b"]
if moving_average:
for i, arg in enumerate(args):
if len(arg) == 0:
continue
ax1.plot(arg, colors[i] + ".", alpha=0.2)
for i, arg in enumerate(args):
if len(arg) == 0:
continue
ax1.plot(util.moving_average(arg), colors[i])
else:
for i, arg in enumerate(args):
if len(arg) == 0:
continue
ax1.plot(arg, colors[i])
return figure_to_image(fig)
def count_steps(data):
print 'count_steps'
# Different Algo
num_steps = 0
plot_data(data)
plot_mag(vector_magnitude(data))
plot_mag(moving_average((data),230))
'''
This function counts the number of steps in data and returns the number of steps
'''
return num_steps
def run():
# Get data
file_name = "data.txt" # Change to your file name
data = iosParser.get_data(file_name)
# print data
number_of_steps = count_steps(data)
magnitudes = util.vector_magnitude(data)
plot_mag(magnitudes)
moving_avg = util.moving_average(magnitudes, 10)
plot_mag(moving_avg)
print "Number of steps counted are :", number_of_steps
def count_steps(data):
print "Accelerometer data graph"
plot_data(data)
mag = vector_magnitude(data)
plot_mag(mag)
average = moving_average(data, 10)
plot_mag(average)
num_steps = 0
'''
ADD YOUR CODE HERE. This function counts the number of steps in data and returns the number of steps
'''
return num_steps
def _plot_corr(s1, s2, **kwargs):
if not kwargs.get("label"):
kwargs["label"] = "correlation"
d1 = series_to_dict(price_diff(s1))
d2 = series_to_dict(price_diff(s2))
s = []
for a in s1:
if a[0] not in d2:
continue
if d1[a[0]] * d2[a[0]] > 0:
s.append((a[0], 100))
else:
s.append((a[0], 0))
s = moving_average(s, days=28)
plt.plot([a[0] for a in s], [a[1] for a in s], **kwargs)
def get_momentum(vec, winsize=3, mtype=1, matype=1):
mm = np.zeros(len(vec))
vec_sma = None
if matype == 1:
vec_sma = util.moving_average(vec, winsize=winsize)
elif matype == 2:
vec_sma = util.weighted_moving_average(vec, winsize=winsize)
for i in range(len(vec)):
a, b = i - winsize + 1, i + 1
if i < winsize - 1:
a, b = 0, i + 1
if b - a == 1:
mm[i] = 0
else:
if mtype == 1:
mean_diff = 0.0
for i in range(a + 1, b):
mean_diff += (vec_sma[i] - vec_sma[i - 1])
mm[i] = mean_diff / float(b - (a + 1))
elif mtype == 2:
mm[i] = vec_sma[b - 1] - vec_sma[a] #/ float(winsize)
return mm
def other_analysis():
img_url = ''
if request.method == "POST":
if request.form['link'] == 'Moving Average':
if not symbol:
flash('Choose one')
return redirect(url_for('other_analysis'))
img_url = moving_average(symbol)
elif request.form['link'] == 'Rate of Return':
if not symbol:
flash('Choose one')
return redirect(url_for('other_analysis'))
img_url = Rate_of_Return(symbol)
elif request.form['link'] == 'Correlation':
if not symbol:
flash('Choose one')
return redirect(url_for('other_analysis'))
img_url = Correlation(symbol)
elif request.form['link'] == 'Risk and Return':
if not symbol:
flash('Choose one')
return redirect(url_for('other_analysis'))
img_url = Risk_and_Return(symbol)
return render_template('other_analysis.html', img_url=img_url)
normalize(bch, 'btc')
normalize(bsv, 'btc')
draw_custom({'bch': bch.btc_norm, 'bsv': bsv.btc_norm})
if args and args[0] == 'btc,tether':
btc = Coin('bitcoin')
tether = Coin('tether')
draw_custom({'tether supply': tether.supply_norm, 'btc': btc.usd_norm})
if args and args[0] == 'cro,mco':
cro = Coin('crypto-com-coin')
mco = Coin('crypto-com')
draw_custom({
'cro': cro.usd_norm,
'mco': mco.usd_norm,
'r/{}'.format(cro.cmc.sub): cro.subs_norm,
'@{}'.format(cro.cmc.twt): cro.flw_norm,
})
if args and args[0] == 'ntx':
btc = Coin('bitcoin')
btc.n_transactions_squared = [(a[0], a[1]**2) for a in BlockchainCom.fetch_data("n-transactions")]
btc.difficulty = [(a[0], a[1] / 50) for a in BlockchainCom.fetch_data("difficulty") + [BtcCom.get_next_diff()]]
btc.hashrate = [(a[0], a[1] * 2828.42) for a in BlockchainCom.fetch_data("hash-rate")]
draw_custom({
'tx_count_squared': btc.n_transactions_squared,
'tx_count_squared MA365': moving_average(btc.n_transactions_squared, days=365),
'btc_market_cap': [(a[0], a[1] * btc.supply[i][1]) for i, a in enumerate(btc.usd)],
'btc_market_cap MA100': moving_average([(a[0], a[1] * btc.supply[i][1]) for i, a in enumerate(btc.usd)], days=100),
'tether_supply (x25)': [(a[0], a[1] * 25) for a in Coin("tether").supply],
'[--] difficulty (/50)': btc.difficulty,
#'[:] hashrate': btc.hashrate,
'[:] hashrate MA7 (x2828.42)': moving_average(btc.hashrate, days=7)
})
if __name__ == '__main__':
ts = import_data('../data/vale_2019.csv', 'val1', -390 * 1)
ts_norm = normalize_range(ts, -1, 1)
winsize = int(len(ts) / 16)
mm = get_momentum(ts, winsize=winsize, mtype=1, matype=1)
mm2 = get_momentum(ts, winsize=winsize, mtype=1, matype=2)
mm3 = get_momentum(ts, winsize=winsize, mtype=2, matype=1)
mm4 = get_momentum(ts, winsize=winsize, mtype=2, matype=2)
mm_norm = normalize_range(mm, -1, 1)
mm_norm2 = normalize_range(mm2, -1, 1)
mm_norm3 = normalize_range(mm3, -1, 1)
mm_norm4 = normalize_range(mm4, -1, 1)
grads = np.gradient(util.moving_average(
ts, winsize=winsize)) * (1.0 + math.sqrt(5))
grads_norm = util.moving_average(normalize_range(grads, -1, 1),
winsize=winsize)
fig, ax = plt.subplots()
util.plot_bollinger_bands(ts_norm, ax)
ax.plot(ts_norm, alpha=1.0, linewidth=3, label='ts', color='red')
# ax.plot(grads_norm, alpha=0.7, linewidth=3, label='grads')
# ax.plot(mm_norm, alpha=0.7, linewidth=2, label='mm normal sma', color='green')
# ax.plot(mm_norm2, alpha=0.7, linewidth=2, label='mm normal wma', color='blue')
# ax.plot(mm_norm3, alpha=0.5, linewidth=2, label='mm cumdiff sma')
# ax.plot(mm_norm4, alpha=0.5, linewidth=2, label='mm cumdiff wma')
ax.plot(util.weighted_moving_average(ts_norm, winsize=20),
label='sma-20',
color='magenta')
ax.plot(util.weighted_moving_average(ts_norm, winsize=50),
def draw_coin(coin):
fig = plt.figure()
fig.show()
if 'usd' in config.CHART_METRICS:
_plot(coin,
'usd_norm',
label="{}/USD".format(coin.name),
color="blue",
linestyle="-")
if coin.name != 'bitcoin':
if 'btc' in config.CHART_METRICS:
_plot(coin,
'btc_norm',
label="{}/BTC".format(coin.name),
color="blue",
linestyle="--")
if 'supply' in config.CHART_METRICS:
_plot(coin,
'supply_norm',
label="supply".format(coin.name),
color="green",
linestyle="--")
if coin.cmc.sub:
if 'subs' in config.CHART_METRICS:
_plot(coin,
'subs_norm',
label="r/{} subscribers".format(coin.cmc.sub),
color="red",
linestyle="-",
linewidth=2)
if 'asubs' in config.CHART_METRICS:
_plot(coin,
'asubs_norm',
label="r/{} active users".format(coin.cmc.sub),
color="red",
linestyle=":",
linewidth=2)
if coin.cmc.twt:
if 'flw' in config.CHART_METRICS:
_plot(coin,
'flw_norm',
label="@{} followers".format(coin.cmc.twt),
color="cyan",
linestyle="-",
linewidth=2)
if coin.name != 'bitcoin':
if 'btcusd' in config.CHART_METRICS:
_plot(Coin('bitcoin'),
'usd_norm',
label="BTC/USD",
color="orange",
linestyle="--")
if 'ma28' in config.CHART_METRICS:
_plot(coin,
'usd_norm',
label="{}/USD MA28".format(coin.name),
mut=lambda s: moving_average(s, days=28))
if 'ma100' in config.CHART_METRICS:
_plot(coin,
'usd_norm',
label="{}/USD MA100".format(coin.name),
mut=lambda s: moving_average(s, days=100))
if 'xusddiff' in config.CHART_METRICS:
_plot(coin,
'usd',
label="{}/USD diff".format(coin.name),
mut=lambda s: price_diff(s))
if coin.name != 'bitcoin':
if 'btcusddiff' in config.CHART_METRICS:
_plot(Coin('bitcoin'),
'usd',
label="BTC/USD diff",
mut=lambda s: price_diff(s))
if coin.name != 'bitcoin':
if 'btcusdxusdcorr' in config.CHART_METRICS:
_plot_corr(Coin('bitcoin').usd_norm,
coin.usd_norm,
label="{}/USD corr_btc".format(coin.name),
color="black",
linestyle="--")
if 'btcusdxbtccorr' in config.CHART_METRICS:
_plot_corr(Coin('bitcoin').usd_norm,
coin.btc_norm,
label="{}/BTC corr_btc".format(coin.name),
color="black",
linestyle=":")
if 'xusdnextdaycorrbtc' in config.CHART_METRICS:
_plot_corr(Coin('bitcoin').usd_norm,
series_shift(coin.usd_norm, 1),
label="{}/USD next day corr_btc".format(coin.name),
style="y:")
if 'xusdprevdaycorrbtc' in config.CHART_METRICS:
_plot_corr(Coin('bitcoin').usd_norm,
series_shift(coin.usd_norm, -1),
label="{}/USD prev day corr_btc".format(coin.name),
style="b:")
if coin.name == 'bitcoin':
if 'tethersupply' in config.CHART_METRICS:
_plot(Coin('tether'),
"supply_norm",
label="Tether supply",
color="green",
linestyle=":")
if 'ntx' in config.CHART_METRICS or 'ntxsquared' in config.CHART_METRICS:
coin.n_transactions = [
a for a in BlockchainCom.fetch_data("n-transactions")
if a[0] >= config.DATE_START
]
normalize(coin, "n_transactions")
coin.n_transactions_squared = [(a[0], a[1]**2)
for a in coin.n_transactions]
normalize(coin, "n_transactions_squared")
if 'ntx' in config.CHART_METRICS:
_plot(coin,
"n_transactions_norm",
label="n_transactions",
color="violet",
linestyle=":")
if 'ntxsquared' in config.CHART_METRICS:
_plot(coin,
"n_transactions_squared_norm",
label="n_transactions_squared",
color="violet",
linestyle=":")
if 'difficulty' in config.CHART_METRICS:
coin.difficulty = [
a for a in BlockchainCom.fetch_data("difficulty")
if a[0] >= config.DATE_START
]
coin.difficulty.append(BtcCom.get_next_diff())
normalize(coin, "difficulty")
_plot(coin,
"difficulty_norm",
label="difficulty",
color="brown",
linestyle="--")
if 'hashrate' in config.CHART_METRICS:
coin.hash_rate = [
a for a in BlockchainCom.fetch_data("hash-rate")
if a[0] >= config.DATE_START
]
normalize(coin, "hash_rate")
_plot(coin,
"hash_rate_norm",
label="hash_rate",
color="brown",
linestyle=":")
_draw_end(fig)
})
if args and args[0] == 'ntx':
btc = Coin('bitcoin')
btc.n_transactions_squared = [
(a[0], a[1]**2) for a in BlockchainCom.fetch_data("n-transactions")
]
btc.difficulty = [(a[0], a[1] / 50)
for a in BlockchainCom.fetch_data("difficulty") +
[BtcCom.get_next_diff()]]
btc.hashrate = [(a[0], a[1] * 2828.42)
for a in BlockchainCom.fetch_data("hash-rate")]
draw_custom({
'tx_count_squared':
btc.n_transactions_squared,
'tx_count_squared MA365':
moving_average(btc.n_transactions_squared, days=365),
'btc_market_cap':
[(a[0], a[1] * btc.supply[i][1]) for i, a in enumerate(btc.usd)],
'btc_market_cap MA100':
moving_average([(a[0], a[1] * btc.supply[i][1])
for i, a in enumerate(btc.usd)],
days=100),
'tether_supply (x25)':
[(a[0], a[1] * 25) for a in Coin("tether").supply],
'[--] difficulty (/50)':
btc.difficulty,
#'[:] hashrate': btc.hashrate,
'[:] hashrate MA7 (x2828.42)':
moving_average(btc.hashrate, days=7)
})
def GSR_preprocess(GSR, fs):
b, a = signal.butter(2, 1 * 2 / fs, 'lowpass')
GSR = signal.filtfilt(b, a, GSR)
GSR = signal.medfilt(GSR, 61)
GSR = moving_average(GSR, 61)
return GSR
def extract_GSR_fea(GSR, fs, ampThresh=1):
#These are the timing threshold defined
tThreshLow = 0.1
tThreshUp = 20
plt.subplot(211)
plt.plot(np.arange(0, GSR.shape[0]) / fs, GSR)
# b, a = signal.butter(2, 1 * 2 / fs, 'lowpass')
# GSR = signal.filtfilt(b, a, GSR)
# GSR = signal.medfilt(GSR,61)
# GSR = moving_average(GSR,61)
#Search low and high peaks
#low peaks are the GSR appex reactions (highest sudation)
#High peaks are used as starting points for the reaction
# dN = np.array(np.diff(GSR) <= 0 ,dtype=int)
diff = moving_average(np.diff(GSR), 50)
dN = np.array(diff <= 0, dtype=int)
plt.subplot(212)
plt.plot(np.arange(0, GSR.shape[0]) / fs, GSR)
# plt.show()
dN = np.diff(dN)
idxL = np.where(dN < 0)[0] + 1
#+1 to account for the double derivative
idxH = np.where(dN > 0)[0] + 1
#For each low peaks find it's nearest high peak and check that there is no
#low peak between them, if there is then reject the peak (OR SEARCH FOR CURVATURE)
riseTime = np.array([]) #vector of rise time for each detected peak
ampPeaks = np.array([]) #vector of amplitude for each detected peak
posPeaks = np.array(
[]
) #final indexes of low peaks (not used but could be usefull for plot puposes)
for iP in range(0, len(idxL)):
#get list of high peak before the current low peak
nearestHP = idxH[idxH < idxL[iP]]
#if no high peak before (this is certainly the first peak detected in
#the signal) don't do anything else process peak
if len(nearestHP) > 0:
#Get nearest high peak
nearestHP = nearestHP[-1]
#check if there is not other low peak between the nearest high and
#the current low peaks. If not the case than compute peak features
if not any((idxL > nearestHP) & (idxL < idxL[iP])):
rt = (idxL[iP] - nearestHP) / fs
amp = GSR[nearestHP] - GSR[idxL[iP]]
#if rise time and amplitude fits threshold then the peak is
#considered and stored
if (rt >= tThreshLow) and (rt <= tThreshUp) and (amp >=
ampThresh):
riseTime = np.append(riseTime, rt)
ampPeaks = np.append(ampPeaks, amp)
posPeaks = np.append(posPeaks, idxL[iP])
#Compute the number of positive peaks
nbPeaks = len(posPeaks)
#retype the arrays
posPeaks = np.array(posPeaks, dtype=int)
# print(nbPeaks, ampPeaks, riseTime, posPeaks)
#return the values
# rise time ratio
rtr = (np.sum(riseTime) / GSR.shape[0] / fs)
# top3 risetime average
rt_sorted = sorted(riseTime) # sort the fisher from small to large
rt_sorted = rt_sorted[::-1] # arrange from large to small
if len(rt_sorted) < 3:
top_rt = np.mean(rt_sorted)
else:
top_rt = np.mean((rt_sorted[:3]))
# top3 amp average
amp_sorted = sorted(riseTime) # sort the fisher from small to large
amp_sorted = amp_sorted[::-1] # arrange from large to small
if len(amp_sorted) < 3:
top_amp = np.mean(amp_sorted)
else:
top_amp = np.mean((amp_sorted[:3]))
features = [nbPeaks, rtr, top_rt, top_amp]
return features
model_fname = env.run("model.ckpt")
saver = tf.train.Saver()
if os.path.exists(model_fname):
print "Restoring from {}".format(model_fname)
saver.restore(sess, model_fname)
epochs = 0
else:
sess.run(tf.global_variables_initializer())
tmp_dir = pj(os.environ["HOME"], "tmp", "tf_pics")
[ os.remove(pj(tmp_dir, f)) for f in os.listdir(tmp_dir) if f[-4:] == ".png" ]
input_v = moving_average(np.random.randn(seq_size), 50).reshape(1, seq_size, batch_size, 1)
epochs = 1000
for e in xrange(epochs):
input_filtered_v, net_out_v, z_v, input_hat_v, KLD_v, BCE_v, loss_v, _ = sess.run([
input_filtered, net_out, z, input_hat, KLD, BCE, loss, train_step
], {
input: input_v,
state: np.zeros((batch_size, net_size))
})
sl(input_v, input_hat_v, (input_v-input_hat_v) ** 2, file=pj(tmp_dir, "rec_{}.png".format(e)))
sm(z_v, file=pj(tmp_dir, "latent_{}.png".format(e)))
print "Epoch {}, loss {}, KLD {}".format(e, loss_v, np.mean(KLD_v))
'cro': cro.usd_norm,
'mco': mco.usd_norm,
'r/{}'.format(cro.cmc.sub): cro.subs_norm,
'@{}'.format(cro.cmc.twt): cro.flw_norm,
})
if args and args[0] == 'ntx':
btc = Coin('bitcoin')
btc.n_transactions_squared = [
(a[0], a[1]**2) for a in BlockchainCom.fetch_data("n-transactions")
]
btc.difficulty = [(a[0], a[1] / 50)
for a in BlockchainCom.fetch_data("difficulty") +
[BtcCom.get_next_diff()]]
btc.hashrate = [(a[0], a[1] * 2828.42)
for a in BlockchainCom.fetch_data("hash-rate")]
draw_custom({
'tx_count_squared':
btc.n_transactions_squared,
'tx_count_squared MA365':
moving_average(btc.n_transactions_squared, days=365),
'btc_market_cap':
[(a[0], a[1] * btc.supply[i][1]) for i, a in enumerate(btc.usd)],
'tether_supply (x25)':
[(a[0], a[1] * 25) for a in Coin("tether").supply],
'[--] difficulty (/50)':
btc.difficulty,
#'[:] hashrate': btc.hashrate,
'[:] hashrate MA7 (x2828.42)':
moving_average(btc.hashrate, days=7)
})
sess = tf.Session()
saver = tf.train.Saver()
model_fname = env.run("fhn_model.ckpt")
if os.path.exists(model_fname):
print "Restoring from {}".format(model_fname)
saver.restore(sess, model_fname)
epochs = 0
else:
sess.run(tf.global_variables_initializer())
tmp_dir = pj(os.environ["HOME"], "tmp", "tf_pics")
[os.remove(pj(tmp_dir, f)) for f in os.listdir(tmp_dir) if f[-4:] == ".png"]
inputs_v = moving_average(np.random.randn(seq_size),
10).reshape(batch_size, seq_size, 1, 1)
epochs = 1000
for e in xrange(epochs):
state_v = FHNStateTuple(
np.zeros((batch_size, net_size)),
np.zeros((batch_size, net_size)),
)
sess_out = sess.run([
net_out, finstate, apply_grads, loss, input_n, output_n, grads,
grads_raw
], {
input: inputs_v,
state: state_v,
})
net_out, finstate, _ = tf.nn.raw_rnn(cell, rnn_with_hist_loop_fn(input_n, target_n, sequence_length, state, L))
spikes_ta, a_ta, output_ta, loss_ta = net_out
spikes = spikes_ta.stack()
a = a_ta.stack()
output = output_ta.stack()
loss = loss_ta.stack()
# l2_loss = tf.nn.l2_loss(input_n - result)
## RUNNING
inputs_v = moving_average(np.random.randn(seq_size), 10).reshape(seq_size, batch_size, 1)
sess = tf.Session()
saver = tf.train.Saver()
model_fname = env.run("glm_model.ckpt")
if os.path.exists(model_fname):
print "Restoring from {}".format(model_fname)
saver.restore(sess, model_fname)
epochs = 0
else:
sess.run(tf.global_variables_initializer())
reward_v, reward_mean_v = None, None
tmp_dir = pj(os.environ["HOME"], "tmp")
