def getSignals(self):
signals = []
for (inWidgetInstance, outName, inName,
X) in self.signalManager.links.get(self.outWidget.instance(), []):
if inWidgetInstance == self.inWidget.instance:
signals.append((outName, inName))
#print 'Signals collected, ', signals
return signals
def load_s3_data(files, features, signal_fn, start_hour=None, end_hour=None):
"""Reads given filenames from s3, returns:
- feature matrix (based on the features argument)
- output signal (based on given signal_fn)
- times
- bids
- offers
- currency pair"""
print "Loading datasets..."
datasets = load_s3_files(files)
print "Flattening datasets into feature matrices..."
matrices = []
signals = []
times = []
bids = []
offers = []
currencies = []
for d in datasets:
print "Processing dataset", d.hdf
start_idx = None if start_hour is None else hour_to_idx(d.t, start_hour)
end_idx = None if end_hour is None else hour_to_idx(d.t, end_hour)
print "start_idx", start_idx, "end_idx", end_idx
start_ok = start_idx is None or start_idx < len(d.t)
end_ok = end_idx is None or (end_idx < len(d.t) and end_idx > start_idx)
if start_ok and end_ok:
mat = dataset_to_feature_matrix(d, features, start_idx=start_idx, end_idx=end_idx)
matrices.append(mat)
times.append(d.t[start_idx:end_idx])
bids.append(d['bid/100ms'][start_idx:end_idx])
offers.append(d['offer/100ms'][start_idx:end_idx])
currencies.append(d.currency_pair)
print "Generating output signal"
signals.append(signal_fn(d, start_idx=start_idx, end_idx=end_idx))
feature_data = np.concatenate(matrices)
print "Checking data validity..."
check_data(feature_data)
signal = np.concatenate(signals)
times = np.concatenate(times)
bids = np.concatenate(bids)
offers = np.concatenate(offers)
print "Deleting local files..."
for d in datasets:
d.hdf.close()
return feature_data, signal, times, bids, offers, currencies
def getOutConnectedSignalNames(self):
signals = []
for line in self.outLines:
for (outSignal, inSignal) in line.getSignals():
if outSignal not in signals: signals.append(outSignal)
return signals
def getOutConnectedSignalNames(self):
signals = []
for line in self.outLines:
for (outSignal, inSignal) in line.getSignals():
if outSignal not in signals: signals.append(outSignal)
return signals
