def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
newest_t = float(dataio.csv_get_last(pair)['time'])
else:
newest_t = self.__find_start_trade_time(pair, start)
while newest_t < end:
# old -> new
r = self.__get_slice(pair, newest_t + 1e-4)
# list to dict
r = self.__to_dict(r)
# save to file
dataio.csv_append(pair, r)
# break condition
if len(r) < self.__MAX_LIMIT:
break
# prepare next iteration
newest_t = float(r[-1]['time'])
print('Kraken\t| {} : {}'.format(timeutil.unix_to_iso(newest_t),
pair))
print('Kraken\t| Download complete : {}'.format(pair))
def get_trades(self, pair, start, end):
"""Get trade data from .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to fetch.
end (int): End UNIX of trade data to fetch.
Returns:
List of trade events, from old to new data.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
data = dataio.csv_get(pair)
else:
raise ValueError(
'Bitmex\t| No trades downloaded: {}'.format(pair))
# look for start and end row indices
start_index = 0
end_index = len(data['time']) - 1
for i, timestamp in enumerate(data['time']):
if float(timestamp) <= float(start):
start_index = i
if float(timestamp) <= float(end):
end_index = i
# filter by requested date range
new_data = {}
for col_name in data:
new_data[col_name] = data[col_name][start_index: (end_index + 1)]
return new_data
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_id = int(last_row['trade_id']) + 1
newest_t = int(last_row['time'])
else:
newest_id = self.__find_start_trade_id(pair, start)
newest_t = 0
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_id)
# old -> new, add unix timestamp
new_r = []
for row in r:
row['time'] = row['T'] // 1000
row['date'] = timeutil.unix_to_iso(row['time'])
row['price'] = row['p']
row['size'] = row['q']
row['side'] = 'sell' if row['m'] == True else 'buy'
row['best_price_match'] = row['M']
row['trade_id'] = row['a']
row.pop('a', None)
row.pop('p', None)
row.pop('q', None)
row.pop('f', None)
row.pop('l', None)
row.pop('T', None)
row.pop('m', None)
row.pop('M', None)
new_r.append(row)
# save to file
dataio.csv_append(pair, new_r)
# break condition
if len(r) < self.__MAX_LIMIT:
break
# prepare next iteration
newest_id = new_r[-1]['trade_id'] + 1
newest_t = new_r[-1]['time']
print('Binance\t| {} : {}'.format(timeutil.unix_to_iso(newest_t),
pair))
print('Binance\t| Download complete : {}'.format(pair))
def data_preprocess(params):
### Record Concatenation
dataio = DataIO(params['input_path'], params['result_path'])
dataio.read_data()
ctn = Concatenation(dataio)
patient_info, n_feature = ctn.get_concatenation() # patient id: Patient
# static feature and dynamic feature
# dynamic feature{time:feature_value}
### Data Imputation
imp_method = 'simple'
imp = Imputation(patient_info, n_feature)
patient_array = imp.get_imputation(imp_method)
return (dataio, patient_info, patient_array)
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_id = int(last_row['trade_id']) + 1
newest_t = int(last_row['time'])
else:
newest_id = self.__find_start_trade_id(pair, start)
newest_t = 0
last_trade_id = self.__find_last_trade_id(pair)
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_id + self.__MAX_LIMIT)
# break condition
to_break = False
# old -> new, add unix timestamp
new_r = []
for row in reversed(r):
if row['trade_id'] > newest_id:
row['date'] = row['time']
row['time'] = timeutil.iso_to_unix(row['time'])
new_r.append(row)
if row['trade_id'] == last_trade_id:
to_break = True
# save to file
dataio.csv_append(pair, new_r)
# break condition
if to_break:
break
# prepare next iteration
newest_id = new_r[-1]['trade_id']
newest_t = new_r[-1]['time']
print('GDAX\t| {} : {}'.format(timeutil.unix_to_iso(newest_t),
pair))
print('GDAX\t| Download complete : {}'.format(pair))
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_id = int(last_row['trade_id']) + 1
newest_t = int(last_row['time'])
else:
newest_id = self.__find_start_trade_id(pair, start)
newest_t = 0
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_id)
# old -> new, add unix timestamp
new_r = []
for i, row in enumerate(r):
row['time'] = timeutil.iso_to_unix(row['timestamp'])
row['date'] = row['timestamp']
row['trade_id'] = newest_id + i
row['side'] = row['side'].lower()
row.pop('timestamp', None)
row.pop('symbol', None)
new_r.append(row)
# save to file
dataio.csv_append(pair, new_r)
# break condition
if len(r) < self.__MAX_LIMIT:
break
# prepare next iteration
newest_id = new_r[-1]['trade_id'] + 1
newest_t = new_r[-1]['time']
print('Bitmex\t| {} : {}'.format(
timeutil.unix_to_iso(newest_t), pair))
print('Bitmex\t| Download complete : {}'.format(pair))
def download_data(self, pair, start, end):
"""Download trade data and store as .csv file.
Args:
pair (str): Currency pair.
start (int): Start UNIX of trade data to download.
end (int): End UNIX of trade data to download.
"""
dataio = DataIO(savedir=self._savedir, fieldnames=self.FIELDNAMES)
last_row = None
if dataio.csv_check(pair):
last_row = dataio.csv_get_last(pair)
newest_t = int(last_row['time'])
else:
newest_t = self.__find_start_trade_time(pair, start) - 1
# break condition
last_trade_time = self.__find_last_trade_time(pair)
while newest_t < end:
# new -> old
r = self.__get_slice(pair, newest_t)
# old -> new; remove duplicate data by trade ID
new_r = []
for row in reversed(r):
if last_row is not None:
if int(last_row['tradeID']) >= row['tradeID']:
continue # remove duplicates
last_row = row
row['time'] = timeutil.iso_to_unix(row['date'])
new_r.append(row)
if newest_t > last_trade_time:
break
# save to file
dataio.csv_append(pair, new_r)
# prepare next iteration
newest_t += self.__MAX_RANGE
print('Poloniex| {} : {}'.format(
timeutil.unix_to_iso(newest_t), pair))
print('Poloniex| Download complete : {}'.format(pair))
def data_preprocess(params):
### Record Concatenation
dataio = DataIO(params['input_path'], params['map_path'], params['domain'])
dataio.read_data()
dataio.read_label()
ctn = Concatenation(dataio, params['domain'])
patient_info, n_feature, feature_list, feature_range = ctn.get_concatenation()
# patient id: Patient
# static feature and dynamic feature
# dynamic feature{time:feature_value}
### Data Imputation
imp_method = 'simple'
imp = Imputation(patient_info, n_feature)
patient_array, patient_time = imp.get_imputation(imp_method)
### Clinical Data with DTI Generation
cli = CliGen(feature_list, feature_range, ctn.dti_time)
subject_array = cli.get_data(patient_array, patient_time, params['time'])
if True == params['binary']: # only works for discrete clinical features
subject_array = cli.get_binarization()
subject_label = cli.get_label(patient_info, params['labels'], params['time'])
return subject_array, subject_label
def main():
result_path = 'results/'
subtype_method = "Algorithm"
K = 3 # number of subtypes(clusters)
############################## LOAD DATA ######################################
print('patients loading...')
dataio = DataIO(K)
dataio.load_demographics('../ufm/patient.csv')
dataio.load_feature('Motor', 'MDS UPDRS PartI')
dataio.load_feature('Motor', 'MDS UPDRS PartII')
dataio.load_feature('Motor', 'MDS UPDRS PartIII')
dataio.load_feature('Motor', 'MDS UPDRS PartIV')
dataio.load_feature('Non-Motor', 'BJLO')
dataio.load_feature('Non-Motor', 'ESS')
dataio.load_feature('Non-Motor', 'GDS')
dataio.load_feature('Non-Motor', 'HVLT')
dataio.load_feature('Non-Motor', 'LNS')
dataio.load_feature('Non-Motor', 'MoCA')
dataio.load_feature('Non-Motor', 'QUIP')
dataio.load_feature('Non-Motor', 'RBD')
dataio.load_feature('Non-Motor', 'SCOPA-AUT')
dataio.load_feature('Non-Motor', 'SF')
dataio.load_feature('Non-Motor', 'STAI')
dataio.load_feature('Non-Motor', 'SDM')
dataio.load_feature('Non-Motor', 'MCI')
dataio.load_feature('Biospecimen', 'DNA')
dataio.load_feature('Biospecimen', 'CSF', 'Total tau')
dataio.load_feature('Biospecimen', 'CSF', 'Abeta 42')
dataio.load_feature('Biospecimen', 'CSF', 'p-Tau181P')
dataio.load_feature('Biospecimen', 'CSF', 'CSF Alpha-synuclein')
dataio.load_feature('Image', 'DaTScan SBR')
dataio.load_feature('Image', 'MRI')
dataio.load_feature('Medication', 'MED USE')
suffix = 'normalized_clusters_Deep'
dataio.load_clustering_result('input/clustering_by_lstm.csv')
############################# STATISTICS ######################################
print('-----------------------')
print('statistics analyzing...')
var = Variable(K)
ftype = 'demographics'
p = var.get_variables(dataio, ftype)
var.p_value.extend(p)
ftype = 'motor'
_ = var.get_variables(dataio, ftype, 'MDS UPDRS PartI')
_ = var.get_variables(dataio, ftype, 'MDS UPDRS PartII')
_ = var.get_variables(dataio, ftype, 'MDS UPDRS PartIII', 'MDS-UPDRS')
_ = var.get_variables(dataio, ftype, 'MDS UPDRS PartIII', 'H&Y')
p = var.get_variables(dataio, ftype, 'MDS UPDRS PartIV')
var.p_value.extend(p)
ftype = 'nonmotor'
_ = var.get_variables(dataio, ftype, 'BJLO')
_ = var.get_variables(dataio, ftype, 'ESS')
_ = var.get_variables(dataio, ftype, 'GDS')
_ = var.get_variables(dataio, ftype, 'HVLT', 'Immediate Recall')
_ = var.get_variables(dataio, ftype, 'HVLT', 'Discrimination Recognition')
_ = var.get_variables(dataio, ftype, 'HVLT', 'Retention')
_ = var.get_variables(dataio, ftype, 'LNS')
print(var.pat_edu)
_ = var.get_variables(dataio, ftype, 'MoCA', pat_edu=var.pat_edu)
_ = var.get_variables(dataio, ftype, 'QUIP')
_ = var.get_variables(dataio, ftype, 'RBD')
_ = var.get_variables(dataio, ftype, 'SCOPA-AUT')
_ = var.get_variables(dataio, ftype, 'SF')
_ = var.get_variables(dataio, ftype, 'STAI')
_ = var.get_variables(dataio, ftype, 'SDM')
p = var.get_variables(dataio, ftype, 'MCI')
var.p_value.extend(p)
ftype = 'biospecimen'
var.get_variables(dataio, ftype, 'DNA')
_ = var.get_variables(dataio, ftype, 'CSF', 'Total tau')
_ = var.get_variables(dataio, ftype, 'CSF', 'Abeta 42')
_ = var.get_variables(dataio, ftype, 'CSF', 'p-Tau181P')
p = var.get_variables(dataio, ftype, 'CSF', 'CSF Alpha-synuclein')
var.p_value.extend(p)
ftype = 'image'
_ = var.get_variables(dataio, ftype, 'DaTScan SBR', 'CAUDATE RIGHT')
_ = var.get_variables(dataio, ftype, 'DaTScan SBR', 'CAUDATE LEFT')
_ = var.get_variables(dataio, ftype, 'DaTScan SBR', 'PUTAMEN RIGHT')
_ = var.get_variables(dataio, ftype, 'DaTScan SBR', 'PUTAMEN LEFT')
p = var.get_variables(dataio, ftype, 'MRI')
var.p_value.extend(p)
ftype = 'medication'
p = var.get_variables(dataio, ftype, 'MED USE')
var.p_value.extend(p)
################################# DISPLAY ######################################
print('-----------------------')
print('value displaying...')
ds = Display(var)
print('heatmap of the final mean value')
figurename = 'results/heatmap_clustering_by_' + subtype_method.lower(
) + '_' + suffix + '.pdf'
ds.heatmap(figurename, is_progress=False, is_rotate=False)
print('heatmap of the first order difference mean value')
figurename = 'results/heatmap_clustering_by_' + subtype_method.lower(
) + '_progression_' + suffix + '.pdf'
ds.heatmap(figurename, is_progress=True, is_rotate=False)
############################## SAVE RESULTS ####################################
print('-----------------------')
filename = result_path + 'statistics_clustering_by_' + subtype_method.lower(
) + '_' + suffix + '.csv'
dataio.save_result(var, filename)
print('done!')
from dataio import DataIO, ImageHandler
from unet_model import UNet
from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession
import tensorflow as tf
# Solving CUDNN Issues
config = ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
session = InteractiveSession(config=config)
# create all instances here
dat = DataIO().load_matfile_images_first('retina_training_STARE.mat')
display = ImageHandler()
# Run U-Net Model
model = UNet('Test_Model')
model.create_UNet_retina()
model.fit_model(*dat, nepochs=2000)
# get plot for training accuracy and loss
plot1 = model.plot_accuracy()
plot1.plot()
plot1.show()
plot2 = model.plot_loss()
plot2.plot()
plot2.show()
import numpy
from dataio import DataIO
if __name__ == '__main__':
filename = 'test'
fieldnames = ['time', 'low', 'high']
csvio = DataIO('test', fieldnames)
csvio.csv_newfile(filename)
test_row1 = {'time': 1.0, 'low': 100, 'high': 101}
test_row2 = {'time': 2.0, 'low': 101, 'high': 102}
test_rows = [{'time': 3.0, 'low': 102, 'high': 103},
{'time': 4.0, 'low': 103, 'high': 104}]
exists = csvio.csv_check(filename)
print(exists)
print(len(numpy.shape(test_row1)))
print(len(numpy.shape(test_rows)))
csvio.csv_append(filename, test_row1)
csvio.csv_append(filename, test_row2)
csvio.csv_append(filename, test_rows)
data = csvio.csv_get(filename)
print(data)
csvio.csv_rename(filename, 'test2')
