def __init__(self):
self.id = str(uuid.uuid4())
"""str: Unique identifier for this session"""
self.folder = SESSION_FOLDER + '/' + self.id
"""str: main session folder with all generated files"""
self.data_folder = self.folder + '/Data'
"""str: subfolder for session data (phase, fluor and params file)"""
self.params = Parameters()
"""Parameters: parameters object to share with ehooke instance"""
self.params_file = None
"""str: parameters file name"""
self.fluor_file = None
"""str: fluor file name, to override parameters so that params can be imported from other sessions"""
self.phase_file = None
"""str: phase file name, to override parameters so that params can be imported from other sessions"""
self.name = 'Session'
"""str: session name, user defined"""
self.description = ''
"""str: session description, user defined"""
self.mask_image = None
"""str: filename for the current mask image, none if no mask is computed"""
self.ehooke = None
"""EHooke: to be initialized upon starting the session (only with images and parameters"""
class EHooke:
"""Encapsulates all the code for processing a fluorescence frame"""
def __init__(self,params_obj=None,param_file=None):
"""Creates FluorFrame object, sets up parameters and loads images
It makes no sense to create a EHooke object without data or parameters.
So one must either supply params, a Parameters object, or param_file, in
which case EHooke loads the parameters file.
"""
self.params = None
"""Parameters: this can be a reference to an external object to be
changed for recomputing any step.
"""
if params_obj is None:
self.params = Parameters()
self.params.load_parameters(param_file)
else:
self.params = params_obj
self.fluor_frame = FluorFrame()
"""FluorFrame: manager for the fluor and phase images, plus masks"""
def load_images(self):
"""checks which images to load and loads them into the fluor_frame
"""
ffparams = self.params.fluor_frame_params
self.fluor_frame.load_fluor(ffparams)
if ffparams.phase_file is not None:
self.fluor_frame.load_phase(ffparams)
def create_masks(self):
"""creates masks using the current parameters"""
mparams = self.params.mask_params
self.fluor_frame.create_masks(mparams)
def align_fluor_to_phase(self):
"""aligns the fluorescence image to the phase mask, if a phase file exists"""
if ffparams.phase_file is not None:
self.fluor_frame.align_fluor(self.params.fluor_frame_params)
def save_mask_overlay(self, fname, back=(0,0,1), fore=(1,1,0), mask='phase',image='phase'):
"""saves the mask overlay image to a file"""
img = self.fluor_frame.mask_overlay(back, fore, mask,image)
imsave(fname,img)
def save_mask_contour(self, fname, mask='phase',image='phase', color=(1,1,0)):
"""saves the mask contour image to a file"""
img = self.fluor_frame.contour_overlay(mask,image,color)
imsave(fname,img)
def main():
"""
:return:
"""
params = Parameters()
if not params.SAVE_MODEL_PATH.exists():
params.SAVE_MODEL_PATH.mkdir()
print("Write params in JSON")
json_string = Parameters().to_json()
params_path = params.SAVE_MODEL_PATH / "params.json"
with open(params_path.as_posix(), "w") as f:
f.write(json_string)
ohlcv_histories, technical_indicators, next_day_open_values, unscaled_y, y_normaliser = csv_to_dataset(
params.CSV_DATA_PATH, params.num_history_points)
print("----", ohlcv_histories.shape, technical_indicators.shape,
next_day_open_values.shape)
n = int(ohlcv_histories.shape[0] * params.train_split)
ohlcv_train = ohlcv_histories[:n]
tech_ind_train = technical_indicators[:n]
y_true_train = next_day_open_values[:n]
ohlcv_test = ohlcv_histories[n:]
tech_ind_test = technical_indicators[n:]
# y_true_test = next_day_open_values[n:]
unscaled_y_test = unscaled_y[n:]
print("train_val split", ohlcv_train.shape)
print("test split", ohlcv_test.shape)
callbacks: List[tf.keras.callbacks.Callback] = [
tf.keras.callbacks.TensorBoard(params.SAVE_MODEL_PATH.as_posix()),
tf.keras.callbacks.ModelCheckpoint(
get_best_model_path(params),
monitor='val_loss',
verbose=1,
save_best_only=True,
)
]
# model = tech_net(technical_indicators.shape[1:], params)
#
# adam = optimizers.Adam(lr=params.LR)
# model.compile(optimizer=adam, loss='mse')
#
# model.fit(x=[ohlcv_train, tech_ind_train], y=y_true_train, batch_size=params.BATCH_SIZE, epochs=params.EPOCHS,
# shuffle=True, validation_split=params.val_split_out_of_train,
# callbacks=callbacks)
evaluate(params, ohlcv_test, tech_ind_test, unscaled_y_test, y_normaliser)
def main():
init_logging()
# init computation
prms = Parameters()
Run(run_name=config['Name'], tset_path=config['TraininSetPath']).index = 'DENS-0000'
trset = TrainingSet(config['TraininSetPath'], config['TraininSetName'])
# MolSet.p_call_mol_energy('full')
# print('NEEDEDMOL:',list(map(lambda x: x.id,trset.container[0].container[0].needed_mol)))
# print('UNIENERGY:',list(map(lambda x: x.uni_energy,trset.container[0].container[0].needed_mol)))
# print('FULLENERGY:',list(map(lambda x: x.full_energy,trset.container[0].container[0].needed_mol)))
# print('FUNCENERGY:',list(map(lambda x: x.func_energy,trset.container[0].container[0].needed_mol)))
# sys.exit()
# MolSet.p_call_mol_energy('func')
# for mol in MolSet.container:
# print(mol.myprm_full.sprms)
# print(mol.myprm_func.sprms)
prms.prms = dict(tta=[13.300000190734863],
ttb=[1.5299999713897705],
cxhf=[0.157706],
omega=[0.3],
cx_aa=[0.842294, 0.726479, 1.04760, -5.70635, 13.2794],
cc_aa=[1.000000, -4.33879, 18.2308, -31.7430, 17.2901],
cc_ab=[1.000000, 2.37031, -11.3995, 6.58405, -3.78132])
print(prms._parameters)
# prms.prms = {'cx_aa_0': 0.5, 'cc_aa_2':12000, 'tta':100}
# print(trset.compute_MAE('full'))
# print(prms._parameters)
prms.optim = ['tta', 'cx_aa_0','cx_aa_1','cx_aa_2','cx_aa_3','cx_aa_4']
print(prms.optim)
x0_=[13.3, 0.842294, 0.726479, 1.0476, -5.70635, 13.2794]
# x1_=[13.3, 0.04334172, 2.14371968, 2.0473701, -5.10893517, 13.63496978]
# x2_=[13.4, 0.05334172, 2.14371968, 2.0473701, -5.10893517, 13.63496978]
def printer(xc):
print('END of STEP')
print(xc)
bnds=((None,None),(0,None),(None,None),(None,None),(None,None),(None,None))
print(trset.optimizer(x0_,'full','MAE'))
# print(trset.optimizer(x1_,'func','MAE'))
# print(trset.optimizer(x2_,'func','MAE'))
# print(minimize(trset.optimizer,x0_,args=('func','MAE'),method='L-BFGS-B', callback=printer, bounds=bnds, options={'disp': True} ))
print(minimize(trset.optimizer,x0_,args=('func','MAE'), method='TNC', bounds=bnds, callback=printer, options={'disp': True,'ftol': 1e-2 }))
def __init__(self):
self.id=str(uuid.uuid4())
"""str: Unique identifier for this session"""
self.folder = SESSION_FOLDER+'/'+self.id
"""str: main session folder with all generated files"""
self.data_folder = self.folder+'/Data'
"""str: subfolder for session data (phase, fluor and params file)"""
self.params = Parameters()
"""Parameters: parameters object to share with ehooke instance"""
self.params_file = None
"""str: parameters file name"""
self.fluor_file = None
"""str: fluor file name, to override parameters so that params can be imported from other sessions"""
self.phase_file = None
"""str: phase file name, to override parameters so that params can be imported from other sessions"""
self.name = 'Session'
"""str: session name, user defined"""
self.description = ''
"""str: session description, user defined"""
self.mask_image = None
"""str: filename for the current mask image, none if no mask is computed"""
self.ehooke = None
"""EHooke: to be initialized upon starting the session (only with images and parameters"""
def __init__(self, params_obj=None, param_file=None):
"""Creates FluorFrame object, sets up parameters and loads images
It makes no sense to create a EHooke object without data or parameters.
So one must either supply params, a Parameters object, or param_file, in
which case EHooke loads the parameters file.
"""
self.params = None
"""Parameters: this can be a reference to an external object to be
changed for recomputing any step.
"""
if params_obj is None:
self.params = Parameters()
self.params.load_parameters(param_file)
else:
self.params = params_obj
self.fluor_frame = FluorFrame()
"""FluorFrame: manager for the fluor and phase images, plus masks"""
def __init__(self, scene_file=None, params=None):
"""
Create a new simulation.
"""
self.on_step_functions = []
# All the effects added in the simulation. keys are strings of the effect name, values are the effect objects
if not params:
self.params = Parameters()
else:
self.params = params
if scene_file:
self.params.scene_file = scene_file
self.scene = Scene()
def main():
parser = argparse.ArgumentParser(description='Car recogniser.')
parser.add_argument(
'-c', '--configuration', default='conf.yml', type=str, help='Configuration yaml file.')
args = parser.parse_args()
if args.configuration:
p = Path(args.configuration)
if not p.exists():
raise OSError('Configuration file {} does not exist'.format(args.configuration))
with p.open() as f:
conf = yaml.load(f)
params = Parameters(**conf)
hogs = [
hog
for f in itertools.chain(
Path(params.positive_training_dir).iterdir(),
Path(params.negative_training_dir).iterdir())
for hog in hogs_for_image(read_image(f), params)
]
vocabulary = generate_vocabulary(hogs, params)
def cross_validate(samples, labels):
X_train, X_test, y_train, y_test = train_test_split(
samples, labels, test_size=0.4, random_state=0)
print(len(X_train), len(X_test))
print(len(y_train), len(y_test))
clf = get_classifier(params).fit(X_train, y_train)
return clf.score(X_test, y_test)
samples, labels = generate_data(vocabulary, params, kind='training')
clf = get_classifier(params).fit(samples, labels)
testing_samples, testing_labels = generate_data(vocabulary, params, kind='testing')
print(clf.score(testing_samples, testing_labels))
def __init__(self,params_obj=None,param_file=None):
"""Creates FluorFrame object, sets up parameters and loads images
It makes no sense to create a EHooke object without data or parameters.
So one must either supply params, a Parameters object, or param_file, in
which case EHooke loads the parameters file.
"""
self.params = None
"""Parameters: this can be a reference to an external object to be
changed for recomputing any step.
"""
if params_obj is None:
self.params = Parameters()
self.params.load_parameters(param_file)
else:
self.params = params_obj
self.fluor_frame = FluorFrame()
"""FluorFrame: manager for the fluor and phase images, plus masks"""
def __init__(self, scene_file=None, params=None):
"""
Create a new simulation.
"""
self.on_step_functions = []
self.on_pedestrian_exit_functions = []
self.on_pedestrian_init_functions = []
self.finish_functions = []
# All the effects added in the simulation. keys are strings of the effect name, values are the effect objects
self.effects = {}
self.populations = []
if not params:
self.params = Parameters()
else:
self.params = params
if scene_file:
self.scene_file = scene_file
self.inflow = False
self.store_positions = False
self.logger = None
self.collect_data = None
self.visual_backend = True
functions.EPS = self.params.tolerance # TODO: Remove
self.scene = Scene()
def main():
init_logging()
lg.debug('test')
testing = ['params_Parameters']
# testing = ['trset_DataSet']
# testing = ['trset_System']
# Testing make_input
if 'make_input' in testing:
newinput = mkinp.Input('test.xyz')
del newinput
# Testing Molecule
if 'trset_Molecule' in testing:
molpath = '../../example/trset-tree-example/S-022/geometry/001.xyz'
mol = trset.Molecule(molpath)
lg.debug(str(mol))
if 'trset_System' in testing:
rulep = ('../../example/trset-tree-example/S-022/rule.dat')
with open(rulep, 'r') as rulef:
line = rulef.readline()
while line[0] == '#':
line = rulef.readline()
system = trset.System(line, 'S-022')
lg.info(str(system))
lg.info(system.fulldft_energy_error())
lg.info(system.fulldft_energy())
system.set_fulldft()
lg.info(system.fulldft_energy())
if 'trset_DataSet' in testing:
path = ('../../example/trset-tree-example/S-022')
dset = trset.DataSet(path)
print(dset.get_fulldftlist())
print(dset.get_blacklist())
dset.add_to_blacklist(['1', '2', '3', '4'])
dset.add_to_fulldftlist(['5', '6', '3', '4'])
print(dset.get_fulldftlist())
print(list(map(lambda x: x.name, dset.get_blacklist())))
print(list(map(lambda x: x.name, dset.get_fulldftlist())))
if 'trset_TrainingSet' in testing:
path = ('../../example/trset-tree-example')
tset = trset.TrainingSet(path, 'test-tset.dat')
# print(list(map(str, tset.container)))
print(tset.get_blacklist())
print(tset.get_fulldftlist())
# print(tset.p_compute_MAE('func'))
print(tset.p_compute_MAE('func'))
if 'params_Parameters' in testing:
x = list(range(1, 20))
# print(x)
prm = Prm()
prm1 = Prm()
print('prm: ', prm.prms)
print('prm1: ', prm1.prms)
prm.prms = dict(tta=[1000])
print('prm: ', prm.prms)
print('prm1: ', prm1.prms)
print('sprm1: ', prm1.sprms)
prm1.sprms = dict(tta=[2])
print('--------------------')
print('prm: ', prm.prms)
print('sprm: ', prm.sprms)
print('prm validity: ', prm.check_prms())
print('--------------------')
print('prm1: ', prm1.prms)
print('sprm1: ', prm1.sprms)
print('prm1 validity: ', prm1.check_prms())
print('--------------------')
prm1.refresh()
print('--------------------')
print('prm1: ', prm1.prms)
print('sprm1: ', prm1.sprms)
print('prm1 validity: ', prm1.check_prms())
print('--------------------')
print('prm1: ', prm1.prms)
print('prm1_old: ', prm1.prms_old)
print('prm1 check old: ', prm1.check_old())
print('--------------------')
prm1.prms = dict(tta=[2])
print('prm1: ', prm1.prms)
print('prm1_old: ', prm1.prms_old)
print('prm1 check old: ', prm1.check_old())
prm1.optim = dict(tta=504, cc_aa_1=505)
print(prm.prms)
print(prm.check_old())
print(prm.optim)
class Session:
"""eHooke session
This class manages the folder with data and report files (in the server path)
and a EHooke instance for running the computations
"""
def __init__(self):
self.id=str(uuid.uuid4())
"""str: Unique identifier for this session"""
self.folder = SESSION_FOLDER+'/'+self.id
"""str: main session folder with all generated files"""
self.data_folder = self.folder+'/Data'
"""str: subfolder for session data (phase, fluor and params file)"""
self.params = Parameters()
"""Parameters: parameters object to share with ehooke instance"""
self.params_file = None
"""str: parameters file name"""
self.fluor_file = None
"""str: fluor file name, to override parameters so that params can be imported from other sessions"""
self.phase_file = None
"""str: phase file name, to override parameters so that params can be imported from other sessions"""
self.name = 'Session'
"""str: session name, user defined"""
self.description = ''
"""str: session description, user defined"""
self.mask_image = None
"""str: filename for the current mask image, none if no mask is computed"""
self.ehooke = None
"""EHooke: to be initialized upon starting the session (only with images and parameters"""
#TODO change this to work with multiprocessing
#<LK 2015-06-30>
def setup(self):
"""creates session folder (add other setup stuff here)"""
if not os.path.exists(self.folder):
os.mkdir(self.folder)
if not os.path.exists(self.data_folder):
os.mkdir(self.data_folder)
def set_phase(self,file_name):
"""sets the phase file without computing anything
Overrides the file name in the parameters file. This is necessary
in order to allow importing parameter files to the session
"""
self.phase_file = file_name
self.params.fluor_frame_params.phase_file = file_name
def set_fluor(self,file_name):
"""sets the fluorescence file without computing anything
Overrides the file name in the parameters file. This is necessary
in order to allow importing parameter files to the session
"""
self.fluor_file = file_name
self.params.fluor_frame_params.fluor_file = file_name
def set_parameters_file(self, file_name):
"""sets and reloads parameters
(but overrides fluor_file and phase_file in order to allow parameter
updates after uploading the image files
"""
self.params_file = file_name
self.params.load_parameters(file_name)
if self.fluor_file != self.params.fluor_frame_params.fluor_file or \
self.phase_file != self.params.fluor_frame_params.phase_file:
self.params.fluor_frame_params.fluor_file = self.fluor_file
self.params.fluor_frame_params.phase_file = self.phase_file
self.params.save_parameters(file_name)
def start_ehooke(self):
"""Initiates the ehooke process and loads images
if there is no fluorescence image specified, returns false and error message
otherwise returns true,''
if ehooke is not none does nothing and returns ok
"""
if self.ehooke is not None:
return (True,'')
if self.fluor_file is None:
return (False,'Cannot start eHooke without a fluorescence image')
self.ehooke = EHooke(self.params)
self.ehooke.load_images()
return (True,'')
def save_overlay(self, back=(1,1,1), fore=(0,1,1), mask='phase',image='phase'):
"""saves the overlay image to overlay.png"""
self.ehooke.save_mask_overlay(self.folder+'/overlay.png',back, fore, mask,image)
self.mask_image = self.folder+'/overlay.png'
def recompute_mask(self):
res,msg = self.start_ehooke()
if res:
self.ehooke.create_masks()
self.save_overlay()
return (res,msg)
""" Parameter definitions. Could have easily been put into the configuration yaml """
defaultsecgrp = 'secgrp2'
keyuser = '******'
""" Set the configuration for AWS to us-east-1 to enable use of the public AMI's in that region """
awsconfig = Config(region_name='us-east-1',
signature_version='v4',
retries={
'max_attempts': 10,
'mode': 'standard'
})
""" Simple ArgParse """
parse = argparse.ArgumentParser(
description="AWS Command Line w/configuration file")
parse.add_argument("-f", "--config_file", help="AWS Configuration file")
arg1 = parse.parse_args()
config = Parameters('awsconfig.yaml')
""" Formulation of part I of the user_data scripts based upon the users in the configuration file """
user_data = '#!/bin/bash\n'
for u in config.server.users:
user_data = user_data + 'useradd ' + u['login'] + '\nmkdir -p /home/' + u[
'login'] + '/.ssh\nchown ' + u['login'] + ':' + u['login'] + ' /home/' + u[
'login'] + '/.ssh\necho "' + u['ssh_key'] + '" >> /home/' + u[
'login'] + '/.ssh/authorized_keys\nchmod 700 /home/' + u[
'login'] + '/.ssh\nchown ' + u['login'] + ':' + u[
'login'] + ' /home/' + u[
'login'] + '/.ssh/authorized_keys\nchmod 600 /home/' + u[
'login'] + '/.ssh/authorized_keys\n'
""" AMI Index Dictionary. To be added to later for more root volume options. Speeds up the deployment """
amiindex = {'ext4': 'ami-04108d64a147ddb9a', 'xfs': 'ami-048f6ed62451373d9'}
""" Setting the AMI and formulating the volumes needed for launch """
""" and formulation of part II of the user_data scripts based upon the volumes configured """
action="store_true")
trace.add_argument(
"-s",
"--sleep",
type=int,
help="time interval (in secs) between the intermediate prints")
trace.add_argument("-d",
"--duration",
type=int,
help="duration of the test (in secs)")
trace.add_argument("-i", "--input", help="input file path")
trace.add_argument("-o", "--output", help="output file path")
args = parser.parse_args()
parameters = Parameters()
element = args.func(args, parameters)
start_time = time.time() # for the elapsed time
if parameters.brief is not True:
element.intro_print(parameters)
if parameters.verbose is True:
element.verbose_print(parameters)
element.make_bpf_program(parameters)
b = BPF(text=parameters.prog)
for f in parameters.active_functions:
class Session:
"""eHooke session
This class manages the folder with data and report files (in the server path)
and a EHooke instance for running the computations
"""
def __init__(self):
self.id = str(uuid.uuid4())
"""str: Unique identifier for this session"""
self.folder = SESSION_FOLDER + '/' + self.id
"""str: main session folder with all generated files"""
self.data_folder = self.folder + '/Data'
"""str: subfolder for session data (phase, fluor and params file)"""
self.params = Parameters()
"""Parameters: parameters object to share with ehooke instance"""
self.params_file = None
"""str: parameters file name"""
self.fluor_file = None
"""str: fluor file name, to override parameters so that params can be imported from other sessions"""
self.phase_file = None
"""str: phase file name, to override parameters so that params can be imported from other sessions"""
self.name = 'Session'
"""str: session name, user defined"""
self.description = ''
"""str: session description, user defined"""
self.mask_image = None
"""str: filename for the current mask image, none if no mask is computed"""
self.ehooke = None
"""EHooke: to be initialized upon starting the session (only with images and parameters"""
#TODO change this to work with multiprocessing
#<LK 2015-06-30>
def setup(self):
"""creates session folder (add other setup stuff here)"""
if not os.path.exists(self.folder):
os.mkdir(self.folder)
if not os.path.exists(self.data_folder):
os.mkdir(self.data_folder)
def set_phase(self, file_name):
"""sets the phase file without computing anything
Overrides the file name in the parameters file. This is necessary
in order to allow importing parameter files to the session
"""
self.phase_file = file_name
self.params.fluor_frame_params.phase_file = file_name
def set_fluor(self, file_name):
"""sets the fluorescence file without computing anything
Overrides the file name in the parameters file. This is necessary
in order to allow importing parameter files to the session
"""
self.fluor_file = file_name
self.params.fluor_frame_params.fluor_file = file_name
def set_parameters_file(self, file_name):
"""sets and reloads parameters
(but overrides fluor_file and phase_file in order to allow parameter
updates after uploading the image files
"""
self.params_file = file_name
self.params.load_parameters(file_name)
if self.fluor_file != self.params.fluor_frame_params.fluor_file or \
self.phase_file != self.params.fluor_frame_params.phase_file:
self.params.fluor_frame_params.fluor_file = self.fluor_file
self.params.fluor_frame_params.phase_file = self.phase_file
self.params.save_parameters(file_name)
def start_ehooke(self):
"""Initiates the ehooke process and loads images
if there is no fluorescence image specified, returns false and error message
otherwise returns true,''
if ehooke is not none does nothing and returns ok
"""
if self.ehooke is not None:
return (True, '')
if self.fluor_file is None:
return (False, 'Cannot start eHooke without a fluorescence image')
self.ehooke = EHooke(self.params)
self.ehooke.load_images()
return (True, '')
def save_overlay(self,
back=(1, 1, 1),
fore=(0, 1, 1),
mask='phase',
image='phase'):
"""saves the overlay image to overlay.png"""
self.ehooke.save_mask_overlay(self.folder + '/overlay.png', back, fore,
mask, image)
self.mask_image = self.folder + '/overlay.png'
def recompute_mask(self):
res, msg = self.start_ehooke()
if res:
self.ehooke.create_masks()
self.save_overlay()
return (res, msg)
def main():
"""TODO:
"""
params = Parameters()
model = tf.keras.models.load_model(get_best_model_path(params), compile=True)
ohlcv_histories, technical_indicators, next_day_open_values, unscaled_y, y_normaliser = \
csv_to_dataset(params.CSV_DATA_PATH, params.num_history_points)
n = int(ohlcv_histories.shape[0] * params.train_split)
ohlcv_test = ohlcv_histories[n:]
tech_ind_test = technical_indicators[n:]
y_test = next_day_open_values[n:]
unscaled_y_test = unscaled_y[n:]
y_test_predicted = model.predict([ohlcv_test, tech_ind_test])
y_test_predicted = y_normaliser.inverse_transform(y_test_predicted)
buys = []
sells = []
# thresh = 0.7
thresh = 0.1
start = 0
end = -1
x = -1
holding_stocks = False
ratio_treshold = 0.005
def compute_earnings(buys_, sells_):
# purchase amount
purchase_amt = 10
stock = 0
balance = 0
while len(buys_) > 0 and len(sells_) > 0:
if buys_[0][0] < sells_[0][0]:
# time to buy $10 worth of stock
balance -= purchase_amt
stock += purchase_amt / buys_[0][1]
buys_.pop(0)
else:
# time to sell all of our stock
balance += stock * sells_[0][1]
stock = 0
sells_.pop(0)
print(f"earnings: ${balance}")
for ohlcv, ind in zip(ohlcv_test[start: end], tech_ind_test[start: end]):
normalised_price_today = ohlcv[-1][0]
normalised_price_today = np.array([[normalised_price_today]])
price_today = y_normaliser.inverse_transform(normalised_price_today)
predicted_price_tomorrow = np.squeeze(y_normaliser.inverse_transform(model.predict([[ohlcv], [ind]])))
###### HIS CODE ########
# delta = predicted_price_tomorrow - price_today
# price_today = price_today[0][0]
#
# ratio = predicted_price_tomorrow / price_today
# ratio_threshold = 0.02
#
# if ratio > 1 + ratio_threshold:
# buys.append((x, price_today))
# elif ratio < 1 - ratio_threshold:
# sells.append((x, price_today))
#
# if delta > thresh and ratio > 1 + ratio_threshold:
# buys.append((x, price_today))
# elif delta < -thresh and ratio < 1 - ratio_threshold:
# sells.append((x, price_today))
##### START STEFAN ######
abs_price_today = price_today[0][0]
if holding_stocks:
price = buys[-1][1]
else:
price = abs_price_today
# Absolute diff of price
ratio = predicted_price_tomorrow / price
if abs_price_today / price >= 1.3 and holding_stocks:
sells.append((x, abs_price_today))
holding_stocks = False
elif ratio >= ratio_treshold and not holding_stocks:
buys.append((x, abs_price_today))
holding_stocks = True
print("BUY: price today", abs_price_today, " predicted price tomorrow",
predicted_price_tomorrow)
elif ratio < 1 - ratio_treshold and holding_stocks:
sells.append((x, abs_price_today))
holding_stocks = False
###################################################
x += 1
print(f"buys: {len(buys)}")
print(f"sells: {len(sells)}")
# we create new lists so we don't modify the original
compute_earnings([b for b in buys], [s for s in sells])
import matplotlib.pyplot as plt
plt.gcf().set_size_inches(22, 15, forward=True)
real = plt.plot(unscaled_y_test[start:end], label='real')
pred = plt.plot(y_test_predicted[start:end], label='predicted')
if len(buys) > 0:
plt.scatter(list(list(zip(*buys))[0]), list(list(zip(*buys))[1]), c='#00ff00', s=50)
if len(sells) > 0:
plt.scatter(list(list(zip(*sells))[0]), list(list(zip(*sells))[1]), c='#ff0000', s=50)
# TRAIN TEST
# ohlcv_train = ohlcv_histories[:n]
# tech_ind_train = technical_indicators[:n]
# y_train = next_day_open_values[:n]
# real = plt.plot(unscaled_y[start:end], label='real')
# pred = plt.plot(y_predicted[start:end], label='predicted')
plt.legend(['Real', 'Predicted', 'Buy', 'Sell'])
plt.show()
def main():
parser = argparse.ArgumentParser(description='Car recogniser.')
parser.add_argument('-c',
'--configuration',
default='conf.yml',
type=str,
help='Configuration yaml file.')
args = parser.parse_args()
if args.configuration:
p = Path(args.configuration)
if not p.exists():
raise OSError('Configuration file {} does not exist'.format(
args.configuration))
with p.open() as f:
conf = yaml.load(f)
params = Parameters(**conf)
positive_features = np.asarray([
features_for_image(read_image(f), params)
for f in Path(params.positive_training_dir).iterdir()
])
negative_features = np.asarray([
features_for_image(img, params)
for f in Path(params.negative_training_dir).iterdir()
for img in partition_negative_image(read_image(f), params)
])
features = np.concatenate((positive_features, negative_features))
labels = np.asarray([1] * len(positive_features) +
[-1] * len(negative_features))
print(
'Training the classifier with {} positives samples and {} negative samples'
.format(len(positive_features), len(negative_features)))
csf = svm.LinearSVC(verbose=True).fit(features, labels)
def train_for_hard_negatives():
def false_positives() -> ndarray:
pred = csf.predict(negative_features)
fp = [i for i in range(len(negative_features)) if pred[i] != -1]
return np.asarray(fp)
fp = false_positives()
print('Running hard negative samples', end='')
while len(fp) > 0:
features = np.concatenate((features, negative_features[fp]))
labels = np.concatenate((labels, [-1] * len(fp)))
csf.fit(features, labels)
fp = false_positives()
print('.', end='', flush=True)
def cross_validate(samples, labels):
X_train, X_test, y_train, y_test = train_test_split(samples,
labels,
test_size=0.4,
random_state=0)
clf = svm.LinearSVC().fit(X_train, y_train)
return clf.score(X_test, y_test)
print('cross validation performance: ', (cross_validate(features, labels)))
for f in Path(params.test_dir).iterdir():
detections_for_image(read_image(f), params, csf)
from bilstm_att import MultiNLIModel
from torchtext import data, vocab, datasets
import sys
sys.path.append('..')
from params import Parameters
if __name__ == '__main__':
params = Parameters()
inputs = data.Field(lower=True, tokenize='spacy')
answers = data.Field(sequential=False)
train, val, test = datasets.MultiNLI.splits(text_field=inputs,
label_field=answers)
inputs.build_vocab(train, val, test)
if params.word_vectors:
inputs.vocab.load_vectors(vocab.Vectors(params.glove_path, cache="."))
answers.build_vocab(train)
params.n_embed = len(inputs.vocab)
params.d_out = len(answers.vocab)
print(f"Unique tokens in inputs vocabulary: {params.n_embed}")
print(f"Unique tokens in answers vocabulary: {params.d_out}")
class EHooke:
"""Encapsulates all the code for processing a fluorescence frame"""
def __init__(self, params_obj=None, param_file=None):
"""Creates FluorFrame object, sets up parameters and loads images
It makes no sense to create a EHooke object without data or parameters.
So one must either supply params, a Parameters object, or param_file, in
which case EHooke loads the parameters file.
"""
self.params = None
"""Parameters: this can be a reference to an external object to be
changed for recomputing any step.
"""
if params_obj is None:
self.params = Parameters()
self.params.load_parameters(param_file)
else:
self.params = params_obj
self.fluor_frame = FluorFrame()
"""FluorFrame: manager for the fluor and phase images, plus masks"""
def load_images(self):
"""checks which images to load and loads them into the fluor_frame
"""
ffparams = self.params.fluor_frame_params
self.fluor_frame.load_fluor(ffparams)
if ffparams.phase_file is not None:
self.fluor_frame.load_phase(ffparams)
def create_masks(self):
"""creates masks using the current parameters"""
mparams = self.params.mask_params
self.fluor_frame.create_masks(mparams)
def align_fluor_to_phase(self):
"""aligns the fluorescence image to the phase mask, if a phase file exists"""
if ffparams.phase_file is not None:
self.fluor_frame.align_fluor(self.params.fluor_frame_params)
def save_mask_overlay(self,
fname,
back=(0, 0, 1),
fore=(1, 1, 0),
mask='phase',
image='phase'):
"""saves the mask overlay image to a file"""
img = self.fluor_frame.mask_overlay(back, fore, mask, image)
imsave(fname, img)
def save_mask_contour(self,
fname,
mask='phase',
image='phase',
color=(1, 1, 0)):
"""saves the mask contour image to a file"""
img = self.fluor_frame.contour_overlay(mask, image, color)
imsave(fname, img)
from time import time
from time import gmtime
def get_time():
hours = gmtime()[3]+2
minutes = gmtime()[4]
seconds = gmtime()[5]
return "{}:{}:{}".format(hours, minutes, seconds)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ['CUDA_VISIBLE_DEVICES']='4,5,6,7'#,8,9'#,1,2,3'
"""
Hyperparameters
"""
args = Parameters()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if args.save:
print('Results will be saved in ./results/vqvae_' + args.filename + '.pth')
"""
Load data and define batch data loaders
"""
training_loader = create_data_loader(args.batch_size, split=False)
'''
transform = transforms.Compose([transforms.ToTensor()])
mnist_trainset = datasets.MNIST(root='./data', train=True, download=True, transform=transform)
print(color + "~> " + cmd + "\n")
def print_header(color, commands, message):
if len(commands) > 0:
print(color + message + "\n")
parser = argparse.ArgumentParser(description="Get a PostgreSQL dump file and load it as parquet files on S3")
parser.add_argument('-c', '--command', help="Set command plan or apply", required=True)
parser.add_argument('-f', '--file', help="Path to the file to process", required=True)
parser.add_argument('-u', '--user', help="User to connect in PG", required=True)
args = parser.parse_args()
file = args.file
command = args.command
pm = Parameters(args.file, args.user)
conn = OpenConnection(pm.publication, pm.subscription)
publication = Publication(pm, conn.get_connection_pub())
publication.run()
subscription = Subscription(pm, conn.get_connection_sub(), publication.has_any_changes())
subscription.run()
if publication.has_any_changes():
print( Fore.CYAN + '\npg-logical-form will perform the following actions in PUBLICATION:\n')
print_header(Fore.LIGHTGREEN_EX, publication.commands_add, 'Add things of Publication:')
print_messages(Fore.GREEN, publication.commands_add)
print_header(Fore.LIGHTYELLOW_EX, publication.commands_change, 'Change things of Publication:')
