def __init__(self, port_path): self.port_path = port_path self.serial = serial.Serial(port_path) self.serial.bytesize = serial.EIGHTBITS self.serial.baudrate = 19200 self.serial.timeout = 1 self.devices = {} actions.run('onInsteonModemInitialized', modem=self)
def test_breakout_up(self): """3A. Users can't call actions above the actions directory. Tests both a relative and a literal path. """ options = "hi" for arg in ("../echo", "/bin/echo"): with self.assertRaises(ValueError): run(arg, options)
def test_breakout_up(self): """3A. Users can't call actions above the actions directory. Tests both a relative and a literal path. """ options="hi" for arg in ("../echo", "/bin/echo"): with self.assertRaises(ValueError): run(arg, options)
def test_multiple_options(self): """4. Multiple options can be provided as a list. """ self.assertEqual( subprocess.check_output(shlex.split("id -ur")).strip(), run("id", ["-u" ,"-r"])[0].strip())
def test_quest(self): s = State({'red': Player()}) q = Quest(Release(Resources(fighters=3)), Supply(Resources(vp=6))) s = actions.run(q.action('red', s)) r = s.players['red'].resources self.assertEqual([r.fighters, r.vp], [-3, 6])
def _run(arguments, superuser=True): """Run an given command and raise exception if there was an error""" command = 'pagekite-configure' LOGGER.info('Running command - %s, %s, %s', command, arguments, superuser) if superuser: output = actions.superuser_run(command, arguments) else: output = actions.run(command, arguments) return output
def process_form(self, **kwargs): checkedinfo = { 'inband_enable' : False, } opts = [] for k in kwargs.keys(): if 'on' == kwargs[k]: shortk = k.split("xmpp_").pop() checkedinfo[shortk] = True for key in checkedinfo.keys(): if checkedinfo[key]: opts.append(key) else: opts.append('no'+key) actions.run("xmpp-setup", opts) main = self.main(checkedinfo['inband_enable']) return self.fill_template(title="XMPP Server Configuration", main=main, sidebar_left=self.sidebar_left, sidebar_right=self.sidebar_right)
def test_breakout_option_string(self): """3D. Option strings can't be used to run other actions. Verify that shell control characters aren't interpreted. """ action = "echo" # counting is safer than actual badness. options = "good; echo $((1+1))" output, error = run(action, options) self.assertFalse("2" in output)
def test_building(self): players = {'red': OpenPlayer(4), 'blue': Opponent(4)} s = State(players) b = Building(5, [Supply(Resources(wizards=3))], [Supply(Resources(clerics=2)), DrawFaceDown(QualityResources(intrigues=1))]) b.owner = 'blue' s = actions.run(b.action('red', s)) self.assertEqual([s.players['red'].resources.wizards, s.players['blue'].resources.clerics, s.players['blue'].intriguesN], [3, 2, 1])
def test_breakout_option_list(self): """3D. Option lists can't be used to run other actions. Verify that only a string of options is accepted and that we can't just tack additional shell control characters onto the list. """ action = "echo" # counting is safer than actual badness. options = ["good", ";", "echo $((1+1))"] output, error = run(action, options) # we'd better not evaluate the data. self.assertFalse("2" in output)
def test_breakout_actions(self): """3C. Actions can't be used to run other actions. If multiple actions are specified, bail out. """ # counting is safer than actual badness. actions = ("echo ''; echo $((1+1))", "echo '' && echo $((1+1))", "echo '' || echo $((1+1))") options = ("good", "") for action in actions: for option in options: with self.assertRaises(ValueError): output = run(action, option) # if it somewhow doesn't error, we'd better not evaluate # the data. self.assertFalse("2" in output[0])
def get_modules_available(): """Return list of all modules""" output = actions.run('module-manager', ['list-available']) return output.split()
def get_status(): """Return the current status""" output = actions.run('owncloud-setup', 'status') return {'enabled': 'enable' in output.split()}
def get_status(): """Return the current status""" output = actions.run('xmpp-setup', 'status') return {'inband_enabled': 'inband_enable' in output.split()}
def get_modules_enabled(): """Return list of all modules""" output = actions.run('module-manager', ['list-enabled']) return output.split()
def perform_experiment(): # Initializing some variables to suppress linter warnings data_package = shared_layers = individual_layers = MSHE = MSHE_oos = MAPHE = MAPHE_oos = None N_train = N_val = X_train = X_val = data = scaler_X = model_name = None try: with pd.ExcelFile(paths['results-excel']) as reader: runID = reader.parse("RunData")['runID'].max() + 1 except FileNotFoundError: runID = 1 if not os.path.exists(paths['all_models']): os.makedirs(paths['all_models']) if run_BS != 'only_BS': msg = 'Evaluating {} different settings with {} feature combinations, in {} windows, ' \ 'each {} times for a total of {} runs.' print( msg.format( int(settings_combi_count / len(active_feature_combinations)), len(active_feature_combinations), window_combi_count, identical_reruns, settings_combi_count * window_combi_count * identical_reruns)) watches = [ 'model_name', 'time', 'optimizer', 'lr', 'epochs', 'features', 'activation', 'layers', 'nodes', 'batch_normalization', 'loss', 'loss_oos', 'used_synth', 'normalize', 'dropout', 'batch_size', 'failed', 'loss_mean', 'loss_std', 'val_loss_mean', 'val_loss_std', 'stock', 'dt_start', 'dt_middle', 'dt_end', 'duration', 'N_train', 'N_val', 'regularizer', 'useEarlyStopping', 'loss_func', 'MSHE', 'MSHE_oos', 'MAPHE', 'MAPHE_oos' ] cols = {col: [] for col in watches} i = 0 for settings in itertools.product( *settings_list): # equivalent to a bunch of nested for-loops i += 1 SSD_distribution_train = [] SSD_distribution_val = [] (act, n, layers, optimizer, include_synthetic_data, dropout_rate, normalization, batch_size, regularizer, c) = settings used_features = full_feature_combination_list[c] if type(layers) is tuple: shared_layers, individual_layers = layers layers = shared_layers + individual_layers j = 0 for window in itertools.product(*windows_list): j += 1 stock, date_tuple, rerun_id = window dt_start, dt_middle, dt_end = date_tuple print('{}.{}'.format(i, j), end=' ', flush=True) # We reinitialize these variables to None because they will be appended to cols loss_oos = last_losses_mean = last_losses_std = last_val_losses_mean =\ last_val_losses_std = None pattern = 'c{}_act{}_lf{}_l{}_n{}_o{}_bn{}_do{}_s{}_no{}_bs{}_r{}' model_name = pattern.format(c, act, loss_func, layers, n, optimizer, int(batch_normalization), int(dropout_rate * 10), int(include_synthetic_data), normalization, batch_size, regularizer) if multi_target: model_name = 'multit_' + model_name print(model_name, end=': ', flush=True) model = multitask_model( input_dim=len(used_features), shared_layers=shared_layers, individual_layers=individual_layers, nodes_per_layer=n, activation=act, use_batch_normalization=batch_normalization, optimizer=optimizer) else: model_name = 'full_' + model_name print(model_name, end=': ', flush=True) model = full_model( input_dim=len(used_features), num_layers=layers, nodes_per_layer=n, loss=loss_func, activation=act, optimizer=optimizer, use_batch_normalization=batch_normalization, dropout_rate=dropout_rate, regularizer=regularizer) model.name = model_name # when rerun_id is 0, that means we just switched to a new stock or date, or setting # that is when we need to get the data again if rerun_id == 0: data_package = get_data_package( model=model, columns=used_features, include_synth=include_synthetic_data, normalize=normalization, stock=stock, start_date=dt_start, end_train_start_val_date=dt_middle, end_val_date=dt_end) N_train = len(data_package[0][0]) N_val = len(data_package[0][2]) if lr is not None: K.set_value(model.optimizer.lr, lr) actual_epochs = epochs starting_time = datetime.now() starting_time_str = '{:%Y-%m-%d_%H-%M}'.format(starting_time) annResult = run_and_store_ann( model=model, in_sample=True, model_name='i_' + model_name + '_inSample', nb_epochs=separate_initial_epochs, reset='yes', columns=used_features, include_synth=include_synthetic_data, normalize=normalization, batch_size=batch_size, data_package=data_package, starting_time_str=starting_time_str) initial_hist = annResult.history initial_loss = annResult.last_loss # During early experimentation it was useful to quickly abort models that failed to converge if initial_loss > required_precision: print('FAILED', end=' ') cols['failed'].append(int(True)) loss = initial_loss if useEarlyStopping: actual_epochs = len(initial_hist.history['loss']) else: cols['failed'].append(int(False)) annResult = run_and_store_ann( model=model, in_sample=True, model_name=model_name + '_inSample', nb_epochs=epochs - separate_initial_epochs, reset='continue', columns=used_features, get_deltas=True, include_synth=include_synthetic_data, normalize=normalization, batch_size=batch_size, data_package=data_package, starting_time_str=starting_time_str) hist, loss, loss_tuple, MSHE, MAPHE = annResult annResult = run_and_store_ann( model=model, in_sample=False, model_name=model_name + '_outSample', reset='reuse', columns=used_features, get_deltas=True, normalize=normalization, batch_size=batch_size, data_package=data_package, starting_time_str=starting_time_str) loss_oos = annResult.last_loss MSHE_oos = annResult.MSHE MAPHE_oos = annResult.MAPHE if useEarlyStopping: actual_epochs = len(hist.history['loss']) + len( initial_hist.history['loss']) (last_losses_mean, last_losses_std, last_val_losses_mean, last_val_losses_std) = loss_tuple data = data_package.data scaler_X = data_package.scaler_X X_train = data[0] X_val = data[2] SSD_train = get_ssd(model, X_train) SSD_val = get_ssd(model, X_val) SSD_distribution_train.append(SSD_train) SSD_distribution_val.append(SSD_val) model_end_time = datetime.now() feature_string = '_'.join(used_features) pos_fff = feature_string.find("_ff_ind") # reducing the long string of fama & french factors down if pos_fff > -1: feature_string = feature_string[0:pos_fff] + "_fff" cols['model_name'].append(model_name) cols['time'].append(datetime.now()) cols['duration'].append(model_end_time - starting_time) cols['N_train'].append(N_train) cols['N_val'].append(N_val) cols['stock'].append(stock) cols['dt_start'].append(dt_start) cols['dt_middle'].append(dt_middle) cols['dt_end'].append(dt_end) cols['loss'].append(loss) cols['loss_oos'].append(loss_oos) cols['loss_mean'].append(last_losses_mean) cols['loss_std'].append(last_losses_std) cols['val_loss_mean'].append(last_val_losses_mean) cols['val_loss_std'].append(last_val_losses_std) cols['MSHE'].append(MSHE) cols['MSHE_oos'].append(MSHE_oos) cols['MAPHE'].append(MAPHE) cols['MAPHE_oos'].append(MAPHE_oos) cols['epochs'].append(actual_epochs) cols['optimizer'].append(optimizer) cols['lr'].append(lr) cols['features'].append(feature_string) cols['activation'].append(act) cols['layers'].append(layers) cols['nodes'].append(n) cols['batch_normalization'].append(batch_normalization) cols['loss_func'].append(loss_func) cols['used_synth'].append(int(include_synthetic_data)) cols['normalize'].append(normalization) cols['dropout'].append(dropout_rate) cols['batch_size'].append(batch_size) cols['regularizer'].append(regularizer) cols['useEarlyStopping'].append(int(useEarlyStopping)) print((model_end_time - starting_time).seconds, end=' - ') print(loss) filename = model_name + '_' + starting_time_str + '.h5' model.save(os.path.join(paths['all_models'], filename)) if i == 1 and j == i: # sample model to be particularly investigated loss, Y_prediction, history = run( model, data=data, reset='reuse', plot_prediction=False, segment_plot=False, verbose=0, model_name=model_name, in_sample=False, batch_size=batch_size, starting_time_str=starting_time_str) model.save(paths['sample_model']) with pd.HDFStore(paths['sample_data']) as store: store['X_train'] = X_train store['X_test'] = X_val store['Y_train'] = data[1] store['Y_test'] = data[3] store['Y_prediction'] = pd.Series( Y_prediction.flatten()) featureCounts_to_record = [ len(full_feature_combination_list[-1]) ] is_All_or_None_Run = len( used_features) in featureCounts_to_record if collect_gradients_data and is_All_or_None_Run: gradient_df_columns = [ 'model_name', 'time', 'sample', 'feature', 'feature_value', 'gradient', 'stock', 'dt_start', 'runID', 'num_features', 'moneyness' ] grad_data = {key: [] for key in gradient_df_columns} sampling_dict = dict(train=X_train, test=X_val) for sample_key, points in sampling_dict.items(): gradients = get_gradients(model, points) rescaled = scaler_X.inverse_transform(points) points = pd.DataFrame(rescaled, index=points.index, columns=points.columns) for feature_iloc, feature_name in enumerate( points.columns): iterator = zip(points.iloc[:, feature_iloc], gradients[:, feature_iloc], points.loc[:, 'moneyness']) for value, gradient, moneyness in iterator: grad_data['model_name'].append(model_name) grad_data['time'].append(starting_time) grad_data['sample'].append(sample_key) grad_data['feature'].append(feature_name) grad_data['feature_value'].append(value) grad_data['gradient'].append(gradient) grad_data['stock'].append(stock) grad_data['dt_start'].append(dt_start) grad_data['runID'].append(runID) grad_data['num_features'].append( len(points.columns)) grad_data['moneyness'].append(moneyness) gradients_df = pd.DataFrame(grad_data) if limit_windows != 'mock-testing': with pd.HDFStore(paths['gradients_data'], mode='a') as store: store.append('gradients_data', gradients_df, index=False, data_columns=True) K.clear_session() tf.reset_default_graph() if j >= 5: if not onCluster and len(used_features) > 4: boxplot_SSD_distribution(SSD_distribution_train, used_features, 'Training Data', model_name) boxplot_SSD_distribution(SSD_distribution_val, used_features, 'Validation Data', model_name) if saveResultsForLatex: SSDD_df_train = pd.DataFrame(SSD_distribution_train, columns=used_features) SSDD_df_val = pd.DataFrame(SSD_distribution_val, columns=used_features) SSDD_df_train['sample'] = 'train' SSDD_df_val['sample'] = 'test' merged_results = pd.concat([SSDD_df_train, SSDD_df_val]) merged_results['runID'] = runID merged_results['used_synth'] = include_synthetic_data try: with pd.HDFStore(paths['data_for_latex']) as store: previous_results = store['SSDD_df'] merged_results = pd.concat( [merged_results, previous_results]) except FileNotFoundError: pass with pd.HDFStore(paths['data_for_latex']) as store: store['SSDD_df'] = merged_results results_df = pd.DataFrame(cols) results_df['runID'] = runID if limit_windows != 'mock-testing': try: with pd.ExcelFile(paths['results-excel']) as reader: previous_results = reader.parse("RunData") merged_results = pd.concat([results_df, previous_results]) except FileNotFoundError: merged_results = results_df with pd.ExcelWriter(paths['results-excel']) as writer: merged_results.to_excel(writer, 'RunData') writer.save() print('ANN calculations done') if not onCluster: if not cols['failed'][-1]: get_and_plot( [model_name + '_inSample', model_name + '_outSample'], variable='prediction') get_and_plot( [model_name + '_inSample', model_name + '_outSample'], variable='error') get_and_plot( [model_name + '_inSample', model_name + '_outSample'], variable='calculated_delta') get_and_plot( [model_name + '_inSample', model_name + '_outSample'], variable='scaled_option_price') if run_BS in ['yes', 'only_BS']: # not 'no' print('Running Black Scholes Benchmark') BS_watches = [ 'stock', 'dt_start', 'dt_middle', 'dt_end', 'vol_proxy', 'MSE', 'MAE', 'MAPE', 'MSHE', 'MAPHE' ] BS_cols = {col: [] for col in BS_watches} i = 0 for vol_proxy in vol_proxies: i += 1 j = 0 for window in itertools.product(*windows_list): j += 1 print('{}.{}'.format(i, j), end=' ', flush=True) stock, date_tuple, rerun_id = window dt_start, dt_middle, dt_end = date_tuple data_package = get_data_package( model='BS', columns=[ 'days', 'moneyness', 'impl_volatility', 'v60', 'r' ], stock=stock, start_date=dt_start, end_train_start_val_date=dt_middle, end_val_date=dt_end) MSE, MAE, MAPE, MSHE, MAPHE = run_black_scholes( data_package, vol_proxy=vol_proxy) print(MSE) BS_cols['stock'].append(stock) BS_cols['dt_start'].append(dt_start) BS_cols['dt_middle'].append(dt_middle) BS_cols['dt_end'].append(dt_end) BS_cols['vol_proxy'].append(vol_proxy) BS_cols['MSE'].append(MSE) BS_cols['MAE'].append(MAE) BS_cols['MAPE'].append(MAPE) BS_cols['MSHE'].append(MSHE) BS_cols['MAPHE'].append(MAPHE) BS_results_df = pd.DataFrame(BS_cols) if limit_windows != 'mock-testing': try: with pd.ExcelFile(paths['results-excel-BS']) as reader: BS_previous_results = reader.parse("RunData") if run_BS == 'only_BS': BS_results_df['runID'] = BS_previous_results.runID.max( ) + 1 else: BS_results_df['runID'] = runID BS_merged_results = pd.concat( [BS_results_df, BS_previous_results]) except FileNotFoundError: BS_results_df['runID'] = 1 BS_merged_results = BS_results_df with pd.ExcelWriter(paths['results-excel-BS']) as writer: BS_merged_results.to_excel(writer, 'RunData') writer.save() print('BS done') print('Close')
def notest_multiple_options(self): """ 4. Multiple options can be provided as a list. """ # TODO: it's not allowed to call a symlink in the actions dir anymore self.assertEqual( subprocess.check_output(shlex.split("id -ur")).strip(), run("id", ["-u", "-r"])[0].strip())
def notest_multiple_options(self): """ 4. Multiple options can be provided as a list. """ # TODO: it's not allowed to call a symlink in the actions dir anymore self.assertEqual( subprocess.check_output(shlex.split("id -ur")).strip(), run("id", ["-u" ,"-r"])[0].strip())
#!/usr/bin/python # -*- coding: UTF-8 -*- import sys import actions if __name__ == '__main__': args_len = len(sys.argv) if args_len < 4: print("Missing enough arguments, at least 3: Uid, Pwd, qmsgkey") exit(1) if args_len > 4: print("Too many arguments") exit(1) Uid = sys.argv[1] Pwd = sys.argv[2] actions.qmsgkey = sys.argv[3] actions.run(Uid, Pwd)
def transform(): code = ui.getCode() if code[1]: actions.run(code[0])