def experiment_ball_point_vary_r(range_count): algo = 'query_ball' dfs = [] q_point = (22.6, 114) cols_remove = cols_remove_dict[algo] kwargs = dict(range_count=range_count, debug=False) bs = 50 settings = dict(repeat_times=300, tr=75, D=3, bs=bs, alpha=10) block_dag = GeneratorDAGchain.generate(**settings) block_dag.super_optimize() for r in range(1, 50, 3): print("=============={}==============".format(r)) # block size experiment t_start = get_ts(2017, 1, 2, 0, 0, 0) t_end = get_ts(2019, 1, 3, 0, 0, 0) df = compare_time_query_ball(block_dag, q_point, r, t_start, t_end, **kwargs) df.drop(cols_remove, axis=1, inplace=True) df = df * 1000 df['r'] = r dfs.append(df) df_k = pd.concat(dfs).groupby('r').median() df_k.plot(title='per r') save_df(df_k, '{}/{}_vary_r.csv'.format(algo, algo), True) return df_k
def exp7(): dataset = 'aids10k' model = 'vj' train_data = load_data(dataset, True) test_data = load_data(dataset, False) m = len(test_data.graphs) n = len(train_data.graphs) ged_mat = np.zeros((m, n)) time_mat = np.zeros((n, n)) outdir = get_root_path() + '/files' file = open('{}/ged_{}_{}_{}.csv'.format( \ outdir, dataset, model, get_ts()), 'w') print_and_log('i,j,i_node,j_node,i_edge,j_edge,ged,time', file) for i in range(m): for j in range(n): g1 = test_data.graphs[i] g2 = train_data.graphs[j] t = time() d = ged(g1, g2, model) t = time() - t s = '{},{},{},{},{},{},{},{:.5f}'.format(i, j, \ g1.number_of_nodes(), g2.number_of_nodes(), \ g1.number_of_edges(), g2.number_of_edges(), \ d, t) print_and_log(s, file) ged_mat[i][j] = d time_mat[i][j] = t file.close() np.save('{}/ged_ged_mat_{}_{}_{}'.format( \ outdir, dataset, model, get_ts()), ged_mat) np.save('{}/ged_time_mat_{}_{}_{}'.format( \ outdir, dataset, model, get_ts()), time_mat)
def __init__(self, sess=None): model_str = self._get_model_str() self.logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str, get_ts()) create_dir_if_not_exists(self.logdir) if sess is not None: self.tw = tf.summary.FileWriter(self.logdir + '/train', sess.graph) self.all_merged = tf.summary.merge_all() self.loss_merged = tf.summary.merge( self._extract_loss_related_summaries_as_list()) self._log_model_info(self.logdir, sess) self.f = open('{}/results_{}.txt'.format(self.logdir, get_ts()), 'w') print('Logging to {}'.format(self.logdir))
def place_piece(self, column, player): self.board, self.latest_move = connect4_utils.place_piece( self.board, column, self.pieces[player]) game_end = None self.winning_moves = connect4_utils.check_win(self.board, column) if self.winning_moves: game_end = 'win' self.game_history['game_state'] = self.pieces[self.current_player] elif connect4_utils.check_tie(self.board): game_end = 'tie' self.game_history['game_state'] = 0 board_url = self.render_board() # Save the players move before game_over gets called and the player is toggled self.game_history['moves'].append({ 'player': self.pieces[self.current_player], 'piece_played': self.latest_move, 'board': copy.deepcopy(self.board), 'rendered_board_url': board_url, 'timestamp': core_utils.get_ts(), }) if game_end is None: # Only toggle the player if game has not ended self.toggle_player() return board_url, game_end
def get_auth_token(tenant, role): """ Gets a JWT token for 'role'. 'role' is either 'battleserver' or 'battledaemon'. """ ts = get_ts() driftbase_tenant = ts.get_table('tenants').find({ 'tenant_name': tenant, 'deployable_name': 'drift-base' })[0] auth_url = get_root_endpoint(tenant) + "/auth" #! TODO: Find user in organization config table if role == 'battleserver': credentials = config.battleserver_credentials elif role == "battledaemon": credentials = config.battledaemon_credentials else: raise RuntimeError("'role' %s not understood." % role) headers = { 'Drift-Api-Key': config.api_key, 'Content-type': 'application/json' } #! TODO: Replace api key with product key r = requests.post(auth_url, data=json.dumps(credentials), headers=headers) r.raise_for_status() return r.json()
def __init__(self, player1_id, player2_id, team_id, channel_id, theme='classic'): self.player1_id = player1_id self.player2_id = player2_id self.team_id = team_id self.channel_id = channel_id self.pieces = { self.player1_id: 1, self.player2_id: 2, } self.theme = theme self.game_id = str(uuid.uuid4()) self.game_history = { 'platform': 'slack', 'game_id': self.game_id, 'start_time': core_utils.get_ts(), 'end_time': None, 'theme': theme, 'player1_id': self.player1_id, 'player2_id': self.player2_id, 'team_id': self.team_id, 'channel_id': self.channel_id, # Url to the finial rendered gif of all moves played 'recap_url': None, # None-game not done; 0-tie; 1-player 1 won; 2-player2 won 'game_state': None, 'moves': [ # Example of whats in a move # { # 'player': 1, # 'latest_move': (6, 2), # 'board': [[0, 0, 0],[0, 0, 0],[0, 0, 0]] ..., # 'rendered_board_url': 'https://...', # 'timestamp': '...', # } ], } self.s3_root_folder = f"connect4/slack/{self.team_id}" self.latest_move = (None, None) self.current_player = self.player1_id self.board = connect4_utils.gen_new_board() self.winning_moves = None
def get_model_info_as_str(): rtn = [] d = vars(FLAGS) for k in sorted_nicely(d.keys()): v = d[k] s = '{0:26} : {1}'.format(k, v) rtn.append(s) rtn.append('{0:26} : {1}'.format('ts', get_ts())) return '\n'.join(rtn)
def __init__(self): model_str = self.get_model_str() self.logdir = join(get_root_path(), 'logs', '{}_{}'.format(model_str, get_ts())) create_dir_if_not_exists(self.logdir) self.model_info_f = self._open('model_info.txt') self._log_model_info() self._save_conf_code() print('Logging to {}'.format(self.logdir))
def get_result(gp, algo): with open('{}/result/temp_{}'.format(gp, get_ts())) as f: lines = f.readlines() ln = 23 if 'beam' in algo else 22 rtn = float(lines[ln]) * 2 # alpha=0.5 --> / 2 assert (rtn - int(rtn) == 0) rtn = int(rtn) if rtn < 0: rtn = -1 # in case rtn == -2 return rtn
def extend_df(df, repeat_times=10, first_time=get_ts(2018, 1, 1), bs=20): a = pd.concat([df] * repeat_times, ignore_index=True) MS = 1000 dt = MS / bs new_times = np.arange(first_time * MS, first_time * MS + dt * a.shape[0], dt) / MS new_times = new_times[:a.shape[0]] new_times = new_times.astype(int) a.ts = new_times print(df.shape, '-(rept{})>'.format(repeat_times), a.shape) return a
def get_model_info_as_str(model_info_table=None): rtn = [] d = FLAGS.flag_values_dict() for k in sorted_nicely(d.keys()): v = d[k] s = '{0:26} : {1}'.format(k, v) rtn.append(s) if model_info_table: model_info_table.append([k, '**{}**'.format(v)]) rtn.append('{0:26} : {1}'.format('ts', get_ts())) return '\n'.join(rtn)
def start(self, player1_name, player2_name): banner_url = self.render_player_banner(player1_name, player2_name) board_url = self.render_board() self.game_history['moves'].append({ 'player': 0, 'piece_played': (None, None), 'board': copy.deepcopy(self.board), 'rendered_board_url': board_url, 'timestamp': core_utils.get_ts(), }) return banner_url, board_url
def __init__(self, sess): if FLAGS.log: model_str = self._get_model_str() logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str, get_ts()) create_dir_if_not_exists(logdir) self.tw = tf.summary.FileWriter(logdir + '/train', sess.graph) self.vw = tf.summary.FileWriter(logdir + '/val', sess.graph) self.merged = tf.summary.merge_all() self._log_model_info(logdir, sess) print('Logging to {}'.format(logdir))
def get_root_endpoint(tenant): ts = get_ts() driftbase_tenant = ts.get_table('tenants').find({ 'tenant_name': tenant, 'deployable_name': 'drift-base' })[0] endpoint = driftbase_tenant.get('root_endpoint') if not endpoint: endpoint = DEFAULT_ROOT_ENDPOINT.format(tenant) return endpoint
def game_over(self): self.game_history['end_time'] = core_utils.get_ts() recap_url = self._generate_recap(self.game_history['moves']) self.game_history['recap_url'] = recap_url s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', None)) s3.put_object(Body=json.dumps(self.game_history).encode('utf-8'), Bucket=os.environ['GAME_HISTORY_BUCKET'], Key=f"{self.s3_root_folder}/{self.game_id}_history.json", ContentType='application/json') return recap_url
def ged(g1, g2, algo): # https://github.com/dan-zam/graph-matching-toolkit gp = get_gmt_path() src, tp = setup_temp_folder(gp) meta1 = write_to_temp(g1, tp, algo, 'g1') meta2 = write_to_temp(g2, tp, algo, 'g2') if meta1 != meta2: raise RuntimeError('Different meta data {} vs {}'.format(meta1, meta2)) setup_property_file(src, gp, meta1) if not exec( 'cd {} && java -classpath {}/src/graph-matching-toolkit/bin algorithms.GraphMatching ./properties/properties_temp_{}.prop' .format(gp, get_root_path(), get_ts()), timeout=1000): return -1 return get_result(gp, algo)
def make_move(self, move): self.board, game_state = mastermind_utils.make_move(self.board, move) if game_state is not None: # Game over self.game_history['board'] = self.board self.game_history['game_state'] = game_state self.game_history['end_time'] = core_utils.get_ts() s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', None)) s3.put_object( Body=json.dumps(self.game_history).encode('utf-8'), Bucket=os.environ['GAME_HISTORY_BUCKET'], Key=f"{self.s3_root_folder}/{self.game_id}_history.json", ContentType='application/json') return self.render_board(), game_state
def __init__(self, player_id, player_name, team_id, channel_id, theme='classic'): self.player_id = player_id self.player_name = player_name self.team_id = team_id self.channel_id = channel_id self.theme = theme self.game_id = str(uuid.uuid4()) self.num_holes = 4 self.num_colors = 6 self.num_guesses = 10 self.s3_root_folder = f"mastermind/slack/{self.team_id}" self.board = mastermind_utils.gen_new_board( self.num_holes, self.num_colors, self.num_guesses, ) self.game_history = { 'platform': 'slack', 'game_id': self.game_id, 'start_time': core_utils.get_ts(), 'end_time': None, 'theme': theme, 'player_id': self.player_id, 'team_id': self.team_id, 'channel_id': self.channel_id, # None-game not done; 1-player won; 2-player lost 'game_state': None, 'num_colors': self.num_colors, 'board': {}, }
def exp4(): ms = [ 'astar', 'beam5', 'beam10', 'beam20', 'beam40', 'beam80', 'hungarian', 'vj' ] fn = '_'.join(ms) file = open(get_root_path() + '/files/ged_{}_{}.csv'.format(fn, get_ts()), 'w') xs = [10] ys = list(range(10, 141, 10)) cnt = 10 ged_s = ','.join(['ged_' + i for i in ms]) time_s = ','.join(['time_' + i for i in ms]) print_and_log('g1_node,g2_node,g1_edge,g2_edge,{},{}'.format( \ ged_s, time_s), file) for x in xs: for y in ys: for i in range(cnt): g1 = generate_random_graph(x) g2 = generate_random_graph(y) ds = [] ts = [] for m in ms: t = time() d = ged(g1, g2, m) t = time() - t ds.append(d) ts.append(t) s = '{},{},{},{},{},{}'.format( \ g1.number_of_nodes(), g2.number_of_nodes(), \ g1.number_of_edges(), g2.number_of_edges(), \ ','.join(str(i) for i in ds), ','.join(['{:.5f}'.format(i) for i in ts])) print_and_log(s, file) # if d1 < 0: # exit(-1) file.close()
def get_append_str(g1, g2): return '{}_{}_{}_{}'.format(get_ts(), getpid(), g1.graph['gid'], g2.graph['gid'])
def save_train_val_info(self, train_costs, train_times, val_results_dict): sfn = '{}/train_val_info'.format(self.logdir) flags = FLAGS.flag_values_dict() ts = get_ts() save_as_dict(sfn, train_costs, train_times, val_results_dict, flags, ts)
def real_dataset_run_helper(computer_name, dataset, ds_metric, algo, row_graphs, col_graphs, num_cpu, timeout): if ds_metric == 'ged': func = ged elif ds_metric == 'mcs': func = mcs # For MCS, since the solver can handle labeled and unlabeled graphs, but the compressed # encoding must be labeled (need to tell it to ignore labels or not). # TODO: this should go in some kind of config file specific for mcs if node_has_type_attrib(row_graphs[0]): labeled = True label_key = 'type' print('Has node type') else: labeled = False label_key = '' print('Does not have node type') else: raise RuntimeError('Unknown distance similarity metric {}'.format(ds_metric)) m = len(row_graphs) n = len(col_graphs) ds_mat = np.zeros((m, n)) time_mat = np.zeros((m, n)) outdir = '{}/{}'.format(get_result_path(), dataset) create_dir_if_not_exists(outdir + '/csv') create_dir_if_not_exists(outdir + '/{}'.format(ds_metric)) create_dir_if_not_exists(outdir + '/time') exsiting_csv = prompt('File path to exsiting csv files?') exsiting_entries = load_from_exsiting_csv(exsiting_csv, ds_metric, skip_eval=False) is_symmetric = prompt('Is the ds matrix symmetric? (1/0)', options=['0', '1']) == '1' if is_symmetric: assert (m == n) smart_needed = prompt('Is smart pair sorting needed? (1/0)', options=['0', '1']) == '1' csv_fn = '{}/csv/{}_{}_{}_{}_{}_{}cpus.csv'.format( outdir, ds_metric, dataset, algo, get_ts(), computer_name, num_cpu) file = open(csv_fn, 'w') print('Saving to {}'.format(csv_fn)) if ds_metric == 'ged': print_and_log('i,j,i_gid,j_gid,i_node,j_node,i_edge,j_edge,ged,lcnt,time(msec)', file) else: print_and_log( 'i,j,i_gid,j_gid,i_node,j_node,i_edge,j_edge,mcs,node_mapping,edge_mapping,time(msec)', file) # Multiprocessing. pool = mp.Pool(processes=num_cpu) # Submit to pool workers. results = {} pairs_to_run = get_all_pairs_to_run(row_graphs, col_graphs, smart_needed) for k, (i, j) in enumerate(pairs_to_run): g1, g2 = row_graphs[i], col_graphs[j] i_gid, j_gid = g1.graph['gid'], g2.graph['gid'] if (i_gid, j_gid) in exsiting_entries: continue if is_symmetric and (j_gid, i_gid) in exsiting_entries: continue if ds_metric == 'mcs': results[(i, j)] = pool.apply_async( func, args=(g1, g2, algo, labeled, label_key, True, True, timeout,)) else: results[(i, j)] = pool.apply_async( func, args=(g1, g2, algo, True, True, timeout,)) print_progress(k, m, n, 'submit: {} {} {} {} cpus;'. format(algo, dataset, computer_name, num_cpu)) # Retrieve results from pool workers or a loaded csv file (previous run). for k, (i, j) in enumerate(pairs_to_run): print_progress(k, m, n, 'work: {} {} {} {} {} cpus;'. format(ds_metric, algo, dataset, computer_name, num_cpu)) g1, g2 = row_graphs[i], col_graphs[j] i_gid, j_gid = g1.graph['gid'], g2.graph['gid'] if (i, j) not in results: lcnt, mcs_node_mapping, mcs_edge_mapping = None, None, None tmp = exsiting_entries.get((i_gid, j_gid)) if tmp: if ds_metric == 'ged': i_gid, j_gid, i_node, j_node, ds, lcnt, t = tmp else: i_gid, j_gid, i_node, j_node, ds, mcs_node_mapping, mcs_edge_mapping, t = tmp else: assert (is_symmetric) get_from = exsiting_entries[(j_gid, i_gid)] if ds_metric == 'ged': j_gid, i_gid, j_node, i_node, ds, lcnt, t = \ get_from else: j_gid, i_gid, j_node, i_node, ds, mcs_node_mapping, mcs_edge_mapping, t = \ get_from if ds_metric == 'ged': assert (lcnt is not None) assert (g1.graph['gid'] == i_gid) assert (g2.graph['gid'] == j_gid) assert (g1.number_of_nodes() == i_node) assert (g2.number_of_nodes() == j_node) s = form_ged_print_string(i, j, g1, g2, ds, lcnt, t) else: assert (mcs_node_mapping is not None and mcs_edge_mapping is not None) s = form_mcs_print_string( i, j, g1, g2, ds, mcs_node_mapping, mcs_edge_mapping, t) else: if ds_metric == 'ged': ds, lcnt, g1_a, g2_a, t = results[(i, j)].get() i_gid, j_gid, i_node, j_node = \ g1.graph['gid'], g2.graph['gid'], \ g1.number_of_nodes(), g2.number_of_nodes() assert (g1.number_of_nodes() == g1_a.number_of_nodes()) assert (g2.number_of_nodes() == g2_a.number_of_nodes()) exsiting_entries[(i_gid, j_gid)] = \ (i_gid, j_gid, i_node, j_node, ds, lcnt, t) s = form_ged_print_string(i, j, g1, g2, ds, lcnt, t) else: # MCS ds, mcs_node_mapping, mcs_edge_mapping, t = \ results[(i, j)].get() exsiting_entries[(i_gid, j_gid)] = \ (ds, mcs_node_mapping, mcs_edge_mapping, t) s = form_mcs_print_string( i, j, g1, g2, ds, mcs_node_mapping, mcs_edge_mapping, t) print_and_log(s, file) if ds_metric == 'mcs' and (i_gid, j_gid) in exsiting_entries: # Save memory, clear the mappings since they're saved to file. exsiting_entries[(i_gid, j_gid)] = list(exsiting_entries[(i_gid, j_gid)]) exsiting_entries[(i_gid, j_gid)][1] = {} exsiting_entries[(i_gid, j_gid)][2] = {} ds_mat[i][j] = ds time_mat[i][j] = t file.close() save_as_np(outdir, ds_metric, ds_mat, time_mat, get_ts(), dataset, row_graphs, col_graphs, algo, computer_name, num_cpu)
def setup_temp_folder(gp): tp = gp + '/data/temp_' + get_ts() exec('rm -rf {} && mkdir {}'.format(tp, tp)) src = get_root_path() + '/src/gmk_files' exec('cp {}/temp.xml {}/temp_{}.xml'.format(src, tp, get_ts())) return src, tp
from utils_siamese import get_siamese_dir, get_model_info_as_str from utils import get_ts from main import main from config import FLAGS import numpy as np import tensorflow as tf dataset = 'aids700nef' file_name = '{}/logs/parameter_tuning_{}_{}.csv'.format( get_siamese_dir(), dataset, get_ts()) header = [ 'valid_percentage', 'node_feat_name', 'node_feat_encoder', 'dist_metric', 'dist_algo', 'sampler', 'sample_num', 'sampler_duplicate_removal', 'batch_size', 'dist_norm', 'pos_thresh', 'neg_thresh', 'graph_loss', 'graph_loss_alpha', 'dropout', 'weight_decay', 'learning_rate', 'iters', 'iters_val', 'plot_results', 'best_train_loss', 'best_train_iter', 'best_val_loss', 'best_val_iter', 'train_time', 'acc_norm', 'time_norm' ] # header = sorted(header) model = 'siamese_classification' batch_size_range = [32, 64, 128] pn_thresh_range = [[0.63, 1.63], [0.95, 0.95]] graph_loss_range = [None] # what is the other option? dropout_range = [0] lr_range = [0.01, 0.05] def tune_structure(FLAGS, structure_info): f = setup_file() csv_record(header + ['structure_info'], f)
# for fold in range(10): # if _train_model(..., saver): # ... raise NotImplementedError() else: assert False overall_time = convert_long_time_to_str(time() - t) print(overall_time) print(saver.get_log_dir()) print(basename(saver.get_log_dir())) saver.save_overall_time(overall_time) saver.close() if __name__ == '__main__': t = time() print(get_model_info_as_str()) check_flags() saver = Saver() try: main() except: traceback.print_exc() s = '\n'.join(traceback.format_exc(limit=-1).split('\n')[1:]) saver.save_exception_msg(traceback.format_exc()) slack_notify('{}@{}: model {} {} error \n{}'.format( FLAGS.user, FLAGS.hostname, FLAGS.model, get_ts(), s)) else: slack_notify('{}@{}: model {} {} complete'.format( FLAGS.user, FLAGS.hostname, FLAGS.model, get_ts()))
def experiment_range_growing_blockchain(range_count=4, algo='range'): def settings_size_generator(max_mult): for mult in range(50, max_mult + 1, 50): yield dict(repeat_times=mult, tr=70, D=3, bs=50, alpha=20) cols_remove = cols_remove_dict[algo] # mn = (22.20, 114.03) # mx = (22.31, 114.05) mn = (22.20, 113.89) mx = (22.31, 113.95) q_point = (22.6, 114) t_start = get_ts(2018, 1, 1, 0, 0, 0) t_end = get_ts(2018, 1, 1, 0, 0, 0) kwargs = dict(range_count=range_count, debug=False) def execute(block_dag): t_start = block_dag.chain[1]['timestamp'] # + (block_dag.chain[-1]['timestamp'] - block_dag.chain[1]['timestamp']) // 2 hours = 2 t_end = t_start + int(60 * 60 * hours) print(algo, (t_end - t_start) / (60 * 60), 'hours') df = None if algo == 'range': block_dag.optimize() df = compare_time_range(block_dag, mn, mx, t_start, t_end, **kwargs) elif algo == 'knn': block_dag.super_optimize() count_nn = 15 df = compare_time_knn(block_dag, q_point, count_nn, t_start, t_end, **kwargs) elif algo == 'knn_bound': block_dag.super_optimize() bound = 12 count_nn = 15 df = compare_time_knn_bound(block_dag, q_point, count_nn, bound, t_start, t_end, **kwargs) elif algo == 'query_ball': block_dag.super_optimize() r = 20 df = compare_time_query_ball(block_dag, q_point, r, t_start, t_end, **kwargs) df.drop(cols_remove, axis=1, inplace=True) df = df * 1000 return df dfs = [] for settings in settings_size_generator(301): print("=============={}==============".format(settings['bs'])) block_dag = GeneratorDAGchain.generate(**settings) df = execute(block_dag) df['trx_count'] = 18732 * settings['repeat_times'] dfs.append(df) del block_dag df = pd.concat(dfs).groupby('trx_count').median() df.plot(title='growing block-DAG') save_df(df, '{}/{}_growing.csv'.format(algo, algo), True) return df
def setup_property_file(src, gp, meta): file = '{}/properties/properties_temp_{}.prop'.format(gp, get_ts()) exec('cp {}/{}.prop {}'.format(src, meta, file)) for line in fileinput.input(file, inplace=True): print(line.rstrip().replace('temp', 'temp_' + get_ts()))
def experiment_bs_and_spatiotemporal(range_count, algo='range'): def settings_generator(bs_mn, bs_max, bs_step=1): for bs in range(bs_mn, bs_max + 1, bs_step): yield dict(repeat_times=300, tr=bs + 30, D=3, bs=bs, alpha=20) dfs_bs = [] dfs = [] cols_remove = cols_remove_dict[algo] # mn = (22.20, 114.03) # mx = (22.31, 114.05) mn = (22.20, 113.89) mx = (22.31, 113.95) q_point = (22.6, 114) kwargs = dict(range_count=range_count, debug=False) def execute(block_dag, t_start, t_end): print(algo, (t_end - t_start) / (60 * 60), 'hours') df = None if algo == 'range': block_dag.optimize() df = compare_time_range(block_dag, mn, mx, t_start, t_end, **kwargs) elif algo == 'knn': block_dag.super_optimize() count_nn = 15 df = compare_time_knn(block_dag, q_point, count_nn, t_start, t_end, **kwargs) elif algo == 'knn_bound': block_dag.super_optimize() bound = 12 count_nn = 15 df = compare_time_knn_bound(block_dag, q_point, count_nn, bound, t_start, t_end, **kwargs) elif algo == 'query_ball': block_dag.super_optimize() r = 20 df = compare_time_query_ball(block_dag, q_point, r, t_start, t_end, **kwargs) df.drop(cols_remove, axis=1, inplace=True) df = df * 1000 return df bs_consider = [40, 100] for settings in settings_generator(30, 110, bs_step=10): print("=============={}==============".format(settings['bs'])) block_dag = GeneratorDAGchain.generate(**settings) # block size experiment t_start = get_ts(2017, 1, 2, 0, 0, 0) t_end = get_ts(2019, 1, 3, 0, 0, 0) df_res = execute(block_dag, t_start, t_end) df_res['bs'] = settings['bs'] dfs_bs.append(df_res) if settings['bs'] not in bs_consider: continue # spatiotemporal query t_start = block_dag.chain[1]['timestamp'] # + (block_dag.chain[-1]['timestamp'] - block_dag.chain[1]['timestamp']) // 2 for hours in [0.5] + list(range(1, 12, 1)): t_end = t_start + int(60 * 60 * hours) df = execute(block_dag, t_start, t_end) df['bs'] = settings['bs'] df['hours'] = hours dfs.append(df) del block_dag # trnasform to milliseconds df_bs = pd.concat(dfs_bs).groupby('bs').median() df = pd.concat(dfs).groupby(['bs', 'hours'], as_index=False).median() df_bs.plot(title='per bs') for bs in bs_consider: df[df['bs'] == bs].drop(['bs'], axis=1).plot(x='hours', title='bs: ' + str(bs)) save_df(df_bs, '{}/{}_bs.csv'.format(algo, algo), True) save_df(df, '{}/{}_spatiotemporal.csv'.format(algo, algo)) return df_bs, df