def experiment_ball_point_vary_r(range_count):
algo = 'query_ball'
dfs = []
q_point = (22.6, 114)
cols_remove = cols_remove_dict[algo]
kwargs = dict(range_count=range_count, debug=False)
bs = 50
settings = dict(repeat_times=300, tr=75, D=3, bs=bs, alpha=10)
block_dag = GeneratorDAGchain.generate(**settings)
block_dag.super_optimize()
for r in range(1, 50, 3):
print("=============={}==============".format(r))
# block size experiment
t_start = get_ts(2017, 1, 2, 0, 0, 0)
t_end = get_ts(2019, 1, 3, 0, 0, 0)
df = compare_time_query_ball(block_dag, q_point, r, t_start, t_end,
**kwargs)
df.drop(cols_remove, axis=1, inplace=True)
df = df * 1000
df['r'] = r
dfs.append(df)
df_k = pd.concat(dfs).groupby('r').median()
df_k.plot(title='per r')
save_df(df_k, '{}/{}_vary_r.csv'.format(algo, algo), True)
return df_k
def exp7():
dataset = 'aids10k'
model = 'vj'
train_data = load_data(dataset, True)
test_data = load_data(dataset, False)
m = len(test_data.graphs)
n = len(train_data.graphs)
ged_mat = np.zeros((m, n))
time_mat = np.zeros((n, n))
outdir = get_root_path() + '/files'
file = open('{}/ged_{}_{}_{}.csv'.format( \
outdir, dataset, model, get_ts()), 'w')
print_and_log('i,j,i_node,j_node,i_edge,j_edge,ged,time', file)
for i in range(m):
for j in range(n):
g1 = test_data.graphs[i]
g2 = train_data.graphs[j]
t = time()
d = ged(g1, g2, model)
t = time() - t
s = '{},{},{},{},{},{},{},{:.5f}'.format(i, j, \
g1.number_of_nodes(),
g2.number_of_nodes(), \
g1.number_of_edges(),
g2.number_of_edges(), \
d, t)
print_and_log(s, file)
ged_mat[i][j] = d
time_mat[i][j] = t
file.close()
np.save('{}/ged_ged_mat_{}_{}_{}'.format( \
outdir, dataset, model, get_ts()), ged_mat)
np.save('{}/ged_time_mat_{}_{}_{}'.format( \
outdir, dataset, model, get_ts()), time_mat)
def __init__(self, sess=None):
model_str = self._get_model_str()
self.logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str,
get_ts())
create_dir_if_not_exists(self.logdir)
if sess is not None:
self.tw = tf.summary.FileWriter(self.logdir + '/train', sess.graph)
self.all_merged = tf.summary.merge_all()
self.loss_merged = tf.summary.merge(
self._extract_loss_related_summaries_as_list())
self._log_model_info(self.logdir, sess)
self.f = open('{}/results_{}.txt'.format(self.logdir, get_ts()), 'w')
print('Logging to {}'.format(self.logdir))
def place_piece(self, column, player):
self.board, self.latest_move = connect4_utils.place_piece(
self.board, column, self.pieces[player])
game_end = None
self.winning_moves = connect4_utils.check_win(self.board, column)
if self.winning_moves:
game_end = 'win'
self.game_history['game_state'] = self.pieces[self.current_player]
elif connect4_utils.check_tie(self.board):
game_end = 'tie'
self.game_history['game_state'] = 0
board_url = self.render_board()
# Save the players move before game_over gets called and the player is toggled
self.game_history['moves'].append({
'player':
self.pieces[self.current_player],
'piece_played':
self.latest_move,
'board':
copy.deepcopy(self.board),
'rendered_board_url':
board_url,
'timestamp':
core_utils.get_ts(),
})
if game_end is None:
# Only toggle the player if game has not ended
self.toggle_player()
return board_url, game_end
def get_auth_token(tenant, role):
"""
Gets a JWT token for 'role'.
'role' is either 'battleserver' or 'battledaemon'.
"""
ts = get_ts()
driftbase_tenant = ts.get_table('tenants').find({
'tenant_name':
tenant,
'deployable_name':
'drift-base'
})[0]
auth_url = get_root_endpoint(tenant) + "/auth"
#! TODO: Find user in organization config table
if role == 'battleserver':
credentials = config.battleserver_credentials
elif role == "battledaemon":
credentials = config.battledaemon_credentials
else:
raise RuntimeError("'role' %s not understood." % role)
headers = {
'Drift-Api-Key': config.api_key,
'Content-type': 'application/json'
} #! TODO: Replace api key with product key
r = requests.post(auth_url, data=json.dumps(credentials), headers=headers)
r.raise_for_status()
return r.json()
def __init__(self,
player1_id,
player2_id,
team_id,
channel_id,
theme='classic'):
self.player1_id = player1_id
self.player2_id = player2_id
self.team_id = team_id
self.channel_id = channel_id
self.pieces = {
self.player1_id: 1,
self.player2_id: 2,
}
self.theme = theme
self.game_id = str(uuid.uuid4())
self.game_history = {
'platform':
'slack',
'game_id':
self.game_id,
'start_time':
core_utils.get_ts(),
'end_time':
None,
'theme':
theme,
'player1_id':
self.player1_id,
'player2_id':
self.player2_id,
'team_id':
self.team_id,
'channel_id':
self.channel_id,
# Url to the finial rendered gif of all moves played
'recap_url':
None,
# None-game not done; 0-tie; 1-player 1 won; 2-player2 won
'game_state':
None,
'moves': [
# Example of whats in a move
# {
# 'player': 1,
# 'latest_move': (6, 2),
# 'board': [[0, 0, 0],[0, 0, 0],[0, 0, 0]] ...,
# 'rendered_board_url': 'https://...',
# 'timestamp': '...',
# }
],
}
self.s3_root_folder = f"connect4/slack/{self.team_id}"
self.latest_move = (None, None)
self.current_player = self.player1_id
self.board = connect4_utils.gen_new_board()
self.winning_moves = None
def get_model_info_as_str():
rtn = []
d = vars(FLAGS)
for k in sorted_nicely(d.keys()):
v = d[k]
s = '{0:26} : {1}'.format(k, v)
rtn.append(s)
rtn.append('{0:26} : {1}'.format('ts', get_ts()))
return '\n'.join(rtn)
def __init__(self):
model_str = self.get_model_str()
self.logdir = join(get_root_path(), 'logs',
'{}_{}'.format(model_str, get_ts()))
create_dir_if_not_exists(self.logdir)
self.model_info_f = self._open('model_info.txt')
self._log_model_info()
self._save_conf_code()
print('Logging to {}'.format(self.logdir))
def get_result(gp, algo):
with open('{}/result/temp_{}'.format(gp, get_ts())) as f:
lines = f.readlines()
ln = 23 if 'beam' in algo else 22
rtn = float(lines[ln]) * 2 # alpha=0.5 --> / 2
assert (rtn - int(rtn) == 0)
rtn = int(rtn)
if rtn < 0:
rtn = -1 # in case rtn == -2
return rtn
def extend_df(df, repeat_times=10, first_time=get_ts(2018, 1, 1), bs=20):
a = pd.concat([df] * repeat_times, ignore_index=True)
MS = 1000
dt = MS / bs
new_times = np.arange(first_time * MS,
first_time * MS + dt * a.shape[0], dt) / MS
new_times = new_times[:a.shape[0]]
new_times = new_times.astype(int)
a.ts = new_times
print(df.shape, '-(rept{})>'.format(repeat_times), a.shape)
return a
def get_model_info_as_str(model_info_table=None):
rtn = []
d = FLAGS.flag_values_dict()
for k in sorted_nicely(d.keys()):
v = d[k]
s = '{0:26} : {1}'.format(k, v)
rtn.append(s)
if model_info_table:
model_info_table.append([k, '**{}**'.format(v)])
rtn.append('{0:26} : {1}'.format('ts', get_ts()))
return '\n'.join(rtn)
def start(self, player1_name, player2_name):
banner_url = self.render_player_banner(player1_name, player2_name)
board_url = self.render_board()
self.game_history['moves'].append({
'player': 0,
'piece_played': (None, None),
'board': copy.deepcopy(self.board),
'rendered_board_url': board_url,
'timestamp': core_utils.get_ts(),
})
return banner_url, board_url
def __init__(self, sess):
if FLAGS.log:
model_str = self._get_model_str()
logdir = '{}/logs/{}_{}'.format(get_siamese_dir(), model_str,
get_ts())
create_dir_if_not_exists(logdir)
self.tw = tf.summary.FileWriter(logdir + '/train', sess.graph)
self.vw = tf.summary.FileWriter(logdir + '/val', sess.graph)
self.merged = tf.summary.merge_all()
self._log_model_info(logdir, sess)
print('Logging to {}'.format(logdir))
def get_root_endpoint(tenant):
ts = get_ts()
driftbase_tenant = ts.get_table('tenants').find({
'tenant_name':
tenant,
'deployable_name':
'drift-base'
})[0]
endpoint = driftbase_tenant.get('root_endpoint')
if not endpoint:
endpoint = DEFAULT_ROOT_ENDPOINT.format(tenant)
return endpoint
def game_over(self):
self.game_history['end_time'] = core_utils.get_ts()
recap_url = self._generate_recap(self.game_history['moves'])
self.game_history['recap_url'] = recap_url
s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', None))
s3.put_object(Body=json.dumps(self.game_history).encode('utf-8'),
Bucket=os.environ['GAME_HISTORY_BUCKET'],
Key=f"{self.s3_root_folder}/{self.game_id}_history.json",
ContentType='application/json')
return recap_url
def ged(g1, g2, algo):
# https://github.com/dan-zam/graph-matching-toolkit
gp = get_gmt_path()
src, tp = setup_temp_folder(gp)
meta1 = write_to_temp(g1, tp, algo, 'g1')
meta2 = write_to_temp(g2, tp, algo, 'g2')
if meta1 != meta2:
raise RuntimeError('Different meta data {} vs {}'.format(meta1, meta2))
setup_property_file(src, gp, meta1)
if not exec(
'cd {} && java -classpath {}/src/graph-matching-toolkit/bin algorithms.GraphMatching ./properties/properties_temp_{}.prop'
.format(gp, get_root_path(), get_ts()),
timeout=1000):
return -1
return get_result(gp, algo)
def make_move(self, move):
self.board, game_state = mastermind_utils.make_move(self.board, move)
if game_state is not None:
# Game over
self.game_history['board'] = self.board
self.game_history['game_state'] = game_state
self.game_history['end_time'] = core_utils.get_ts()
s3 = boto3.client('s3',
endpoint_url=os.getenv('S3_ENDPOINT', None))
s3.put_object(
Body=json.dumps(self.game_history).encode('utf-8'),
Bucket=os.environ['GAME_HISTORY_BUCKET'],
Key=f"{self.s3_root_folder}/{self.game_id}_history.json",
ContentType='application/json')
return self.render_board(), game_state
def __init__(self,
player_id,
player_name,
team_id,
channel_id,
theme='classic'):
self.player_id = player_id
self.player_name = player_name
self.team_id = team_id
self.channel_id = channel_id
self.theme = theme
self.game_id = str(uuid.uuid4())
self.num_holes = 4
self.num_colors = 6
self.num_guesses = 10
self.s3_root_folder = f"mastermind/slack/{self.team_id}"
self.board = mastermind_utils.gen_new_board(
self.num_holes,
self.num_colors,
self.num_guesses,
)
self.game_history = {
'platform': 'slack',
'game_id': self.game_id,
'start_time': core_utils.get_ts(),
'end_time': None,
'theme': theme,
'player_id': self.player_id,
'team_id': self.team_id,
'channel_id': self.channel_id,
# None-game not done; 1-player won; 2-player lost
'game_state': None,
'num_colors': self.num_colors,
'board': {},
}
def exp4():
ms = [
'astar', 'beam5', 'beam10', 'beam20', 'beam40', 'beam80', 'hungarian',
'vj'
]
fn = '_'.join(ms)
file = open(get_root_path() + '/files/ged_{}_{}.csv'.format(fn, get_ts()),
'w')
xs = [10]
ys = list(range(10, 141, 10))
cnt = 10
ged_s = ','.join(['ged_' + i for i in ms])
time_s = ','.join(['time_' + i for i in ms])
print_and_log('g1_node,g2_node,g1_edge,g2_edge,{},{}'.format( \
ged_s, time_s), file)
for x in xs:
for y in ys:
for i in range(cnt):
g1 = generate_random_graph(x)
g2 = generate_random_graph(y)
ds = []
ts = []
for m in ms:
t = time()
d = ged(g1, g2, m)
t = time() - t
ds.append(d)
ts.append(t)
s = '{},{},{},{},{},{}'.format( \
g1.number_of_nodes(), g2.number_of_nodes(), \
g1.number_of_edges(), g2.number_of_edges(), \
','.join(str(i) for i in ds),
','.join(['{:.5f}'.format(i) for i in ts]))
print_and_log(s, file)
# if d1 < 0:
# exit(-1)
file.close()
def get_append_str(g1, g2):
return '{}_{}_{}_{}'.format(get_ts(), getpid(), g1.graph['gid'],
g2.graph['gid'])
def save_train_val_info(self, train_costs, train_times,
val_results_dict):
sfn = '{}/train_val_info'.format(self.logdir)
flags = FLAGS.flag_values_dict()
ts = get_ts()
save_as_dict(sfn, train_costs, train_times, val_results_dict, flags, ts)
def real_dataset_run_helper(computer_name, dataset, ds_metric, algo, row_graphs, col_graphs,
num_cpu, timeout):
if ds_metric == 'ged':
func = ged
elif ds_metric == 'mcs':
func = mcs
# For MCS, since the solver can handle labeled and unlabeled graphs, but the compressed
# encoding must be labeled (need to tell it to ignore labels or not).
# TODO: this should go in some kind of config file specific for mcs
if node_has_type_attrib(row_graphs[0]):
labeled = True
label_key = 'type'
print('Has node type')
else:
labeled = False
label_key = ''
print('Does not have node type')
else:
raise RuntimeError('Unknown distance similarity metric {}'.format(ds_metric))
m = len(row_graphs)
n = len(col_graphs)
ds_mat = np.zeros((m, n))
time_mat = np.zeros((m, n))
outdir = '{}/{}'.format(get_result_path(), dataset)
create_dir_if_not_exists(outdir + '/csv')
create_dir_if_not_exists(outdir + '/{}'.format(ds_metric))
create_dir_if_not_exists(outdir + '/time')
exsiting_csv = prompt('File path to exsiting csv files?')
exsiting_entries = load_from_exsiting_csv(exsiting_csv, ds_metric, skip_eval=False)
is_symmetric = prompt('Is the ds matrix symmetric? (1/0)', options=['0', '1']) == '1'
if is_symmetric:
assert (m == n)
smart_needed = prompt('Is smart pair sorting needed? (1/0)', options=['0', '1']) == '1'
csv_fn = '{}/csv/{}_{}_{}_{}_{}_{}cpus.csv'.format(
outdir, ds_metric, dataset, algo, get_ts(), computer_name, num_cpu)
file = open(csv_fn, 'w')
print('Saving to {}'.format(csv_fn))
if ds_metric == 'ged':
print_and_log('i,j,i_gid,j_gid,i_node,j_node,i_edge,j_edge,ged,lcnt,time(msec)',
file)
else:
print_and_log(
'i,j,i_gid,j_gid,i_node,j_node,i_edge,j_edge,mcs,node_mapping,edge_mapping,time(msec)',
file)
# Multiprocessing.
pool = mp.Pool(processes=num_cpu)
# Submit to pool workers.
results = {}
pairs_to_run = get_all_pairs_to_run(row_graphs, col_graphs, smart_needed)
for k, (i, j) in enumerate(pairs_to_run):
g1, g2 = row_graphs[i], col_graphs[j]
i_gid, j_gid = g1.graph['gid'], g2.graph['gid']
if (i_gid, j_gid) in exsiting_entries:
continue
if is_symmetric and (j_gid, i_gid) in exsiting_entries:
continue
if ds_metric == 'mcs':
results[(i, j)] = pool.apply_async(
func, args=(g1, g2, algo, labeled, label_key, True, True, timeout,))
else:
results[(i, j)] = pool.apply_async(
func, args=(g1, g2, algo, True, True, timeout,))
print_progress(k, m, n, 'submit: {} {} {} {} cpus;'.
format(algo, dataset, computer_name, num_cpu))
# Retrieve results from pool workers or a loaded csv file (previous run).
for k, (i, j) in enumerate(pairs_to_run):
print_progress(k, m, n, 'work: {} {} {} {} {} cpus;'.
format(ds_metric, algo, dataset, computer_name, num_cpu))
g1, g2 = row_graphs[i], col_graphs[j]
i_gid, j_gid = g1.graph['gid'], g2.graph['gid']
if (i, j) not in results:
lcnt, mcs_node_mapping, mcs_edge_mapping = None, None, None
tmp = exsiting_entries.get((i_gid, j_gid))
if tmp:
if ds_metric == 'ged':
i_gid, j_gid, i_node, j_node, ds, lcnt, t = tmp
else:
i_gid, j_gid, i_node, j_node, ds, mcs_node_mapping, mcs_edge_mapping, t = tmp
else:
assert (is_symmetric)
get_from = exsiting_entries[(j_gid, i_gid)]
if ds_metric == 'ged':
j_gid, i_gid, j_node, i_node, ds, lcnt, t = \
get_from
else:
j_gid, i_gid, j_node, i_node, ds, mcs_node_mapping, mcs_edge_mapping, t = \
get_from
if ds_metric == 'ged':
assert (lcnt is not None)
assert (g1.graph['gid'] == i_gid)
assert (g2.graph['gid'] == j_gid)
assert (g1.number_of_nodes() == i_node)
assert (g2.number_of_nodes() == j_node)
s = form_ged_print_string(i, j, g1, g2, ds, lcnt, t)
else:
assert (mcs_node_mapping is not None and
mcs_edge_mapping is not None)
s = form_mcs_print_string(
i, j, g1, g2, ds, mcs_node_mapping, mcs_edge_mapping, t)
else:
if ds_metric == 'ged':
ds, lcnt, g1_a, g2_a, t = results[(i, j)].get()
i_gid, j_gid, i_node, j_node = \
g1.graph['gid'], g2.graph['gid'], \
g1.number_of_nodes(), g2.number_of_nodes()
assert (g1.number_of_nodes() == g1_a.number_of_nodes())
assert (g2.number_of_nodes() == g2_a.number_of_nodes())
exsiting_entries[(i_gid, j_gid)] = \
(i_gid, j_gid, i_node, j_node, ds, lcnt, t)
s = form_ged_print_string(i, j, g1, g2, ds, lcnt, t)
else: # MCS
ds, mcs_node_mapping, mcs_edge_mapping, t = \
results[(i, j)].get()
exsiting_entries[(i_gid, j_gid)] = \
(ds, mcs_node_mapping, mcs_edge_mapping, t)
s = form_mcs_print_string(
i, j, g1, g2, ds, mcs_node_mapping, mcs_edge_mapping, t)
print_and_log(s, file)
if ds_metric == 'mcs' and (i_gid, j_gid) in exsiting_entries:
# Save memory, clear the mappings since they're saved to file.
exsiting_entries[(i_gid, j_gid)] = list(exsiting_entries[(i_gid, j_gid)])
exsiting_entries[(i_gid, j_gid)][1] = {}
exsiting_entries[(i_gid, j_gid)][2] = {}
ds_mat[i][j] = ds
time_mat[i][j] = t
file.close()
save_as_np(outdir, ds_metric, ds_mat, time_mat, get_ts(),
dataset, row_graphs, col_graphs, algo, computer_name, num_cpu)
def setup_temp_folder(gp):
tp = gp + '/data/temp_' + get_ts()
exec('rm -rf {} && mkdir {}'.format(tp, tp))
src = get_root_path() + '/src/gmk_files'
exec('cp {}/temp.xml {}/temp_{}.xml'.format(src, tp, get_ts()))
return src, tp
from utils_siamese import get_siamese_dir, get_model_info_as_str
from utils import get_ts
from main import main
from config import FLAGS
import numpy as np
import tensorflow as tf
dataset = 'aids700nef'
file_name = '{}/logs/parameter_tuning_{}_{}.csv'.format(
get_siamese_dir(), dataset, get_ts())
header = [
'valid_percentage', 'node_feat_name', 'node_feat_encoder', 'dist_metric',
'dist_algo', 'sampler', 'sample_num', 'sampler_duplicate_removal',
'batch_size', 'dist_norm', 'pos_thresh', 'neg_thresh', 'graph_loss',
'graph_loss_alpha', 'dropout', 'weight_decay', 'learning_rate', 'iters',
'iters_val', 'plot_results', 'best_train_loss', 'best_train_iter',
'best_val_loss', 'best_val_iter', 'train_time', 'acc_norm', 'time_norm'
]
# header = sorted(header)
model = 'siamese_classification'
batch_size_range = [32, 64, 128]
pn_thresh_range = [[0.63, 1.63], [0.95, 0.95]]
graph_loss_range = [None] # what is the other option?
dropout_range = [0]
lr_range = [0.01, 0.05]
def tune_structure(FLAGS, structure_info):
f = setup_file()
csv_record(header + ['structure_info'], f)
# for fold in range(10):
# if _train_model(..., saver):
# ...
raise NotImplementedError()
else:
assert False
overall_time = convert_long_time_to_str(time() - t)
print(overall_time)
print(saver.get_log_dir())
print(basename(saver.get_log_dir()))
saver.save_overall_time(overall_time)
saver.close()
if __name__ == '__main__':
t = time()
print(get_model_info_as_str())
check_flags()
saver = Saver()
try:
main()
except:
traceback.print_exc()
s = '\n'.join(traceback.format_exc(limit=-1).split('\n')[1:])
saver.save_exception_msg(traceback.format_exc())
slack_notify('{}@{}: model {} {} error \n{}'.format(
FLAGS.user, FLAGS.hostname, FLAGS.model, get_ts(), s))
else:
slack_notify('{}@{}: model {} {} complete'.format(
FLAGS.user, FLAGS.hostname, FLAGS.model, get_ts()))
def experiment_range_growing_blockchain(range_count=4, algo='range'):
def settings_size_generator(max_mult):
for mult in range(50, max_mult + 1, 50):
yield dict(repeat_times=mult, tr=70, D=3, bs=50, alpha=20)
cols_remove = cols_remove_dict[algo]
# mn = (22.20, 114.03)
# mx = (22.31, 114.05)
mn = (22.20, 113.89)
mx = (22.31, 113.95)
q_point = (22.6, 114)
t_start = get_ts(2018, 1, 1, 0, 0, 0)
t_end = get_ts(2018, 1, 1, 0, 0, 0)
kwargs = dict(range_count=range_count, debug=False)
def execute(block_dag):
t_start = block_dag.chain[1]['timestamp']
# + (block_dag.chain[-1]['timestamp'] - block_dag.chain[1]['timestamp']) // 2
hours = 2
t_end = t_start + int(60 * 60 * hours)
print(algo, (t_end - t_start) / (60 * 60), 'hours')
df = None
if algo == 'range':
block_dag.optimize()
df = compare_time_range(block_dag, mn, mx, t_start, t_end,
**kwargs)
elif algo == 'knn':
block_dag.super_optimize()
count_nn = 15
df = compare_time_knn(block_dag, q_point, count_nn, t_start, t_end,
**kwargs)
elif algo == 'knn_bound':
block_dag.super_optimize()
bound = 12
count_nn = 15
df = compare_time_knn_bound(block_dag, q_point, count_nn, bound,
t_start, t_end, **kwargs)
elif algo == 'query_ball':
block_dag.super_optimize()
r = 20
df = compare_time_query_ball(block_dag, q_point, r, t_start, t_end,
**kwargs)
df.drop(cols_remove, axis=1, inplace=True)
df = df * 1000
return df
dfs = []
for settings in settings_size_generator(301):
print("=============={}==============".format(settings['bs']))
block_dag = GeneratorDAGchain.generate(**settings)
df = execute(block_dag)
df['trx_count'] = 18732 * settings['repeat_times']
dfs.append(df)
del block_dag
df = pd.concat(dfs).groupby('trx_count').median()
df.plot(title='growing block-DAG')
save_df(df, '{}/{}_growing.csv'.format(algo, algo), True)
return df
def setup_property_file(src, gp, meta):
file = '{}/properties/properties_temp_{}.prop'.format(gp, get_ts())
exec('cp {}/{}.prop {}'.format(src, meta, file))
for line in fileinput.input(file, inplace=True):
print(line.rstrip().replace('temp', 'temp_' + get_ts()))
def experiment_bs_and_spatiotemporal(range_count, algo='range'):
def settings_generator(bs_mn, bs_max, bs_step=1):
for bs in range(bs_mn, bs_max + 1, bs_step):
yield dict(repeat_times=300, tr=bs + 30, D=3, bs=bs, alpha=20)
dfs_bs = []
dfs = []
cols_remove = cols_remove_dict[algo]
# mn = (22.20, 114.03)
# mx = (22.31, 114.05)
mn = (22.20, 113.89)
mx = (22.31, 113.95)
q_point = (22.6, 114)
kwargs = dict(range_count=range_count, debug=False)
def execute(block_dag, t_start, t_end):
print(algo, (t_end - t_start) / (60 * 60), 'hours')
df = None
if algo == 'range':
block_dag.optimize()
df = compare_time_range(block_dag, mn, mx, t_start, t_end,
**kwargs)
elif algo == 'knn':
block_dag.super_optimize()
count_nn = 15
df = compare_time_knn(block_dag, q_point, count_nn, t_start, t_end,
**kwargs)
elif algo == 'knn_bound':
block_dag.super_optimize()
bound = 12
count_nn = 15
df = compare_time_knn_bound(block_dag, q_point, count_nn, bound,
t_start, t_end, **kwargs)
elif algo == 'query_ball':
block_dag.super_optimize()
r = 20
df = compare_time_query_ball(block_dag, q_point, r, t_start, t_end,
**kwargs)
df.drop(cols_remove, axis=1, inplace=True)
df = df * 1000
return df
bs_consider = [40, 100]
for settings in settings_generator(30, 110, bs_step=10):
print("=============={}==============".format(settings['bs']))
block_dag = GeneratorDAGchain.generate(**settings)
# block size experiment
t_start = get_ts(2017, 1, 2, 0, 0, 0)
t_end = get_ts(2019, 1, 3, 0, 0, 0)
df_res = execute(block_dag, t_start, t_end)
df_res['bs'] = settings['bs']
dfs_bs.append(df_res)
if settings['bs'] not in bs_consider:
continue
# spatiotemporal query
t_start = block_dag.chain[1]['timestamp']
# + (block_dag.chain[-1]['timestamp'] - block_dag.chain[1]['timestamp']) // 2
for hours in [0.5] + list(range(1, 12, 1)):
t_end = t_start + int(60 * 60 * hours)
df = execute(block_dag, t_start, t_end)
df['bs'] = settings['bs']
df['hours'] = hours
dfs.append(df)
del block_dag
# trnasform to milliseconds
df_bs = pd.concat(dfs_bs).groupby('bs').median()
df = pd.concat(dfs).groupby(['bs', 'hours'], as_index=False).median()
df_bs.plot(title='per bs')
for bs in bs_consider:
df[df['bs'] == bs].drop(['bs'], axis=1).plot(x='hours',
title='bs: ' + str(bs))
save_df(df_bs, '{}/{}_bs.csv'.format(algo, algo), True)
save_df(df, '{}/{}_spatiotemporal.csv'.format(algo, algo))
return df_bs, df
