def flow_based_algorithm(server_types, demand_profile, use_flow=True):
'''
>>> from set_cover_reduction import construct_server_types, construct_demand_profile
>>> number_of_elements = 3
>>> sets = [[1, 2], [1, 3], [2, 3]]
>>> server_types = construct_server_types(number_of_elements, sets)
>>> demand_profile = construct_demand_profile(number_of_elements)
>>> schedule = flow_based_algorithm(server_types, demand_profile)
>>> schedule.total_energy
167.0045222223457
'''
network = construct_network(server_types, demand_profile)
tau = len(server_types)
if use_flow:
solver = MinimumCostTwoCommodityFlowSolver(network)
solver.silent()
fractional_cost = solver.solve()
d = {
# scale up the flow of commodity 1
(i, k): min(int(tau * network.edges[(u(i, k), u(i, k + 1))].flow1),
server_types[i].m)
for k in demand_profile for i in server_types
}
else:
solver = DPMSolver(server_types, demand_profile, variable_type='C')
solver.silent()
fractional_cost = solver.solve()
solution = solver.solution
d = {(i, k): min(int(tau * solution[x(i, 0, k)]), server_types[i].m)
for k in demand_profile for i in server_types}
schedule = construct_schedule(server_types, demand_profile, d)
return schedule, fractional_cost
def split_network(origin_net_dir, train_net_dir, eval_net_dir, data_prefixname,
data_format, isdirected, train_ratio):
net_origin = network.construct_network(
data_path=os.path.join(origin_net_dir,
data_prefixname + "." + data_format),
data_format=data_format,
net_info_path=os.path.join(origin_net_dir, "net.info"),
isdirected=isdirected)
net_train, net_eval = net_origin.split_by_edges(train_ratio=train_ratio)
net_train.save_network(train_net_dir, data_prefixname, data_format)
# net_train.print_net_info(edges_file=os.path.join(train_net_dir, datafilename), file_path=os.path.join(train_net_dir, "net.info"))
net_eval.save_network(eval_net_dir, data_prefixname, data_format)
# net_eval.print_net_info(edges_file=os.path.join(eval_net_dir, datafilename), file_path=os.path.join(eval_net_dir, "net.info"))
del net_origin, net_train, net_eval
def train_vectors(options):
if not utils.check_rebuild(options.vectors_path,
descrip='vectors',
always_rebuild=options.always_rebuild):
return
train_vec_dir = os.path.split(options.vectors_path)[0]
if not os.path.exists(train_vec_dir):
os.makedirs(train_vec_dir)
# construct network
net = network.construct_network(options)
Kstep = 2
# train info
logger.info('Train info:')
logger.info('\t train_model = {}'.format(options.model))
logger.info('\t total embedding nodes = {}'.format(net.get_nodes_size()))
logger.info('\t total edges = {}'.format(net.get_edges_size()))
logger.info('\t embedding size = {}'.format(options.embedding_size))
logger.info('\t Kstep = {}'.format(Kstep))
logger.info('\t vectors_path = {}'.format(options.vectors_path))
fr_vec = open(os.path.join(train_vec_dir, 'embedding.info'), 'w')
fr_vec.write('embedding info:\n')
fr_vec.write('\t train_model = {}\n'.format(options.model))
fr_vec.write('\t total embedding nodes = {}\n'.format(
net.get_nodes_size()))
fr_vec.write('\t total edges = {}\n'.format(net.get_edges_size()))
fr_vec.write('\t embedding size = {}\n'.format(options.embedding_size))
fr_vec.write('\t Kstep = {}\n'.format(Kstep))
fr_vec.write('\t vectors_path = {}\n'.format(options.vectors_path))
fr_vec.close()
# train
logger.info('training...')
time_start = time.time()
grarep = GraRep(net.get_nodes_size(), net.edges, options.embedding_size,
Kstep)
vecs = grarep.train()
save_word2vec_format(options.vectors_path, vecs, net._idx_nodes)
logger.info('train completed in {}s'.format(time.time() - time_start))
return
def process(options):
logger.info("Data preprocessing: network serialization ...")
time_start = time.time()
source_data_dir, data_filename = os.path.split(options.data_path)
data_prefixname = data_filename.split(".")[0]
data_format = options.data_format
isdirected = options.isdirected
target_data_dir = options.target_data_dir
logger.info("\t source_data_dir = {}".format(source_data_dir))
logger.info("\t data_filename = {}".format(data_filename))
logger.info("\t data_format = {}".format(data_format))
logger.info("\t isdirected = {}".format(isdirected))
logger.info("\n\t target_data_dir = {}".format(target_data_dir))
net = network.construct_network(data_path = options.data_path,
data_format = data_format,
net_info_path = os.path.join(source_data_dir, "net.info"),
isdirected = isdirected,
print_net_info = True)
net.make_consistent(remove_isolated = not options.keep_isolated)
target_data_format = ["adjlist", "edgelist"]
for save_format in target_data_format:
net.save_network(target_data_dir, data_prefixname, save_format)
# for label
source_data_path = os.path.join(source_data_dir, data_prefixname + ".labels")
target_data_path = os.path.join(target_data_dir, data_prefixname + ".labels")
if os.path.exists(source_data_path):
with open(target_data_path, "w") as fr:
for line in open(source_data_path):
line = line.strip()
if line:
linelist = line.split('\t')
source_id = int(linelist[0].strip())
if source_id in net._nodes_id:
fr.write("{}".format(net._nodes_id[source_id]))
for label in linelist[1:]:
fr.write("\t{}".format(label.strip()))
fr.write("\n")
logger.info('Data preprocessing: network serialization completed in {}s.'.format(time.time() - time_start))
def eval(options):
if "all" not in options.metapath_path:
metatree = network.construct_meta_tree(
metapaths_filename=options.metapath_path)
flag0 = False
flag1 = False
for each in metatree.nodes():
if options.eval_edge_type[0] == metatree.nodes[each]["type"]:
flag0 = True
if options.eval_edge_type[1] == metatree.nodes[each]["type"]:
flag1 = True
if not (flag0 and flag1):
return (flag0 and flag1)
flag0 = True
flag1 = True
net = network.construct_network(options, isHIN=True, print_net_info=False)
if options.eval_online:
eval_online(options, net)
else:
eval_once(options, net)
return (flag0 and flag1)
def train_vectors(options):
# check vectors and ckpt
checkpoint = '0'
train_vec_dir = os.path.split(options.vectors_path)[0]
ckpt_dir = os.path.join(train_vec_dir, 'ckpt')
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if ckpt and ckpt.model_checkpoint_path:
cur_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
logger.info(
"model and vectors already exists, checkpoint step = {}".format(
cur_step))
checkpoint = input(
"please input 0 to start a new train, or input a choosed ckpt to restore (-1 for latest ckpt)"
)
if checkpoint == '0':
if ckpt:
tf.gfile.DeleteRecursively(ckpt_dir)
logger.info('start a new embedding train using tensorflow ...')
elif checkpoint == '-1':
logger.info(
'restore a embedding train using tensorflow from latest ckpt...')
else:
logger.info(
'restore a embedding train using tensorflow from ckpt-%s...' %
checkpoint)
if not os.path.exists(train_vec_dir):
os.makedirs(train_vec_dir)
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
# construct network
net = network.construct_network(options)
lr_file = os.path.join(train_vec_dir, "lr.info")
np.savetxt(lr_file,
np.asarray([
options.learning_rate, options.decay_epochs,
options.decay_rate, options.iter_epoches
],
dtype=np.float32),
fmt="%.6f")
# train info
logger.info('Train info:')
logger.info('\t train_model = {}'.format(options.model))
logger.info('\t order = {}'.format(options.order))
logger.info('\t total embedding nodes = {}'.format(net.get_nodes_size()))
logger.info('\t total edges = {}'.format(net.get_edges_size()))
logger.info('\t embedding size = {}'.format(options.embedding_size))
logger.info('\t negative = {}'.format(options.negative))
logger.info('\t distortion_power = {}\n'.format(options.distortion_power))
logger.info('\t batch_size = {}'.format(options.batch_size))
logger.info('\t iter_epoches = {}'.format(options.iter_epoches))
logger.info('\t init_learning_rate = {}'.format(options.learning_rate))
logger.info('\t decay_epochs = {}'.format(options.decay_epochs))
logger.info('\t decay_interval = {}'.format(options.decay_interval))
logger.info('\t decay_rate = {}'.format(options.decay_rate))
logger.info('\t loss_interval = {}s'.format(options.loss_interval))
logger.info('\t summary_steps = {}'.format(options.summary_steps))
logger.info('\t summary_interval = {}s'.format(options.summary_interval))
logger.info('\t ckpt_epochs = {}'.format(options.ckpt_epochs))
logger.info('\t ckpt_interval = {}s\n'.format(options.ckpt_interval))
logger.info('\t using_gpu = {}'.format(options.using_gpu))
logger.info('\t visible_device_list = {}'.format(
options.visible_device_list))
logger.info('\t log_device_placement = {}'.format(
options.log_device_placement))
logger.info('\t allow_soft_placement = {}'.format(
options.allow_soft_placement))
logger.info('\t gpu_memory_fraction = {}'.format(
options.gpu_memory_fraction))
logger.info('\t gpu_memory_allow_growth = {}'.format(options.allow_growth))
logger.info('\t train_workers = {}\n'.format(options.train_workers))
logger.info('\t ckpt_dir = {}'.format(ckpt_dir))
logger.info('\t vectors_path = {}'.format(options.vectors_path))
logger.info('\t learning_rate_path = {}'.format(lr_file))
fr_vec = open(os.path.join(train_vec_dir, 'embedding.info'), 'w')
fr_vec.write('embedding info:\n')
fr_vec.write('\t train_model = {}\n'.format(options.model))
fr_vec.write('\t order = {}\n'.format(options.order))
fr_vec.write('\t total embedding nodes = {}\n'.format(
net.get_nodes_size()))
fr_vec.write('\t total edges = {}\n'.format(net.get_edges_size()))
fr_vec.write('\t embedding size = {}\n'.format(options.embedding_size))
fr_vec.write('\t negative = {}\n'.format(options.negative))
fr_vec.write('\t distortion_power = {}\n\n'.format(
options.distortion_power))
fr_vec.write('\t batch_size = {}\n'.format(options.batch_size))
fr_vec.write('\t iter_epoches = {}\n'.format(options.iter_epoches))
fr_vec.write('\t init_learning_rate = {}\n'.format(options.learning_rate))
fr_vec.write('\t decay_epochs = {}\n'.format(options.decay_epochs))
fr_vec.write('\t decay_interval = {}\n'.format(options.decay_interval))
fr_vec.write('\t decay_rate = {}\n'.format(options.decay_rate))
fr_vec.write('\t loss_interval = {}s\n'.format(options.loss_interval))
fr_vec.write('\t summary_steps = {}\n'.format(options.summary_steps))
fr_vec.write('\t summary_interval = {}s\n'.format(
options.summary_interval))
fr_vec.write('\t ckpt_epochs = {}\n'.format(options.ckpt_epochs))
fr_vec.write('\t ckpt_interval = {}s\n\n'.format(options.ckpt_interval))
fr_vec.write('\t using_gpu = {}\n'.format(options.using_gpu))
fr_vec.write('\t visible_device_list = {}\n'.format(
options.visible_device_list))
fr_vec.write('\t log_device_placement = {}\n'.format(
options.log_device_placement))
fr_vec.write('\t allow_soft_placement = {}\n'.format(
options.allow_soft_placement))
fr_vec.write('\t gpu_memory_fraction = {}\n'.format(
options.gpu_memory_fraction))
fr_vec.write('\t gpu_memory_allow_growth = {}\n'.format(
options.allow_growth))
fr_vec.write('\t train_workers = {}\n\n'.format(options.train_workers))
fr_vec.write('\t ckpt_dir = {}\n'.format(ckpt_dir))
fr_vec.write('\t vectors_path = {}\n'.format(options.vectors_path))
fr_vec.write('\t learning_rate_path = {}\n'.format(lr_file))
fr_vec.close()
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
if options.using_gpu:
visible_devices = str(options.visible_device_list[0])
for dev in options.visible_device_list[1:]:
visible_devices = visible_devices + ',%s' % dev
os.environ['CUDA_VISIBLE_DEVICES'] = visible_devices
else:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
# set log_level for gpu:
console_log_level = options.log.upper()
if console_log_level == "CRITICAL":
gpu_log = '3'
elif console_log_level == "ERROR":
gpu_log = '2'
elif console_log_level == "WARNING":
gpu_log = '1'
else:
gpu_log = '0'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = gpu_log
# train
logger.info('training...')
time_start = time.time()
train(net=net,
vectors_path=options.vectors_path,
lr_file=lr_file,
ckpt_dir=ckpt_dir,
checkpoint=checkpoint,
order=options.order,
embedding_size=options.embedding_size,
neg_sampled=options.negative,
batch_size=options.batch_size,
distortion_power=options.distortion_power,
initial_learning_rate=options.learning_rate,
decay_epochs=options.decay_epochs,
decay_rate=options.decay_rate,
iter_epochs=options.iter_epoches,
allow_soft_placement=options.allow_soft_placement,
log_device_placement=options.log_device_placement,
gpu_memory_fraction=options.gpu_memory_fraction,
using_gpu=options.using_gpu,
allow_growth=options.allow_growth,
loss_interval=options.loss_interval,
summary_steps=options.summary_steps,
ckpt_interval=options.ckpt_interval,
ckpt_epochs=options.ckpt_epochs,
summary_interval=options.summary_interval,
decay_interval=options.decay_interval,
train_workers=options.train_workers)
logger.info('train completed in {}s'.format(time.time() - time_start))
return
def eval_once(options):
global features_matrix, net_eval, net_except, SAMPLE_NODES, SAMPLE_RULE, METIRC, PREC_K
if not utils.check_rebuild(options.link_prediction_path,
descrip='link_prediction',
always_rebuild=options.always_rebuild):
return
logger.info('eval case: link-prediction ...')
logger.info('\t save_path: {}'.format(options.link_prediction_path))
logger.info('\t eval_data_path: {}'.format(options.eval_data_path))
logger.info('\t except_data_path: {}'.format(options.except_data_path))
logger.info('\t data_format: {}'.format(options.data_format))
logger.info('\t metrics: MAP and precise@K')
logger.info('\t max_index for precise@K: {}'.format(
options.precK_max_index))
logger.info('\t similarity_metric: {}'.format(options.similarity_metric))
logger.info('\t eval_online: {}'.format(options.eval_online))
logger.info('\t eval_interval: {}s'.format(options.eval_interval))
logger.info('\t sample_nodes: {}'.format(options.sample_nodes))
logger.info('\t sample_nodes_rule: {}'.format(options.sample_nodes_rule))
logger.info('\t repeat {} times'.format(options.repeated_times))
logger.info('\t eval_workers: {}'.format(options.eval_workers))
logger.info("constructing eval network ...")
net_eval = network.construct_network(data_path=options.eval_data_path,
data_format=options.data_format,
print_net_info=False,
isdirected=options.isdirected)
eval_net_nodes_size = net_eval.get_nodes_size()
eval_net_edges_size = net_eval.get_edges_size()
logger.info("eval_net_nodes_size = {}".format(eval_net_nodes_size))
logger.info("eval_net_edges_size = {}".format(eval_net_edges_size))
logger.info("constructing except(train) network ...")
net_except = network.construct_network(data_path=options.except_data_path,
data_format=options.data_format,
print_net_info=False,
isdirected=options.isdirected)
except_net_nodes_size = net_except.get_nodes_size()
except_net_edges_size = net_except.get_edges_size()
logger.info("except_net_nodes_size = {}".format(except_net_nodes_size))
logger.info("except_net_edges_size = {}".format(except_net_edges_size))
id_list = list(range(eval_net_nodes_size)) # must be [0,1,2,3,...]
SAMPLE_NODES = options.sample_nodes
SAMPLE_RULE = options.sample_nodes_rule
METIRC = options.similarity_metric
PREC_K = options.precK_max_index
# loading features_matrix(already trained)
logger.info('\t reading embedding vectors from file {}'.format(
options.vectors_path))
time_start = time.time()
features_matrix = utils.get_vectors(
utils.get_KeyedVectors(options.vectors_path), id_list)
logger.info(
'\t reading embedding vectors completed in {}s'.format(time.time() -
time_start))
logger.info('total loaded nodes: {}'.format(
np.size(features_matrix, axis=0)))
logger.info('the embedding dimension: {}'.format(
np.size(features_matrix, axis=1)))
fr = open(options.link_prediction_path, 'w')
fr.write('eval case: link-prediction ...\n')
fr.write('\t save_path: {}\n'.format(options.link_prediction_path))
fr.write('\t eval_data_path: {}\n'.format(options.eval_data_path))
fr.write('\t except_data_path: {}\n'.format(options.except_data_path))
fr.write('\t data_format: {}\n'.format(options.data_format))
fr.write('\t metrics: MAP and precise@K\n')
fr.write('\t max_index for precise@K: {}\n'.format(
options.precK_max_index))
fr.write('\t similarity_metric: {}\n'.format(options.similarity_metric))
fr.write('\t eval_online: {}\n'.format(options.eval_online))
fr.write('\t eval_interval: {}s\n'.format(options.eval_interval))
fr.write('\t sample_nodes: {}\n'.format(options.sample_nodes))
fr.write('\t sample_nodes_rule: {}\n'.format(options.sample_nodes_rule))
fr.write('\t repeat {} times\n'.format(options.repeated_times))
fr.write('\t eval_workers: {}\n'.format(options.eval_workers))
fr.write("eval_net_nodes_size = {}\n".format(eval_net_nodes_size))
fr.write("eval_net_edges_size = {}\n".format(eval_net_edges_size))
fr.write("except_net_nodes_size = {}\n".format(except_net_nodes_size))
fr.write("except_net_edges_size = {}\n".format(except_net_edges_size))
fr.write('total loaded nodes: {}\n'.format(np.size(features_matrix,
axis=0)))
fr.write('the embedding dimension: {}\n'.format(
np.size(features_matrix, axis=1)))
if options.sample_nodes > 0:
if options.eval_workers > 1 and options.repeated_times > 1:
# speed up by using multi-process
logger.info("\t allocating repeat_times to workers ...")
if options.repeated_times <= options.eval_workers:
times_per_worker = [1 for _ in range(options.repeated_times)]
else:
div, mod = divmod(options.repeated_times, options.eval_workers)
times_per_worker = [div for _ in range(options.eval_workers)]
for idx in range(mod):
times_per_worker[idx] = times_per_worker[idx] + 1
assert sum(
times_per_worker
) == options.repeated_times, 'workers allocating failed: %d != %d' % (
sum(times_per_worker), options.repeated_times)
logger.info("\t using {} processes for evaling:".format(
len(times_per_worker)))
for idx, rep_times in enumerate(times_per_worker):
logger.info("\t process-{}: repeat {} times".format(
idx, rep_times))
ret_list = [] # [[MAP, precisionK_list], ... ]
with ProcessPoolExecutor(
max_workers=options.eval_workers) as executor:
for ret in executor.map(_sample_thread_body, times_per_worker):
ret_list.extend(ret)
if len(ret_list) != options.repeated_times:
logger.warning(
"warning: eval unmatched repeated_times: {} != {}".format(
len(ret_list), options.repeated_times))
else:
ret_list = _sample_thread_body(options.repeated_times)
else:
# no sampling, no repeat!
ret_list = [_eval(net_eval, net_except)] # [[MAP, precisionK_list]]
if options.sample_nodes > 0:
fr.write(
'expected repeated_times: {}, actual repeated_times: {}, mean results as follows:\n'
.format(options.repeated_times, len(ret_list)))
else:
fr.write(
'due to the sample nodes = {}, so actual repeated_times = {}, results as follows:\n'
.format(options.sample_nodes, len(ret_list)))
mean_MAP = np.mean([ret[0] for ret in ret_list])
mean_precisionK = np.mean([ret[1] for ret in ret_list], axis=0)
fr.write('\t\t MAP = {}\n'.format(mean_MAP))
for k in range(options.precK_max_index):
if k < len(mean_precisionK):
fr.write('\t\t precisionK_{} = {}\n'.format(
k + 1, mean_precisionK[k]))
else:
fr.write('\t\t precisionK_{} = None\n'.format(k + 1))
fr.write('details:\n')
for repeat in range(len(ret_list)):
fr.write('\t repeated {}/{}:\n'.format(repeat + 1, len(ret_list)))
MAP = ret_list[repeat][0]
precisionK_list = ret_list[repeat][1]
fr.write('\t\t MAP = {}\n'.format(MAP))
for k in range(options.precK_max_index):
if k < len(precisionK_list):
fr.write('\t\t precisionK_{} = {}\n'.format(
k + 1, precisionK_list[k]))
else:
fr.write('\t\t precisionK_{} = None\n'.format(k + 1))
fr.write(
'\neval case: link_prediction completed in {}s.'.format(time.time() -
time_start))
fr.close()
logger.info(
'eval case: link_prediction completed in {}s.'.format(time.time() -
time_start))
return
def eval_online(options):
global features_matrix, net_eval, net_except, SAMPLE_NODES, SAMPLE_RULE, METIRC, PREC_K
link_prediction_dir = os.path.split(options.link_prediction_path)[0]
if not utils.check_rebuild(link_prediction_dir,
descrip='link_prediction',
always_rebuild=options.always_rebuild):
return
if not os.path.exists(link_prediction_dir):
os.makedirs(link_prediction_dir)
logger.info('eval case: link-prediction ...')
logger.info('\t save_path: {}'.format(options.link_prediction_path))
logger.info('\t eval_data_path: {}'.format(options.eval_data_path))
logger.info('\t except_data_path: {}'.format(options.except_data_path))
logger.info('\t data_format: {}'.format(options.data_format))
logger.info('\t metrics: MAP and precise@K')
logger.info('\t max_index for precise@K: {}'.format(
options.precK_max_index))
logger.info('\t similarity_metric: {}'.format(options.similarity_metric))
logger.info('\t eval_online: {}'.format(options.eval_online))
logger.info('\t eval_interval: {}s'.format(options.eval_interval))
logger.info('\t sample_nodes: {}'.format(options.sample_nodes))
logger.info('\t sample_nodes_rule: {}'.format(options.sample_nodes_rule))
logger.info('\t repeat {} times'.format(options.repeated_times))
logger.info('\t eval_workers: {}'.format(options.eval_workers))
logger.info("constructing eval network ...")
net_eval = network.construct_network(data_path=options.eval_data_path,
data_format=options.data_format,
print_net_info=False,
isdirected=options.isdirected)
eval_net_nodes_size = net_eval.get_nodes_size()
eval_net_edges_size = net_eval.get_edges_size()
logger.info("eval_net_nodes_size = {}".format(eval_net_nodes_size))
logger.info("eval_net_edges_size = {}".format(eval_net_edges_size))
logger.info("constructing except(train) network ...")
net_except = network.construct_network(data_path=options.except_data_path,
data_format=options.data_format,
print_net_info=False,
isdirected=options.isdirected)
except_net_nodes_size = net_except.get_nodes_size()
except_net_edges_size = net_except.get_edges_size()
logger.info("except_net_nodes_size = {}".format(except_net_nodes_size))
logger.info("except_net_edges_size = {}".format(except_net_edges_size))
id_list = list(range(eval_net_nodes_size)) # must be [0,1,2,3,...]
SAMPLE_NODES = options.sample_nodes
SAMPLE_RULE = options.sample_nodes_rule
METIRC = options.similarity_metric
PREC_K = options.precK_max_index
metric_prec_k_list = [1]
decimal_number = 10
while metric_prec_k_list[-1] < options.precK_max_index:
if decimal_number <= options.precK_max_index:
metric_prec_k_list.append(decimal_number)
else:
break
if 2 * decimal_number <= options.precK_max_index:
metric_prec_k_list.append(2 * decimal_number)
else:
break
if 5 * decimal_number <= options.precK_max_index:
metric_prec_k_list.append(5 * decimal_number)
else:
break
decimal_number = decimal_number * 10
if options.sample_nodes > 0:
if options.eval_workers > 1 and options.repeated_times > 1:
# speed up by using multi-process
logger.info("\t allocating repeat_times to workers ...")
if options.repeated_times <= options.eval_workers:
times_per_worker = [1 for _ in range(options.repeated_times)]
else:
div, mod = divmod(options.repeated_times, options.eval_workers)
times_per_worker = [div for _ in range(options.eval_workers)]
for idx in range(mod):
times_per_worker[idx] = times_per_worker[idx] + 1
assert sum(
times_per_worker
) == options.repeated_times, 'workers allocating failed: %d != %d' % (
sum(times_per_worker), options.repeated_times)
logger.info("\t using {} processes for evaling:".format(
len(times_per_worker)))
for idx, rep_times in enumerate(times_per_worker):
logger.info("\t process-{}: repeat {} times".format(
idx, rep_times))
fr_total = open(options.link_prediction_path, 'w')
fr_total.write('eval case: link-prediction ...\n')
fr_total.write('\t save_path: {}\n'.format(options.link_prediction_path))
fr_total.write('\t eval_data_path: {}\n'.format(options.eval_data_path))
fr_total.write('\t except_data_path: {}\n'.format(
options.except_data_path))
fr_total.write('\t data_format: {}\n'.format(options.data_format))
fr_total.write('\t metrics: MAP and precise@K\n')
fr_total.write('\t max_index for precise@K: {}\n'.format(
options.precK_max_index))
fr_total.write('\t similarity_metric: {}\n'.format(
options.similarity_metric))
fr_total.write('\t eval_online: {}\n'.format(options.eval_online))
fr_total.write('\t eval_interval: {}s\n'.format(options.eval_interval))
fr_total.write('\t sample_nodes: {}\n'.format(options.sample_nodes))
fr_total.write('\t sample_nodes_rule: {}\n'.format(
options.sample_nodes_rule))
fr_total.write('\t repeat {} times\n'.format(options.repeated_times))
fr_total.write('\t eval_workers: {}\n'.format(options.eval_workers))
fr_total.write("eval_net_nodes_size = {}\n".format(eval_net_nodes_size))
fr_total.write("eval_net_edges_size = {}\n".format(eval_net_edges_size))
fr_total.write(
"except_net_nodes_size = {}\n".format(except_net_nodes_size))
fr_total.write(
"except_net_edges_size = {}\n".format(except_net_edges_size))
fr_total.write(
'\t results:\n=============================================================\n'
)
fr_total.write('finish_time\tckpt\tMAP\t')
for v in metric_prec_k_list:
fr_total.write('\tPr@{}'.format(v))
fr_total.write("\n")
last_step = 0
summary_writer = tf.summary.FileWriter(link_prediction_dir, tf.Graph())
summary = tf.Summary()
summary.value.add(tag='MAP', simple_value=0.)
for v in metric_prec_k_list:
summary.value.add(tag='Pr_{}'.format(v), simple_value=0.)
summary_writer.add_summary(summary, last_step)
best_MAP = 0
ckpt_dir = os.path.join(os.path.split(options.vectors_path)[0], 'ckpt')
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
while (not (ckpt and ckpt.model_checkpoint_path)):
logger.info("\t model and vectors not exist, waiting ...")
time.sleep(options.eval_interval)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
reading = options.vectors_path + ".reading_link_prediction"
writing = options.vectors_path + ".writing"
while (options.eval_online):
while True:
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
cur_step = int(
ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
if cur_step <= last_step or (not os.path.exists(
options.vectors_path)) or os.path.exists(writing):
if os.path.exists(
os.path.join(
os.path.split(options.vectors_path)[0],
"RUN_SUCCESS")):
return
time.sleep(options.eval_interval)
continue
# ready for reading
logger.info("\t declare for reading ...")
open(reading, "w") # declare
time.sleep(30)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
cur_step = int(
ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
if cur_step <= last_step or (not os.path.exists(
options.vectors_path)) or os.path.exists(writing):
os.remove(reading) # undeclare
logger.info("\t confliction! undeclare and waiting ...")
time.sleep(options.eval_interval)
continue
break
logger.info("\t eval ckpt-{}.......".format(cur_step))
# loading features_matrix(already trained)
logger.info('\t reading embedding vectors from file {}'.format(
options.vectors_path))
time_start = time.time()
features_matrix = utils.get_vectors(
utils.get_KeyedVectors(options.vectors_path), id_list)
os.remove(reading)
logger.info("\t done for reading ...")
logger.info('\t reading embedding vectors completed in {}s'.format(
time.time() - time_start))
logger.info('total loaded nodes: {}'.format(
np.size(features_matrix, axis=0)))
logger.info('the embedding dimension: {}'.format(
np.size(features_matrix, axis=1)))
#
fr = open(options.link_prediction_path + '.{}'.format(cur_step), 'w')
fr.write('eval case: link-prediction ...\n')
fr.write('\t save_path: {}\n'.format(options.link_prediction_path))
fr.write('\t eval_data_path: {}\n'.format(options.eval_data_path))
fr.write('\t except_data_path: {}\n'.format(options.except_data_path))
fr.write('\t data_format: {}\n'.format(options.data_format))
fr.write('\t metrics: MAP and precise@K\n')
fr.write('\t max_index for precise@K: {}\n'.format(
options.precK_max_index))
fr.write('\t similarity_metric: {}\n'.format(
options.similarity_metric))
fr.write('\t eval_online: {}\n'.format(options.eval_online))
fr.write('\t eval_interval: {}s\n'.format(options.eval_interval))
fr.write('\t sample_nodes: {}\n'.format(options.sample_nodes))
fr.write('\t sample_nodes_rule: {}\n'.format(
options.sample_nodes_rule))
fr.write('\t repeat {} times\n'.format(options.repeated_times))
fr.write('\t eval_workers: {}\n'.format(options.eval_workers))
fr.write("eval_net_nodes_size = {}\n".format(eval_net_nodes_size))
fr.write("eval_net_edges_size = {}\n".format(eval_net_edges_size))
fr.write("except_net_nodes_size = {}\n".format(except_net_nodes_size))
fr.write("except_net_edges_size = {}\n".format(except_net_edges_size))
fr.write('total loaded nodes: {}\n'.format(
np.size(features_matrix, axis=0)))
fr.write('the embedding dimension: {}\n'.format(
np.size(features_matrix, axis=1)))
if options.sample_nodes > 0:
if options.eval_workers > 1 and options.repeated_times > 1:
# speed up by using multi-process
ret_list = [] # [[MAP, precisionK_list], ... ]
with ProcessPoolExecutor(
max_workers=options.eval_workers) as executor:
for ret in executor.map(_sample_thread_body,
times_per_worker):
ret_list.extend(ret)
if len(ret_list) != options.repeated_times:
logger.warning(
"warning: eval unmatched repeated_times: {} != {}".
format(len(ret_list), options.repeated_times))
else:
ret_list = _sample_thread_body(options.repeated_times)
else:
# no sampling, no repeat!
ret_list = [_eval(net_eval,
net_except)] # [[MAP, precisionK_list]]
fr_total.write('%s ckpt-%-9d: ' % (time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time())), cur_step))
summary = tf.Summary()
if options.sample_nodes > 0:
fr.write(
'expected repeated_times: {}, actual repeated_times: {}, mean results as follows:\n'
.format(options.repeated_times, len(ret_list)))
else:
fr.write(
'due to the sample nodes = {}, so actual repeated_times = {}, results as follows:\n'
.format(options.sample_nodes, len(ret_list)))
mean_MAP = np.mean([ret[0] for ret in ret_list])
mean_precisionK = np.mean([ret[1] for ret in ret_list], axis=0)
fr.write('\t\t MAP = {}\n'.format(mean_MAP))
for k in range(options.precK_max_index):
if k < len(mean_precisionK):
fr.write('\t\t precisionK_{} = {}\n'.format(
k + 1, mean_precisionK[k]))
else:
fr.write('\t\t precisionK_{} = None\n'.format(k + 1))
fr.write('details:\n')
for repeat in range(len(ret_list)):
fr.write('\t repeated {}/{}:\n'.format(repeat + 1, len(ret_list)))
MAP = ret_list[repeat][0]
precisionK_list = ret_list[repeat][1]
fr.write('\t\t MAP = {}\n'.format(MAP))
for k in range(options.precK_max_index):
if k < len(precisionK_list):
fr.write('\t\t precisionK_{} = {}\n'.format(
k + 1, precisionK_list[k]))
else:
fr.write('\t\t precisionK_{} = None\n'.format(k + 1))
fr.write('\neval case: link_prediction completed in {}s.'.format(
time.time() - time_start))
fr.close()
fr_total.write('%.4f' % mean_MAP)
summary.value.add(tag='MAP', simple_value=mean_MAP)
for v in metric_prec_k_list:
fr_total.write('\t%.4f' % mean_precisionK[v - 1])
summary.value.add(tag='Pr_{}'.format(v),
simple_value=mean_precisionK[v - 1])
fr_total.write("\n")
fr_total.flush()
summary_writer.add_summary(summary, cur_step)
summary_writer.flush()
logger.info(
'eval case: ret_list completed in {}s.\n================================='
.format(time.time() - time_start))
# copy ckpt-files according to last mean_Micro_F1 (0.9 ratio).
if mean_MAP > best_MAP:
best_MAP = mean_MAP
ckptIsExists = os.path.exists(
os.path.join(ckpt_dir, 'model.ckpt-%d.index' % cur_step))
if ckptIsExists:
fr_best = open(
os.path.join(link_prediction_dir, 'best_ckpt.info'), 'w')
else:
fr_best = open(
os.path.join(link_prediction_dir, 'best_ckpt.info'), 'a')
fr_best.write(
"Note:the model.ckpt-best is the remainings of last best_ckpt!\n"
"the current best_ckpt model is loss, but the result is:\n"
)
fr_best.write("best_MAP: {}\n".format(best_MAP))
fr_best.write("best_ckpt: ckpt-{}\n".format(cur_step))
fr_best.close()
if ckptIsExists:
sourceFile = os.path.join(
ckpt_dir, 'model.ckpt-%d.data-00000-of-00001' % cur_step)
targetFile = os.path.join(
link_prediction_dir, 'model.ckpt-best.data-00000-of-00001')
if os.path.exists(targetFile):
os.remove(targetFile)
shutil.copy(sourceFile, targetFile)
sourceFile = os.path.join(ckpt_dir,
'model.ckpt-%d.index' % cur_step)
targetFile = os.path.join(link_prediction_dir,
'model.ckpt-best.index')
if os.path.exists(targetFile):
os.remove(targetFile)
shutil.copy(sourceFile, targetFile)
sourceFile = os.path.join(ckpt_dir,
'model.ckpt-%d.meta' % cur_step)
targetFile = os.path.join(link_prediction_dir,
'model.ckpt-best.meta')
if os.path.exists(targetFile):
os.remove(targetFile)
shutil.copy(sourceFile, targetFile)
last_step = cur_step
fr_total.close()
summary_writer.close()
def build_walk_corpus(options, net=None):
global walker
# check walk info and record
if not utils.check_rebuild(options.corpus_store_path,
descrip='walk corpus',
always_rebuild=options.always_rebuild):
return
if options.model == "DeepWalk":
random_walker = "uniform"
elif options.model == "Node2Vec":
random_walker = "bias"
else:
logger.error("Unknown model or it cann't build walk corpus: '%s'." %
options.model)
sys.exit()
if net == None:
net = network.construct_network(options)
logger.info('Corpus bulid: walk info:')
logger.info('\t random_walker = {}'.format(random_walker))
logger.info('\t walk times = {}'.format(options.walk_times))
logger.info('\t walk length = {}'.format(options.walk_length))
if random_walker == "uniform":
logger.info('\t walk restart = {}'.format(options.walk_restart))
elif random_walker == "bias":
logger.info('\t return_parameter (p) = {}'.format(options.p))
logger.info('\t in-out_parameter (q) = {}'.format(options.q))
logger.info('\t max walk workers = {}'.format(options.walk_workers))
logger.info('\t walk to memory = {}'.format(str(options.walk_to_memory)))
if options.walk_to_memory:
logger.info('\t donot store corpus = {}'.format(
str(options.not_store_corpus)))
if not options.not_store_corpus:
logger.info('\t corpus store path = {}'.format(
options.corpus_store_path))
else:
logger.info('\t corpus store path = {}'.format(
options.corpus_store_path))
fr_walks = open(
os.path.join(
os.path.split(options.corpus_store_path)[0], 'walks.info'), 'w')
fr_walks.write('Corpus walk info:\n')
fr_walks.write('\t random_walker = {}\n'.format(random_walker))
fr_walks.write('\t walk times = {}\n'.format(options.walk_times))
fr_walks.write('\t walk length = {}\n'.format(options.walk_length))
if random_walker == "uniform":
fr_walks.write('\t walk restart = {}\n'.format(options.walk_restart))
elif random_walker == "bias":
fr_walks.write('\t return_parameter (p) = {}\n'.format(options.p))
fr_walks.write('\t in-out_parameter (q) = {}\n'.format(options.q))
fr_walks.write('\t max walk workers = {}\n'.format(options.walk_workers))
fr_walks.write('\t walk to memory = {}\n'.format(
str(options.walk_to_memory)))
if options.walk_to_memory:
fr_walks.write('\t donot store corpus = {}\n'.format(
str(options.not_store_corpus)))
if not options.not_store_corpus:
fr_walks.write('\t corpus store path = {}\n'.format(
options.corpus_store_path))
else:
fr_walks.write('\t corpus store path = {}\n'.format(
options.corpus_store_path))
fr_walks.close()
walker = Walker(net,
random_walker=random_walker,
walk_length=options.walk_length,
p=options.p,
q=options.q)
if random_walker == "bias":
# walker.preprocess_transition_probs(options.walk_workers)
walker.preprocess_transition_probs(net_info_path=options.net_info_path)
walk_corpus = None
if options.walk_to_memory:
walk_corpus = build_walk_corpus_to_memory(
options.walk_times, max_num_workers=options.walk_workers)
if not options.not_store_corpus:
store_walk_corpus(options.corpus_store_path,
walk_corpus,
always_rebuild=options.always_rebuild)
else:
# walk to files
walk_files = build_walk_corpus_to_files(
options.corpus_store_path,
options.walk_times,
headflag_of_index_file=options.headflag_of_index_file,
max_num_workers=options.walk_workers,
always_rebuild=options.always_rebuild)
if "train" in options.task:
if options.load_from_memory:
walk_corpus = load_walks_corpus(walk_files)
else:
walk_corpus = WalksCorpus(walk_files)
del walker
gc.collect()
return walk_corpus
def build_walk_corpus(options):
global walker
# check walk info and record
if not utils.check_rebuild(options.corpus_store_path,
descrip='walk corpus',
always_rebuild=options.always_rebuild):
return
if options.model == "DeepWalk":
random_walker = "uniform"
net = network.construct_network(options, isHIN=False)
elif options.model == "SpaceyWalk":
random_walker = "spacey"
net = network.construct_network(options, isHIN=True)
elif options.model == "MetatreeWalk":
random_walker = "metatreewalk"
net = network.construct_network(options, isHIN=True)
else:
logger.error("Unknown model or it cann't build walk corpus: '%s'." %
options.model)
sys.exit()
logger.info('Corpus bulid: walk info:')
logger.info('\t data_dir = {}'.format(options.data_dir))
logger.info('\t data_name = {}'.format(options.data_name))
logger.info('\t isdirected = {}\n'.format(options.isdirected))
logger.info('\t random_walker = {}'.format(random_walker))
logger.info('\t walk_times = {}'.format(options.walk_times))
logger.info('\t walk_length = {}'.format(options.walk_length))
logger.info('\t max_walk_workers = {}'.format(options.walk_workers))
logger.info('\t walk_to_memory = {}'.format(options.walk_to_memory))
logger.info('\t seed = {}'.format(options.seed))
logger.info('\t alpha = {}'.format(options.alpha))
logger.info('\t window_size = {}'.format(options.window_size))
logger.info('\t sample_size = {}'.format(options.sample_size))
if options.walk_to_memory:
logger.info('\t donot store corpus = {}'.format(
str(options.not_store_corpus)))
if not options.not_store_corpus:
logger.info('\t corpus store path = {}'.format(
options.corpus_store_path))
else:
logger.info('\t corpus store path = {}'.format(
options.corpus_store_path))
fr_walks = open(
os.path.join(
os.path.split(options.corpus_store_path)[0], 'walks.info'), 'w')
fr_walks.write('Corpus walk info:\n')
fr_walks.write('\t data_dir = {}\n'.format(options.data_dir))
fr_walks.write('\t data_name = {}\n'.format(options.data_name))
fr_walks.write('\t isdirected = {}\n\n'.format(options.isdirected))
fr_walks.write('\t random_walker = {}\n'.format(random_walker))
fr_walks.write('\t walk times = {}\n'.format(options.walk_times))
fr_walks.write('\t walk length = {}\n'.format(options.walk_length))
fr_walks.write('\t max walk workers = {}\n'.format(options.walk_workers))
fr_walks.write('\t seed = {}\n'.format(options.seed))
fr_walks.write('\t alpha = {}\n'.format(options.alpha))
fr_walks.write('\t window_size = {}\n'.format(options.window_size))
fr_walks.write('\t sample_size = {}\n'.format(options.sample_size))
fr_walks.write('\t walk to memory = {}\n'.format(
str(options.walk_to_memory)))
if options.walk_to_memory:
fr_walks.write('\t donot store corpus = {}\n'.format(
str(options.not_store_corpus)))
if not options.not_store_corpus:
fr_walks.write('\t corpus store path = {}\n'.format(
options.corpus_store_path))
else:
fr_walks.write('\t corpus store path = {}\n'.format(
options.corpus_store_path))
fr_walks.close()
if options.model == "SpaceyWalk":
if options.using_metapath == "metagraph":
metagraph = network.construct_meta_graph(
options.metapath_path, isdirected=options.isdirected)
elif options.using_metapath == "metatree":
metagraph = network.construct_meta_tree(options.metapath_path,
isdirected=True)
elif options.using_metapath == "metaschema":
metagraph = None
else:
logger.error("Unknown feature : '%s'." % options.using_metapath)
sys.exit()
walker = Walker(net,
random_walker=random_walker,
walk_length=options.walk_length,
metagraph=metagraph,
using_metapath=options.using_metapath,
history_position=options.history_position,
task="walk",
alpha=options.alpha)
elif options.model == "MetatreeWalk":
metagraph = network.construct_meta_tree(options.metapath_path,
isdirected=True)
walker = Walker(net,
random_walker=random_walker,
walk_length=options.walk_length,
metagraph=metagraph,
task="walk")
corpus_store_dir = os.path.split(options.corpus_store_path)[0]
if not os.path.exists(corpus_store_dir):
os.makedirs(corpus_store_dir)
logger.info(
'Corpus bulid: walking and computing (using %d workers for multi-process)...'
% options.walk_workers)
time_start = time.time()
if options.walk_times <= options.walk_workers:
times_per_worker = [1 for _ in range(options.walk_times)]
else:
div, mod = divmod(options.walk_times, options.walk_workers)
times_per_worker = [div for _ in range(options.walk_workers)]
for idx in range(mod):
times_per_worker[idx] = times_per_worker[idx] + 1
assert sum(
times_per_worker
) == options.walk_times, 'workers allocating failed: %d != %d' % (
sum(times_per_worker), options.walk_times)
nodes_total = list(range(walker.nodes_size))
sp_random = random.Random(options.seed)
sp_random.shuffle(nodes_total)
nodes_total = nodes_total[0:options.sample_size]
nodes_total.insert(0, 8407)
nodes_total.insert(0, 9891)
nodes_total.insert(0, 8354)
nodes_total.insert(0, 8798)
for node in nodes_total:
args_list = []
begin = 0
for cnt in times_per_worker:
args_list.append((corpus_store_dir, node, begin + 1, begin + cnt,
options.window_size))
begin += cnt
with ProcessPoolExecutor(max_workers=options.walk_workers) as executor:
executor.map(_construct_walk_corpus_and_write_singprocess,
args_list)
logger.info('Corpus bulid: walk completed in {}s'.format(time.time() -
time_start))
del walker
gc.collect()
return
def build_walk_corpus(options):
global walker
# check walk info and record
if not utils.check_rebuild(options.corpus_store_path,
descrip='walk corpus',
always_rebuild=options.always_rebuild):
return
if options.model == "DeepWalk":
random_walker = "uniform"
net = network.construct_network(options, isHIN=False)
elif options.model == "SpaceyWalk":
random_walker = "spacey"
net = network.construct_network(options, isHIN=True)
elif options.model == "MetatreeWalk":
random_walker = "metatreewalk"
net = network.construct_network(options, isHIN=True)
else:
logger.error("Unknown model or it cann't build walk corpus: '%s'." %
options.model)
sys.exit()
logger.info('Corpus bulid: walk info:')
logger.info('\t data_dir = {}'.format(options.data_dir))
logger.info('\t data_name = {}'.format(options.data_name))
logger.info('\t isdirected = {}\n'.format(options.isdirected))
logger.info('\t random_walker = {}'.format(random_walker))
logger.info('\t walk_times = {}'.format(options.walk_times))
logger.info('\t walk_length = {}'.format(options.walk_length))
logger.info('\t max_walk_workers = {}'.format(options.walk_workers))
logger.info('\t walk_to_memory = {}'.format(options.walk_to_memory))
logger.info('\t alpha = {}'.format(options.alpha))
if options.walk_to_memory:
logger.info('\t donot store corpus = {}'.format(
str(options.not_store_corpus)))
if not options.not_store_corpus:
logger.info('\t corpus store path = {}'.format(
options.corpus_store_path))
else:
logger.info('\t corpus store path = {}'.format(
options.corpus_store_path))
fr_walks = open(
os.path.join(
os.path.split(options.corpus_store_path)[0], 'walks.info'), 'w')
fr_walks.write('Corpus walk info:\n')
fr_walks.write('\t data_dir = {}\n'.format(options.data_dir))
fr_walks.write('\t data_name = {}\n'.format(options.data_name))
fr_walks.write('\t isdirected = {}\n\n'.format(options.isdirected))
fr_walks.write('\t random_walker = {}\n'.format(random_walker))
fr_walks.write('\t walk times = {}\n'.format(options.walk_times))
fr_walks.write('\t walk length = {}\n'.format(options.walk_length))
fr_walks.write('\t max walk workers = {}\n'.format(options.walk_workers))
fr_walks.write('\t walk to memory = {}\n'.format(
str(options.walk_to_memory)))
if options.walk_to_memory:
fr_walks.write('\t donot store corpus = {}\n'.format(
str(options.not_store_corpus)))
if not options.not_store_corpus:
fr_walks.write('\t corpus store path = {}\n'.format(
options.corpus_store_path))
else:
fr_walks.write('\t corpus store path = {}\n'.format(
options.corpus_store_path))
fr_walks.close()
if options.model == "DeepWalk":
walker = Walker(net,
random_walker=random_walker,
walk_length=options.walk_length)
elif options.model == "SpaceyWalk":
if options.using_metapath == "metagraph":
metagraph = network.construct_meta_graph(
options.metapath_path, isdirected=options.isdirected)
elif options.using_metapath == "metatree":
metagraph = network.construct_meta_tree(options.metapath_path,
isdirected=True)
elif options.using_metapath == "metaschema":
metagraph = None
else:
logger.error("Unknown feature : '%s'." % options.using_metapath)
sys.exit()
walker = Walker(net,
random_walker=random_walker,
walk_length=options.walk_length,
metagraph=metagraph,
using_metapath=options.using_metapath,
history_position=options.history_position,
task="walk",
alpha=options.alpha)
elif options.model == "MetatreeWalk":
metagraph = network.construct_meta_tree(options.metapath_path,
isdirected=True)
walker = Walker(net,
random_walker=random_walker,
walk_length=options.walk_length,
metagraph=metagraph,
task="walk")
walk_corpus = None
if options.walk_to_memory:
walk_corpus = build_walk_corpus_to_memory(
options.walk_times, max_num_workers=options.walk_workers)
if not options.not_store_corpus:
store_walk_corpus(options.corpus_store_path,
walk_corpus,
always_rebuild=options.always_rebuild)
else:
# walk to files
walk_files = build_walk_corpus_to_files(
options.corpus_store_path,
options.walk_times,
headflag_of_index_file=options.headflag_of_index_file,
max_num_workers=options.walk_workers,
always_rebuild=options.always_rebuild)
if "train" in options.task:
if options.load_from_memory:
walk_corpus = load_walks_corpus(walk_files)
else:
walk_corpus = WalksCorpus(walk_files)
del walker
gc.collect()
return walk_corpus
def process(options):
# parser.add_argument("--train_data_dir", dest="train_data_dir", default="./splited_train_0.8_repeat_1",
# help="the train network data path. like: origin_data_dir/splited_train_${train_ratio}_repeat_${repeat_th}")
# parser.add_argument("--eval_data_dir", dest="eval_data_dir", default="./splited_eval_0.2_repeat_1",
# help="the eval network data path. like: origin_data_dir/splited_eval_${eval_ratio}_repeat_${repeat_th}")
logger.info("Data preprocessing: network split ...")
time_start = time.time()
origin_data_dir, data_filename = os.path.split(options.data_path)
data_prefixname = data_filename.split(".")[0]
data_format = options.data_format
isdirected = options.isdirected
if options.train_ratio > 0:
train_ratio_list = [options.train_ratio]
else:
train_ratio_list = [v / 10.0 for v in range(9, 0, -1)]
logger.info("\t origin_data_dir = {}".format(origin_data_dir))
logger.info("\t data_filename = {}".format(data_filename))
logger.info("\t data_format = {}".format(data_format))
logger.info("\t isdirected = {}".format(isdirected))
logger.info("\t train_ratio = {}".format(train_ratio_list))
logger.info("\t repeat_from = {}".format(options.repeat_from))
logger.info("\t repeat_to = {}".format(options.repeat_to))
logger.info("\t log_name = {}".format(options.log_name))
logger.info("\t re_direction_path = {}".format(options.re_direction_path))
net_origin = network.construct_network(data_path=options.data_path,
data_format=data_format,
print_net_info=False,
isdirected=isdirected)
for train_ratio in train_ratio_list:
for repeat_th in range(options.repeat_from, options.repeat_to + 1):
logger.info(
"\ntrain_ratio = {}, repeat_th = {}, spliting ...".format(
train_ratio, repeat_th))
# train_data_dir = os.path.join(origin_data_dir, "splited_train_{}_repeat_{}".format(train_ratio, repeat_th))
# eval_data_dir = os.path.join(origin_data_dir, "splited_eval_{}_repeat_{}".format(round(1-train_ratio, 1), repeat_th))
train_data_dir = os.path.join(
origin_data_dir, "splited_train_{}_repeat_{}_train".format(
train_ratio, repeat_th))
eval_data_dir = os.path.join(
origin_data_dir, "splited_train_{}_repeat_{}_eval".format(
train_ratio, repeat_th))
logger.info("\t train_data_dir = {}".format(train_data_dir))
logger.info("\t eval_data_dir = {}".format(eval_data_dir))
if os.path.exists(
os.path.join(train_data_dir,
data_prefixname + "." + data_format)):
logger.info(
"train_ratio = {}, repeat_th = {}, already splited, skiped!!!\n"
.format(train_ratio, repeat_th))
continue
if not os.path.exists(train_data_dir):
os.mkdir(train_data_dir)
if not os.path.exists(eval_data_dir):
os.mkdir(eval_data_dir)
net_train, net_eval = net_origin.split_by_edges(
train_ratio=train_ratio)
net_train.save_network(train_data_dir, data_prefixname,
data_format)
# net_train.print_net_info(edges_file=os.path.join(train_net_dir, datafilename), file_path=os.path.join(train_net_dir, "net.info"))
net_eval.save_network(eval_data_dir, data_prefixname, data_format)
# net_eval.print_net_info(edges_file=os.path.join(eval_net_dir, datafilename), file_path=os.path.join(eval_net_dir, "net.info"))
# eval_utils.split_network(origin_net_dir = origin_data_dir,
# train_net_dir = train_data_dir,
# eval_net_dir = eval_data_dir,
# data_prefixname = data_prefixname,
# data_format = data_format,
# isdirected = isdirected,
# train_ratio = train_ratio)
logger.info(
"train_ratio = {}, repeat_th = {}, split succssed!!!\n".format(
train_ratio, repeat_th))
logger.info('Data preprocessing: network split completed in {}s.'.format(
time.time() - time_start))
def train_vectors(options):
# check vectors and ckpt
checkpoint = '0'
train_vec_dir = os.path.split(options.vectors_path)[0]
ckpt_dir = os.path.join(train_vec_dir, 'ckpt')
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if ckpt and ckpt.model_checkpoint_path:
cur_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
logger.info(
"model and vectors already exists, checkpoint step = {}".format(
cur_step))
checkpoint = input(
"please input 0 to start a new train, or input a choosed ckpt to restore (-1 for latest ckpt)"
)
if checkpoint == '0':
if ckpt:
tf.gfile.DeleteRecursively(ckpt_dir)
logger.info('start a new embedding train using tensorflow ...')
elif checkpoint == '-1':
logger.info(
'restore a embedding train using tensorflow from latest ckpt...')
else:
logger.info(
'restore a embedding train using tensorflow from ckpt-%s...' %
checkpoint)
if not os.path.exists(train_vec_dir):
os.makedirs(train_vec_dir)
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
# construct network
net = network.construct_network(options)
lr_file = os.path.join(train_vec_dir, "lr.info")
np.savetxt(lr_file,
np.asarray([
options.learning_rate, options.decay_epochs,
options.decay_rate, options.iter_epoches
],
dtype=np.float32),
fmt="%.6f")
dataset = DataSet(nodes_size=net.get_nodes_size(),
edges_list=net.edges,
shuffled=not options.unshuffled)
# train info
logger.info('Train info:')
logger.info('\t train_model = {}'.format(options.model))
logger.info('\t total embedding nodes = {}'.format(net.get_nodes_size()))
logger.info('\t total edges = {}'.format(net.get_edges_size()))
logger.info('\t embedding size = {}'.format(options.embedding_size))
logger.info('\t struct = {}'.format(options.struct))
logger.info('\t alpha = {}'.format(options.alpha))
logger.info('\t beta = {}'.format(options.beta))
logger.info('\t gamma = {}'.format(options.gamma))
logger.info('\t reg = {}\n'.format(options.reg))
logger.info('\t shuffled in training = {}'.format(not options.unshuffled))
logger.info('\t sparse_dot = {}'.format(options.sparse_dot))
logger.info('\t batch_size = {}'.format(options.batch_size))
logger.info('\t iter_epoches = {}'.format(options.iter_epoches))
logger.info('\t init_learning_rate = {}'.format(options.learning_rate))
logger.info('\t decay_epochs = {}'.format(options.decay_epochs))
logger.info('\t decay_interval = {}'.format(options.decay_interval))
logger.info('\t decay_rate = {}'.format(options.decay_rate))
logger.info('\t loss_interval = {}s'.format(options.loss_interval))
logger.info('\t summary_steps = {}'.format(options.summary_steps))
logger.info('\t summary_interval = {}s'.format(options.summary_interval))
logger.info('\t ckpt_epochs = {}'.format(options.ckpt_epochs))
logger.info('\t ckpt_interval = {}s\n'.format(options.ckpt_interval))
logger.info('\t dbn_initial = {}'.format(options.dbn_initial))
logger.info('\t dbn_epochs = {}'.format(options.dbn_epochs))
logger.info('\t dbn_batchsize = {}'.format(options.dbn_batchsize))
logger.info('\t dbn_learning_rate = {}'.format(options.dbn_learning_rate))
logger.info('\t active_function = {}\n'.format(options.active_function))
logger.info('\t using_gpu = {}'.format(options.using_gpu))
logger.info('\t visible_device_list = {}'.format(
options.visible_device_list))
logger.info('\t log_device_placement = {}'.format(
options.log_device_placement))
logger.info('\t allow_soft_placement = {}'.format(
options.allow_soft_placement))
logger.info('\t gpu_memory_fraction = {}'.format(
options.gpu_memory_fraction))
logger.info('\t gpu_memory_allow_growth = {}\n'.format(
options.allow_growth))
logger.info('\t ckpt_dir = {}'.format(ckpt_dir))
logger.info('\t vectors_path = {}'.format(options.vectors_path))
logger.info('\t learning_rate_path = {}'.format(lr_file))
fr_vec = open(os.path.join(train_vec_dir, 'embedding.info'), 'w')
fr_vec.write('embedding info:\n')
fr_vec.write('\t train_model = {}\n'.format(options.model))
fr_vec.write('\t total embedding nodes = {}\n'.format(
net.get_nodes_size()))
fr_vec.write('\t total edges = {}\n'.format(net.get_edges_size()))
fr_vec.write('\t embedding size = {}\n'.format(options.embedding_size))
fr_vec.write('\t struct = {}\n'.format(options.struct))
fr_vec.write('\t alpha = {}\n'.format(options.alpha))
fr_vec.write('\t beta = {}\n'.format(options.beta))
fr_vec.write('\t gamma = {}\n'.format(options.gamma))
fr_vec.write('\t reg = {}\n\n'.format(options.reg))
fr_vec.write(
'\t shuffled in training = {}\n'.format(not options.unshuffled))
fr_vec.write('\t sparse_dot = {}\n'.format(options.sparse_dot))
fr_vec.write('\t batch_size = {}\n'.format(options.batch_size))
fr_vec.write('\t iter_epoches = {}\n'.format(options.iter_epoches))
fr_vec.write('\t init_learning_rate = {}\n'.format(options.learning_rate))
fr_vec.write('\t decay_epochs = {}\n'.format(options.decay_epochs))
fr_vec.write('\t decay_interval = {}\n'.format(options.decay_interval))
fr_vec.write('\t decay_rate = {}\n'.format(options.decay_rate))
fr_vec.write('\t loss_interval = {}s\n'.format(options.loss_interval))
fr_vec.write('\t summary_steps = {}\n'.format(options.summary_steps))
fr_vec.write('\t summary_interval = {}s\n'.format(
options.summary_interval))
fr_vec.write('\t ckpt_epochs = {}\n'.format(options.ckpt_epochs))
fr_vec.write('\t ckpt_interval = {}s\n\n'.format(options.ckpt_interval))
fr_vec.write('\t dbn_initial = {}s\n'.format(options.dbn_initial))
fr_vec.write('\t dbn_epochs = {}s\n'.format(options.dbn_epochs))
fr_vec.write('\t dbn_batchsize = {}s\n'.format(options.dbn_batchsize))
fr_vec.write('\t dbn_learning_rate = {}s\n'.format(
options.dbn_learning_rate))
fr_vec.write('\t active_function = {}\n'.format(options.active_function))
fr_vec.write('\t using_gpu = {}\n'.format(options.using_gpu))
fr_vec.write('\t visible_device_list = {}\n'.format(
options.visible_device_list))
fr_vec.write('\t log_device_placement = {}\n'.format(
options.log_device_placement))
fr_vec.write('\t allow_soft_placement = {}\n'.format(
options.allow_soft_placement))
fr_vec.write('\t gpu_memory_fraction = {}\n'.format(
options.gpu_memory_fraction))
fr_vec.write('\t gpu_memory_allow_growth = {}\n\n'.format(
options.allow_growth))
fr_vec.write('\t ckpt_dir = {}\n'.format(ckpt_dir))
fr_vec.write('\t vectors_path = {}\n'.format(options.vectors_path))
fr_vec.write('\t learning_rate_path = {}\n'.format(lr_file))
fr_vec.close()
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
if options.using_gpu:
visible_devices = str(options.visible_device_list[0])
for dev in options.visible_device_list[1:]:
visible_devices = visible_devices + ',%s' % dev
os.environ['CUDA_VISIBLE_DEVICES'] = visible_devices
else:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
# train
logger.info('training...')
time_start = time.time()
train(dataset=dataset,
vectors_path=options.vectors_path,
lr_file=lr_file,
ckpt_dir=ckpt_dir,
checkpoint=checkpoint,
embedding_size=options.embedding_size,
struct=options.struct,
alpha=options.alpha,
beta=options.beta,
gamma=options.gamma,
reg=options.reg,
sparse_dot=options.sparse_dot,
batch_size=options.batch_size,
initial_learning_rate=options.learning_rate,
decay_epochs=options.decay_epochs,
decay_rate=options.decay_rate,
iter_epochs=options.iter_epoches,
allow_soft_placement=options.allow_soft_placement,
log_device_placement=options.log_device_placement,
gpu_memory_fraction=options.gpu_memory_fraction,
using_gpu=options.using_gpu,
allow_growth=options.allow_growth,
loss_interval=options.loss_interval,
summary_steps=options.summary_steps,
ckpt_interval=options.ckpt_interval,
ckpt_epochs=options.ckpt_epochs,
summary_interval=options.summary_interval,
decay_interval=options.decay_interval,
dbn_initial=options.dbn_initial,
dbn_epochs=options.dbn_epochs,
dbn_batchsize=options.dbn_batchsize,
dbn_learning_rate=options.dbn_learning_rate,
active_function=options.active_function)
logger.info('train completed in {}s'.format(time.time() - time_start))
return
