'bow': 0.8},
{'topics': 1.0},
{'bow': 1.0},
]
for weights in different_weights:
meta_graph_kws = {
'distance_weights': weights,
}
g = IU.get_topic_meta_graph(
interactions,
lda_model=lda_model,
dictionary=dictionary,
undirected=False,
given_topics=False,
decompose_interactions=False,
dist_func=cosine,
preprune_secs=timedelta(weeks=4).total_seconds(),
apply_pagerank=False,
**meta_graph_kws
)
print('weights: {}\n'.format(weights))
out_degrees = g.out_degree(g.nodes())
sorted_nodes = sorted(out_degrees,
key=lambda k: out_degrees[k],
reverse=True)
print('\n'.join(map(lambda n: g.node[n]['subject'], sorted_nodes)[:10]))
node = sorted_nodes[5]
{
'bow': 1.0
},
]
for weights in different_weights:
meta_graph_kws = {
'distance_weights': weights,
}
g = IU.get_topic_meta_graph(
interactions,
lda_model=lda_model,
dictionary=dictionary,
undirected=False,
given_topics=False,
decompose_interactions=False,
dist_func=cosine,
preprune_secs=timedelta(weeks=4).total_seconds(),
apply_pagerank=False,
**meta_graph_kws)
print('weights: {}\n'.format(weights))
out_degrees = g.out_degree(g.nodes())
sorted_nodes = sorted(out_degrees,
key=lambda k: out_degrees[k],
reverse=True)
print('\n'.join(map(lambda n: g.node[n]['subject'], sorted_nodes)[:10]))
node = sorted_nodes[5]
def run(gen_tree_func,
msg_ids_path,
root_sampling_method='random',
interaction_path=os.path.join(CURDIR, 'data/enron.json'),
lda_model_path=os.path.join(CURDIR, 'models/model-4-50.lda'),
corpus_dict_path=os.path.join(CURDIR, 'models/dictionary.pkl'),
meta_graph_pkl_path_prefix=os.path.join(CURDIR, 'data/enron'),
meta_graph_pkl_suffix='',
cand_tree_number=None, # higher priority than percentage
cand_tree_percent=0.1,
result_pkl_path_prefix=os.path.join(CURDIR, 'tmp/results'),
result_suffix='',
all_paths_pkl_prefix='',
all_paths_pkl_suffix='',
true_events_path='',
meta_graph_kws={
'dist_func': cosine,
'preprune_secs': timedelta(weeks=4),
'distance_weights': {'topics': 0.2,
'bow': 0.8},
# 'timestamp_converter': lambda s: s
},
gen_tree_kws={
'timespan': timedelta(weeks=4),
'U': 0.5,
'dijkstra': False
},
convert_time=True,
roots=None,
calculate_graph=False,
given_topics=False,
print_summary=False,
should_binarize_dag=False):
if isinstance(gen_tree_kws['timespan'], timedelta):
timespan = gen_tree_kws['timespan'].total_seconds()
else:
timespan = gen_tree_kws['timespan']
U = gen_tree_kws['U']
if interaction_path.endswith(".json"):
try:
interactions = json.load(open(interaction_path))
except ValueError:
interactions = load_json_by_line(interaction_path)
elif interaction_path.endswith(".pkl"):
interactions = pickle.load(open(interaction_path))
else:
raise ValueError("invalid path extension: {}".format(interaction_path))
logger.info('loading lda from {}'.format(lda_model_path))
if not given_topics:
lda_model = gensim.models.wrappers.LdaMallet.load(
os.path.join(CURDIR, lda_model_path)
)
dictionary = gensim.corpora.dictionary.Dictionary.load(
os.path.join(CURDIR, corpus_dict_path)
)
else:
lda_model = None
dictionary = None
meta_graph_pkl_path = "{}--{}{}.pkl".format(
meta_graph_pkl_path_prefix,
experiment_signature(**meta_graph_kws),
meta_graph_pkl_suffix
)
logger.info('meta_graph_pkl_path: {}'.format(meta_graph_pkl_path))
if calculate_graph or not os.path.exists(meta_graph_pkl_path):
# we want to calculate the graph or
# it's not there so we have to
logger.info('calculating meta_graph...')
meta_graph_kws_copied = copy.deepcopy(meta_graph_kws)
with open(msg_ids_path) as f:
msg_ids = [l.strip() for l in f]
if isinstance(meta_graph_kws_copied['preprune_secs'], timedelta):
meta_graph_kws_copied['preprune_secs'] = meta_graph_kws['preprune_secs'].total_seconds()
g = IU.get_topic_meta_graph(
interactions,
msg_ids=msg_ids,
lda_model=lda_model,
dictionary=dictionary,
undirected=False, # deprecated
given_topics=given_topics,
decompose_interactions=False,
convert_time=convert_time,
**meta_graph_kws_copied
)
logger.info('pickling...')
nx.write_gpickle(
IU.compactize_meta_graph(g, map_nodes=False),
meta_graph_pkl_path
)
else:
logger.info('loading pickle...')
g = nx.read_gpickle(meta_graph_pkl_path)
if print_summary:
logger.debug(get_summary(g))
assert g.number_of_nodes() > 0, 'empty graph!'
if not roots:
cand_tree_number, cand_tree_percent = get_number_and_percentage(
g.number_of_nodes(),
cand_tree_number,
cand_tree_percent
)
if root_sampling_method == 'random':
root_sampler = RandomSampler(g, timespan)
elif root_sampling_method == 'upperbound':
root_sampler = UBSampler(g, U, timespan)
else:
logger.info('init AdaptiveSampler...')
root_sampler = AdaptiveSampler(g, U, timespan)
else:
logger.info('Roots given')
cand_tree_number = len(roots)
root_sampler = DeterministicSampler(g, roots, timespan)
logger.info('#roots: {}'.format(cand_tree_number))
logger.info('#cand_tree_percent: {}'.format(
cand_tree_number / float(g.number_of_nodes()))
)
trees = []
dags = []
for i in xrange(cand_tree_number):
logger.info("sampling root...")
try:
root, dag = root_sampler.take()
except IndexError:
logger.warn('not enough root to take, terminate')
break
dags.append(dag)
start = datetime.now()
tree = calc_tree(i, root, dag, U,
gen_tree_func,
gen_tree_kws,
print_summary,
should_binarize_dag=should_binarize_dag)
tree.graph['calculation_time'] = (datetime.now() - start).total_seconds()
trees.append(tree)
logger.info("updating sampler states...")
root_sampler.update(root, tree)
def make_detailed_path(prefix, suffix):
return "{}--{}----{}----{}{}.pkl".format(
prefix,
experiment_signature(**gen_tree_kws),
experiment_signature(**meta_graph_kws),
experiment_signature(
cand_tree_percent=cand_tree_percent,
root_sampling=root_sampling_method
),
suffix
)
result_pkl_path = make_detailed_path(result_pkl_path_prefix,
result_suffix)
logger.info('result_pkl_path: {}'.format(result_pkl_path))
pickle.dump(trees,
open(result_pkl_path, 'w'),
protocol=pickle.HIGHEST_PROTOCOL)
if False:
# for debugging purpose
pickle.dump(dags,
open(result_pkl_path+'.dag', 'w'),
protocol=pickle.HIGHEST_PROTOCOL)
all_paths_pkl_path = make_detailed_path(all_paths_pkl_prefix,
all_paths_pkl_suffix)
logger.info('Dumping the paths info to {}'.format(all_paths_pkl_path))
paths_dict = {'interactions': interaction_path,
'meta_graph': meta_graph_pkl_path,
'result': result_pkl_path,
'true_events': true_events_path,
'self': all_paths_pkl_path
}
pickle.dump(
paths_dict,
open(all_paths_pkl_path, 'w')
)
return paths_dict
def run(
gen_tree_func,
msg_ids_path,
root_sampling_method='random',
interaction_path=os.path.join(CURDIR, 'data/enron.json'),
lda_model_path=os.path.join(CURDIR, 'models/model-4-50.lda'),
corpus_dict_path=os.path.join(CURDIR, 'models/dictionary.pkl'),
meta_graph_pkl_path_prefix=os.path.join(CURDIR, 'data/enron'),
meta_graph_pkl_suffix='',
cand_tree_number=None, # higher priority than percentage
cand_tree_percent=0.1,
result_pkl_path_prefix=os.path.join(CURDIR, 'tmp/results'),
result_suffix='',
all_paths_pkl_prefix='',
all_paths_pkl_suffix='',
true_events_path='',
meta_graph_kws={
'dist_func': cosine,
'preprune_secs': timedelta(weeks=4),
'distance_weights': {
'topics': 0.2,
'bow': 0.8
},
# 'timestamp_converter': lambda s: s
},
gen_tree_kws={
'timespan': timedelta(weeks=4),
'U': 0.5,
'dijkstra': False
},
convert_time=True,
roots=None,
calculate_graph=False,
given_topics=False,
print_summary=False,
should_binarize_dag=False):
if isinstance(gen_tree_kws['timespan'], timedelta):
timespan = gen_tree_kws['timespan'].total_seconds()
else:
timespan = gen_tree_kws['timespan']
U = gen_tree_kws['U']
if interaction_path.endswith(".json"):
try:
interactions = json.load(open(interaction_path))
except ValueError:
interactions = load_json_by_line(interaction_path)
elif interaction_path.endswith(".pkl"):
interactions = pickle.load(open(interaction_path))
else:
raise ValueError("invalid path extension: {}".format(interaction_path))
logger.info('loading lda from {}'.format(lda_model_path))
if not given_topics:
lda_model = gensim.models.wrappers.LdaMallet.load(
os.path.join(CURDIR, lda_model_path))
dictionary = gensim.corpora.dictionary.Dictionary.load(
os.path.join(CURDIR, corpus_dict_path))
else:
lda_model = None
dictionary = None
meta_graph_pkl_path = "{}--{}{}.pkl".format(
meta_graph_pkl_path_prefix, experiment_signature(**meta_graph_kws),
meta_graph_pkl_suffix)
logger.info('meta_graph_pkl_path: {}'.format(meta_graph_pkl_path))
if calculate_graph or not os.path.exists(meta_graph_pkl_path):
# we want to calculate the graph or
# it's not there so we have to
logger.info('calculating meta_graph...')
meta_graph_kws_copied = copy.deepcopy(meta_graph_kws)
with open(msg_ids_path) as f:
msg_ids = [l.strip() for l in f]
if isinstance(meta_graph_kws_copied['preprune_secs'], timedelta):
meta_graph_kws_copied['preprune_secs'] = meta_graph_kws[
'preprune_secs'].total_seconds()
g = IU.get_topic_meta_graph(
interactions,
msg_ids=msg_ids,
lda_model=lda_model,
dictionary=dictionary,
undirected=False, # deprecated
given_topics=given_topics,
decompose_interactions=False,
convert_time=convert_time,
**meta_graph_kws_copied)
logger.info('pickling...')
nx.write_gpickle(IU.compactize_meta_graph(g, map_nodes=False),
meta_graph_pkl_path)
else:
logger.info('loading pickle...')
g = nx.read_gpickle(meta_graph_pkl_path)
if print_summary:
logger.debug(get_summary(g))
assert g.number_of_nodes() > 0, 'empty graph!'
if not roots:
cand_tree_number, cand_tree_percent = get_number_and_percentage(
g.number_of_nodes(), cand_tree_number, cand_tree_percent)
if root_sampling_method == 'random':
root_sampler = RandomSampler(g, timespan)
elif root_sampling_method == 'upperbound':
root_sampler = UBSampler(g, U, timespan)
else:
logger.info('init AdaptiveSampler...')
root_sampler = AdaptiveSampler(g, U, timespan)
else:
logger.info('Roots given')
cand_tree_number = len(roots)
root_sampler = DeterministicSampler(g, roots, timespan)
logger.info('#roots: {}'.format(cand_tree_number))
logger.info('#cand_tree_percent: {}'.format(cand_tree_number /
float(g.number_of_nodes())))
trees = []
dags = []
for i in xrange(cand_tree_number):
logger.info("sampling root...")
try:
root, dag = root_sampler.take()
except IndexError:
logger.warn('not enough root to take, terminate')
break
dags.append(dag)
start = datetime.now()
tree = calc_tree(i,
root,
dag,
U,
gen_tree_func,
gen_tree_kws,
print_summary,
should_binarize_dag=should_binarize_dag)
tree.graph['calculation_time'] = (datetime.now() -
start).total_seconds()
trees.append(tree)
logger.info("updating sampler states...")
root_sampler.update(root, tree)
def make_detailed_path(prefix, suffix):
return "{}--{}----{}----{}{}.pkl".format(
prefix, experiment_signature(**gen_tree_kws),
experiment_signature(**meta_graph_kws),
experiment_signature(cand_tree_percent=cand_tree_percent,
root_sampling=root_sampling_method), suffix)
result_pkl_path = make_detailed_path(result_pkl_path_prefix, result_suffix)
logger.info('result_pkl_path: {}'.format(result_pkl_path))
pickle.dump(trees,
open(result_pkl_path, 'w'),
protocol=pickle.HIGHEST_PROTOCOL)
if False:
# for debugging purpose
pickle.dump(dags,
open(result_pkl_path + '.dag', 'w'),
protocol=pickle.HIGHEST_PROTOCOL)
all_paths_pkl_path = make_detailed_path(all_paths_pkl_prefix,
all_paths_pkl_suffix)
logger.info('Dumping the paths info to {}'.format(all_paths_pkl_path))
paths_dict = {
'interactions': interaction_path,
'meta_graph': meta_graph_pkl_path,
'result': result_pkl_path,
'true_events': true_events_path,
'self': all_paths_pkl_path
}
pickle.dump(paths_dict, open(all_paths_pkl_path, 'w'))
return paths_dict