def compute_proxEmbedBySubgraph(
wordsEmbeddings=None,
wordsEmbeddings_path=None,
word_dimension=0,
dimension=0,
wordsSize=0,
subpaths_map=None,
subpaths_file=None,
subgraphs_file='',
maxlen_subpaths=1000,
maxlen=100, # Sequence longer then this get ignored
test_data_file='',
top_num=10,
ideal_data_file='',
func=None,
):
model_options = locals().copy()
if wordsEmbeddings is None:
if wordsEmbeddings_path is not None:
wordsEmbeddings,word_dimension,wordsSize=dataProcessTools.getWordsEmbeddings(wordsEmbeddings_path)
else:
exit(0)
subgraphs_map=dataProcessTools.readAllSubgraphDependencyAndSequencesWithLengths(subgraphs_file)
line_count=0
test_map={}
print 'Compute MAP and nDCG for file ',test_data_file
with open(test_data_file) as f:
for l in f:
arr=l.strip().split()
query=int(arr[0])
map={}
for i in range(1,len(arr)):
candidate=int(arr[i])
sequences_data, mask_data, lens_data, subgraph_lens_data, buffer_tensor_data,nodesLens_data=dataProcessTools.prepareDataForTestForSubgraphSingleSequenceWithLengthsAsymmetric(query, candidate, subgraphs_map, dimension)
if sequences_data is None and mask_data is None and lens_data is None:
map[candidate]=-1000.
else:
value=func(sequences_data, mask_data, lens_data, subgraph_lens_data, wordsEmbeddings, buffer_tensor_data, nodesLens_data)
map[candidate]=value
tops_in_line=toolsFunction.mapSortByValueDESC(map, top_num)
test_map[line_count]=tops_in_line
line_count+=1
line_count=0
ideal_map={}
with open(ideal_data_file) as f:
for l in f:
arr=l.strip().split()
arr=[int(x) for x in arr]
ideal_map[line_count]=arr[1:]
line_count+=1
MAP=evaluateTools.get_MAP(top_num, ideal_map, test_map)
MnDCG=evaluateTools.get_MnDCG(top_num, ideal_map, test_map)
return MAP,MnDCG
def compute_proxEmbed(
wordsEmbeddings=None, # words embeddings
wordsEmbeddings_path=None, # the file path of words embeddings
word_dimension=0, # dimension of words embeddings
dimension=0, # the dimension of paths embeddings
wordsSize=0, # the size of words vocabulary
subpaths_map=None, # contains sub-paths
subpaths_file=None, # the file which contains sub-paths
maxlen_subpaths=1000, # the max length for sub-paths
maxlen=100, # Sequence longer then this get ignored
test_data_file='', # the file path of test data
top_num=10, # the top num to predict
ideal_data_file='', # ground truth
func=None, # model function
):
"""
compute the result of the model
"""
model_options = locals().copy()
if wordsEmbeddings is None:
if wordsEmbeddings_path is not None:
wordsEmbeddings, dimension, wordsSize = dataProcessTools.getWordsEmbeddings(
wordsEmbeddings_path)
else:
print 'There is not path for wordsEmbeddings, exit!!!'
exit(0)
if subpaths_map is None:
if subpaths_file is not None:
subpaths_map = dataProcessTools.loadAllSubPaths(
subpaths_file, maxlen_subpaths)
else:
print 'There is not path for sub-paths, exit!!!'
exit(0)
line_count = 0
test_map = {}
print 'Compute MAP and nDCG for file ', test_data_file
with open(test_data_file) as f:
for l in f:
arr = l.strip().split()
query = int(arr[0])
map = {}
for i in range(1, len(arr)):
candidate = int(arr[i])
subPaths_matrix_data, subPaths_mask_data, subPaths_lens_data = dataProcessTools.prepareDataForTest(
query, candidate, subpaths_map)
if subPaths_matrix_data is None and subPaths_mask_data is None and subPaths_lens_data is None:
map[candidate] = -1000.
else:
value = func(subPaths_matrix_data, subPaths_mask_data,
subPaths_lens_data, wordsEmbeddings)
map[candidate] = value
tops_in_line = toolsFunction.mapSortByValueDESC(map, top_num)
test_map[line_count] = tops_in_line
line_count += 1
line_count = 0
ideal_map = {}
with open(ideal_data_file) as f:
for l in f:
arr = l.strip().split()
arr = [int(x) for x in arr]
ideal_map[line_count] = arr[1:]
line_count += 1
MAP = evaluateTools.get_MAP(top_num, ideal_map, test_map)
MnDCG = evaluateTools.get_MnDCG(top_num, ideal_map, test_map)
return MAP, MnDCG
def compute_metagraphAttention(
wordsEmbeddings=None, # words embeddings
wordsEmbeddings_path=None, # the file path of words embeddings
metagraphEmbeddings_path=None, # the file path of metagraph embeddings
wordsSize=0, # the size of words vocabulary
subpaths_map=None, # contains sub-paths
subpaths_file=None, # the file which contains sub-paths
maxlen_subpaths=1000, # the max length for sub-paths
test_data_file='', # test data file
top_num=10, # top num in experiments
ideal_data_file='', # ideal data file
func=None, # the MPE process model
):
"""
evaluate the MPE model
"""
model_options = locals().copy()
if wordsEmbeddings is None:
if wordsEmbeddings_path is not None:
wordsEmbeddings,dimension,wordsSize=dataProcessTools.getWordsEmbeddings(wordsEmbeddings_path)
else:
print 'There is not path for wordsEmbeddings, exit!!!'
exit(0)
if subpaths_map is None:
if subpaths_file is not None:
subpaths_map=dataProcessTools.loadAllSubPathsRomove0Path(subpaths_file, maxlen_subpaths, wordsEmbeddings)
else:
print 'There is not path for sub-paths, exit!!!'
exit(0)
metagraphEmbedding_data, metagraphDimension, metagraphSize=dataProcessTools.getMetagraphEmbeddings(metagraphEmbeddings_path)
line_count=0
test_map={}
print 'Compute MAP and nDCG for file ',test_data_file
with open(test_data_file) as f:
for l in f:
arr=l.strip().split()
query=int(arr[0])
map={}
for i in range(1,len(arr)):
candidate=int(arr[i])
subPaths_matrix_data,subPaths_mask_data,subPaths_lens_data=dataProcessTools.prepareDataForTest(query, candidate, subpaths_map)
if subPaths_matrix_data is None and subPaths_mask_data is None and subPaths_lens_data is None:
map[candidate]=-1000.
else:
value=func(metagraphEmbedding_data, subPaths_matrix_data, subPaths_mask_data, wordsEmbeddings)
map[candidate]=value
del subPaths_matrix_data
del subPaths_mask_data
del subPaths_lens_data
tops_in_line=toolsFunction.mapSortByValueDESC(map, top_num)
test_map[line_count]=tops_in_line
line_count+=1
map=None
gc.collect()
line_count=0
ideal_map={}
with open(ideal_data_file) as f:
for l in f:
arr=l.strip().split()
arr=[int(x) for x in arr]
ideal_map[line_count]=arr[1:]
line_count+=1
MAP=evaluateTools.get_MAP(top_num, ideal_map, test_map)
MnDCG=evaluateTools.get_MnDCG(top_num, ideal_map, test_map)
return MAP,MnDCG
def compute_path2vec(
wordsEmbeddings=None,
wordsEmbeddings_path='None',
typesEmbeddings=None,
typesEmbeddings_path='None',
word_dimension=0,
type_dimension=0,
dimension=0,
attention_dimension=0,
wordsSize=0,
subpaths_map=None,
subpaths_file='',
sequences_map=None,
sequences_file='',
maxlen_subpaths=1000,
maxlen=100, # Sequence longer then this get ignored
alpha=0,
beta=0,
gamma=0,
test_data_file='',
top_num=10,
ideal_data_file='',
func=None,
):
model_options = locals().copy()
if wordsEmbeddings is None:
if wordsEmbeddings_path is not None:
wordsEmbeddings, dimension, wordsSize = dataProcessTools.getWordsEmbeddings(
wordsEmbeddings_path)
else:
print 'Exit...'
exit(0)
if typesEmbeddings is None:
if typesEmbeddings_path is not None:
typesEmbeddings, type_dimension, wordsSize = dataProcessTools.getTypesEmbeddings(
typesEmbeddings_path)
else:
print 'Exit...'
exit(0)
sequences_data = dataProcessTools.readAllSequencesFromFile(sequences_file)
errCount = 0
line_count = 0
test_map = {}
print 'Compute MAP and nDCG for file ', test_data_file
with open(test_data_file) as f:
for l in f:
arr = l.strip().split()
query = int(arr[0])
map = {}
candidates = []
for i in range(1, len(arr)):
key1 = arr[0] + '-' + arr[i]
key2 = arr[i] + '-' + arr[0]
if key1 in sequences_data or key2 in sequences_data:
candidates.append(int(arr[i]))
else:
map[int(arr[i])] = -1000.
errCount += 1
sequences_matrix, dependency_matrix, dependWeight_matrix, sequencesLen_vector, discountSeq_matrix, discountForEachNode_matrix, masks_matrix, group_tensor = dataProcessTools.prepareDataForTestBatch(
query, candidates, sequences_data, alpha, beta, gamma)
if len(sequences_matrix) > 0:
scores = func(sequences_matrix, dependency_matrix,
dependWeight_matrix, sequencesLen_vector,
discountSeq_matrix, discountForEachNode_matrix,
wordsEmbeddings, typesEmbeddings, masks_matrix,
group_tensor)
for index in range(len(candidates)):
map[candidates[index]] = scores[index]
else:
for i in range(1, len(arr)):
map[int(arr[i])] = -1.
tops_in_line = toolsFunction.mapSortByValueDESC(map, top_num)
test_map[line_count] = tops_in_line
line_count += 1
if line_count % 500 == 0:
print '+',
if line_count % 5000 == 0:
print ' time ==', time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
line_count = 0
ideal_map = {}
with open(ideal_data_file) as f:
for l in f:
arr = l.strip().split()
arr = [int(x) for x in arr]
ideal_map[line_count] = arr[1:]
line_count += 1
MAP = evaluateTools.get_MAP(top_num, ideal_map, test_map)
MnDCG = evaluateTools.get_MnDCG(top_num, ideal_map, test_map)
print 'errCount =', errCount
return MAP, MnDCG
def compute_path2vec(
wordsEmbeddings=None,
wordsEmbeddings_path='None',
typesEmbeddings=None,
typesEmbeddings_path='None',
word_dimension=0,
type_dimension=0,
dimension=0,
attention_dimension=0,
wordsSize=0,
subpaths_map=None,
subpaths_file='',
sequences_map=None,
sequences_file='',
maxlen_subpaths=1000,
maxlen=100,
alpha=0,
beta=0,
gamma=0,
test_data_file='',
top_num=10,
ideal_data_file='',
func=None,
):
model_options = locals().copy()
if wordsEmbeddings is None:
if wordsEmbeddings_path is not None:
wordsEmbeddings,dimension,wordsSize=dataProcessTools.getWordsEmbeddings(wordsEmbeddings_path)
else:
exit(0)
if typesEmbeddings is None:
if typesEmbeddings_path is not None:
typesEmbeddings,type_dimension,wordsSize=dataProcessTools.getTypesEmbeddings(typesEmbeddings_path)
else:
exit(0)
sequences_data=dataProcessTools.readAllSequencesFromFile(sequences_file)
errCount=0
line_count=0
test_map={}
print 'Compute MAP and nDCG for file ',test_data_file
with open(test_data_file) as f:
for l in f:
arr=l.strip().split()
query=int(arr[0])
map={}
for i in range(1,len(arr)):
candidate=int(arr[i])
sequences_matrix, dependency_matrix, dependWeight_matrix, sequencesLen_vector, discountSeq_matrix, discountForEachNode_matrix=dataProcessTools.prepareDataForTest(query, candidate, sequences_data, alpha, beta, gamma)
if sequences_matrix is None or len(sequences_matrix)==0:
map[candidate]=-1000.
errCount+=1
else:
value=func(sequences_matrix, dependency_matrix, dependWeight_matrix, sequencesLen_vector, discountSeq_matrix, discountForEachNode_matrix,wordsEmbeddings,typesEmbeddings)
map[candidate]=value
tops_in_line=toolsFunction.mapSortByValueDESC(map, top_num)
test_map[line_count]=tops_in_line
line_count+=1
line_count=0
ideal_map={}
with open(ideal_data_file) as f:
for l in f:
arr=l.strip().split()
arr=[int(x) for x in arr]
ideal_map[line_count]=arr[1:]
line_count+=1
MAP=evaluateTools.get_MAP(top_num, ideal_map, test_map)
MnDCG=evaluateTools.get_MnDCG(top_num, ideal_map, test_map)
print 'errCount =',errCount
return MAP,MnDCG
def compute_proxEmbed(
wordsEmbeddings=None, # words embeddings
wordsEmbeddings_path=None, # the file path of words embeddings
word_dimension=0, # dimension of words embeddings
dimension=0, # the dimension of paths embeddings
wordsSize=0, # the size of words vocabulary
subpaths_map=None, # contains sub-paths
subpaths_file=None, # the file which contains sub-paths
maxlen_subpaths=1000, # the max length for sub-paths
maxlen=100, # Sequence longer then this get ignored
test_data_file='', # the file path of test data
top_num=10, # the top num to predict
ideal_data_file='', # ground truth
func=None, # model function
):
"""
compute the result of the model
"""
model_options = locals().copy()
if wordsEmbeddings is None:
if wordsEmbeddings_path is not None:
wordsEmbeddings, dimension, wordsSize = dataProcessTools.getWordsEmbeddings(wordsEmbeddings_path)
else:
print 'There is not path for wordsEmbeddings, exit!!!'
exit(0)
if subpaths_map is None:
if subpaths_file is not None:
subpaths_map = dataProcessTools.loadAllSubPaths(subpaths_file, maxlen_subpaths)
else:
print 'There is not path for sub-paths, exit!!!'
exit(0)
line_count = 0
test_map = {}
print 'Compute MAP and nDCG for file ', test_data_file
with open(test_data_file) as f:
for l in f:
arr = l.strip().split()
query = int(arr[0])
map = {}
count_none = 0
count_have = 0
for i in range(1, len(arr)):
candidate = int(arr[i])
subPaths_matrix_data, subPaths_mask_data, subPaths_lens_data = dataProcessTools.prepareDataForTestAsymmetric(
query, candidate, subpaths_map)
if subPaths_matrix_data is None and subPaths_mask_data is None and subPaths_lens_data is None:
map[candidate] = -1000.
# print(candidate, "subPaths_data is None")
count_none += 1
else:
count_have += 1
value = func(subPaths_matrix_data, subPaths_mask_data, subPaths_lens_data, wordsEmbeddings)
map[candidate] = value
print("count_none", count_none, "count_have", count_have)
tops_in_line = toolsFunction.mapSortByValueDESC(map, top_num)
test_map[line_count] = tops_in_line
line_count += 1
# print("map:", map)
line_count = 0
ideal_map = {}
with open(ideal_data_file) as f:
for l in f:
arr = l.strip().split()
arr = [int(x) for x in arr]
ideal_map[line_count] = arr[1:]
line_count += 1
# MAP=evaluateTools.get_MAP(top_num, ideal_map, test_map)
MnDCG10 = evaluateTools.get_MnDCG(10, ideal_map, test_map)
MnDCG20 = evaluateTools.get_MnDCG(20, ideal_map, test_map)
# print("top_num:", top_num)
# print("ideal_map:", ideal_map)
# print("test_map:", test_map)
return MnDCG10, MnDCG20