def start_ranker(self, isAsync, isRepeat, text_boost=1, title_boost=1000,
bm25_b=0.75, bm25_k1=2):
settings = {
"text_boost": text_boost,
"title_boost": title_boost,
"bm25_b": bm25_b,
"bm25_k1": bm25_k1
}
def _check_crawler(event):
event.set()
while event.is_set():
time.sleep(0.2)
print "Ranker finished"
def _check_repeat_crawler(_event_ranker, _event_stop_repeat):
_event_stop_repeat.clear()
_event_ranker.clear()
proc = None
while not _event_stop_repeat.is_set():
if _event_ranker.is_set():
time.sleep(0.5)
else:
_event_ranker.set()
if proc is not None and proc.is_alive():
proc.terminate()
proc = Ranker.Ranker(settings, _event_ranker)
proc.start()
else:
print "Ranker finished"
if not isAsync and isRepeat:
raise Exception("You will be in deadlock, if you use that configure (Sync and Repeat)")
event_finish_ranker = Event()
if isAsync:
if isRepeat:
repeat_crawler_process = Process(target=_check_repeat_crawler,
args=(event_finish_ranker, self.event_repeat))
repeat_crawler_process.start()
else:
process = Ranker.Ranker(settings, event_finish_ranker)
process.start()
check_process = Process(target=_check_crawler, args=(event_finish_ranker,))
check_process.start()
else:
process = Ranker.Ranker(settings, event_finish_ranker)
process.start()
_check_crawler(event_finish_ranker)
print "You may continue work"
def match_fuzzy(self, uni_query='', idx={}):
if not idx: return []
segmenter = Segmenter.get_seg(id=idx['seg'])
prefix = idx['prefix']
list_s = segmenter(uni_query)
list_bitset, list_docid, len_list_docid = [], [], 0
for s in list_s:
str_s = redis_zero.hget(prefix, s)
if str_s:
yhBitset = YhBitset.YhBitset()
yhBitset.frombytes(str_s)
list_bitset.append(yhBitset)
logger.error('%s matched len %s' % (s, yhBitset.length()))
else:
logger.error('%s filtered' % s)
bitset = YhBitset.YhBitset()
if list_bitset:
bitset = list_bitset[0]
for bs in list_bitset[1:]:
test = bitset.anditem(bs)
if test.length() <= 10:
break
bitset = test
logger.error('test_length %s' % test.length())
#logger.error('match_title seg %s len %s ids %s' % ('|'.join(list_s), len(list_docid), list_docid[:3]))
list_docid = bitset.search(200, 1)
list_docid = ['%s' % id for id in list_docid]
list_docid = Ranker.Ranker().getRank(name='unigram_rank',
list_id=list_docid)
logger.error('match_fuzzy %s' % list_docid)
return list_docid[:200]
def full_match(self, uni_query="", idx={}):
if not idx: return []
prefix = idx['prefix']
list_s = [uni_query]
logger.error("================lists:%s" % ("|".join(list_s)))
list_bitset, list_docid, len_list_docid = [], [], 0
for s in list_s:
str_s = redis_zero.hget(prefix, s)
if str_s:
yhBitset = YhBitset.YhBitset()
yhBitset.frombytes(str_s)
list_bitset.append(yhBitset)
logger.error('%s matched len %s' % (s, yhBitset.length()))
else:
logger.error('%s filtered' % s)
bitset = YhBitset.YhBitset()
if list_bitset:
bitset = list_bitset[0]
for bs in list_bitset[1:]:
test = bitset.anditem(bs)
if test.length() <= 0:
break
bitset = test
list_docid = bitset.search(200, 1)
list_docid = ['%s' % id for id in list_docid]
list_docid = Ranker.Ranker().getRank(name='unigram_rank',
list_id=list_docid)
logger.error('match [%s] [%s] [%s]' % (uni_query, list_s, list_docid))
return list_docid[:200]
def _check_repeat_crawler(_event_ranker, _event_stop_repeat):
_event_stop_repeat.clear()
_event_ranker.clear()
proc = None
while not _event_stop_repeat.is_set():
if _event_ranker.is_set():
time.sleep(0.5)
else:
_event_ranker.set()
if proc is not None and proc.is_alive():
proc.terminate()
proc = Ranker.Ranker(settings, _event_ranker)
proc.start()
else:
print "Ranker finished"
def search(posting_path, query, stemmer, query_source_path, list_of_language,
list_of_city, semantic):
Parse.set_stop_words_file(posting_path + "/stop_words.txt")
list_save_queries = Parse.parse_queries(query_source_path, posting_path,
query, stemmer, semantic)
res = {}
for query_post in list_save_queries:
fileName = posting_path + "/" + query_post + ".pkl"
file = open(fileName, "rb+")
querie_term_dictionary = pickle.load(file)
file.close()
os.remove(fileName)
query_name = query_post.replace('post', "")
res[query_name] = Ranker.rank(posting_path, stemmer,
querie_term_dictionary, list_of_language,
list_of_city)
return res
def match(self, uni_query='', idx={}, full=0):
if not idx: return []
segmenter = Segmenter.get_seg(id=idx['seg'])
prefix = idx['prefix']
list_s = segmenter(uni_query)
logger.error("================lists:[%s]\t[%s]\t[%s]" %
(uni_query, idx['seg'], "|".join(list_s)))
list_bitset, list_docid, len_list_docid = [], [], 0
for s in list_s:
str_s = redis_zero.hget(prefix, s)
if str_s:
yhBitset = YhBitset.YhBitset()
yhBitset.frombytes(str_s)
list_bitset.append(yhBitset)
logger.error('%s matched len %s' % (s, yhBitset.length()))
else:
logger.error('%s filtered' % s)
bitset = YhBitset.YhBitset()
bitset_join_len = 0
if list_bitset:
bitset = list_bitset[0]
bitset_join_len += 1
for bs in list_bitset[1:]:
test = bitset.anditem(bs)
if test.length() <= 0:
break
bitset = test
bitset_join_len += 1
if full:
logger.error('===============match full %s %s' %
(bitset_join_len, len(list_s)))
if bitset_join_len < floor(0.8 * len(list_s)):
logger.error('===============match not enough long %s %s' %
(bitset_join_len, len(list_s)))
return []
#bitset= bitset.oritem(bitset_right)
list_docid = bitset.search(200, 1)
list_docid = ['%s' % id for id in list_docid]
if list_docid:
list_docid = Ranker.Ranker().getRank(name='unigram_rank',
list_id=list_docid)
logger.error('match [%s] [%s] [%s]' % (uni_query, list_s, list_docid))
return list_docid[:200]
def eval_batch(fts,
captioner,
retriever,
args,
train_mode=False,
optimizer=None):
criterion = nn.TripletMarginLoss(reduction='mean',
margin=args.margin).to(device)
# generate a mapping for dev, to ensure sampling bias is reduced
num_target = len(fts['asins'])
batch_size = args.batch_size
ranker = Ranker.Ranker(device)
total_step = math.floor(num_target / batch_size)
ranking_tracker = [0] * args.num_dialog_turns
loss_tracker = [0] * args.num_dialog_turns
with open('data/shuffled.{}.{}.json'.format(args.data_set, 'val')) as f:
first_candidate_set = json.load(f)
with torch.no_grad():
retriever.eval()
ranker.update_emb(fts, args.batch_size, retriever)
if train_mode:
retriever.train()
else:
retriever.eval()
for step in tqdm.tqdm(range(total_step)):
# sample target
if train_mode:
target_ids = torch.tensor([0] * args.batch_size,
device=device,
dtype=torch.long)
target_ids.random_(0, num_target)
else:
target_ids = torch.tensor([
i for i in range(step * batch_size, (step + 1) * batch_size)
]).to(device=device, dtype=torch.long)
# sample first batch of candidates
if train_mode:
candidate_ids = torch.tensor([0] * args.batch_size,
device=device,
dtype=torch.long)
candidate_ids.random_(0, num_target)
else:
candidate_ids = torch.tensor([
first_candidate_set[i]
for i in range(step * batch_size, (step + 1) * batch_size)
],
device=device,
dtype=torch.long)
# target_ids.random_(0, num_target)
target_img_ft = utils.get_image_batch(fts, target_ids)
target_img_ft = target_img_ft.to(device)
target_img_emb = retriever.encode_image(target_img_ft)
target_attr = utils.get_attribute_batch(fts, target_ids)
target_attr = target_attr.to(device)
# clean up dialog history tracker
retriever.init_hist()
# history_hidden = history_hidden.expand_as(target_img_emb)
# history_hidden = None
loss = 0
for d_turn in range(args.num_dialog_turns):
# get candidate image features
candidate_img_ft = utils.get_image_batch(fts, candidate_ids)
candidate_img_ft = candidate_img_ft.to(device)
candidate_attr = utils.get_attribute_batch(fts, candidate_ids)
candidate_attr = candidate_attr.to(device)
# generate captions from model
with torch.no_grad():
sentence_ids = captioner.get_caption(target_img_ft,
candidate_img_ft,
target_attr,
candidate_attr)
sentence_ids = sentence_ids.to(device)
candidate_img_ft = candidate_img_ft.to(device)
history_hidden = retriever.forward(text=sentence_ids,
image=candidate_img_ft,
attribute=candidate_attr)
# sample negatives, update tracker's output to
# match targets via triplet loss
negative_ids = torch.tensor([0] * args.batch_size,
device=device,
dtype=torch.long)
negative_ids.random_(0, num_target)
negative_img_ft = utils.get_image_batch(fts, negative_ids)
negative_img_ft = negative_img_ft.to(device)
negative_img_emb = retriever.encode_image(negative_img_ft)
# accumulate loss
loss_tmp = criterion(history_hidden, target_img_emb,
negative_img_emb)
loss += loss_tmp
loss_tracker[d_turn] += loss_tmp.item()
# generate new candidates, compute ranking information
with torch.no_grad():
candidate_ids = ranker.nearest_neighbors(history_hidden)
ranking = ranker.compute_rank(history_hidden, target_ids)
ranking_tracker[d_turn] += (ranking.mean().item() /
(num_target * 1.0))
# update weights
if train_mode:
optimizer.zero_grad()
# loss = loss / args.num_dialog_turns
loss.backward()
# clip_grad_norm_(retriever.parameters(), args.clip_norm)
optimizer.step()
with torch.no_grad():
retriever.eval()
ranker.update_emb(fts, args.batch_size, retriever)
retriever.train()
loss = loss.item() / total_step
for i in range(args.num_dialog_turns):
ranking_tracker[i] /= total_step
loss_tracker[i] /= total_step
metrics = {
'loss': loss,
'score': 5 - sum(ranking_tracker),
'loss_tracker': loss_tracker,
'ranking_tracker': ranking_tracker
}
return metrics
def test(self, passages):
keywords = getKeyWords(self.desc, STOP_WORDS)
return Ranker.passage_rankings(self.desc, passages, keywords)
def top5(self, passages):
keywords = getKeyWords(self.desc, STOP_WORDS)
return zip(*Ranker.rank_passages(self.desc, passages, keywords)[:5])[1]
def eval_batch(fts, captioner, retriever, args):
criterion = nn.TripletMarginLoss(reduction='mean',
margin=args.margin).to(device)
# generate a mapping for dev, to ensure sampling bias is reduced
num_target = len(fts['asins'])
batch_size = args.batch_size
ranker = Ranker.Ranker(device)
total_step = math.floor(num_target / batch_size)
ranking_tracker = [0] * args.num_dialog_turns
loss_tracker = [0] * args.num_dialog_turns
with open('data/shuffled.{}.{}.json'.format(args.data_set, 'test')) as f:
first_candidate_set = json.load(f)
with torch.no_grad():
retriever.eval()
ranker.update_emb(fts, args.batch_size, retriever)
retriever.eval()
ret_results = {}
total_time = 0
for step in tqdm.tqdm(range(total_step)):
# sample target
target_ids = torch.tensor([
i for i in range(step * batch_size, (step + 1) * batch_size)
]).to(device=device, dtype=torch.long)
# sample first batch of candidates
candidate_ids = torch.tensor([
first_candidate_set[i]
for i in range(step * batch_size, (step + 1) * batch_size)
],
device=device,
dtype=torch.long)
# keep track of results
ret_result = {}
for batch_id in range(target_ids.size(0)):
idx = target_ids[batch_id].cpu().item()
ret_result[idx] = {}
ret_result[idx]['candidate'] = []
ret_result[idx]['ranking'] = []
ret_result[idx]['caption'] = []
target_img_ft = utils.get_image_batch(fts, target_ids)
target_img_ft = target_img_ft.to(device)
target_img_emb = retriever.encode_image(target_img_ft)
target_attr = utils.get_attribute_batch(fts, target_ids)
target_attr = target_attr.to(device)
# clean up dialog history tracker
retriever.init_hist()
# history_hidden = history_hidden.expand_as(target_img_emb)
loss = 0
for d_turn in range(args.num_dialog_turns):
last_timer = int(round(time.time() * 1000))
# get candidate image features
candidate_img_ft = utils.get_image_batch(fts, candidate_ids)
candidate_img_ft = candidate_img_ft.to(device)
candidate_attr = utils.get_attribute_batch(fts, candidate_ids)
candidate_attr = candidate_attr.to(device)
# generate captions from model
total_time += (int(round(time.time() * 1000)) - last_timer)
with torch.no_grad():
sentence_ids, caps = captioner.get_caption(target_img_ft,
candidate_img_ft,
target_attr,
candidate_attr,
return_cap=True)
last_timer = int(round(time.time() * 1000))
sentence_ids = sentence_ids.to(device)
candidate_img_ft = candidate_img_ft.to(device)
history_hidden = retriever.forward(text=sentence_ids,
image=candidate_img_ft,
attribute=candidate_attr)
# sample negatives, update tracker's output to
# match targets via triplet loss
negative_ids = torch.tensor([0] * args.batch_size,
device=device,
dtype=torch.long)
negative_ids.random_(0, num_target)
negative_img_ft = utils.get_image_batch(fts, negative_ids)
negative_img_ft = negative_img_ft.to(device)
negative_img_emb = retriever.encode_image(negative_img_ft)
# accumulate loss
loss_tmp = criterion(history_hidden, target_img_emb,
negative_img_emb)
loss += loss_tmp
loss_tracker[d_turn] += loss_tmp.item()
# generate new candidates, compute ranking information
with torch.no_grad():
candidate_ids = ranker.nearest_neighbors(history_hidden)
ranking = ranker.compute_rank(history_hidden, target_ids)
ranking_tracker[d_turn] += (ranking.mean().item() /
(num_target * 1.0))
for batch_id in range(target_ids.size(0)):
idx = target_ids[batch_id].cpu().item()
ret_result[idx]['caption'].append(caps[batch_id])
ret_result[idx]['candidate'].append(
candidate_ids[batch_id].item())
ret_result[idx]['ranking'].append(ranking[batch_id].item())
total_time += (int(round(time.time() * 1000)) - last_timer)
ret_results.update(ret_result)
loss = loss.item() / total_step
for i in range(args.num_dialog_turns):
ranking_tracker[i] /= total_step
loss_tracker[i] /= total_step
metrics = {
'loss': loss,
'score': 5 - sum(ranking_tracker),
'loss_tracker': loss_tracker,
'ranking_tracker': ranking_tracker,
'retrieve_time': total_time / float(num_target)
}
return metrics, ret_results
def main(argv):
collectionFile = ''
tokenizerType = ''
queriesFile = ''
rankType = ''
start = []
end = []
try:
opts, args = getopt.getopt(argv, "hf:t:q:r:", ["collectionFile=", "tokenizerType=", "queriesFilePath=",
"rankType="])
except getopt.GetoptError:
print('main.py -f <collectionFile> -t <tokenizerType: 0 - Simple, 1 - Better> -q <queriesFilePath> '
'-r <rankType: 0 - TF-IDF, 1 - BM25>')
sys.exit()
if len(opts) != 4:
print('main.py -f <collectionFile> -t <tokenizerType: 0 - Simple, 1 - Better> -q <queriesFilePath> '
'-r <rankType: 0 - TF-IDF, 1 - BM25>')
sys.exit()
for opt, arg in opts:
if opt == '-h':
print('main.py -f <collectionFile> -t <tokenizerType: 0 - Simple, 1 - Better> -q <queriesFilePath> '
'-r <rankType: 0 - TF-IDF, 1 - BM25>')
sys.exit()
elif opt in ("-f", "--collectionFile"):
if not path.exists(arg):
print('Incorrect path to collection file.')
sys.exit()
collectionFile = arg
elif opt in ("-t", "--tokenizerType"):
if arg != '0' and arg != '1':
print('Incorrect tokenizer type. Simple tokenizer: 0, Better tokenizer: 1.')
sys.exit()
tokenizerType = arg
elif opt in ("-q", "--queriesFilePath"):
if not path.exists(arg):
print('Incorrect path to queries file.')
sys.exit()
queriesFile = arg
elif opt in ("-r", "--rankType"):
if arg != '0' and arg != '1':
print('Incorrect rank type. TF-IDF: 0, BM25: 1.')
sys.exit()
rankType = arg
# Indexer
(Indexer(collectionFile, tokenizerType)).writeIndexToFile('index')
f = open(queriesFile, 'r')
queries = f.read().splitlines()
f.close()
scores = []
if tokenizerType == '0': # simple
tokenizer = Tokenizer.SimpleTokenizer('')
else: # better
tokenizer = Tokenizer.BetterTokenizer('')
for query in queries:
# Query Operations
tokenizer.changeText(query)
queryTerms = tokenizer.getTerms()
# Searcher
documentsInfo, avgDocLen = Searcher.searchDocuments(queryTerms, 'index')
# Ranker
ranker = Ranker(documentsInfo, avgDocLen)
# Start time (latency purpose)
start.append(timer())
# If rankType = 0 (tf-idf)
if rankType == '0':
scores += [ranker.lnc_ltc()]
# If rankType = 1 (BM25)
else:
scores += [ranker.bm25(1.2, 0.75)]
# End time (latency purpose)
end.append(timer())
# Evaluation
Evaluation.getResults('./data/queries.relevance.txt', queries, scores, start, end)
def main(argv):
# ----------------------------------------- HANDLING PROGRAM INPUT -------------------------------------------------
collectionFile = ''
tokenizerType = ''
queriesFile = ''
rankType = ''
storePos = ''
proximity = ''
try:
opts, args = getopt.getopt(argv, "hf:t:q:r:p:b:", ["collectionFile=", "tokenizerType=", "queriesFilePath=",
"rankType=", "storePositions=", "proximityBoost="])
except getopt.GetoptError:
print('main.py -f <collectionFile> -t <tokenizerType: 0 - Simple, 1 - Better> -q <queriesFilePath> '
'-r <rankType: 0 - TF-IDF, 1 - BM25> -p <storePositions: 0 - No, 1 - Yes> '
'-b <proximityBoost: 0 - No, 1 - Yes>')
sys.exit()
if len(opts) != 6:
print('main.py -f <collectionFile> -t <tokenizerType: 0 - Simple, 1 - Better> -q <queriesFilePath> '
'-r <rankType: 0 - TF-IDF, 1 - BM25> -p <storePositions: 0 - No, 1 - Yes> '
'-b <proximityBoost: 0 - No, 1 - Yes>')
sys.exit()
for opt, arg in opts:
if opt == '-h':
print('main.py -f <collectionFile> -t <tokenizerType: 0 - Simple, 1 - Better> -q <queriesFilePath> '
'-r <rankType: 0 - TF-IDF, 1 - BM25> -p <storePositions: 0 - No, 1 - Yes> '
'-b <proximityBoost: 0 - No, 1 - Yes>')
sys.exit()
elif opt in ("-f", "--collectionFile"):
if not path.exists(arg):
print('Incorrect path to collection file.')
sys.exit()
collectionFile = arg
elif opt in ("-t", "--tokenizerType"):
if arg != '0' and arg != '1':
print('Incorrect tokenizer type. Simple tokenizer: 0, Better tokenizer: 1.')
sys.exit()
tokenizerType = arg
elif opt in ("-q", "--queriesFilePath"):
if not path.exists(arg):
print('Incorrect path to queries file.')
sys.exit()
queriesFile = arg
elif opt in ("-r", "--rankType"):
if arg != '0' and arg != '1':
print('Incorrect rank type. TF-IDF: 0, BM25: 1.')
sys.exit()
rankType = arg
elif opt in ("-p", "--storePositions"):
if arg != '0' and arg != '1':
print('\nIncorrect store positions choice. No: 0, Yes: 1.')
sys.exit()
storePos = arg
elif opt in ("-b", "--proximityBoost"):
if arg != '0' and arg != '1':
print('\nIncorrect proximity boost choice. No: 0, Yes: 1.')
sys.exit()
proximity = arg
# ----------------------------------------------- INDEXER ----------------------------------------------------------
indexer = Indexer(collectionFile, tokenizerType, True if storePos=='1' else False)
start = timeit.default_timer()
indexer.index()
stop = timeit.default_timer()
print('Indexing total time - {} tokenizer: {} min and {} seconds'.format("simple" if tokenizerType == "0" else "better", (stop - start)//60, (stop - start) % 60))
f = open(queriesFile, 'r')
queries = f.read().splitlines()
f.close()
scores = []
if tokenizerType == '0': # simple
tokenizer = Tokenizer.SimpleTokenizer('')
else: # better
tokenizer = Tokenizer.BetterTokenizer('')
start_queries = []
end_queries = []
time_searcher = 0
time_ranker = 0
for query in queries:
# --------------------------------------- QUERY OPERATIONS -----------------------------------------------------
tokenizer.changeText(query)
#queryTerms, queryTermsPositions = tokenizer.getTerms(withPositions=True if storePos == '1' else False)
queryTerms = tokenizer.getTerms(withPositions=False)
# ------------------------------------------- SEARCHER ---------------------------------------------------------
start = timeit.default_timer()
documentsInfo, avgDocLen = Searcher.searchDocuments(queryTerms, 'index', True if storePos == '1' else False)
stop = timeit.default_timer()
time_searcher = time_searcher + stop - start
# -------------------------------------------- RANKER ----------------------------------------------------------'
start = timeit.default_timer()
ranker = Ranker(documentsInfo, avgDocLen)
# Start time (latency purpose)
start_queries.append(timer())
# If rankType = 0 (tf-idf)
if rankType == '0':
# If proximity = 1 (Proximity Boost)
if proximity == '1':
scores += [ranker.proximity_boost(ranker.lnc_ltc(), queryTerms)]
else:
scores += [ranker.lnc_ltc()]
# If rankType = 1 (BM25)
else:
# If proximity = 1 (Proximity Boost)
if proximity == '1':
scores += [ranker.proximity_boost(ranker.bm25(1.2, 0.75), queryTerms)]
else:
scores += [ranker.bm25(1.2, 0.75)]
stop = timeit.default_timer()
time_ranker = time_ranker + stop - start
# End time (latency purpose)
end_queries.append(timer())
print('Searching time for all queries: {} min and {} seconds'.format(time_searcher // 60, time_searcher % 60))
print('Ranking time for all queries: {} min and {} seconds'.format(time_ranker // 60, time_ranker % 60))
# Evaluation
Evaluation.getResults('./data/queries.relevance.txt', queries, scores, start_queries, end_queries)
def write_out(self,
epoch,
names_test,
num_img_test,
inp_feat_test,
rank_algo,
out_file='results_nn'):
start_idx = -1
end_idx = 0
pred_file = open(out_file, 'r+')
pred_file.seek(0)
for i in range(len(num_img_test)):
print "Predicting for video: ", i + 78
vid_name = "video_" + str(i + 78)
start_idx = end_idx
end_idx = start_idx + num_img_test[i]
interest_vec_sp = 0.0 * np.array(range(num_img_test[i]))
interest_vec_pp = 0.0 * np.array(range(num_img_test[i]))
binary_int = 0 * np.array(range(num_img_test[i]))
num_int = int(0.12 * num_img_test[i]) + 1
cur_PPM = np.zeros([num_img_test[i], num_img_test[i]])
for j in range(start_idx, end_idx, 1):
for k in range(j + 1, end_idx, 1):
pred = self.model.predict(
np.append(inp_feat_test[j],
inp_feat_test[k]).reshape(1, -1))[0][0]
pred_int = int(np.round(pred))
cur_PPM[j - start_idx, k - start_idx] = pred_int
interest_vec_pp[pred_int * (k - start_idx) +
(1 - pred_int) *
(j - start_idx)] += abs(pred - 0.5)
interest_vec_sp[
pred_int * (k - start_idx) + (1 - pred_int) *
(j - start_idx
)] += 1 #Less interesting image gets 1 added in score
if (rank_algo == 'sp'):
interest_vec = interest_vec_sp
temp = interest_vec.argsort()
ranks = np.arange(len(interest_vec))[temp.argsort()]
ranks = ranks + 1
elif (rank_algo == 'pp'):
interest_vec = interest_vec_pp
temp = interest_vec.argsort()
ranks = np.arange(len(interest_vec))[temp.argsort()]
ranks = ranks + 1
elif (rank_algo == 'mih_to'):
ranks = Ranker.mih_to(cur_PPM)
elif (rank_algo == 'mih_ro'):
ranks = Ranker.mih_ro(cur_PPM)
binary_int = (ranks <= num_int).astype(int)
print "Predictions made for video: ", i + 78
for j in range(num_img_test[i]):
str_line = vid_name + ',' + str(names_test[i][j]) + ',' + str(
binary_int[j]) + ',' + str(1.0 / ranks[j])
pred_file.write(str_line + '\n')
pred_file.close()
return
def cycle_1_min():
#Pass the timestamp to the various programs
now = datetime.datetime.now()
#File components
iteration = now
iteration_str = "{year} {month} {day} {hour} {minute}".format(
year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute)
iteration_file_puller = "Puller {year} {month} {day} {hour} {minute}.sqlite".format(
year=now.year,
month=now.month,
day=now.day,
hour=now.hour,
minute=now.minute)
#Do Puller
pullerids = [
"%23soapbox -filter:retweets -filter:media",
"@internetsoapbox -filter:retweets -filter:media",
]
print("Pulling file = ", iteration_file_puller)
startpull = Puller.Scrape(iteration_file_puller, pullerids)
startpull.main()
print("Pulling complete!")
pullerfinish = datetime.datetime.now() - now
print("Time to Pull: ", pullerfinish)
#Do Alltime Ranker
print("Ranking alltime...")
alltime = Ranker.Ranker(iteration, iteration_str, "alltime")
alltime.main()
print("Ranking alltime complete!")
#Do Daily Ranker
print("Ranking daily...")
daily = Ranker.Ranker(iteration, iteration_str, "daily")
daily.main()
print("Ranking daily complete!")
#Do Hourly Ranker
print("Ranking hourly...")
hourly = Ranker.Ranker(iteration, iteration_str, "hourly")
hourly.main()
print("Ranking hourly complete!")
#Program Time
finishtime = datetime.datetime.now() - now
print("Time to Complete: ", finishtime)
#return startvar and stable iterations
global stable_iteration
stable_iteration = iteration
global stable_iteration_str
stable_iteration_str = iteration_str
global regen_popcorn_flag
regen_popcorn_flag = 1
global first_start_var
first_start_var = 1
