def search(page=0):
try:
novels = []
words = util.encode_string(request.args['words']).split()
user_query = g.db_session.query(User).filter(User.status == True)
novel_query = g.db_session.query(Novel).filter(Novel.status == True)
tag_query = g.db_session.query(Tag).filter(Tag.status == True)
for word in words:
user_query = user_query.filter(User.name.like('%' + word+ '%'))
title_query = novel_query.filter(Novel.title.like('%' + word + '%'))
summary_query = novel_query.filter(Novel.summary.like('%' + word + '%'))
tag_query = tag_query.filter(Tag.tag.like('%' + word +'%'))
for user in user_query:
novels += [novel for novel in user.novel_list if novel.status]
novels += title_query.all() + summary_query.all()
tags = tag_query.all()
novels += [tag.novel for tag in tags if tag.novel.status]
if not novels:
return page_not_found(u'検索結果がないです')
novellist = list(set(novels))
novels = sorted(util.search_result(page, novellist),
key=lambda x: x.id)
page += 1
return render_template('search.htm', novels=novels, page=page,
reqpage="search", words=request.args['words'],
pnum=int(len(novellist) / page), conf=g.config)
except:
return internal_server_error(u'リロードしてみてください')
def print_final_result(result_list, process_gather=False):
if process_gather:
final_result = {}
for result_dict_form_rank in result_list:
if result_dict_form_rank is None:
continue
for grid_id in result_dict_form_rank:
if grid_id not in final_result:
rs = search_result()
rs.id = grid_id
final_result[grid_id] = rs
final_result[grid_id].add_hash_tags(
result_dict_form_rank[grid_id].hash_tags)
final_result[grid_id].add_num_of_post(
result_dict_form_rank[grid_id].num_of_post)
else:
final_result = result_list
# Process The Result
for grid_id in final_result:
final_result[grid_id].process_result()
final_result = [v for v in final_result.values()]
final_result = sorted(final_result,
key=lambda x: x.num_of_post,
reverse=True)
# Print total number of Twitter posts
print(('=' * 30) + ' total number of Twitter posts' + ('=' * 30))
for item in final_result:
if item.top_5_string is not None:
disply = '{:1s}: {:6d} posts - {:10s}'.format(
item.id, item.num_of_post, item.top_5_string)
else:
disply = '{:1s}: {:6d} posts - None'.format(
item.id, item.num_of_post)
print(disply)
def tag_search(tag, page=0):
try:
tags = g.db_session.query(Tag).filter(Tag.tag == tag).filter(Tag.status\
== True).all()
tags = util.search_result(page, tags)
if not tags:
return page_not_found(u'検索結果がないです')
novels = [tag.novel for tag in tags if tag.novel.status]
novels = sorted(novels, key=lambda x: x.id)
page += 1
return render_template('search.htm', novels=novels,
page=page, conf=g.config)
except:
return internal_server_error(u'リロードしてみてください')
def main():
global logger, start, end
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size() # new: gives number of ranks in comm
# Setup Logger
extra = {'Process_ID': rank}
if args.debug:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(
level=level,
format='%(asctime)s-Process-%(Process_ID)s-%(levelname)s-%(message)s')
logger = logging.getLogger(__name__)
logger = logging.LoggerAdapter(logger, extra)
# Boardcast the gird data
grid_data_list = util.load_grid(args.grid_file_path)
grid_data_list = comm.bcast(grid_data_list, root=0)
# Initialze the result dictionary
rs_dict = {}
for item in grid_data_list:
rs = search_result()
rs.id = item.id
rs_dict[item.id] = rs
if size == 1:
program_timmer_start()
rs_dict = util.load_twitter_data_and_search(
file_path=args.twitter_data_file_path,
grid_data_list=grid_data_list,
logger=logger,
rs_dict=rs_dict)
logger.info('Total Number of Cores: %d' % (size))
search_end = datetime.datetime.now()
logger.info('Search takes: %s ' % (str((search_end - start))))
print_final_result(rs_dict)
program_timmer_end()
else:
next_target = 1
# Root Process handle IO
if rank == 0:
program_timmer_start()
# Process the twitter data and send to other process by batch
data_list = []
total_count = 0
with open(args.twitter_data_file_path) as f:
for line in f:
data_list.append(line)
total_count = total_count + 1
if len(data_list) >= args.batch_size:
comm.send(data_list, dest=next_target, tag=11)
next_target = next_target + 1
if next_target == size:
next_target = 1
data_list = []
if len(data_list) > 0:
comm.send(data_list, dest=next_target, tag=11)
for index in range(1, size):
comm.send([], dest=index, tag=11)
end = datetime.datetime.now()
logger.info('Total Number of Cores: %d' % (size))
logger.info('Sent total of %d lines of Data entries' %
(total_count))
logger.info('IO takes: %s ' % (str((end - start))))
rs_dict = None
else:
# Other Child Process Handles the search
total_count = 0
while True:
data = comm.recv(source=0, tag=11)
logger.debug('Processing of %d lines of data' % (len(data)))
rs_dict = util.search(grid_data_list=grid_data_list,
twitter_data_list=data,
logger=logger,
rs_dict=rs_dict)
if len(data) == 0:
break
total_count = total_count + len(data)
logger.info('Recv Total of %d lines of data' % (total_count))
# Gather the result
result_list = comm.gather(rs_dict, root=0)
if rank == 0:
# Merge the results into singal dintionary
print_final_result(result_list, process_gather=True)
program_timmer_end()
return
