def search(page=0): try: novels = [] words = util.encode_string(request.args['words']).split() user_query = g.db_session.query(User).filter(User.status == True) novel_query = g.db_session.query(Novel).filter(Novel.status == True) tag_query = g.db_session.query(Tag).filter(Tag.status == True) for word in words: user_query = user_query.filter(User.name.like('%' + word+ '%')) title_query = novel_query.filter(Novel.title.like('%' + word + '%')) summary_query = novel_query.filter(Novel.summary.like('%' + word + '%')) tag_query = tag_query.filter(Tag.tag.like('%' + word +'%')) for user in user_query: novels += [novel for novel in user.novel_list if novel.status] novels += title_query.all() + summary_query.all() tags = tag_query.all() novels += [tag.novel for tag in tags if tag.novel.status] if not novels: return page_not_found(u'検索結果がないです') novellist = list(set(novels)) novels = sorted(util.search_result(page, novellist), key=lambda x: x.id) page += 1 return render_template('search.htm', novels=novels, page=page, reqpage="search", words=request.args['words'], pnum=int(len(novellist) / page), conf=g.config) except: return internal_server_error(u'リロードしてみてください')
def print_final_result(result_list, process_gather=False): if process_gather: final_result = {} for result_dict_form_rank in result_list: if result_dict_form_rank is None: continue for grid_id in result_dict_form_rank: if grid_id not in final_result: rs = search_result() rs.id = grid_id final_result[grid_id] = rs final_result[grid_id].add_hash_tags( result_dict_form_rank[grid_id].hash_tags) final_result[grid_id].add_num_of_post( result_dict_form_rank[grid_id].num_of_post) else: final_result = result_list # Process The Result for grid_id in final_result: final_result[grid_id].process_result() final_result = [v for v in final_result.values()] final_result = sorted(final_result, key=lambda x: x.num_of_post, reverse=True) # Print total number of Twitter posts print(('=' * 30) + ' total number of Twitter posts' + ('=' * 30)) for item in final_result: if item.top_5_string is not None: disply = '{:1s}: {:6d} posts - {:10s}'.format( item.id, item.num_of_post, item.top_5_string) else: disply = '{:1s}: {:6d} posts - None'.format( item.id, item.num_of_post) print(disply)
def tag_search(tag, page=0): try: tags = g.db_session.query(Tag).filter(Tag.tag == tag).filter(Tag.status\ == True).all() tags = util.search_result(page, tags) if not tags: return page_not_found(u'検索結果がないです') novels = [tag.novel for tag in tags if tag.novel.status] novels = sorted(novels, key=lambda x: x.id) page += 1 return render_template('search.htm', novels=novels, page=page, conf=g.config) except: return internal_server_error(u'リロードしてみてください')
def main(): global logger, start, end comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() # new: gives number of ranks in comm # Setup Logger extra = {'Process_ID': rank} if args.debug: level = logging.DEBUG else: level = logging.INFO logging.basicConfig( level=level, format='%(asctime)s-Process-%(Process_ID)s-%(levelname)s-%(message)s') logger = logging.getLogger(__name__) logger = logging.LoggerAdapter(logger, extra) # Boardcast the gird data grid_data_list = util.load_grid(args.grid_file_path) grid_data_list = comm.bcast(grid_data_list, root=0) # Initialze the result dictionary rs_dict = {} for item in grid_data_list: rs = search_result() rs.id = item.id rs_dict[item.id] = rs if size == 1: program_timmer_start() rs_dict = util.load_twitter_data_and_search( file_path=args.twitter_data_file_path, grid_data_list=grid_data_list, logger=logger, rs_dict=rs_dict) logger.info('Total Number of Cores: %d' % (size)) search_end = datetime.datetime.now() logger.info('Search takes: %s ' % (str((search_end - start)))) print_final_result(rs_dict) program_timmer_end() else: next_target = 1 # Root Process handle IO if rank == 0: program_timmer_start() # Process the twitter data and send to other process by batch data_list = [] total_count = 0 with open(args.twitter_data_file_path) as f: for line in f: data_list.append(line) total_count = total_count + 1 if len(data_list) >= args.batch_size: comm.send(data_list, dest=next_target, tag=11) next_target = next_target + 1 if next_target == size: next_target = 1 data_list = [] if len(data_list) > 0: comm.send(data_list, dest=next_target, tag=11) for index in range(1, size): comm.send([], dest=index, tag=11) end = datetime.datetime.now() logger.info('Total Number of Cores: %d' % (size)) logger.info('Sent total of %d lines of Data entries' % (total_count)) logger.info('IO takes: %s ' % (str((end - start)))) rs_dict = None else: # Other Child Process Handles the search total_count = 0 while True: data = comm.recv(source=0, tag=11) logger.debug('Processing of %d lines of data' % (len(data))) rs_dict = util.search(grid_data_list=grid_data_list, twitter_data_list=data, logger=logger, rs_dict=rs_dict) if len(data) == 0: break total_count = total_count + len(data) logger.info('Recv Total of %d lines of data' % (total_count)) # Gather the result result_list = comm.gather(rs_dict, root=0) if rank == 0: # Merge the results into singal dintionary print_final_result(result_list, process_gather=True) program_timmer_end() return