def _save_answer(self, update): """ Get the current task and associate it with the answer provided. If the latter does not exist, add it, otherwise increase its total. Mark the task as completed if the same answer was given 5 times before. """ string = update.message.text.upper() with session_scope(self._session) as session: task_id = session.query(User).filter_by( id=update.effective_user.id).one().current_task answer = session.query(Answer).filter_by( task_id=task_id, string=string).one_or_none() if answer is None: session.add(Answer(task_id=task_id, string=string)) else: answer.total += 1 if answer.total > 5: session.query(Task).filter_by( id=task_id).one().lemma = string session.query(User).filter_by( id=update.effective_user.id).one().tasks_done += 1
def _get_landmark_point(command): # search landmark by name landmark_name = re.search(r"'(.*?)'", command, re.DOTALL).group(1) with session_scope() as session: landmark = session.query(Landmark).filter_by( name=landmark_name).scalar() return landmark.coordinate
def process_list(self, page): queue = self.application.queue logging.debug('Processing list %s' % page.url) processed = 0 if page.state == Page.State.PARSED: with session_scope(self.application.Session) as session: for url in page.get_contents(): (page_exists, ), = session.query( exists().where(Page.url == url)) if not page_exists: if 'http' not in url: url = self.application.config.SCRAPE_ROOT_PATH + url subpage = Page(url=url) session.add(subpage) session.commit() queue.add_page(subpage.page_id) processed += 1 page.state = Page.State.PROCESSED logging.debug('Processed list %s' % page.url) print( 'Processed %s urls! Fetching movies & getting more movie urls...' % processed) else: logging.debug('Aleady processed list %s' % page.url)
def insert(info, danmu, video=None): """持久化到mysql""" # 信息表 stat = info['data']['stat'] data = info['data'] i = VideoInfo(title=data['title'], pic=data['pic'], danmaku=stat['danmaku'], dislike=stat['dislike'], like=stat['like'], view=stat['view'], reply=stat['reply'], share=stat['share'], coin=stat['coin'], favorite=stat['favorite'], aid=stat['aid'], tname=data['tname'], owner_id=data['owner']['mid'], owner_face=data['owner']['face'], owner_name=data['owner']['name'], danmaku_id=data['cid'], pubdate=data['pubdate']) # 弹幕表 res = [] for item in danmu: d = Danmu(danmu=item['txt'], info=i) res.append(d) if video: # 视频表 v = Video( title=video['title'], path=video['path'], url=video['url'], info=i, ) with session_scope(Session) as session: session.add(i) session.add_all(res) session.add(v) with session_scope(Session) as session: session.add(i) session.add_all(res)
def build_index(self): print('Building index...') with session_scope(self.application.Session) as session: for page in session.query(Page).filter( Page.state == Page.State.PROCESSED).filter( Page.page_type == Page.PageType.DETAIL).all(): self.ingest(page) self.application.index.built = True print('Index built!')
def get_rating(chat_id): with session_scope(engine) as session: rating = '<b>Топ:</b>\n\n\n' for i, user in enumerate( session.query(User).filter_by(chat_id=chat_id).order_by( User.points).all()): rating += f'<b>{i + 1} место</b> {user.username} [{user.points} балла]\n' return rating
def handler(args): # loaddata if args.loaddata: route_handler = RouteParser() stop_point = route_handler.parse_routing_points(args.loaddata) print(stop_point) with session_scope() as session: route = Route() session.add(route) session.flush() sp_obj = [] for i in stop_point: routing_point = RoutingPoint(start_point=i[0], end_point=i[1], base_route=route.id) sp_obj.append(routing_point) session.add_all(sp_obj) # landmark func elif args.landmark: dr_data, error = LandmarkSerializer().load({ 'coordinate': args.landmark[0], 'name': args.landmark[1] }) if error: print(error) sys.exit(0) with session_scope() as session: landmark = Landmark(**dr_data) session.add(landmark) # robot elif args.run: print('<<START>>') loop = asyncio.get_event_loop() loop.run_until_complete(processor()) try: loop.run_forever() except KeyboardInterrupt: pass finally: print('<<END>>') loop.close()
def get_daily_rating(chat_id): with session_scope(engine) as session: query = session.query(Vote.msg_id, func.avg(Vote.mark), Vote.user_id).filter_by(chat_id=chat_id) query = query.filter( Vote.chat_id == chat_id, Vote.voted_at > datetime.utcnow() - timedelta(hours=24)) query = query.group_by(Vote.mark) query = query.order_by(func.avg(Vote.mark)) yield query.all()[::-1]
def get_last_meme(chat_id=None, user_id=None): with session_scope(engine) as session: query = session.query(Meme) if user_id is not None: query = query.filter(Meme.user_id == user_id) if chat_id is not None: query = query.filter(Meme.chat_id == chat_id) last_meme = query.order_by(Meme.posted_at.desc()).first() return last_meme
def process(self, page_id): with session_scope(self.application.Session) as session: page = session.query(Page).get(page_id) if page.state == Page.State.UNFETCHED: self.fetch(page) self.application.queue.add_page(page_id) elif page.state == Page.State.FETCHED: self.parse(page) self.application.queue.add_page(page_id) elif page.state == Page.State.PARSED: if page.page_type == Page.PageType.LIST: self.process_list(page)
def _start_callback(self, update, context): LOGGER.info('User {} started bot'.format( update.effective_user.username)) self._send_message(update, context, self._answers['start']) with session_scope(self._session) as session: if not session.query(User).filter_by( id=update.effective_user.id).count(): session.add( User(id=update.effective_user.id, name=update.effective_user.username)) self._send_task(update, context)
def fetch(self, page): logging.debug('Fetching %s' % page.url) try: resource = urllib.request.urlopen(page.url) except Exception as e: print('COULD NOT OPEN', page.url) raise (e) with session_scope(self.application.Session) as session: contents = resource.read().decode( resource.headers.get_content_charset()) page.save_raw_contents(contents) page.state = Page.State.FETCHED logging.debug('Fetched %s' % page.url)
def search(self, search_str, top_n=20): # relevance: # tf-idf # boost matches that contain all words # boost title, director, and stars fields in text search_str = search_str.lower().strip() if self.application.index.built is False: print('Index missing!') self.build_index() doc_scores = defaultdict(float) wc_index = self.application.index.wc global_wc_index = self.application.index.global_wc doc_count = self.application.index.docs_count for n in range(11): for ngram in find_ngrams(search_str.split(' '), n): if ngram in wc_index: for doc in wc_index[ngram]: doc_scores[doc] += wc_index[ngram][doc] * math.log( doc_count / len(global_wc_index.get(ngram, [1]))) # every word in search query needs to be in a returned doc missing_filter = set() for doc in list(doc_scores.keys()): for word in search_str.split(' '): if doc not in wc_index[word]: missing_filter.add(doc) top_results = [ x[1] for x in sorted([(doc_scores[doc], doc) for doc in doc_scores], reverse=True)[:top_n] ] second_pass = [] for doc in top_results: if doc not in missing_filter: second_pass.append(doc) for doc in top_results: if doc in missing_filter: second_pass.append(doc) with session_scope(self.application.Session) as session: top_results_pages = session.query(Page).filter( Page.page_id.in_(second_pass)).all() top_results_full = [ x.get_contents()['title'] for x in top_results_pages ] print("Search results:") for title in top_results_full: print('-', title)
async def producer(): while True: with session_scope() as session: route = session.query(Route).options(joinedload(Route.routing_point)).filter_by( is_finished=False).order_by(Route.created_date.desc()).first() robot = Robot() if route: for rp in route.routing_point: robot.move(x=rp.start_point, y=rp.end_point) print('x: {x}\ny: {y}'.format(x=rp.start_point, y=rp.end_point)) await asyncio.sleep(2) route.is_finished = True session.add(route) else: break
def simple_show(): with session_scope(sql_db.session()) as session: # 清表 session.query(User).delete() # 增 dic1 = {"name": "Adward", "age": 18} dic2 = {"name": "Dragon", "age": 18} session.add(User(**dic1)) session.add(User(**dic2)) # 删 session.query(User).filter(User.name == "Dragon").delete() # 改 session.query(User).filter(User.name == "Adward").update({"age": 17}) # 查 result = session.query(User).all() print([ each.__dict__ for each in result if each.__dict__.pop("_sa_instance_state") ])
def clear(self): self.index.reset() meta = Base.metadata with session_scope(self.Session) as session: for table in reversed(meta.sorted_tables): session.execute(table.delete()) session.commit() Base.metadata.create_all(self.engine) dir_path = os.path.dirname(os.path.realpath(__file__)) folder = "%s/datastore/object_store" % dir_path for cur_file in os.listdir(folder): file_path = os.path.join(folder, cur_file) try: if os.path.isfile(file_path): os.unlink(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: print(e)
def init(self): pages_added = 0 with session_scope(self.Session) as session: if session.query(Page).count() == 0: for i in self.config.ROOT_NODES: page = Page(url=i) session.add(page) session.commit() self.queue.add_page(page.page_id) pages_added += 1 else: for page in session.query(Page).filter(Page.state != Page.State.PROCESSED).all(): self.queue.add_page(page.page_id) pages_added += 1 session.expunge_all() if pages_added != 0: print('No movie data in our system. We need to scrape IMDB for data...') print('Started pipeline! Added %s root pages to processing queue' % pages_added) self.queue.join() print("Finished processing!") self.search_module.build_index()
def _send_task(self, update, context): """ Get a random unfinished task and assign it to the current user. """ with session_scope(self._session) as session: task = session.query(Task).filter(Task.lemma.is_(None)).order_by( func.random()).first() if task is None: self._send_message(update, context, self._answers['all_tasks_complete']) return word = session.query(Word).filter_by(task_id=task.id).order_by( func.random()).first() session.query(User).filter_by( id=update.effective_user.id).one().current_task = task.id self._send_message( update, context, '{} ({})'.format(word.string, ', '.join(word.morph)), word.options) LOGGER.info('User {} received task {}'.format( update.effective_user.username, word.string))
def is_exists(chat_id, telegram_id): with session_scope(engine) as session: user = session.query(User).filter_by(telegram_id=telegram_id, chat_id=chat_id).first() return user is not None
def is_voted(user_id, meme_id): with session_scope(engine) as session: query = session.query(Vote).filter_by(user_id=str(user_id), meme_id=meme_id) is_marked = query.count() > 0 return is_marked
def save(self): with session_scope(engine) as session: session.add(self) session.commit() return self
def get_by_msg_id(msg_id): with session_scope(engine) as session: meme = session.query(Meme).filter_by(msg_id=msg_id).first() return meme
def get_last(user_id): with session_scope(engine) as session: meme = session.query(Meme).filter_by( user_id=str(user_id)).order_by(desc(Meme.posted_at)).first() return meme
def get(user_id): with session_scope(engine) as session: return session.query(User).filter_by(id=str(user_id)).first()
def add_points(self, points): with session_scope(engine) as session: self.points += points session.add(self)
def get_meme(chat_id, msg_id): with session_scope(engine) as session: return session.query(Meme).filter_by(chat_id=chat_id, msg_id=msg_id).first()
def is_voted(user_id, meme_id): with session_scope(engine) as session: query = session.query(Vote).join(Meme).filter( Meme.user_id == user_id, Meme.id == meme_id) is_marked = query.count() > 0 return is_marked
def add_points(user_id, points): with session_scope(engine) as session: user = session.query(User).filter_by(id=user_id).first() user.points += points
def get_chat_ids(): with session_scope(engine) as session: chats = session.query(Vote.chat_id).group_by(Vote.chat_id).all() return [chat_id[0] for chat_id in chats]
def get_votes(chat_id): with session_scope(engine) as session: votes = session.query(Vote).filter_by(chat_id=chat_id).all() return votes