def _save_answer(self, update):
"""
Get the current task and associate it with the answer provided.
If the latter does not exist, add it, otherwise increase its total.
Mark the task as completed if the same answer was given 5 times before.
"""
string = update.message.text.upper()
with session_scope(self._session) as session:
task_id = session.query(User).filter_by(
id=update.effective_user.id).one().current_task
answer = session.query(Answer).filter_by(
task_id=task_id, string=string).one_or_none()
if answer is None:
session.add(Answer(task_id=task_id, string=string))
else:
answer.total += 1
if answer.total > 5:
session.query(Task).filter_by(
id=task_id).one().lemma = string
session.query(User).filter_by(
id=update.effective_user.id).one().tasks_done += 1
def _get_landmark_point(command):
# search landmark by name
landmark_name = re.search(r"'(.*?)'", command, re.DOTALL).group(1)
with session_scope() as session:
landmark = session.query(Landmark).filter_by(
name=landmark_name).scalar()
return landmark.coordinate
def process_list(self, page):
queue = self.application.queue
logging.debug('Processing list %s' % page.url)
processed = 0
if page.state == Page.State.PARSED:
with session_scope(self.application.Session) as session:
for url in page.get_contents():
(page_exists, ), = session.query(
exists().where(Page.url == url))
if not page_exists:
if 'http' not in url:
url = self.application.config.SCRAPE_ROOT_PATH + url
subpage = Page(url=url)
session.add(subpage)
session.commit()
queue.add_page(subpage.page_id)
processed += 1
page.state = Page.State.PROCESSED
logging.debug('Processed list %s' % page.url)
print(
'Processed %s urls! Fetching movies & getting more movie urls...'
% processed)
else:
logging.debug('Aleady processed list %s' % page.url)
def insert(info, danmu, video=None):
"""持久化到mysql"""
# 信息表
stat = info['data']['stat']
data = info['data']
i = VideoInfo(title=data['title'],
pic=data['pic'],
danmaku=stat['danmaku'],
dislike=stat['dislike'],
like=stat['like'],
view=stat['view'],
reply=stat['reply'],
share=stat['share'],
coin=stat['coin'],
favorite=stat['favorite'],
aid=stat['aid'],
tname=data['tname'],
owner_id=data['owner']['mid'],
owner_face=data['owner']['face'],
owner_name=data['owner']['name'],
danmaku_id=data['cid'],
pubdate=data['pubdate'])
# 弹幕表
res = []
for item in danmu:
d = Danmu(danmu=item['txt'], info=i)
res.append(d)
if video:
# 视频表
v = Video(
title=video['title'],
path=video['path'],
url=video['url'],
info=i,
)
with session_scope(Session) as session:
session.add(i)
session.add_all(res)
session.add(v)
with session_scope(Session) as session:
session.add(i)
session.add_all(res)
def build_index(self):
print('Building index...')
with session_scope(self.application.Session) as session:
for page in session.query(Page).filter(
Page.state == Page.State.PROCESSED).filter(
Page.page_type == Page.PageType.DETAIL).all():
self.ingest(page)
self.application.index.built = True
print('Index built!')
def get_rating(chat_id):
with session_scope(engine) as session:
rating = '<b>Топ:</b>\n\n\n'
for i, user in enumerate(
session.query(User).filter_by(chat_id=chat_id).order_by(
User.points).all()):
rating += f'<b>{i + 1} место</b> {user.username} [{user.points} балла]\n'
return rating
def handler(args):
# loaddata
if args.loaddata:
route_handler = RouteParser()
stop_point = route_handler.parse_routing_points(args.loaddata)
print(stop_point)
with session_scope() as session:
route = Route()
session.add(route)
session.flush()
sp_obj = []
for i in stop_point:
routing_point = RoutingPoint(start_point=i[0],
end_point=i[1],
base_route=route.id)
sp_obj.append(routing_point)
session.add_all(sp_obj)
# landmark func
elif args.landmark:
dr_data, error = LandmarkSerializer().load({
'coordinate':
args.landmark[0],
'name':
args.landmark[1]
})
if error:
print(error)
sys.exit(0)
with session_scope() as session:
landmark = Landmark(**dr_data)
session.add(landmark)
# robot
elif args.run:
print('<<START>>')
loop = asyncio.get_event_loop()
loop.run_until_complete(processor())
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
print('<<END>>')
loop.close()
def get_daily_rating(chat_id):
with session_scope(engine) as session:
query = session.query(Vote.msg_id, func.avg(Vote.mark),
Vote.user_id).filter_by(chat_id=chat_id)
query = query.filter(
Vote.chat_id == chat_id,
Vote.voted_at > datetime.utcnow() - timedelta(hours=24))
query = query.group_by(Vote.mark)
query = query.order_by(func.avg(Vote.mark))
yield query.all()[::-1]
def get_last_meme(chat_id=None, user_id=None):
with session_scope(engine) as session:
query = session.query(Meme)
if user_id is not None:
query = query.filter(Meme.user_id == user_id)
if chat_id is not None:
query = query.filter(Meme.chat_id == chat_id)
last_meme = query.order_by(Meme.posted_at.desc()).first()
return last_meme
def process(self, page_id):
with session_scope(self.application.Session) as session:
page = session.query(Page).get(page_id)
if page.state == Page.State.UNFETCHED:
self.fetch(page)
self.application.queue.add_page(page_id)
elif page.state == Page.State.FETCHED:
self.parse(page)
self.application.queue.add_page(page_id)
elif page.state == Page.State.PARSED:
if page.page_type == Page.PageType.LIST:
self.process_list(page)
def _start_callback(self, update, context):
LOGGER.info('User {} started bot'.format(
update.effective_user.username))
self._send_message(update, context, self._answers['start'])
with session_scope(self._session) as session:
if not session.query(User).filter_by(
id=update.effective_user.id).count():
session.add(
User(id=update.effective_user.id,
name=update.effective_user.username))
self._send_task(update, context)
def fetch(self, page):
logging.debug('Fetching %s' % page.url)
try:
resource = urllib.request.urlopen(page.url)
except Exception as e:
print('COULD NOT OPEN', page.url)
raise (e)
with session_scope(self.application.Session) as session:
contents = resource.read().decode(
resource.headers.get_content_charset())
page.save_raw_contents(contents)
page.state = Page.State.FETCHED
logging.debug('Fetched %s' % page.url)
def search(self, search_str, top_n=20):
# relevance:
# tf-idf
# boost matches that contain all words
# boost title, director, and stars fields in text
search_str = search_str.lower().strip()
if self.application.index.built is False:
print('Index missing!')
self.build_index()
doc_scores = defaultdict(float)
wc_index = self.application.index.wc
global_wc_index = self.application.index.global_wc
doc_count = self.application.index.docs_count
for n in range(11):
for ngram in find_ngrams(search_str.split(' '), n):
if ngram in wc_index:
for doc in wc_index[ngram]:
doc_scores[doc] += wc_index[ngram][doc] * math.log(
doc_count / len(global_wc_index.get(ngram, [1])))
# every word in search query needs to be in a returned doc
missing_filter = set()
for doc in list(doc_scores.keys()):
for word in search_str.split(' '):
if doc not in wc_index[word]:
missing_filter.add(doc)
top_results = [
x[1]
for x in sorted([(doc_scores[doc], doc) for doc in doc_scores],
reverse=True)[:top_n]
]
second_pass = []
for doc in top_results:
if doc not in missing_filter:
second_pass.append(doc)
for doc in top_results:
if doc in missing_filter:
second_pass.append(doc)
with session_scope(self.application.Session) as session:
top_results_pages = session.query(Page).filter(
Page.page_id.in_(second_pass)).all()
top_results_full = [
x.get_contents()['title'] for x in top_results_pages
]
print("Search results:")
for title in top_results_full:
print('-', title)
async def producer():
while True:
with session_scope() as session:
route = session.query(Route).options(joinedload(Route.routing_point)).filter_by(
is_finished=False).order_by(Route.created_date.desc()).first()
robot = Robot()
if route:
for rp in route.routing_point:
robot.move(x=rp.start_point, y=rp.end_point)
print('x: {x}\ny: {y}'.format(x=rp.start_point, y=rp.end_point))
await asyncio.sleep(2)
route.is_finished = True
session.add(route)
else:
break
def simple_show():
with session_scope(sql_db.session()) as session:
# 清表
session.query(User).delete()
# 增
dic1 = {"name": "Adward", "age": 18}
dic2 = {"name": "Dragon", "age": 18}
session.add(User(**dic1))
session.add(User(**dic2))
# 删
session.query(User).filter(User.name == "Dragon").delete()
# 改
session.query(User).filter(User.name == "Adward").update({"age": 17})
# 查
result = session.query(User).all()
print([
each.__dict__ for each in result
if each.__dict__.pop("_sa_instance_state")
])
def clear(self):
self.index.reset()
meta = Base.metadata
with session_scope(self.Session) as session:
for table in reversed(meta.sorted_tables):
session.execute(table.delete())
session.commit()
Base.metadata.create_all(self.engine)
dir_path = os.path.dirname(os.path.realpath(__file__))
folder = "%s/datastore/object_store" % dir_path
for cur_file in os.listdir(folder):
file_path = os.path.join(folder, cur_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path): shutil.rmtree(file_path)
except Exception as e:
print(e)
def init(self):
pages_added = 0
with session_scope(self.Session) as session:
if session.query(Page).count() == 0:
for i in self.config.ROOT_NODES:
page = Page(url=i)
session.add(page)
session.commit()
self.queue.add_page(page.page_id)
pages_added += 1
else:
for page in session.query(Page).filter(Page.state != Page.State.PROCESSED).all():
self.queue.add_page(page.page_id)
pages_added += 1
session.expunge_all()
if pages_added != 0:
print('No movie data in our system. We need to scrape IMDB for data...')
print('Started pipeline! Added %s root pages to processing queue' % pages_added)
self.queue.join()
print("Finished processing!")
self.search_module.build_index()
def _send_task(self, update, context):
"""
Get a random unfinished task and assign it to the current user.
"""
with session_scope(self._session) as session:
task = session.query(Task).filter(Task.lemma.is_(None)).order_by(
func.random()).first()
if task is None:
self._send_message(update, context,
self._answers['all_tasks_complete'])
return
word = session.query(Word).filter_by(task_id=task.id).order_by(
func.random()).first()
session.query(User).filter_by(
id=update.effective_user.id).one().current_task = task.id
self._send_message(
update, context, '{} ({})'.format(word.string,
', '.join(word.morph)),
word.options)
LOGGER.info('User {} received task {}'.format(
update.effective_user.username, word.string))
def is_exists(chat_id, telegram_id):
with session_scope(engine) as session:
user = session.query(User).filter_by(telegram_id=telegram_id,
chat_id=chat_id).first()
return user is not None
def is_voted(user_id, meme_id):
with session_scope(engine) as session:
query = session.query(Vote).filter_by(user_id=str(user_id),
meme_id=meme_id)
is_marked = query.count() > 0
return is_marked
def save(self):
with session_scope(engine) as session:
session.add(self)
session.commit()
return self
def get_by_msg_id(msg_id):
with session_scope(engine) as session:
meme = session.query(Meme).filter_by(msg_id=msg_id).first()
return meme
def get_last(user_id):
with session_scope(engine) as session:
meme = session.query(Meme).filter_by(
user_id=str(user_id)).order_by(desc(Meme.posted_at)).first()
return meme
def get(user_id):
with session_scope(engine) as session:
return session.query(User).filter_by(id=str(user_id)).first()
def add_points(self, points):
with session_scope(engine) as session:
self.points += points
session.add(self)
def get_meme(chat_id, msg_id):
with session_scope(engine) as session:
return session.query(Meme).filter_by(chat_id=chat_id,
msg_id=msg_id).first()
def is_voted(user_id, meme_id):
with session_scope(engine) as session:
query = session.query(Vote).join(Meme).filter(
Meme.user_id == user_id, Meme.id == meme_id)
is_marked = query.count() > 0
return is_marked
def add_points(user_id, points):
with session_scope(engine) as session:
user = session.query(User).filter_by(id=user_id).first()
user.points += points
def get_chat_ids():
with session_scope(engine) as session:
chats = session.query(Vote.chat_id).group_by(Vote.chat_id).all()
return [chat_id[0] for chat_id in chats]
def get_votes(chat_id):
with session_scope(engine) as session:
votes = session.query(Vote).filter_by(chat_id=chat_id).all()
return votes
