def find_all_episodes():
session = DBUtil.open_session()
try:
result = session.query(Episode).all()
return result
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def find_all_bangumis():
session = DBUtil.open_session()
try:
result = session.query(Bangumi).all()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def find_all_senders():
session = DBUtil.open_session()
try:
result = session.query(func.distinct(
Danmaku.sender_id)).select_from(Danmaku).all()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def find_danmakus_by_sender_ids(sender_ids):
session = DBUtil.open_session()
try:
result = session.query(Danmaku).filter(
Danmaku.sender_id.in_(sender_ids)).all()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def find_danmakus_by_episode(episode_id):
session = DBUtil.open_session()
try:
result = session.query(Danmaku).filter(
Danmaku.episode_id == episode_id).order_by('playback_time').all()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def find_danmaku(danmaku):
session = DBUtil.open_session()
try:
result = session.query(Danmaku).filter(
Danmaku.raw_id == danmaku.raw_id).first()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def update_batch(batch):
session = DBUtil.open_session()
try:
session.merge(batch)
session.commit()
return True
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def add_episode(episode):
session = DBUtil.open_session()
try:
session.merge(episode)
session.commit()
return True
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def find_episodes_by_bangumi(bangumi_season_id):
session = DBUtil.open_session()
try:
result = session.query(Episode).filter(
Episode.season_id == bangumi_season_id).order_by("index").all()
return result
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def find_bangumi_by_id(season_id):
session = DBUtil.open_session()
try:
result = session.query(Bangumi).filter(
Bangumi.season_id == season_id).first()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def find_episodes_by_ids(episode_id_list):
session = DBUtil.open_session()
try:
result = session.query(Episode).filter(
Episode.episode_id.in_(episode_id_list)).all()
return result
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def add_danmakus(danmaku_list):
session = DBUtil.open_session()
try:
for danmaku in danmaku_list:
session.merge(danmaku)
session.commit()
return True
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def find_fetch_logs_by_episode(episode_id):
session = DBUtil.open_session()
try:
result = session.query(FetchLog).filter(
FetchLog.episode_id == episode_id).order_by(
FetchLog.batch.asc()).all()
return result
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def add_fetch_logs(fetch_log_list):
session = DBUtil.open_session()
try:
for fetch_log in fetch_log_list:
session.merge(fetch_log)
session.commit()
return True
except Exception as e:
print e
session.rollback()
return False
finally:
DBUtil.close_session(session)
def add_batch(batch):
session = DBUtil.open_session()
try:
session.add(batch)
session.commit()
session.refresh(batch)
return batch
except Exception as e:
print e
session.rollback()
return None
finally:
DBUtil.close_session(session)
def __init__(self, handlers, tp, sp, ui):
super().__init__(handlers,
template_path=tp,
static_path=sp,
ui_modules=ui,
debug=True)
self.dbutil = DBUtil(**mysettings.settings['dbsetting'])
self.q = multiprocessing.Queue()
for _ in range(mysettings.settings['crawl_num']):
p = multiprocessing.Process(target=crawlbyWord, args=(self.q, ))
p.daemon = True
p.start()
def __init__(self, handlers, modules, tpath, spath):
super().__init__(handlers=handlers,
ui_modules=modules,
template_path=tpath,
static_path=spath)
self.db = DBUtil(**myconfigs.configs.get('dbconfig'))
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
from util.dbutil import DBUtil
import logging
DBUtil.init_db()
logger = logging.getLogger("logger")
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
from tasks import index_page
from util.dbutil import DBUtil
from apscheduler.schedulers.blocking import BlockingScheduler
if __name__ == "__main__":
db_util = DBUtil()
db_util.init_db()
index_page.index_page_spider()
scheduler = BlockingScheduler()
scheduler.add_job(index_page.index_page_spider, 'interval', minutes=120)
scheduler.start()