def test_use_commit_no_error(self):
with db_test_session() as session:
# run
with db_session() as session:
dao = NicoruDAO(session)
dao.nicoru('sm9', '1')
# verify
with db_session() as session:
dao = NicoruDAO(session)
stored = dao.find_by_video_id_and_comment_id('sm9', '1')
assert stored.video_id == 'sm9'
assert stored.comment_id == '1'
def test_use_commit_error(self):
with db_test_session() as session:
# run
try:
with db_session() as session:
dao = NicoruDAO(session)
dao.nicoru('sm9', '1')
raise Exception
except:
pass
# verify
with db_session() as session:
dao = NicoruDAO(session)
stored = dao.find_by_video_id_and_comment_id('sm9', '1')
assert stored is None
def db_test_session():
app.config[
'SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@db/test_nicoru'
configured_db.init_app(app)
try:
with db_session() as session:
Nicoru.__table__.create(configured_db.engine, checkfirst=True)
Video.__table__.create(configured_db.engine, checkfirst=True)
Comment.__table__.create(configured_db.engine, checkfirst=True)
JobLog.__table__.create(configured_db.engine, checkfirst=True)
IrregularVideoId.__table__.create(configured_db.engine,
checkfirst=True)
IrregularCommentId.__table__.create(configured_db.engine,
checkfirst=True)
yield session
finally:
with app.app_context():
Nicoru.__table__.drop(configured_db.engine, checkfirst=True)
Video.__table__.drop(configured_db.engine, checkfirst=True)
Comment.__table__.drop(configured_db.engine, checkfirst=True)
JobLog.__table__.drop(configured_db.engine, checkfirst=True)
IrregularVideoId.__table__.drop(configured_db.engine,
checkfirst=True)
IrregularCommentId.__table__.drop(configured_db.engine,
checkfirst=True)
def get(self, user_id: str):
"""get nicoru data which targets are comments posted by given user.
:param user_id: niconico user id
:return: Nicorareta JSON
"""
with db_session() as session:
dao = NicoruDAO(session)
return json.dumps(dao.get_nicorareta(user_id))
def execute(cls) -> int:
with db_session() as session:
try:
job_log_dao = JobLogDAO(session)
job_log = job_log_dao.find_by_type(cls.TYPE)
# exit if previous process is running
if cls.previous_process_is_running(job_log):
return cls.ReturnCode.PREVIOUS_PROCESS_IS_RUNNING
try:
incomplete_data_key = cls.get_incomplete_data_key(session)
except IncompleteDataGetter.NoIncompleteDataError:
# exit if there is no data to get
logger.debug('there is no data to get.')
return cls.ReturnCode.NO_INCOMPLETE_DATA
# wait to run next process
cls.wait_to_run_next_process(job_log)
# mark the job as running
job_log_dao.add_or_update(cls.TYPE, JobLogStatus.RUNNING)
session.commit()
# get and register data
cls.get_and_register_data(session, incomplete_data_key)
session.commit()
# mark the job as running
job_log_dao.add_or_update(cls.TYPE, JobLogStatus.DONE)
session.commit()
return cls.ReturnCode.SUCCESS
except Exception:
if session:
session.rollback()
with db_session() as session_for_error:
# mark the job as aborted
JobLogDAO(session_for_error).add_or_update(
cls.TYPE, JobLogStatus.ABORTED)
session_for_error.commit()
raise
def export_public_data(cls):
with db_session() as session:
dao = NicoruDAO(session)
records = dao.get_data_for_export()
with open(HardConstants.App.REPORT_CSV, 'w') as f:
writer = csv.writer(f, delimiter=",", lineterminator='\n', quotechar='"', quoting=csv.QUOTE_ALL)
writer.writerow(('動画ID', 'コメ番', 'コメント', '擬似ニコる', '公式ニコる'))
for record in records:
if cls.row_is_valid(record):
writer.writerow(record)
def get(self, video_id: str):
"""get nicoru of video by comment ids.
:param video_id: video id
:return: Nicoru JSON
e.g. if comment #100 has 1 nicoru, #101 has 2 nicoru:
{"100": 1, "101": 2}
"""
with db_session() as session:
dao = NicoruDAO(session)
return dao.get_nicoru_for_video(video_id)
def execute(cls):
with db_session() as session:
dao = JobLogDAO(session)
try:
dao.add_or_update(JobLogType.UPLOAD_TO_STORAGE,
JobLogStatus.RUNNING)
session.commit()
cls.upload()
except Exception as e:
dao.add_or_update(JobLogType.UPLOAD_TO_STORAGE,
JobLogStatus.ABORTED)
session.commit()
raise e
else:
dao.add_or_update(JobLogType.UPLOAD_TO_STORAGE,
JobLogStatus.DONE)
def execute(cls):
with db_session() as session:
dao = JobLogDAO(session)
try:
dao.add_or_update(JobLogType.DB_DATA_EXPORT, JobLogStatus.RUNNING)
session.commit()
os.makedirs(HardConstants.App.DB_DUMP_DIR, exist_ok=True)
cls.export_public_data()
cls.export_data_for_restore()
cls.compress_exported_data()
except Exception as e:
dao.add_or_update(JobLogType.DB_DATA_EXPORT, JobLogStatus.ABORTED)
session.commit()
raise e
else:
dao.add_or_update(JobLogType.DB_DATA_EXPORT, JobLogStatus.DONE)
def put(self, video_id: str):
"""receive nicoru request, update db
:param video_id: 動画 ID
:return: status
"""
data = request.json
comment_id = data.get('cid')
if not NicoruService.post_input_is_valid(video_id, comment_id):
return
with db_session() as session:
dao = NicoruDAO(session)
dao.nicoru(
video_id,
comment_id,
)
return {
'status': 'ok',
}
def make_mail_body(cls) -> str:
with db_session() as session:
logs = JobLogDAO(session).list()
logs_text = "\n".join([
"{}: {}({})".format(x.type, x.updated_at, x.status)
for x in logs
])
video_progress = NicoruDAO(
session).get_status_of_video_data_getting()
video_progress_text = "got: {}, irregular: {}, all(to get): {}, progress: {}".format(
video_progress[0], video_progress[1], video_progress[2],
video_progress[3])
comment_progress = NicoruDAO(
session).get_status_of_comment_data_getting()
comment_progress_text = "got: {}, irregular: {}, irregular2: {}, all(to get): {}, progress: {}".format(
comment_progress[0], comment_progress[1], comment_progress[2],
comment_progress[3], comment_progress[4])
return "\n".join([
"[Job Status]", logs_text, "",
"[Progress of video data completion]", video_progress_text, "",
"[Progress of comment data completion]", comment_progress_text
])
def execute(cls):
with open(HardConstants.App.NICORU_CSV_TO_BE_IMPORTED, 'r') as f:
with db_session() as session:
# print(list(f.readlines()))
lines = csv.reader(f,
delimiter=",",
lineterminator='\n',
quotechar='"')
lines_to_be_committed = 0
dao = NicoruDAO(session)
# do import
for vid, cid, nicoru in lines:
dao.nicoru_from_csv(vid, cid, nicoru)
lines_to_be_committed += 1
if lines_to_be_committed >= 1000:
# commit for each 1000 lines
session.commit()
lines_to_be_committed = 0
# commit last lines
session.commit()
def get(user_id: str):
"""get nicoru data which targets are comments posted by given user."""
with db_session() as session:
dao = NicoruDAO(session)
records = dao.get_nicorareta(user_id)
return render_template('nicorareta.html', records=records)