def test_use_commit_no_error(self): with db_test_session() as session: # run with db_session() as session: dao = NicoruDAO(session) dao.nicoru('sm9', '1') # verify with db_session() as session: dao = NicoruDAO(session) stored = dao.find_by_video_id_and_comment_id('sm9', '1') assert stored.video_id == 'sm9' assert stored.comment_id == '1'
def test_use_commit_error(self): with db_test_session() as session: # run try: with db_session() as session: dao = NicoruDAO(session) dao.nicoru('sm9', '1') raise Exception except: pass # verify with db_session() as session: dao = NicoruDAO(session) stored = dao.find_by_video_id_and_comment_id('sm9', '1') assert stored is None
def db_test_session(): app.config[ 'SQLALCHEMY_DATABASE_URI'] = 'mysql://*****:*****@db/test_nicoru' configured_db.init_app(app) try: with db_session() as session: Nicoru.__table__.create(configured_db.engine, checkfirst=True) Video.__table__.create(configured_db.engine, checkfirst=True) Comment.__table__.create(configured_db.engine, checkfirst=True) JobLog.__table__.create(configured_db.engine, checkfirst=True) IrregularVideoId.__table__.create(configured_db.engine, checkfirst=True) IrregularCommentId.__table__.create(configured_db.engine, checkfirst=True) yield session finally: with app.app_context(): Nicoru.__table__.drop(configured_db.engine, checkfirst=True) Video.__table__.drop(configured_db.engine, checkfirst=True) Comment.__table__.drop(configured_db.engine, checkfirst=True) JobLog.__table__.drop(configured_db.engine, checkfirst=True) IrregularVideoId.__table__.drop(configured_db.engine, checkfirst=True) IrregularCommentId.__table__.drop(configured_db.engine, checkfirst=True)
def get(self, user_id: str): """get nicoru data which targets are comments posted by given user. :param user_id: niconico user id :return: Nicorareta JSON """ with db_session() as session: dao = NicoruDAO(session) return json.dumps(dao.get_nicorareta(user_id))
def execute(cls) -> int: with db_session() as session: try: job_log_dao = JobLogDAO(session) job_log = job_log_dao.find_by_type(cls.TYPE) # exit if previous process is running if cls.previous_process_is_running(job_log): return cls.ReturnCode.PREVIOUS_PROCESS_IS_RUNNING try: incomplete_data_key = cls.get_incomplete_data_key(session) except IncompleteDataGetter.NoIncompleteDataError: # exit if there is no data to get logger.debug('there is no data to get.') return cls.ReturnCode.NO_INCOMPLETE_DATA # wait to run next process cls.wait_to_run_next_process(job_log) # mark the job as running job_log_dao.add_or_update(cls.TYPE, JobLogStatus.RUNNING) session.commit() # get and register data cls.get_and_register_data(session, incomplete_data_key) session.commit() # mark the job as running job_log_dao.add_or_update(cls.TYPE, JobLogStatus.DONE) session.commit() return cls.ReturnCode.SUCCESS except Exception: if session: session.rollback() with db_session() as session_for_error: # mark the job as aborted JobLogDAO(session_for_error).add_or_update( cls.TYPE, JobLogStatus.ABORTED) session_for_error.commit() raise
def export_public_data(cls): with db_session() as session: dao = NicoruDAO(session) records = dao.get_data_for_export() with open(HardConstants.App.REPORT_CSV, 'w') as f: writer = csv.writer(f, delimiter=",", lineterminator='\n', quotechar='"', quoting=csv.QUOTE_ALL) writer.writerow(('動画ID', 'コメ番', 'コメント', '擬似ニコる', '公式ニコる')) for record in records: if cls.row_is_valid(record): writer.writerow(record)
def get(self, video_id: str): """get nicoru of video by comment ids. :param video_id: video id :return: Nicoru JSON e.g. if comment #100 has 1 nicoru, #101 has 2 nicoru: {"100": 1, "101": 2} """ with db_session() as session: dao = NicoruDAO(session) return dao.get_nicoru_for_video(video_id)
def execute(cls): with db_session() as session: dao = JobLogDAO(session) try: dao.add_or_update(JobLogType.UPLOAD_TO_STORAGE, JobLogStatus.RUNNING) session.commit() cls.upload() except Exception as e: dao.add_or_update(JobLogType.UPLOAD_TO_STORAGE, JobLogStatus.ABORTED) session.commit() raise e else: dao.add_or_update(JobLogType.UPLOAD_TO_STORAGE, JobLogStatus.DONE)
def execute(cls): with db_session() as session: dao = JobLogDAO(session) try: dao.add_or_update(JobLogType.DB_DATA_EXPORT, JobLogStatus.RUNNING) session.commit() os.makedirs(HardConstants.App.DB_DUMP_DIR, exist_ok=True) cls.export_public_data() cls.export_data_for_restore() cls.compress_exported_data() except Exception as e: dao.add_or_update(JobLogType.DB_DATA_EXPORT, JobLogStatus.ABORTED) session.commit() raise e else: dao.add_or_update(JobLogType.DB_DATA_EXPORT, JobLogStatus.DONE)
def put(self, video_id: str): """receive nicoru request, update db :param video_id: 動画 ID :return: status """ data = request.json comment_id = data.get('cid') if not NicoruService.post_input_is_valid(video_id, comment_id): return with db_session() as session: dao = NicoruDAO(session) dao.nicoru( video_id, comment_id, ) return { 'status': 'ok', }
def make_mail_body(cls) -> str: with db_session() as session: logs = JobLogDAO(session).list() logs_text = "\n".join([ "{}: {}({})".format(x.type, x.updated_at, x.status) for x in logs ]) video_progress = NicoruDAO( session).get_status_of_video_data_getting() video_progress_text = "got: {}, irregular: {}, all(to get): {}, progress: {}".format( video_progress[0], video_progress[1], video_progress[2], video_progress[3]) comment_progress = NicoruDAO( session).get_status_of_comment_data_getting() comment_progress_text = "got: {}, irregular: {}, irregular2: {}, all(to get): {}, progress: {}".format( comment_progress[0], comment_progress[1], comment_progress[2], comment_progress[3], comment_progress[4]) return "\n".join([ "[Job Status]", logs_text, "", "[Progress of video data completion]", video_progress_text, "", "[Progress of comment data completion]", comment_progress_text ])
def execute(cls): with open(HardConstants.App.NICORU_CSV_TO_BE_IMPORTED, 'r') as f: with db_session() as session: # print(list(f.readlines())) lines = csv.reader(f, delimiter=",", lineterminator='\n', quotechar='"') lines_to_be_committed = 0 dao = NicoruDAO(session) # do import for vid, cid, nicoru in lines: dao.nicoru_from_csv(vid, cid, nicoru) lines_to_be_committed += 1 if lines_to_be_committed >= 1000: # commit for each 1000 lines session.commit() lines_to_be_committed = 0 # commit last lines session.commit()
def get(user_id: str): """get nicoru data which targets are comments posted by given user.""" with db_session() as session: dao = NicoruDAO(session) records = dao.get_nicorareta(user_id) return render_template('nicorareta.html', records=records)