def get_execute_log_details(self, execute_id, after=None): with DBSession() as db: filters = [ExecuteLog.execute_id == execute_id] if after is not None: filters.append(ExecuteLog.id > after) log_entries = db.query(ExecuteLog).filter(*filters).all() return [row2dict(e) for e in log_entries]
def get_watching_topics(self): watching_topics = [] with DBSession() as db: dbtopics = db.query(Topic).all() for dbtopic in dbtopics: try: tracker = self.get_tracker(dbtopic.type) except KeyError: # TODO: Log warning of not existing topic # Need to think, should we return not existing plugin # as just default topic, and show it disabled on UI to # let user ability for delete such topics continue topic = row2dict( dbtopic, None, ['id', 'url', 'display_name', 'last_update', 'paused']) topic['info'] = tracker.get_topic_info(dbtopic) topic['tracker'] = dbtopic.type topic['status'] = dbtopic.status.__str__() if dbtopic.type == 'lostfilm.tv': topic['external_id'] = db.query( tracker.topic_class).filter( tracker.topic_class.id == dbtopic.id).first().cat watching_topics.append(topic) return watching_topics
def get_log_entries(self, skip, take): with DBSession() as db: downloaded_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\ .group_by(ExecuteLog.execute_id, ExecuteLog.level)\ .having(ExecuteLog.level == 'downloaded')\ .subquery() failed_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\ .group_by(ExecuteLog.execute_id, ExecuteLog.level)\ .having(ExecuteLog.level == 'failed')\ .subquery() result_query = db.query(Execute, downloaded_sub_query.c.count, failed_sub_query.c.count)\ .outerjoin(failed_sub_query, Execute.id == failed_sub_query.c.execute_id)\ .outerjoin(downloaded_sub_query, Execute.id == downloaded_sub_query.c.execute_id)\ .order_by(Execute.finish_time.desc())\ .offset(skip)\ .limit(take) result = [] for execute, downloads, fails in result_query.all(): execute_result = row2dict(execute) execute_result['downloaded'] = downloads or 0 execute_result['failed'] = fails or 0 execute_result['is_running'] = execute.id == self._execute_id result.append(execute_result) execute_count = db.query(func.count(Execute.id)).scalar() return result, execute_count
def get_credentials(self): with DBSession() as db: dbcredentials = db.query(self.credentials_class).first() if dbcredentials is None: return None return row2dict(dbcredentials, None, self.credentials_public_fields)
def get_topic(self, id): with DBSession() as db: topic = db.query(self.topic_class).filter(Topic.id == id).first() if topic is None: return None data = row2dict(topic, None, self.topic_public_fields) data['info'] = self.get_topic_info(topic) return data
def create_removed_topic(self): remove_type = TRACKER1_PLUGIN_NAME + ".uk" with DBSession() as db: topic = Topic(display_name=self.DISPLAY_NAME1 + " / Test", url="http://tracker.com/2/", type=remove_type) result = db.execute(topic.__table__.insert(), row2dict(topic, fields=['display_name', 'url', 'type'])) tracker1_id2 = result.inserted_primary_key[0] return tracker1_id2
def create_removed_topic(self): remove_type = TRACKER1_PLUGIN_NAME + ".uk" with DBSession() as db: topic = Topic(display_name=self.DISPLAY_NAME1 + " / Test", url="http://tracker.com/2/", type=remove_type) result = db.execute( topic.__table__.insert(), row2dict(topic, fields=['display_name', 'url', 'type'])) tracker1_id2 = result.inserted_primary_key[0] return tracker1_id2
def test_upgrade_filled_from_version_0(self): topic1 = { 'id': 1, 'url': 'https://www.anilibria.tv/release/seishun-buta-yarou-wa-bunny-girl-senpai-no-yume' '-wo-minai.html', 'display_name': '1', 'type': 'anilibria.tv' } anilibria_topic1 = {'id': 1, 'hash': 'abcdef'} topic2 = { 'id': 2, 'url': 'https://www.anilibria.tv', 'display_name': '2', 'type': 'anilibria.tv' } anilibria_topic2 = {'id': 2, 'hash': 'abcdef'} self._upgrade_from( [[anilibria_topic1, anilibria_topic2], [topic1, topic2]], 0) with DBSession() as db: topics = [ row2dict(t, self.AnilibriaTopic1) for t in db.query(self.AnilibriaTopic1) ] base_topics = [ row2dict(t, self.BaseTopic1) for t in db.query(self.BaseTopic1) ] assert topics[0][ 'format_list'] == u'Серия 1-13 [HDTVRip 1080p],Серия 1-13 [HDTVRip 720p]' assert topics[0]['format'] == u'Серия 1-13 [HDTVRip 1080p]' assert base_topics[0]['status'] == Status.Ok assert base_topics[1]['status'] == Status.Error
def upgrade_1_to_2(operations_factory): m1 = MetaData() rutor_topic_1 = Table('rutororg_topics', m1, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=False)) m2 = MetaData() rutor_topic_2 = Table('rutororg_topics2', m2, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=True)) with operations_factory() as operations: operations.create_table(rutor_topic_2) topics = operations.db.query(rutor_topic_1) for topic in topics: raw_topic = row2dict(topic, rutor_topic_1) operations.db.execute(rutor_topic_2.insert(), raw_topic) operations.drop_table(rutor_topic_1) operations.rename_table(rutor_topic_2.name, rutor_topic_1.name)
def upgrade_0_to_1(operations_factory): m1 = MetaData() unionpeer_topic_table_0 = Table('unionpeerorg_topics', m1, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=False)) m2 = MetaData() unionpeer_topic_table_1 = Table('unionpeerorg_topics1', m2, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=True)) with operations_factory() as operations: operations.create_table(unionpeer_topic_table_1) topics = operations.db.query(unionpeer_topic_table_0) for topic in topics: raw_topic = row2dict(topic, unionpeer_topic_table_0) operations.db.execute(unionpeer_topic_table_1.insert(), raw_topic) operations.drop_table(unionpeer_topic_table_0) operations.rename_table(unionpeer_topic_table_1.name, unionpeer_topic_table_0.name)
def get_watching_topics(self): watching_topics = [] with DBSession() as db: dbtopics = db.query(Topic).all() for dbtopic in dbtopics: try: tracker = self.get_tracker(dbtopic.type) except KeyError: # TODO: Log warning of not existing topic # Need to think, should we return not existing plugin # as just default topic, and show it disabled on UI to # let user ability for delete such topics continue topic = row2dict(dbtopic, None, ['id', 'url', 'display_name', 'last_update', 'paused']) topic['info'] = tracker.get_topic_info(dbtopic) topic['tracker'] = dbtopic.type topic['status'] = dbtopic.status.__str__() watching_topics.append(topic) return watching_topics
def upgrade_0_to_1(operations_factory): m1 = MetaData() unionpeer_topic_table_0 = Table( 'unionpeerorg_topics', m1, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=False)) m2 = MetaData() unionpeer_topic_table_1 = Table( 'unionpeerorg_topics1', m2, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=True)) with operations_factory() as operations: operations.create_table(unionpeer_topic_table_1) topics = operations.db.query(unionpeer_topic_table_0) for topic in topics: raw_topic = row2dict(topic, unionpeer_topic_table_0) operations.db.execute(unionpeer_topic_table_1.insert(), raw_topic) operations.drop_table(unionpeer_topic_table_0) operations.rename_table(unionpeer_topic_table_1.name, unionpeer_topic_table_0.name)
def upgrade_0_to_1(operations_factory): from monitorrent.settings_manager import SettingsManager settings_manager = SettingsManager() tracker_settings = None with operations_factory() as operations: operations.add_column(AnilibriaTvTopic.__tablename__, Column('format', String, nullable=True)) operations.add_column(AnilibriaTvTopic.__tablename__, Column('format_list', String, nullable=True)) topic_values = [] m = MetaData() ani_topics = Table( AnilibriaTvTopic.__tablename__, m, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("hash", String, nullable=True), Column("format", String, nullable=True), Column("format_list", String, nullable=True)) m1 = MetaData() base_topics = Table(Topic.__tablename__, m1, Column("id", Integer, primary_key=True), Column("url", String), Column("type", String), Column('status', String)) topics = operations.db.query(base_topics).filter( base_topics.c.type == PLUGIN_NAME) for topic in topics: raw_topic = row2dict(topic, base_topics) # noinspection PyBroadException try: if tracker_settings is None: tracker_settings = settings_manager.tracker_settings response = requests.get( raw_topic['url'], **tracker_settings.get_requests_kwargs()) soup = get_soup(response.text) format_list = AnilibriaTvTracker._find_format_list(soup) format_list.sort() topic_values.append({ 'id': raw_topic['id'], 'format_list': ",".join(format_list), 'format': format_list[0], 'status': Status.Ok.__str__() }) except: exc_info = sys.exc_info() print(u''.join(traceback.format_exception(*exc_info))) topic_values.append({ 'id': raw_topic['id'], 'status': Status.Error.__str__() }) for upd in topic_values: if 'format' in upd: operations.db.execute( ani_topics.update( whereclause=(ani_topics.c.id == upd['id']), values={ 'format_list': upd['format_list'], 'format': upd['format'] })) operations.db.execute( base_topics.update(whereclause=(base_topics.c.id == upd['id']), values={'status': upd['status']}))
def test_updage_filled_from_version_3(self): Table('lostfilmtv_series4', self.m3, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("cat", Integer, nullable=False), Column("season", Integer, nullable=True), Column("episode", Integer, nullable=True), Column("quality", String, nullable=False)) Table("lostfilmtv_credentials4", self.m3, Column('username', String, primary_key=True), Column('password', String, primary_key=True), Column('session', String), Column('default_quality', String, nullable=False, server_default='SD')) topic1 = {'id': 1, 'url': 'http://www.lostfilm.tv/browse.php?cat=236', 'display_name': u'12 обезьян / 12 Monkeys', 'type': 'lostfilm.tv'} topic2 = {'id': 2, 'url': 'http://www.lostfilm.tv/browse.php?cat=245', 'display_name': u'Mr. Robot', 'type': 'lostfilm.tv'} topic3 = {'id': 3, 'url': 'http://www.lostfilm.tv/browse.php?cat=251', 'display_name': u'Scream', 'type': 'lostfilm.tv'} topic4 = {'id': 4, 'url': 'https://www.lostfilm.tv/browse.php?cat=300', 'display_name': u'Emerald City', 'type': 'lostfilm.tv'} topic5 = {'id': 5, 'url': 'http://www.lostfilm.tv/browse.php?cat=_301', 'display_name': u'Taboo', 'type': 'lostfilm.tv'} topic6 = {'id': 6, 'url': 'http://www.lostfilm.tv/browse.php?cat=131', 'display_name': u'Broadwalk Empire', 'type': 'lostfilm.tv'} topic7 = {'id': 7, 'url': 'http://www.lostfilm.tv/random_url.php?cat=1', 'display_name': u'Random Show', 'type': 'lostfilm.tv'} lostfilm_topic1 = {'id': 1, 'search_name': '1', 'quality': 'SD'} lostfilm_topic2 = {'id': 2, 'search_name': '2', 'season': 2, 'quality': '720p'} lostfilm_topic3 = {'id': 3, 'search_name': '3', 'season': 1, 'episode': 3, 'quality': '1080p'} lostfilm_topic4 = {'id': 4, 'search_name': '4', 'season': 1, 'episode': 3, 'quality': '720p'} lostfilm_topic5 = {'id': 5, 'search_name': '5', 'quality': '720p'} lostfilm_topic6 = {'id': 6, 'search_name': '6', 'quality': '720p'} lostfilm_topic7 = {'id': 7, 'search_name': '7', 'quality': '720p'} cred = {'username': '******', 'password': '******'} topics = [topic1, topic2, topic3, topic4, topic5, topic6, topic7] lostfilm_topics = [lostfilm_topic1, lostfilm_topic2, lostfilm_topic3, lostfilm_topic4, lostfilm_topic5, lostfilm_topic6, lostfilm_topic7] self._upgrade_from([lostfilm_topics, topics, [cred]], 3) with DBSession() as db: lostfilm_topics = [row2dict(t, self.LostFilmTVSeries4) for t in db.query(self.LostFilmTVSeries4)] lostfilm_topics = {t['id']: t for t in lostfilm_topics} topics4 = [row2dict(t, self.Topic4) for t in db.query(self.Topic4)] topics4 = {t['id']: t for t in topics4} assert len(lostfilm_topics) == 7 assert lostfilm_topics[1]['cat'] == 236 assert lostfilm_topics[2]['cat'] == 245 assert lostfilm_topics[3]['cat'] == 251 assert lostfilm_topics[4]['cat'] == 300 assert lostfilm_topics[5]['cat'] == 301 assert lostfilm_topics[6]['cat'] == 131 assert lostfilm_topics[7]['cat'] == 0 assert len(topics4) == 7 assert topics4[1]['url'] == 'https://www.lostfilm.tv/series/12_Monkeys/seasons' assert topics4[1]['status'] == Status.Ok assert topics4[2]['url'] == 'https://www.lostfilm.tv/series/Mr_Robot/seasons' assert topics4[2]['status'] == Status.Ok assert topics4[3]['url'] == 'https://www.lostfilm.tv/series/Scream/seasons' assert topics4[3]['status'] == Status.Ok assert topics4[4]['url'] == 'https://www.lostfilm.tv/series/Emerald_City/seasons' assert topics4[4]['status'] == Status.Ok assert topics4[5]['url'] == 'https://www.lostfilm.tv/series/Taboo/seasons' assert topics4[5]['status'] == Status.Ok assert topics4[6]['url'] == 'http://www.lostfilm.tv/browse.php?cat=131' assert topics4[6]['status'] == Status.Error assert topics4[7]['url'] == 'http://www.lostfilm.tv/random_url.php?cat=1' assert topics4[7]['status'] == Status.Error
def upgrade_3_to_4(engine, operations_factory): # Version 3 m3 = MetaData() lostfilm_series_3 = Table( 'lostfilmtv_series', m3, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("search_name", String, nullable=False), Column("season", Integer, nullable=True), Column("episode", Integer, nullable=True), Column("quality", String, nullable=False)) lostfilm_credentials_3 = Table( "lostfilmtv_credentials", m3, Column('username', String, primary_key=True), Column('password', String, primary_key=True), Column('uid', String), Column('pass', String), Column('usess', String), Column('default_quality', String, nullable=False, server_default='SD')) # Version 4 m4 = MetaData(engine) topic_last = Table('topics', m4, *[c.copy() for c in Topic.__table__.columns]) lostfilm_series_4 = Table( 'lostfilmtv_series4', m4, Column("id", Integer, ForeignKey('topics.id'), primary_key=True), Column("cat", Integer, nullable=False), Column("season", Integer, nullable=True), Column("episode", Integer, nullable=True), Column("quality", String, nullable=False)) lostfilm_credentials_4 = Table( "lostfilmtv_credentials4", m4, Column('username', String, primary_key=True), Column('password', String, primary_key=True), Column('session', String), Column('default_quality', String, nullable=False, server_default='SD')) cat_re = re.compile( six.text_type( r'https?://(www|old)\.lostfilm\.tv/browse\.php\?cat=_?(?P<cat>\d+)' ), re.UNICODE) from monitorrent.settings_manager import SettingsManager settings_manager = SettingsManager() tracker_settings = None with operations_factory() as operations: # if previuos run fails, it can not delete this table if operations.has_table(lostfilm_series_4.name): operations.drop_table(lostfilm_series_4.name) operations.create_table(lostfilm_series_4) lostfilm_topics = operations.db.query(lostfilm_series_3) topics = operations.db.query(topic_last) topics = [row2dict(t, topic_last) for t in topics] topics = {t['id']: t for t in topics} for topic in lostfilm_topics: raw_lostfilm_topic = row2dict(topic, lostfilm_series_3) raw_topic = topics[raw_lostfilm_topic['id']] match = cat_re.match(raw_topic['url']) topic_values = {} if not match: print("can't parse old url: {0}".format(raw_topic['url'])) raw_lostfilm_topic['cat'] = 0 topic_values['status'] = Status.Error else: cat = int(match.group('cat')) raw_lostfilm_topic['cat'] = cat try: if tracker_settings is None: tracker_settings = settings_manager.tracker_settings old_url = 'https://www.lostfilm.tv/browse.php?cat={0}'.format( cat) url_response = scraper.get( old_url, **tracker_settings.get_requests_kwargs()) soup = get_soup(url_response.text) meta_content = soup.find('meta').attrs['content'] redirect_url = meta_content.split(';')[1].strip()[4:] if redirect_url.startswith('/'): redirect_url = redirect_url[1:] redirect_url = u'https://www.lostfilm.tv/{0}'.format( redirect_url) url = LostFilmShow.get_seasons_url(redirect_url) if url is None: raise Exception( "Can't parse url from {0} it was redirected to {1}" .format(old_url, redirect_url)) topic_values['url'] = url except: exc_info = sys.exc_info() print(u''.join(traceback.format_exception(*exc_info))) topic_values['status'] = Status.Error operations.db.execute(lostfilm_series_4.insert(), raw_lostfilm_topic) operations.db.execute( topic_last.update( whereclause=(topic_last.c.id == raw_topic['id']), values=topic_values)) # drop original table operations.drop_table(lostfilm_series_3.name) # rename new created table to old one operations.rename_table(lostfilm_series_4.name, lostfilm_series_3.name) # if previuos run fails, it can not delete this table if operations.has_table(lostfilm_credentials_4.name): operations.drop_table(lostfilm_credentials_4.name) operations.create_table(lostfilm_credentials_4) credentials = list(operations.db.query(lostfilm_credentials_3)) for credential in credentials: raw_credential = row2dict(credential, lostfilm_credentials_3) operations.db.execute(lostfilm_credentials_4.insert(), raw_credential) # drop original table operations.drop_table(lostfilm_credentials_3.name) # rename new created table to old one operations.rename_table(lostfilm_credentials_4.name, lostfilm_credentials_3.name)