def get_execute_log_details(self, execute_id, after=None):
with DBSession() as db:
filters = [ExecuteLog.execute_id == execute_id]
if after is not None:
filters.append(ExecuteLog.id > after)
log_entries = db.query(ExecuteLog).filter(*filters).all()
return [row2dict(e) for e in log_entries]
def get_watching_topics(self):
watching_topics = []
with DBSession() as db:
dbtopics = db.query(Topic).all()
for dbtopic in dbtopics:
try:
tracker = self.get_tracker(dbtopic.type)
except KeyError:
# TODO: Log warning of not existing topic
# Need to think, should we return not existing plugin
# as just default topic, and show it disabled on UI to
# let user ability for delete such topics
continue
topic = row2dict(
dbtopic, None,
['id', 'url', 'display_name', 'last_update', 'paused'])
topic['info'] = tracker.get_topic_info(dbtopic)
topic['tracker'] = dbtopic.type
topic['status'] = dbtopic.status.__str__()
if dbtopic.type == 'lostfilm.tv':
topic['external_id'] = db.query(
tracker.topic_class).filter(
tracker.topic_class.id == dbtopic.id).first().cat
watching_topics.append(topic)
return watching_topics
def get_log_entries(self, skip, take):
with DBSession() as db:
downloaded_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
.group_by(ExecuteLog.execute_id, ExecuteLog.level)\
.having(ExecuteLog.level == 'downloaded')\
.subquery()
failed_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
.group_by(ExecuteLog.execute_id, ExecuteLog.level)\
.having(ExecuteLog.level == 'failed')\
.subquery()
result_query = db.query(Execute, downloaded_sub_query.c.count, failed_sub_query.c.count)\
.outerjoin(failed_sub_query, Execute.id == failed_sub_query.c.execute_id)\
.outerjoin(downloaded_sub_query, Execute.id == downloaded_sub_query.c.execute_id)\
.order_by(Execute.finish_time.desc())\
.offset(skip)\
.limit(take)
result = []
for execute, downloads, fails in result_query.all():
execute_result = row2dict(execute)
execute_result['downloaded'] = downloads or 0
execute_result['failed'] = fails or 0
execute_result['is_running'] = execute.id == self._execute_id
result.append(execute_result)
execute_count = db.query(func.count(Execute.id)).scalar()
return result, execute_count
def get_execute_log_details(self, execute_id, after=None):
with DBSession() as db:
filters = [ExecuteLog.execute_id == execute_id]
if after is not None:
filters.append(ExecuteLog.id > after)
log_entries = db.query(ExecuteLog).filter(*filters).all()
return [row2dict(e) for e in log_entries]
def get_log_entries(self, skip, take):
with DBSession() as db:
downloaded_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
.group_by(ExecuteLog.execute_id, ExecuteLog.level)\
.having(ExecuteLog.level == 'downloaded')\
.subquery()
failed_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
.group_by(ExecuteLog.execute_id, ExecuteLog.level)\
.having(ExecuteLog.level == 'failed')\
.subquery()
result_query = db.query(Execute, downloaded_sub_query.c.count, failed_sub_query.c.count)\
.outerjoin(failed_sub_query, Execute.id == failed_sub_query.c.execute_id)\
.outerjoin(downloaded_sub_query, Execute.id == downloaded_sub_query.c.execute_id)\
.order_by(Execute.finish_time.desc())\
.offset(skip)\
.limit(take)
result = []
for execute, downloads, fails in result_query.all():
execute_result = row2dict(execute)
execute_result['downloaded'] = downloads or 0
execute_result['failed'] = fails or 0
execute_result['is_running'] = execute.id == self._execute_id
result.append(execute_result)
execute_count = db.query(func.count(Execute.id)).scalar()
return result, execute_count
def get_credentials(self):
with DBSession() as db:
dbcredentials = db.query(self.credentials_class).first()
if dbcredentials is None:
return None
return row2dict(dbcredentials, None,
self.credentials_public_fields)
def get_topic(self, id):
with DBSession() as db:
topic = db.query(self.topic_class).filter(Topic.id == id).first()
if topic is None:
return None
data = row2dict(topic, None, self.topic_public_fields)
data['info'] = self.get_topic_info(topic)
return data
def get_topic(self, id):
with DBSession() as db:
topic = db.query(self.topic_class).filter(Topic.id == id).first()
if topic is None:
return None
data = row2dict(topic, None, self.topic_public_fields)
data['info'] = self.get_topic_info(topic)
return data
def create_removed_topic(self):
remove_type = TRACKER1_PLUGIN_NAME + ".uk"
with DBSession() as db:
topic = Topic(display_name=self.DISPLAY_NAME1 + " / Test",
url="http://tracker.com/2/",
type=remove_type)
result = db.execute(topic.__table__.insert(), row2dict(topic, fields=['display_name', 'url', 'type']))
tracker1_id2 = result.inserted_primary_key[0]
return tracker1_id2
def create_removed_topic(self):
remove_type = TRACKER1_PLUGIN_NAME + ".uk"
with DBSession() as db:
topic = Topic(display_name=self.DISPLAY_NAME1 + " / Test",
url="http://tracker.com/2/",
type=remove_type)
result = db.execute(
topic.__table__.insert(),
row2dict(topic, fields=['display_name', 'url', 'type']))
tracker1_id2 = result.inserted_primary_key[0]
return tracker1_id2
def test_upgrade_filled_from_version_0(self):
topic1 = {
'id':
1,
'url':
'https://www.anilibria.tv/release/seishun-buta-yarou-wa-bunny-girl-senpai-no-yume'
'-wo-minai.html',
'display_name':
'1',
'type':
'anilibria.tv'
}
anilibria_topic1 = {'id': 1, 'hash': 'abcdef'}
topic2 = {
'id': 2,
'url': 'https://www.anilibria.tv',
'display_name': '2',
'type': 'anilibria.tv'
}
anilibria_topic2 = {'id': 2, 'hash': 'abcdef'}
self._upgrade_from(
[[anilibria_topic1, anilibria_topic2], [topic1, topic2]], 0)
with DBSession() as db:
topics = [
row2dict(t, self.AnilibriaTopic1)
for t in db.query(self.AnilibriaTopic1)
]
base_topics = [
row2dict(t, self.BaseTopic1) for t in db.query(self.BaseTopic1)
]
assert topics[0][
'format_list'] == u'Серия 1-13 [HDTVRip 1080p],Серия 1-13 [HDTVRip 720p]'
assert topics[0]['format'] == u'Серия 1-13 [HDTVRip 1080p]'
assert base_topics[0]['status'] == Status.Ok
assert base_topics[1]['status'] == Status.Error
def upgrade_1_to_2(operations_factory):
m1 = MetaData()
rutor_topic_1 = Table('rutororg_topics', m1,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=False))
m2 = MetaData()
rutor_topic_2 = Table('rutororg_topics2', m2,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=True))
with operations_factory() as operations:
operations.create_table(rutor_topic_2)
topics = operations.db.query(rutor_topic_1)
for topic in topics:
raw_topic = row2dict(topic, rutor_topic_1)
operations.db.execute(rutor_topic_2.insert(), raw_topic)
operations.drop_table(rutor_topic_1)
operations.rename_table(rutor_topic_2.name, rutor_topic_1.name)
def upgrade_1_to_2(operations_factory):
m1 = MetaData()
rutor_topic_1 = Table('rutororg_topics', m1,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=False))
m2 = MetaData()
rutor_topic_2 = Table('rutororg_topics2', m2,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=True))
with operations_factory() as operations:
operations.create_table(rutor_topic_2)
topics = operations.db.query(rutor_topic_1)
for topic in topics:
raw_topic = row2dict(topic, rutor_topic_1)
operations.db.execute(rutor_topic_2.insert(), raw_topic)
operations.drop_table(rutor_topic_1)
operations.rename_table(rutor_topic_2.name, rutor_topic_1.name)
def upgrade_0_to_1(operations_factory):
m1 = MetaData()
unionpeer_topic_table_0 = Table('unionpeerorg_topics', m1,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=False))
m2 = MetaData()
unionpeer_topic_table_1 = Table('unionpeerorg_topics1', m2,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=True))
with operations_factory() as operations:
operations.create_table(unionpeer_topic_table_1)
topics = operations.db.query(unionpeer_topic_table_0)
for topic in topics:
raw_topic = row2dict(topic, unionpeer_topic_table_0)
operations.db.execute(unionpeer_topic_table_1.insert(), raw_topic)
operations.drop_table(unionpeer_topic_table_0)
operations.rename_table(unionpeer_topic_table_1.name, unionpeer_topic_table_0.name)
def get_watching_topics(self):
watching_topics = []
with DBSession() as db:
dbtopics = db.query(Topic).all()
for dbtopic in dbtopics:
try:
tracker = self.get_tracker(dbtopic.type)
except KeyError:
# TODO: Log warning of not existing topic
# Need to think, should we return not existing plugin
# as just default topic, and show it disabled on UI to
# let user ability for delete such topics
continue
topic = row2dict(dbtopic, None, ['id', 'url', 'display_name', 'last_update', 'paused'])
topic['info'] = tracker.get_topic_info(dbtopic)
topic['tracker'] = dbtopic.type
topic['status'] = dbtopic.status.__str__()
watching_topics.append(topic)
return watching_topics
def upgrade_0_to_1(operations_factory):
m1 = MetaData()
unionpeer_topic_table_0 = Table(
'unionpeerorg_topics', m1,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=False))
m2 = MetaData()
unionpeer_topic_table_1 = Table(
'unionpeerorg_topics1', m2,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=True))
with operations_factory() as operations:
operations.create_table(unionpeer_topic_table_1)
topics = operations.db.query(unionpeer_topic_table_0)
for topic in topics:
raw_topic = row2dict(topic, unionpeer_topic_table_0)
operations.db.execute(unionpeer_topic_table_1.insert(), raw_topic)
operations.drop_table(unionpeer_topic_table_0)
operations.rename_table(unionpeer_topic_table_1.name,
unionpeer_topic_table_0.name)
def get_credentials(self):
with DBSession() as db:
dbcredentials = db.query(self.credentials_class).first()
if dbcredentials is None:
return None
return row2dict(dbcredentials, None, self.credentials_public_fields)
def upgrade_0_to_1(operations_factory):
from monitorrent.settings_manager import SettingsManager
settings_manager = SettingsManager()
tracker_settings = None
with operations_factory() as operations:
operations.add_column(AnilibriaTvTopic.__tablename__,
Column('format', String, nullable=True))
operations.add_column(AnilibriaTvTopic.__tablename__,
Column('format_list', String, nullable=True))
topic_values = []
m = MetaData()
ani_topics = Table(
AnilibriaTvTopic.__tablename__, m,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("hash", String, nullable=True),
Column("format", String, nullable=True),
Column("format_list", String, nullable=True))
m1 = MetaData()
base_topics = Table(Topic.__tablename__, m1,
Column("id", Integer, primary_key=True),
Column("url", String), Column("type", String),
Column('status', String))
topics = operations.db.query(base_topics).filter(
base_topics.c.type == PLUGIN_NAME)
for topic in topics:
raw_topic = row2dict(topic, base_topics)
# noinspection PyBroadException
try:
if tracker_settings is None:
tracker_settings = settings_manager.tracker_settings
response = requests.get(
raw_topic['url'], **tracker_settings.get_requests_kwargs())
soup = get_soup(response.text)
format_list = AnilibriaTvTracker._find_format_list(soup)
format_list.sort()
topic_values.append({
'id': raw_topic['id'],
'format_list': ",".join(format_list),
'format': format_list[0],
'status': Status.Ok.__str__()
})
except:
exc_info = sys.exc_info()
print(u''.join(traceback.format_exception(*exc_info)))
topic_values.append({
'id': raw_topic['id'],
'status': Status.Error.__str__()
})
for upd in topic_values:
if 'format' in upd:
operations.db.execute(
ani_topics.update(
whereclause=(ani_topics.c.id == upd['id']),
values={
'format_list': upd['format_list'],
'format': upd['format']
}))
operations.db.execute(
base_topics.update(whereclause=(base_topics.c.id == upd['id']),
values={'status': upd['status']}))
def test_updage_filled_from_version_3(self):
Table('lostfilmtv_series4', self.m3,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("cat", Integer, nullable=False),
Column("season", Integer, nullable=True),
Column("episode", Integer, nullable=True),
Column("quality", String, nullable=False))
Table("lostfilmtv_credentials4", self.m3,
Column('username', String, primary_key=True),
Column('password', String, primary_key=True),
Column('session', String),
Column('default_quality', String, nullable=False, server_default='SD'))
topic1 = {'id': 1, 'url': 'http://www.lostfilm.tv/browse.php?cat=236',
'display_name': u'12 обезьян / 12 Monkeys', 'type': 'lostfilm.tv'}
topic2 = {'id': 2, 'url': 'http://www.lostfilm.tv/browse.php?cat=245',
'display_name': u'Mr. Robot', 'type': 'lostfilm.tv'}
topic3 = {'id': 3, 'url': 'http://www.lostfilm.tv/browse.php?cat=251',
'display_name': u'Scream', 'type': 'lostfilm.tv'}
topic4 = {'id': 4, 'url': 'https://www.lostfilm.tv/browse.php?cat=300',
'display_name': u'Emerald City', 'type': 'lostfilm.tv'}
topic5 = {'id': 5, 'url': 'http://www.lostfilm.tv/browse.php?cat=_301',
'display_name': u'Taboo', 'type': 'lostfilm.tv'}
topic6 = {'id': 6, 'url': 'http://www.lostfilm.tv/browse.php?cat=131',
'display_name': u'Broadwalk Empire', 'type': 'lostfilm.tv'}
topic7 = {'id': 7, 'url': 'http://www.lostfilm.tv/random_url.php?cat=1',
'display_name': u'Random Show', 'type': 'lostfilm.tv'}
lostfilm_topic1 = {'id': 1, 'search_name': '1', 'quality': 'SD'}
lostfilm_topic2 = {'id': 2, 'search_name': '2', 'season': 2, 'quality': '720p'}
lostfilm_topic3 = {'id': 3, 'search_name': '3', 'season': 1, 'episode': 3, 'quality': '1080p'}
lostfilm_topic4 = {'id': 4, 'search_name': '4', 'season': 1, 'episode': 3, 'quality': '720p'}
lostfilm_topic5 = {'id': 5, 'search_name': '5', 'quality': '720p'}
lostfilm_topic6 = {'id': 6, 'search_name': '6', 'quality': '720p'}
lostfilm_topic7 = {'id': 7, 'search_name': '7', 'quality': '720p'}
cred = {'username': '******', 'password': '******'}
topics = [topic1, topic2, topic3, topic4, topic5, topic6, topic7]
lostfilm_topics = [lostfilm_topic1, lostfilm_topic2, lostfilm_topic3, lostfilm_topic4, lostfilm_topic5,
lostfilm_topic6, lostfilm_topic7]
self._upgrade_from([lostfilm_topics, topics, [cred]], 3)
with DBSession() as db:
lostfilm_topics = [row2dict(t, self.LostFilmTVSeries4) for t in db.query(self.LostFilmTVSeries4)]
lostfilm_topics = {t['id']: t for t in lostfilm_topics}
topics4 = [row2dict(t, self.Topic4) for t in db.query(self.Topic4)]
topics4 = {t['id']: t for t in topics4}
assert len(lostfilm_topics) == 7
assert lostfilm_topics[1]['cat'] == 236
assert lostfilm_topics[2]['cat'] == 245
assert lostfilm_topics[3]['cat'] == 251
assert lostfilm_topics[4]['cat'] == 300
assert lostfilm_topics[5]['cat'] == 301
assert lostfilm_topics[6]['cat'] == 131
assert lostfilm_topics[7]['cat'] == 0
assert len(topics4) == 7
assert topics4[1]['url'] == 'https://www.lostfilm.tv/series/12_Monkeys/seasons'
assert topics4[1]['status'] == Status.Ok
assert topics4[2]['url'] == 'https://www.lostfilm.tv/series/Mr_Robot/seasons'
assert topics4[2]['status'] == Status.Ok
assert topics4[3]['url'] == 'https://www.lostfilm.tv/series/Scream/seasons'
assert topics4[3]['status'] == Status.Ok
assert topics4[4]['url'] == 'https://www.lostfilm.tv/series/Emerald_City/seasons'
assert topics4[4]['status'] == Status.Ok
assert topics4[5]['url'] == 'https://www.lostfilm.tv/series/Taboo/seasons'
assert topics4[5]['status'] == Status.Ok
assert topics4[6]['url'] == 'http://www.lostfilm.tv/browse.php?cat=131'
assert topics4[6]['status'] == Status.Error
assert topics4[7]['url'] == 'http://www.lostfilm.tv/random_url.php?cat=1'
assert topics4[7]['status'] == Status.Error
def upgrade_3_to_4(engine, operations_factory):
# Version 3
m3 = MetaData()
lostfilm_series_3 = Table(
'lostfilmtv_series', m3,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("search_name", String, nullable=False),
Column("season", Integer, nullable=True),
Column("episode", Integer, nullable=True),
Column("quality", String, nullable=False))
lostfilm_credentials_3 = Table(
"lostfilmtv_credentials", m3,
Column('username', String, primary_key=True),
Column('password', String, primary_key=True), Column('uid', String),
Column('pass', String), Column('usess', String),
Column('default_quality', String, nullable=False, server_default='SD'))
# Version 4
m4 = MetaData(engine)
topic_last = Table('topics', m4,
*[c.copy() for c in Topic.__table__.columns])
lostfilm_series_4 = Table(
'lostfilmtv_series4', m4,
Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
Column("cat", Integer, nullable=False),
Column("season", Integer, nullable=True),
Column("episode", Integer, nullable=True),
Column("quality", String, nullable=False))
lostfilm_credentials_4 = Table(
"lostfilmtv_credentials4", m4,
Column('username', String, primary_key=True),
Column('password', String, primary_key=True),
Column('session', String),
Column('default_quality', String, nullable=False, server_default='SD'))
cat_re = re.compile(
six.text_type(
r'https?://(www|old)\.lostfilm\.tv/browse\.php\?cat=_?(?P<cat>\d+)'
), re.UNICODE)
from monitorrent.settings_manager import SettingsManager
settings_manager = SettingsManager()
tracker_settings = None
with operations_factory() as operations:
# if previuos run fails, it can not delete this table
if operations.has_table(lostfilm_series_4.name):
operations.drop_table(lostfilm_series_4.name)
operations.create_table(lostfilm_series_4)
lostfilm_topics = operations.db.query(lostfilm_series_3)
topics = operations.db.query(topic_last)
topics = [row2dict(t, topic_last) for t in topics]
topics = {t['id']: t for t in topics}
for topic in lostfilm_topics:
raw_lostfilm_topic = row2dict(topic, lostfilm_series_3)
raw_topic = topics[raw_lostfilm_topic['id']]
match = cat_re.match(raw_topic['url'])
topic_values = {}
if not match:
print("can't parse old url: {0}".format(raw_topic['url']))
raw_lostfilm_topic['cat'] = 0
topic_values['status'] = Status.Error
else:
cat = int(match.group('cat'))
raw_lostfilm_topic['cat'] = cat
try:
if tracker_settings is None:
tracker_settings = settings_manager.tracker_settings
old_url = 'https://www.lostfilm.tv/browse.php?cat={0}'.format(
cat)
url_response = scraper.get(
old_url, **tracker_settings.get_requests_kwargs())
soup = get_soup(url_response.text)
meta_content = soup.find('meta').attrs['content']
redirect_url = meta_content.split(';')[1].strip()[4:]
if redirect_url.startswith('/'):
redirect_url = redirect_url[1:]
redirect_url = u'https://www.lostfilm.tv/{0}'.format(
redirect_url)
url = LostFilmShow.get_seasons_url(redirect_url)
if url is None:
raise Exception(
"Can't parse url from {0} it was redirected to {1}"
.format(old_url, redirect_url))
topic_values['url'] = url
except:
exc_info = sys.exc_info()
print(u''.join(traceback.format_exception(*exc_info)))
topic_values['status'] = Status.Error
operations.db.execute(lostfilm_series_4.insert(),
raw_lostfilm_topic)
operations.db.execute(
topic_last.update(
whereclause=(topic_last.c.id == raw_topic['id']),
values=topic_values))
# drop original table
operations.drop_table(lostfilm_series_3.name)
# rename new created table to old one
operations.rename_table(lostfilm_series_4.name, lostfilm_series_3.name)
# if previuos run fails, it can not delete this table
if operations.has_table(lostfilm_credentials_4.name):
operations.drop_table(lostfilm_credentials_4.name)
operations.create_table(lostfilm_credentials_4)
credentials = list(operations.db.query(lostfilm_credentials_3))
for credential in credentials:
raw_credential = row2dict(credential, lostfilm_credentials_3)
operations.db.execute(lostfilm_credentials_4.insert(),
raw_credential)
# drop original table
operations.drop_table(lostfilm_credentials_3.name)
# rename new created table to old one
operations.rename_table(lostfilm_credentials_4.name,
lostfilm_credentials_3.name)
