Esempio n. 1
0
 def get_execute_log_details(self, execute_id, after=None):
     with DBSession() as db:
         filters = [ExecuteLog.execute_id == execute_id]
         if after is not None:
             filters.append(ExecuteLog.id > after)
         log_entries = db.query(ExecuteLog).filter(*filters).all()
         return [row2dict(e) for e in log_entries]
Esempio n. 2
0
    def get_watching_topics(self):
        watching_topics = []
        with DBSession() as db:
            dbtopics = db.query(Topic).all()
            for dbtopic in dbtopics:
                try:
                    tracker = self.get_tracker(dbtopic.type)
                except KeyError:
                    # TODO: Log warning of not existing topic
                    #       Need to think, should we return not existing plugin
                    #       as just default topic, and show it disabled on UI to
                    #       let user ability for delete such topics
                    continue
                topic = row2dict(
                    dbtopic, None,
                    ['id', 'url', 'display_name', 'last_update', 'paused'])
                topic['info'] = tracker.get_topic_info(dbtopic)
                topic['tracker'] = dbtopic.type
                topic['status'] = dbtopic.status.__str__()

                if dbtopic.type == 'lostfilm.tv':
                    topic['external_id'] = db.query(
                        tracker.topic_class).filter(
                            tracker.topic_class.id == dbtopic.id).first().cat

                watching_topics.append(topic)
        return watching_topics
Esempio n. 3
0
    def get_log_entries(self, skip, take):
        with DBSession() as db:
            downloaded_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
                .group_by(ExecuteLog.execute_id, ExecuteLog.level)\
                .having(ExecuteLog.level == 'downloaded')\
                .subquery()
            failed_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
                .group_by(ExecuteLog.execute_id, ExecuteLog.level)\
                .having(ExecuteLog.level == 'failed')\
                .subquery()

            result_query = db.query(Execute, downloaded_sub_query.c.count, failed_sub_query.c.count)\
                .outerjoin(failed_sub_query, Execute.id == failed_sub_query.c.execute_id)\
                .outerjoin(downloaded_sub_query, Execute.id == downloaded_sub_query.c.execute_id)\
                .order_by(Execute.finish_time.desc())\
                .offset(skip)\
                .limit(take)

            result = []
            for execute, downloads, fails in result_query.all():
                execute_result = row2dict(execute)
                execute_result['downloaded'] = downloads or 0
                execute_result['failed'] = fails or 0
                execute_result['is_running'] = execute.id == self._execute_id
                result.append(execute_result)

            execute_count = db.query(func.count(Execute.id)).scalar()

        return result, execute_count
Esempio n. 4
0
 def get_execute_log_details(self, execute_id, after=None):
     with DBSession() as db:
         filters = [ExecuteLog.execute_id == execute_id]
         if after is not None:
             filters.append(ExecuteLog.id > after)
         log_entries = db.query(ExecuteLog).filter(*filters).all()
         return [row2dict(e) for e in log_entries]
Esempio n. 5
0
    def get_log_entries(self, skip, take):
        with DBSession() as db:
            downloaded_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
                .group_by(ExecuteLog.execute_id, ExecuteLog.level)\
                .having(ExecuteLog.level == 'downloaded')\
                .subquery()
            failed_sub_query = db.query(ExecuteLog.execute_id, func.count(ExecuteLog.id).label('count'))\
                .group_by(ExecuteLog.execute_id, ExecuteLog.level)\
                .having(ExecuteLog.level == 'failed')\
                .subquery()

            result_query = db.query(Execute, downloaded_sub_query.c.count, failed_sub_query.c.count)\
                .outerjoin(failed_sub_query, Execute.id == failed_sub_query.c.execute_id)\
                .outerjoin(downloaded_sub_query, Execute.id == downloaded_sub_query.c.execute_id)\
                .order_by(Execute.finish_time.desc())\
                .offset(skip)\
                .limit(take)

            result = []
            for execute, downloads, fails in result_query.all():
                execute_result = row2dict(execute)
                execute_result['downloaded'] = downloads or 0
                execute_result['failed'] = fails or 0
                execute_result['is_running'] = execute.id == self._execute_id
                result.append(execute_result)

            execute_count = db.query(func.count(Execute.id)).scalar()

        return result, execute_count
Esempio n. 6
0
 def get_credentials(self):
     with DBSession() as db:
         dbcredentials = db.query(self.credentials_class).first()
         if dbcredentials is None:
             return None
         return row2dict(dbcredentials, None,
                         self.credentials_public_fields)
Esempio n. 7
0
 def get_topic(self, id):
     with DBSession() as db:
         topic = db.query(self.topic_class).filter(Topic.id == id).first()
         if topic is None:
             return None
         data = row2dict(topic, None, self.topic_public_fields)
         data['info'] = self.get_topic_info(topic)
         return data
Esempio n. 8
0
 def get_topic(self, id):
     with DBSession() as db:
         topic = db.query(self.topic_class).filter(Topic.id == id).first()
         if topic is None:
             return None
         data = row2dict(topic, None, self.topic_public_fields)
         data['info'] = self.get_topic_info(topic)
         return data
 def create_removed_topic(self):
     remove_type = TRACKER1_PLUGIN_NAME + ".uk"
     with DBSession() as db:
         topic = Topic(display_name=self.DISPLAY_NAME1 + " / Test",
                       url="http://tracker.com/2/",
                       type=remove_type)
         result = db.execute(topic.__table__.insert(), row2dict(topic, fields=['display_name', 'url', 'type']))
         tracker1_id2 = result.inserted_primary_key[0]
     return tracker1_id2
 def create_removed_topic(self):
     remove_type = TRACKER1_PLUGIN_NAME + ".uk"
     with DBSession() as db:
         topic = Topic(display_name=self.DISPLAY_NAME1 + " / Test",
                       url="http://tracker.com/2/",
                       type=remove_type)
         result = db.execute(
             topic.__table__.insert(),
             row2dict(topic, fields=['display_name', 'url', 'type']))
         tracker1_id2 = result.inserted_primary_key[0]
     return tracker1_id2
    def test_upgrade_filled_from_version_0(self):
        topic1 = {
            'id':
            1,
            'url':
            'https://www.anilibria.tv/release/seishun-buta-yarou-wa-bunny-girl-senpai-no-yume'
            '-wo-minai.html',
            'display_name':
            '1',
            'type':
            'anilibria.tv'
        }
        anilibria_topic1 = {'id': 1, 'hash': 'abcdef'}
        topic2 = {
            'id': 2,
            'url': 'https://www.anilibria.tv',
            'display_name': '2',
            'type': 'anilibria.tv'
        }
        anilibria_topic2 = {'id': 2, 'hash': 'abcdef'}

        self._upgrade_from(
            [[anilibria_topic1, anilibria_topic2], [topic1, topic2]], 0)

        with DBSession() as db:
            topics = [
                row2dict(t, self.AnilibriaTopic1)
                for t in db.query(self.AnilibriaTopic1)
            ]
            base_topics = [
                row2dict(t, self.BaseTopic1) for t in db.query(self.BaseTopic1)
            ]

        assert topics[0][
            'format_list'] == u'Серия 1-13 [HDTVRip 1080p],Серия 1-13 [HDTVRip 720p]'
        assert topics[0]['format'] == u'Серия 1-13 [HDTVRip 1080p]'
        assert base_topics[0]['status'] == Status.Ok

        assert base_topics[1]['status'] == Status.Error
Esempio n. 12
0
def upgrade_1_to_2(operations_factory):
    m1 = MetaData()
    rutor_topic_1 = Table('rutororg_topics', m1,
                          Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
                          Column("hash", String, nullable=False))

    m2 = MetaData()
    rutor_topic_2 = Table('rutororg_topics2', m2,
                          Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
                          Column("hash", String, nullable=True))
    with operations_factory() as operations:
        operations.create_table(rutor_topic_2)
        topics = operations.db.query(rutor_topic_1)
        for topic in topics:
            raw_topic = row2dict(topic, rutor_topic_1)
            operations.db.execute(rutor_topic_2.insert(), raw_topic)
        operations.drop_table(rutor_topic_1)
        operations.rename_table(rutor_topic_2.name, rutor_topic_1.name)
Esempio n. 13
0
def upgrade_1_to_2(operations_factory):
    m1 = MetaData()
    rutor_topic_1 = Table('rutororg_topics', m1,
                          Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
                          Column("hash", String, nullable=False))

    m2 = MetaData()
    rutor_topic_2 = Table('rutororg_topics2', m2,
                          Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
                          Column("hash", String, nullable=True))
    with operations_factory() as operations:
        operations.create_table(rutor_topic_2)
        topics = operations.db.query(rutor_topic_1)
        for topic in topics:
            raw_topic = row2dict(topic, rutor_topic_1)
            operations.db.execute(rutor_topic_2.insert(), raw_topic)
        operations.drop_table(rutor_topic_1)
        operations.rename_table(rutor_topic_2.name, rutor_topic_1.name)
Esempio n. 14
0
def upgrade_0_to_1(operations_factory):
    m1 = MetaData()
    unionpeer_topic_table_0 = Table('unionpeerorg_topics', m1,
                                    Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
                                    Column("hash", String, nullable=False))

    m2 = MetaData()
    unionpeer_topic_table_1 = Table('unionpeerorg_topics1', m2,
                                    Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
                                    Column("hash", String, nullable=True))
    with operations_factory() as operations:
        operations.create_table(unionpeer_topic_table_1)
        topics = operations.db.query(unionpeer_topic_table_0)
        for topic in topics:
            raw_topic = row2dict(topic, unionpeer_topic_table_0)
            operations.db.execute(unionpeer_topic_table_1.insert(), raw_topic)
        operations.drop_table(unionpeer_topic_table_0)
        operations.rename_table(unionpeer_topic_table_1.name, unionpeer_topic_table_0.name)
Esempio n. 15
0
 def get_watching_topics(self):
     watching_topics = []
     with DBSession() as db:
         dbtopics = db.query(Topic).all()
         for dbtopic in dbtopics:
             try:
                 tracker = self.get_tracker(dbtopic.type)
             except KeyError:
                 # TODO: Log warning of not existing topic
                 #       Need to think, should we return not existing plugin
                 #       as just default topic, and show it disabled on UI to
                 #       let user ability for delete such topics
                 continue
             topic = row2dict(dbtopic, None, ['id', 'url', 'display_name', 'last_update', 'paused'])
             topic['info'] = tracker.get_topic_info(dbtopic)
             topic['tracker'] = dbtopic.type
             topic['status'] = dbtopic.status.__str__()
             watching_topics.append(topic)
     return watching_topics
Esempio n. 16
0
def upgrade_0_to_1(operations_factory):
    m1 = MetaData()
    unionpeer_topic_table_0 = Table(
        'unionpeerorg_topics', m1,
        Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
        Column("hash", String, nullable=False))

    m2 = MetaData()
    unionpeer_topic_table_1 = Table(
        'unionpeerorg_topics1', m2,
        Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
        Column("hash", String, nullable=True))
    with operations_factory() as operations:
        operations.create_table(unionpeer_topic_table_1)
        topics = operations.db.query(unionpeer_topic_table_0)
        for topic in topics:
            raw_topic = row2dict(topic, unionpeer_topic_table_0)
            operations.db.execute(unionpeer_topic_table_1.insert(), raw_topic)
        operations.drop_table(unionpeer_topic_table_0)
        operations.rename_table(unionpeer_topic_table_1.name,
                                unionpeer_topic_table_0.name)
Esempio n. 17
0
 def get_credentials(self):
     with DBSession() as db:
         dbcredentials = db.query(self.credentials_class).first()
         if dbcredentials is None:
             return None
         return row2dict(dbcredentials, None, self.credentials_public_fields)
Esempio n. 18
0
def upgrade_0_to_1(operations_factory):
    from monitorrent.settings_manager import SettingsManager
    settings_manager = SettingsManager()
    tracker_settings = None
    with operations_factory() as operations:
        operations.add_column(AnilibriaTvTopic.__tablename__,
                              Column('format', String, nullable=True))
        operations.add_column(AnilibriaTvTopic.__tablename__,
                              Column('format_list', String, nullable=True))
        topic_values = []
        m = MetaData()
        ani_topics = Table(
            AnilibriaTvTopic.__tablename__, m,
            Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
            Column("hash", String, nullable=True),
            Column("format", String, nullable=True),
            Column("format_list", String, nullable=True))
        m1 = MetaData()
        base_topics = Table(Topic.__tablename__, m1,
                            Column("id", Integer, primary_key=True),
                            Column("url", String), Column("type", String),
                            Column('status', String))
        topics = operations.db.query(base_topics).filter(
            base_topics.c.type == PLUGIN_NAME)
        for topic in topics:
            raw_topic = row2dict(topic, base_topics)
            # noinspection PyBroadException
            try:
                if tracker_settings is None:
                    tracker_settings = settings_manager.tracker_settings
                response = requests.get(
                    raw_topic['url'], **tracker_settings.get_requests_kwargs())
                soup = get_soup(response.text)
                format_list = AnilibriaTvTracker._find_format_list(soup)
                format_list.sort()
                topic_values.append({
                    'id': raw_topic['id'],
                    'format_list': ",".join(format_list),
                    'format': format_list[0],
                    'status': Status.Ok.__str__()
                })
            except:
                exc_info = sys.exc_info()
                print(u''.join(traceback.format_exception(*exc_info)))
                topic_values.append({
                    'id': raw_topic['id'],
                    'status': Status.Error.__str__()
                })

        for upd in topic_values:
            if 'format' in upd:
                operations.db.execute(
                    ani_topics.update(
                        whereclause=(ani_topics.c.id == upd['id']),
                        values={
                            'format_list': upd['format_list'],
                            'format': upd['format']
                        }))
            operations.db.execute(
                base_topics.update(whereclause=(base_topics.c.id == upd['id']),
                                   values={'status': upd['status']}))
    def test_updage_filled_from_version_3(self):
        Table('lostfilmtv_series4', self.m3,
              Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
              Column("cat", Integer, nullable=False),
              Column("season", Integer, nullable=True),
              Column("episode", Integer, nullable=True),
              Column("quality", String, nullable=False))

        Table("lostfilmtv_credentials4", self.m3,
              Column('username', String, primary_key=True),
              Column('password', String, primary_key=True),
              Column('session', String),
              Column('default_quality', String, nullable=False, server_default='SD'))

        topic1 = {'id': 1, 'url': 'http://www.lostfilm.tv/browse.php?cat=236',
                  'display_name': u'12 обезьян / 12 Monkeys', 'type': 'lostfilm.tv'}
        topic2 = {'id': 2, 'url': 'http://www.lostfilm.tv/browse.php?cat=245',
                  'display_name': u'Mr. Robot', 'type': 'lostfilm.tv'}
        topic3 = {'id': 3, 'url': 'http://www.lostfilm.tv/browse.php?cat=251',
                  'display_name': u'Scream', 'type': 'lostfilm.tv'}
        topic4 = {'id': 4, 'url': 'https://www.lostfilm.tv/browse.php?cat=300',
                  'display_name': u'Emerald City', 'type': 'lostfilm.tv'}
        topic5 = {'id': 5, 'url': 'http://www.lostfilm.tv/browse.php?cat=_301',
                  'display_name': u'Taboo', 'type': 'lostfilm.tv'}
        topic6 = {'id': 6, 'url': 'http://www.lostfilm.tv/browse.php?cat=131',
                  'display_name': u'Broadwalk Empire', 'type': 'lostfilm.tv'}
        topic7 = {'id': 7, 'url': 'http://www.lostfilm.tv/random_url.php?cat=1',
                  'display_name': u'Random Show', 'type': 'lostfilm.tv'}

        lostfilm_topic1 = {'id': 1, 'search_name': '1', 'quality': 'SD'}
        lostfilm_topic2 = {'id': 2, 'search_name': '2', 'season': 2, 'quality': '720p'}
        lostfilm_topic3 = {'id': 3, 'search_name': '3', 'season': 1, 'episode': 3, 'quality': '1080p'}
        lostfilm_topic4 = {'id': 4, 'search_name': '4', 'season': 1, 'episode': 3, 'quality': '720p'}
        lostfilm_topic5 = {'id': 5, 'search_name': '5', 'quality': '720p'}
        lostfilm_topic6 = {'id': 6, 'search_name': '6', 'quality': '720p'}
        lostfilm_topic7 = {'id': 7, 'search_name': '7', 'quality': '720p'}

        cred = {'username': '******', 'password': '******'}

        topics = [topic1, topic2, topic3, topic4, topic5, topic6, topic7]
        lostfilm_topics = [lostfilm_topic1, lostfilm_topic2, lostfilm_topic3, lostfilm_topic4, lostfilm_topic5,
                           lostfilm_topic6, lostfilm_topic7]

        self._upgrade_from([lostfilm_topics, topics, [cred]], 3)

        with DBSession() as db:
            lostfilm_topics = [row2dict(t, self.LostFilmTVSeries4) for t in db.query(self.LostFilmTVSeries4)]
            lostfilm_topics = {t['id']: t for t in lostfilm_topics}

            topics4 = [row2dict(t, self.Topic4) for t in db.query(self.Topic4)]
            topics4 = {t['id']: t for t in topics4}

        assert len(lostfilm_topics) == 7
        assert lostfilm_topics[1]['cat'] == 236
        assert lostfilm_topics[2]['cat'] == 245
        assert lostfilm_topics[3]['cat'] == 251
        assert lostfilm_topics[4]['cat'] == 300
        assert lostfilm_topics[5]['cat'] == 301
        assert lostfilm_topics[6]['cat'] == 131
        assert lostfilm_topics[7]['cat'] == 0

        assert len(topics4) == 7
        assert topics4[1]['url'] == 'https://www.lostfilm.tv/series/12_Monkeys/seasons'
        assert topics4[1]['status'] == Status.Ok
        assert topics4[2]['url'] == 'https://www.lostfilm.tv/series/Mr_Robot/seasons'
        assert topics4[2]['status'] == Status.Ok
        assert topics4[3]['url'] == 'https://www.lostfilm.tv/series/Scream/seasons'
        assert topics4[3]['status'] == Status.Ok
        assert topics4[4]['url'] == 'https://www.lostfilm.tv/series/Emerald_City/seasons'
        assert topics4[4]['status'] == Status.Ok
        assert topics4[5]['url'] == 'https://www.lostfilm.tv/series/Taboo/seasons'
        assert topics4[5]['status'] == Status.Ok
        assert topics4[6]['url'] == 'http://www.lostfilm.tv/browse.php?cat=131'
        assert topics4[6]['status'] == Status.Error
        assert topics4[7]['url'] == 'http://www.lostfilm.tv/random_url.php?cat=1'
        assert topics4[7]['status'] == Status.Error
Esempio n. 20
0
def upgrade_3_to_4(engine, operations_factory):
    # Version 3
    m3 = MetaData()
    lostfilm_series_3 = Table(
        'lostfilmtv_series', m3,
        Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
        Column("search_name", String, nullable=False),
        Column("season", Integer, nullable=True),
        Column("episode", Integer, nullable=True),
        Column("quality", String, nullable=False))
    lostfilm_credentials_3 = Table(
        "lostfilmtv_credentials", m3,
        Column('username', String, primary_key=True),
        Column('password', String, primary_key=True), Column('uid', String),
        Column('pass', String), Column('usess', String),
        Column('default_quality', String, nullable=False, server_default='SD'))

    # Version 4
    m4 = MetaData(engine)
    topic_last = Table('topics', m4,
                       *[c.copy() for c in Topic.__table__.columns])
    lostfilm_series_4 = Table(
        'lostfilmtv_series4', m4,
        Column("id", Integer, ForeignKey('topics.id'), primary_key=True),
        Column("cat", Integer, nullable=False),
        Column("season", Integer, nullable=True),
        Column("episode", Integer, nullable=True),
        Column("quality", String, nullable=False))
    lostfilm_credentials_4 = Table(
        "lostfilmtv_credentials4", m4,
        Column('username', String, primary_key=True),
        Column('password', String, primary_key=True),
        Column('session', String),
        Column('default_quality', String, nullable=False, server_default='SD'))

    cat_re = re.compile(
        six.text_type(
            r'https?://(www|old)\.lostfilm\.tv/browse\.php\?cat=_?(?P<cat>\d+)'
        ), re.UNICODE)

    from monitorrent.settings_manager import SettingsManager
    settings_manager = SettingsManager()

    tracker_settings = None

    with operations_factory() as operations:
        # if previuos run fails, it can not delete this table
        if operations.has_table(lostfilm_series_4.name):
            operations.drop_table(lostfilm_series_4.name)
        operations.create_table(lostfilm_series_4)

        lostfilm_topics = operations.db.query(lostfilm_series_3)
        topics = operations.db.query(topic_last)
        topics = [row2dict(t, topic_last) for t in topics]
        topics = {t['id']: t for t in topics}
        for topic in lostfilm_topics:
            raw_lostfilm_topic = row2dict(topic, lostfilm_series_3)
            raw_topic = topics[raw_lostfilm_topic['id']]
            match = cat_re.match(raw_topic['url'])

            topic_values = {}

            if not match:
                print("can't parse old url: {0}".format(raw_topic['url']))
                raw_lostfilm_topic['cat'] = 0
                topic_values['status'] = Status.Error
            else:
                cat = int(match.group('cat'))
                raw_lostfilm_topic['cat'] = cat

                try:
                    if tracker_settings is None:
                        tracker_settings = settings_manager.tracker_settings

                    old_url = 'https://www.lostfilm.tv/browse.php?cat={0}'.format(
                        cat)
                    url_response = scraper.get(
                        old_url, **tracker_settings.get_requests_kwargs())

                    soup = get_soup(url_response.text)
                    meta_content = soup.find('meta').attrs['content']
                    redirect_url = meta_content.split(';')[1].strip()[4:]

                    if redirect_url.startswith('/'):
                        redirect_url = redirect_url[1:]

                    redirect_url = u'https://www.lostfilm.tv/{0}'.format(
                        redirect_url)
                    url = LostFilmShow.get_seasons_url(redirect_url)

                    if url is None:
                        raise Exception(
                            "Can't parse url from {0} it was redirected to {1}"
                            .format(old_url, redirect_url))

                    topic_values['url'] = url
                except:
                    exc_info = sys.exc_info()
                    print(u''.join(traceback.format_exception(*exc_info)))
                    topic_values['status'] = Status.Error

            operations.db.execute(lostfilm_series_4.insert(),
                                  raw_lostfilm_topic)
            operations.db.execute(
                topic_last.update(
                    whereclause=(topic_last.c.id == raw_topic['id']),
                    values=topic_values))

        # drop original table
        operations.drop_table(lostfilm_series_3.name)
        # rename new created table to old one
        operations.rename_table(lostfilm_series_4.name, lostfilm_series_3.name)

        # if previuos run fails, it can not delete this table
        if operations.has_table(lostfilm_credentials_4.name):
            operations.drop_table(lostfilm_credentials_4.name)
        operations.create_table(lostfilm_credentials_4)
        credentials = list(operations.db.query(lostfilm_credentials_3))
        for credential in credentials:
            raw_credential = row2dict(credential, lostfilm_credentials_3)
            operations.db.execute(lostfilm_credentials_4.insert(),
                                  raw_credential)

        # drop original table
        operations.drop_table(lostfilm_credentials_3.name)
        # rename new created table to old one
        operations.rename_table(lostfilm_credentials_4.name,
                                lostfilm_credentials_3.name)