Esempio n. 1
0
    def test_entry_errors(self):
        session = create_session()
        col = model.Collection.create(session, name="test")
        session.commit()
        e1 = model.Entry(name="example.com", url="http://www.example.com", parent_id=col.get_root().id,
                         collection=col)
        session.add(e1)
        session.commit()
        e1.set_error("test failure")

        self.assertEqual(e1.error_msg, "test failure")
        now = datetime.datetime.now()
        in_5 = now + datetime.timedelta(minutes=5)
        in_1day = now + datetime.timedelta(days=1)
        self.assertLess(e1.last_failure, e1.next_check)
        lc = e1.next_check
        self.assertLess(e1.next_check, in_5)
        e1.set_error("test failure2")
        self.assertLess(e1.next_check, in_5)
        e1.set_error("test failure2")
        e1.set_error("test failure2")
        self.assertLess(e1.next_check, in_5)
        e1.set_error("test failure2")
        self.assertGreater(e1.next_check, in_5)
        self.assertEqual(e1.error_msg, "test failure2")
        self.assertLess(lc, e1.next_check)
        self.assertEqual(e1.failure_count, 5)

        e1.set_success()
        self.assertEqual(e1.failure_count, 0)
        self.assertGreater(e1.next_check, in_1day)
        self.assertEqual(e1.state, model.EntryState.success)
Esempio n. 2
0
    def exec_entry(self, entry):
        os = object_session(entry)
        os.expunge(entry)

        msg = []
        session = create_session()
        session.add(entry)
        if entry.type != model.EntryType.collection:
            entry.type = model.EntryType.collection
            session.commit()
        self.log.debug("load feed url %s", entry.url)
        feed = feedparser.parse(entry.url)
        self.log.debug("%s done", entry.url)
        if feed['bozo']:
            if feed.get('status', 0) > 400:
                if feed['status'] == 404:
                    msg.append(
                        "HTTP Error 404. File not found. Please check url: %s"
                        % entry.url)
                else:
                    msg.append("HTTP error: %s" % feed['status'])
            msg.append(str(feed['bozo_exception']))
            self.failure(entry, "\n".join(msg))
            return False
        self.log.debug("%s: found %s entries", entry,
                       len(feed.get('entries', ())))
        for fentry in feed.get('entries', ()):

            subentry, created = entry.get_or_create_child(
                fentry['title'], {
                    'name': fentry['title'],
                    'parent_id': entry.id,
                    'collection_id': entry.collection_id,
                    'type': model.EntryType.collection_directory
                })
            #embed()
            msg.append("Found entry: %s" % (subentry.name))
            for link in fentry.get('links', ()):
                if link.get('rel', None) in ['enclosure', 'alternate']:
                    fname = urllib.parse.urlparse(link['href'])
                    bname = utils.basename(fname.path)
                    lnk, created = subentry.get_or_create_child(
                        bname, {
                            "name": bname,
                            "parent_id": subentry.id,
                            "collection_id": subentry.collection_id,
                            "type": model.EntryType.collection_single
                        })
                    msg.append("Found link:%s name:%s rel:%s" %
                               (lnk.url, lnk.name, link['rel']))
                    session.add(lnk)
                    lnk.url = link['href']
                    lnk.set_meta("type", link['type'])
            session.commit()
        self.log.info("successfully checked feed: %s", entry)
        entry.set_success("\n".join(msg))
        return True
Esempio n. 3
0
    def exec_entry(self, entry):
        os = object_session(entry)
        os.expunge(entry)

        msg = []
        session = create_session()
        session.add(entry)
        if entry.type != model.EntryType.collection:
            entry.type = model.EntryType.collection
            session.commit()
        self.log.debug("load feed url %s", entry.url)
        feed = feedparser.parse(entry.url)
        self.log.debug("%s done", entry.url)
        if feed['bozo']:
            if feed.get('status', 0) > 400:
                if feed['status'] == 404:
                    msg.append("HTTP Error 404. File not found. Please check url: %s" %entry.url)
                else:
                    msg.append("HTTP error: %s" %feed['status'])
            msg.append(str(feed['bozo_exception']))
            self.failure(entry, "\n".join(msg))
            return False
        self.log.debug("%s: found %s entries", entry, len(feed.get('entries', ())))
        for fentry in feed.get('entries', ()):

            subentry, created = entry.get_or_create_child(fentry['title'],
                               {'name':          fentry['title'],
                                'parent_id':     entry.id,
                                'collection_id': entry.collection_id,
                                'type':          model.EntryType.collection_directory})
            #embed()
            msg.append("Found entry: %s" %(subentry.name))
            for link in fentry.get('links', ()):
                if link.get('rel', None) in ['enclosure', 'alternate']:
                    fname = urllib.parse.urlparse(link['href'])
                    bname = utils.basename(fname.path)
                    lnk, created = subentry.get_or_create_child(bname,
                                         {"name":          bname,
                                          "parent_id":     subentry.id,
                                          "collection_id": subentry.collection_id,
                                          "type":          model.EntryType.collection_single})
                    msg.append("Found link:%s name:%s rel:%s" %(lnk.url, lnk.name, link['rel']))
                    session.add(lnk)
                    lnk.url = link['href']
                    lnk.set_meta("type", link['type'])
            session.commit()
        self.log.info("successfully checked feed: %s", entry)
        entry.set_success("\n".join(msg))
        return True
Esempio n. 4
0
    def test_entry(self):
        session = create_session()
        col = model.Collection.create(session, name="test")
        session.commit()
        root = col.get_root()

        e1 = model.Entry(name="e1", parent_id=root.id, priority=10, collection=col)
        e2 = model.Entry(name="e2", parent_id=root.id, priority=-10, collection=col)
        session.add_all((e1,e2))
        session.flush()
        e1e = model.Entry(name="child", parent_id=e1.id, priority=10, collection=col)

        u1_name = "Übername for \\ | be"

        u1 = model.Entry(name=u1_name, collection=col)
        session.add(u1)
        session.flush()
        u1c = model.Entry(name="Some Ünicode child", parent_id=u1.id, collection=col)
        session.add(u1c)
        session.commit()

        self.assertEqual(e1e.full_path, "/e1/child")
        self.assertEqual(e1e.system_path, os.path.join("e1","child"))
        self.assertEqual(e1.system_path, "e1")
        self.assertEqual(e1.full_path, "/e1")
        self.assertEqual(u1.full_path, u1_name)
        self.assertEqual(u1c.full_path, "%s/%s" %(u1_name, "Some Ünicode child"))

        u1.set_meta("bla", "blubb")
        session.commit()
        self.assertEqual(u1.get_meta("bla"), "blubb")
        u1.set_meta("i1", 1)
        self.assertEqual(u1.get_meta("i1"), 1)

        u1.set_meta("d1", {"test":"blubb", "haha":"json", "in": 32})
        self.assertEqual(u1.get_meta("d1"), {"test":"blubb", "haha":"json", "in": 32})

        b1 = bytes([0, 1, 2, 3])
        u1.set_meta("b1", b1)
        self.assertEqual(u1.get_meta("b1"), b1)

        u1.set_meta("b1", None)
        self.assertEqual(u1.get_meta("b1"), None)

        u1.set_meta("f1", 0.3)
        self.assertAlmostEqual(u1.get_meta("f1"), 0.3)
Esempio n. 5
0
    def test_download_manager(self):
        man = manager.CollapseManager()
        man.app.options = man.app.parser.parse_known_args([])[0]
        man.app.initialize_app([])
        man.app.configure_logging()
        man.log.setLevel(logging.DEBUG)

        session = create_session()
        col = model.Collection.create(session, name="test")

        example = model.Entry(name="e1", parent_id=col.get_root().id, url="http://www.example.com", priority=10,
                              collection_id=col.id)
        gc = model.Entry(name="git-clone", parent_id=col.get_root().id,
                              url="https://github.com/poelzi/git-clone-test.git",
                              collection_id=col.id)

        session.add(example, gc)

        gc_back, gc_points = man.get_backend_for_entry(gc)
        self.assertEqual(gc_back.name, "git")

        session.commit()
        # test download_mananger
        for name,dm in man._download_managers.items():
            self.log.info("test download_manager %s", name)
            self.assertEqual(name, dm.name)

            man.download_mananger = dm(man, download_path=os.path.join(self.tmpdir, "downloads"))
            man.download_mananger.start()

            out_path = yield from man.download_mananger.prepare_entry(example)
            self.assertEqual(out_path[-12:], os.path.join('downloads', 'e1'))
            self.assertTrue(os.path.exists(out_path))

            example.collection_id = col.id
            out_path2 = yield from man.download_mananger.prepare_entry(example)
            self.assertEqual(out_path, out_path2)

        man.start_daemon(run_forever=False)
        yield from asyncio.sleep(3)