def test_entry_errors(self): session = create_session() col = model.Collection.create(session, name="test") session.commit() e1 = model.Entry(name="example.com", url="http://www.example.com", parent_id=col.get_root().id, collection=col) session.add(e1) session.commit() e1.set_error("test failure") self.assertEqual(e1.error_msg, "test failure") now = datetime.datetime.now() in_5 = now + datetime.timedelta(minutes=5) in_1day = now + datetime.timedelta(days=1) self.assertLess(e1.last_failure, e1.next_check) lc = e1.next_check self.assertLess(e1.next_check, in_5) e1.set_error("test failure2") self.assertLess(e1.next_check, in_5) e1.set_error("test failure2") e1.set_error("test failure2") self.assertLess(e1.next_check, in_5) e1.set_error("test failure2") self.assertGreater(e1.next_check, in_5) self.assertEqual(e1.error_msg, "test failure2") self.assertLess(lc, e1.next_check) self.assertEqual(e1.failure_count, 5) e1.set_success() self.assertEqual(e1.failure_count, 0) self.assertGreater(e1.next_check, in_1day) self.assertEqual(e1.state, model.EntryState.success)
def exec_entry(self, entry): os = object_session(entry) os.expunge(entry) msg = [] session = create_session() session.add(entry) if entry.type != model.EntryType.collection: entry.type = model.EntryType.collection session.commit() self.log.debug("load feed url %s", entry.url) feed = feedparser.parse(entry.url) self.log.debug("%s done", entry.url) if feed['bozo']: if feed.get('status', 0) > 400: if feed['status'] == 404: msg.append( "HTTP Error 404. File not found. Please check url: %s" % entry.url) else: msg.append("HTTP error: %s" % feed['status']) msg.append(str(feed['bozo_exception'])) self.failure(entry, "\n".join(msg)) return False self.log.debug("%s: found %s entries", entry, len(feed.get('entries', ()))) for fentry in feed.get('entries', ()): subentry, created = entry.get_or_create_child( fentry['title'], { 'name': fentry['title'], 'parent_id': entry.id, 'collection_id': entry.collection_id, 'type': model.EntryType.collection_directory }) #embed() msg.append("Found entry: %s" % (subentry.name)) for link in fentry.get('links', ()): if link.get('rel', None) in ['enclosure', 'alternate']: fname = urllib.parse.urlparse(link['href']) bname = utils.basename(fname.path) lnk, created = subentry.get_or_create_child( bname, { "name": bname, "parent_id": subentry.id, "collection_id": subentry.collection_id, "type": model.EntryType.collection_single }) msg.append("Found link:%s name:%s rel:%s" % (lnk.url, lnk.name, link['rel'])) session.add(lnk) lnk.url = link['href'] lnk.set_meta("type", link['type']) session.commit() self.log.info("successfully checked feed: %s", entry) entry.set_success("\n".join(msg)) return True
def exec_entry(self, entry): os = object_session(entry) os.expunge(entry) msg = [] session = create_session() session.add(entry) if entry.type != model.EntryType.collection: entry.type = model.EntryType.collection session.commit() self.log.debug("load feed url %s", entry.url) feed = feedparser.parse(entry.url) self.log.debug("%s done", entry.url) if feed['bozo']: if feed.get('status', 0) > 400: if feed['status'] == 404: msg.append("HTTP Error 404. File not found. Please check url: %s" %entry.url) else: msg.append("HTTP error: %s" %feed['status']) msg.append(str(feed['bozo_exception'])) self.failure(entry, "\n".join(msg)) return False self.log.debug("%s: found %s entries", entry, len(feed.get('entries', ()))) for fentry in feed.get('entries', ()): subentry, created = entry.get_or_create_child(fentry['title'], {'name': fentry['title'], 'parent_id': entry.id, 'collection_id': entry.collection_id, 'type': model.EntryType.collection_directory}) #embed() msg.append("Found entry: %s" %(subentry.name)) for link in fentry.get('links', ()): if link.get('rel', None) in ['enclosure', 'alternate']: fname = urllib.parse.urlparse(link['href']) bname = utils.basename(fname.path) lnk, created = subentry.get_or_create_child(bname, {"name": bname, "parent_id": subentry.id, "collection_id": subentry.collection_id, "type": model.EntryType.collection_single}) msg.append("Found link:%s name:%s rel:%s" %(lnk.url, lnk.name, link['rel'])) session.add(lnk) lnk.url = link['href'] lnk.set_meta("type", link['type']) session.commit() self.log.info("successfully checked feed: %s", entry) entry.set_success("\n".join(msg)) return True
def test_entry(self): session = create_session() col = model.Collection.create(session, name="test") session.commit() root = col.get_root() e1 = model.Entry(name="e1", parent_id=root.id, priority=10, collection=col) e2 = model.Entry(name="e2", parent_id=root.id, priority=-10, collection=col) session.add_all((e1,e2)) session.flush() e1e = model.Entry(name="child", parent_id=e1.id, priority=10, collection=col) u1_name = "Übername for \\ | be" u1 = model.Entry(name=u1_name, collection=col) session.add(u1) session.flush() u1c = model.Entry(name="Some Ünicode child", parent_id=u1.id, collection=col) session.add(u1c) session.commit() self.assertEqual(e1e.full_path, "/e1/child") self.assertEqual(e1e.system_path, os.path.join("e1","child")) self.assertEqual(e1.system_path, "e1") self.assertEqual(e1.full_path, "/e1") self.assertEqual(u1.full_path, u1_name) self.assertEqual(u1c.full_path, "%s/%s" %(u1_name, "Some Ünicode child")) u1.set_meta("bla", "blubb") session.commit() self.assertEqual(u1.get_meta("bla"), "blubb") u1.set_meta("i1", 1) self.assertEqual(u1.get_meta("i1"), 1) u1.set_meta("d1", {"test":"blubb", "haha":"json", "in": 32}) self.assertEqual(u1.get_meta("d1"), {"test":"blubb", "haha":"json", "in": 32}) b1 = bytes([0, 1, 2, 3]) u1.set_meta("b1", b1) self.assertEqual(u1.get_meta("b1"), b1) u1.set_meta("b1", None) self.assertEqual(u1.get_meta("b1"), None) u1.set_meta("f1", 0.3) self.assertAlmostEqual(u1.get_meta("f1"), 0.3)
def test_download_manager(self): man = manager.CollapseManager() man.app.options = man.app.parser.parse_known_args([])[0] man.app.initialize_app([]) man.app.configure_logging() man.log.setLevel(logging.DEBUG) session = create_session() col = model.Collection.create(session, name="test") example = model.Entry(name="e1", parent_id=col.get_root().id, url="http://www.example.com", priority=10, collection_id=col.id) gc = model.Entry(name="git-clone", parent_id=col.get_root().id, url="https://github.com/poelzi/git-clone-test.git", collection_id=col.id) session.add(example, gc) gc_back, gc_points = man.get_backend_for_entry(gc) self.assertEqual(gc_back.name, "git") session.commit() # test download_mananger for name,dm in man._download_managers.items(): self.log.info("test download_manager %s", name) self.assertEqual(name, dm.name) man.download_mananger = dm(man, download_path=os.path.join(self.tmpdir, "downloads")) man.download_mananger.start() out_path = yield from man.download_mananger.prepare_entry(example) self.assertEqual(out_path[-12:], os.path.join('downloads', 'e1')) self.assertTrue(os.path.exists(out_path)) example.collection_id = col.id out_path2 = yield from man.download_mananger.prepare_entry(example) self.assertEqual(out_path, out_path2) man.start_daemon(run_forever=False) yield from asyncio.sleep(3)