def test_entry_errors(self):
session = create_session()
col = model.Collection.create(session, name="test")
session.commit()
e1 = model.Entry(name="example.com", url="http://www.example.com", parent_id=col.get_root().id,
collection=col)
session.add(e1)
session.commit()
e1.set_error("test failure")
self.assertEqual(e1.error_msg, "test failure")
now = datetime.datetime.now()
in_5 = now + datetime.timedelta(minutes=5)
in_1day = now + datetime.timedelta(days=1)
self.assertLess(e1.last_failure, e1.next_check)
lc = e1.next_check
self.assertLess(e1.next_check, in_5)
e1.set_error("test failure2")
self.assertLess(e1.next_check, in_5)
e1.set_error("test failure2")
e1.set_error("test failure2")
self.assertLess(e1.next_check, in_5)
e1.set_error("test failure2")
self.assertGreater(e1.next_check, in_5)
self.assertEqual(e1.error_msg, "test failure2")
self.assertLess(lc, e1.next_check)
self.assertEqual(e1.failure_count, 5)
e1.set_success()
self.assertEqual(e1.failure_count, 0)
self.assertGreater(e1.next_check, in_1day)
self.assertEqual(e1.state, model.EntryState.success)
def exec_entry(self, entry):
os = object_session(entry)
os.expunge(entry)
msg = []
session = create_session()
session.add(entry)
if entry.type != model.EntryType.collection:
entry.type = model.EntryType.collection
session.commit()
self.log.debug("load feed url %s", entry.url)
feed = feedparser.parse(entry.url)
self.log.debug("%s done", entry.url)
if feed['bozo']:
if feed.get('status', 0) > 400:
if feed['status'] == 404:
msg.append(
"HTTP Error 404. File not found. Please check url: %s"
% entry.url)
else:
msg.append("HTTP error: %s" % feed['status'])
msg.append(str(feed['bozo_exception']))
self.failure(entry, "\n".join(msg))
return False
self.log.debug("%s: found %s entries", entry,
len(feed.get('entries', ())))
for fentry in feed.get('entries', ()):
subentry, created = entry.get_or_create_child(
fentry['title'], {
'name': fentry['title'],
'parent_id': entry.id,
'collection_id': entry.collection_id,
'type': model.EntryType.collection_directory
})
#embed()
msg.append("Found entry: %s" % (subentry.name))
for link in fentry.get('links', ()):
if link.get('rel', None) in ['enclosure', 'alternate']:
fname = urllib.parse.urlparse(link['href'])
bname = utils.basename(fname.path)
lnk, created = subentry.get_or_create_child(
bname, {
"name": bname,
"parent_id": subentry.id,
"collection_id": subentry.collection_id,
"type": model.EntryType.collection_single
})
msg.append("Found link:%s name:%s rel:%s" %
(lnk.url, lnk.name, link['rel']))
session.add(lnk)
lnk.url = link['href']
lnk.set_meta("type", link['type'])
session.commit()
self.log.info("successfully checked feed: %s", entry)
entry.set_success("\n".join(msg))
return True
def exec_entry(self, entry):
os = object_session(entry)
os.expunge(entry)
msg = []
session = create_session()
session.add(entry)
if entry.type != model.EntryType.collection:
entry.type = model.EntryType.collection
session.commit()
self.log.debug("load feed url %s", entry.url)
feed = feedparser.parse(entry.url)
self.log.debug("%s done", entry.url)
if feed['bozo']:
if feed.get('status', 0) > 400:
if feed['status'] == 404:
msg.append("HTTP Error 404. File not found. Please check url: %s" %entry.url)
else:
msg.append("HTTP error: %s" %feed['status'])
msg.append(str(feed['bozo_exception']))
self.failure(entry, "\n".join(msg))
return False
self.log.debug("%s: found %s entries", entry, len(feed.get('entries', ())))
for fentry in feed.get('entries', ()):
subentry, created = entry.get_or_create_child(fentry['title'],
{'name': fentry['title'],
'parent_id': entry.id,
'collection_id': entry.collection_id,
'type': model.EntryType.collection_directory})
#embed()
msg.append("Found entry: %s" %(subentry.name))
for link in fentry.get('links', ()):
if link.get('rel', None) in ['enclosure', 'alternate']:
fname = urllib.parse.urlparse(link['href'])
bname = utils.basename(fname.path)
lnk, created = subentry.get_or_create_child(bname,
{"name": bname,
"parent_id": subentry.id,
"collection_id": subentry.collection_id,
"type": model.EntryType.collection_single})
msg.append("Found link:%s name:%s rel:%s" %(lnk.url, lnk.name, link['rel']))
session.add(lnk)
lnk.url = link['href']
lnk.set_meta("type", link['type'])
session.commit()
self.log.info("successfully checked feed: %s", entry)
entry.set_success("\n".join(msg))
return True
def test_entry(self):
session = create_session()
col = model.Collection.create(session, name="test")
session.commit()
root = col.get_root()
e1 = model.Entry(name="e1", parent_id=root.id, priority=10, collection=col)
e2 = model.Entry(name="e2", parent_id=root.id, priority=-10, collection=col)
session.add_all((e1,e2))
session.flush()
e1e = model.Entry(name="child", parent_id=e1.id, priority=10, collection=col)
u1_name = "Übername for \\ | be"
u1 = model.Entry(name=u1_name, collection=col)
session.add(u1)
session.flush()
u1c = model.Entry(name="Some Ünicode child", parent_id=u1.id, collection=col)
session.add(u1c)
session.commit()
self.assertEqual(e1e.full_path, "/e1/child")
self.assertEqual(e1e.system_path, os.path.join("e1","child"))
self.assertEqual(e1.system_path, "e1")
self.assertEqual(e1.full_path, "/e1")
self.assertEqual(u1.full_path, u1_name)
self.assertEqual(u1c.full_path, "%s/%s" %(u1_name, "Some Ünicode child"))
u1.set_meta("bla", "blubb")
session.commit()
self.assertEqual(u1.get_meta("bla"), "blubb")
u1.set_meta("i1", 1)
self.assertEqual(u1.get_meta("i1"), 1)
u1.set_meta("d1", {"test":"blubb", "haha":"json", "in": 32})
self.assertEqual(u1.get_meta("d1"), {"test":"blubb", "haha":"json", "in": 32})
b1 = bytes([0, 1, 2, 3])
u1.set_meta("b1", b1)
self.assertEqual(u1.get_meta("b1"), b1)
u1.set_meta("b1", None)
self.assertEqual(u1.get_meta("b1"), None)
u1.set_meta("f1", 0.3)
self.assertAlmostEqual(u1.get_meta("f1"), 0.3)
def test_download_manager(self):
man = manager.CollapseManager()
man.app.options = man.app.parser.parse_known_args([])[0]
man.app.initialize_app([])
man.app.configure_logging()
man.log.setLevel(logging.DEBUG)
session = create_session()
col = model.Collection.create(session, name="test")
example = model.Entry(name="e1", parent_id=col.get_root().id, url="http://www.example.com", priority=10,
collection_id=col.id)
gc = model.Entry(name="git-clone", parent_id=col.get_root().id,
url="https://github.com/poelzi/git-clone-test.git",
collection_id=col.id)
session.add(example, gc)
gc_back, gc_points = man.get_backend_for_entry(gc)
self.assertEqual(gc_back.name, "git")
session.commit()
# test download_mananger
for name,dm in man._download_managers.items():
self.log.info("test download_manager %s", name)
self.assertEqual(name, dm.name)
man.download_mananger = dm(man, download_path=os.path.join(self.tmpdir, "downloads"))
man.download_mananger.start()
out_path = yield from man.download_mananger.prepare_entry(example)
self.assertEqual(out_path[-12:], os.path.join('downloads', 'e1'))
self.assertTrue(os.path.exists(out_path))
example.collection_id = col.id
out_path2 = yield from man.download_mananger.prepare_entry(example)
self.assertEqual(out_path, out_path2)
man.start_daemon(run_forever=False)
yield from asyncio.sleep(3)