class TestPynab(unittest.TestCase):
def setUp(self):
self.server = None
def test_connect(self):
self.server = Server()
self.server.connect()
self.assertTrue(self.server)
def test_capabilities(self):
self.test_connect()
print(self.server.connection.getcapabilities())
def test_fetch_headers(self):
self.test_connect()
groups = ['alt.binaries.teevee', 'alt.binaries.e-book', 'alt.binaries.moovee']
for group in groups:
(_, _, first, last, _) = self.server.connection.group(group)
for x in range(0, 20000, 10000):
y = x + 10000 - 1
parts = self.server.scan(group, last - y, last - x)
pynab.parts.save_all(parts)
def test_process_binaries(self):
pynab.binaries.process()
def test_process_releases(self):
pynab.releases.process()
def test_all(self):
self.test_fetch_headers()
self.test_process_binaries()
self.test_process_releases()
def test_print_binaries(self):
pprint.pprint([b for b in db.binaries.find()])
def test_day_to_post(self):
self.test_connect()
self.server.day_to_post('alt.binaries.teevee', 5)
def test_group_update(self):
pynab.groups.update('alt.binaries.teevee')
def test_group_backfill(self):
pynab.groups.backfill('alt.binaries.teevee')
def test_tvrage_process(self):
pynab.tvrage.process(100)
def test_omdb_search(self):
print(pynab.imdb.search('South Park Bigger Longer Uncut', '1999'))
def test_omdb_get_details(self):
print(pynab.imdb.get_details('tt1285016'))
def test_nzb_get(self):
release = db.releases.find_one()
pprint.pprint(pynab.nzbs.get_nzb_dict(release['nzb']))
def test_rar_process(self):
pynab.rars.process(5)
def test_nfo_process(self):
pynab.nfos.process(5)
def test_compress(self):
server = Server()
server.connect()
server.scan('alt.binaries.teevee', 563011234, 563031234)
def test_uncompress(self):
server = Server()
server.connect(False)
server.scan('alt.binaries.teevee', 563011234, 563031234)
def tearDown(self):
try:
self.server.connection.quit()
except:
pass
def backfill(group_name, date=None):
log.info('{}: Backfilling group...'.format(group_name))
server = Server()
_, count, first, last, _ = server.group(group_name)
if date:
target_article = server.day_to_post(group_name, server.days_old(date))
else:
target_article = server.day_to_post(group_name, config.site['backfill_days'])
group = db.groups.find_one({'name': group_name})
if group:
# if the group hasn't been updated before, quit
if not group['first']:
log.error('{}: Need to run a normal update prior to backfilling group.'.format(group_name))
if server.connection:
server.connection.quit()
return False
log.info('{0}: Server has {1:d} - {2:d} or ~{3:d} days.'
.format(group_name, first, last, server.days_old(server.post_date(group_name, first)))
)
# if the first article we have is lower than the target
if target_article >= group['first']:
log.info('{}: Nothing to do, we already have the target post.'.format(group_name))
if server.connection:
server.connection.quit()
return True
# or if the target is below the server's first
if target_article < first:
log.warning(
'{}: Backfill target is older than the server\'s retention. Setting target to the first possible article.'.format(
group_name))
target_article = first
total = group['first'] - target_article
end = group['first'] - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
while True:
messages = server.scan(group_name, start, end)
if messages:
if parts.save_all(messages):
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'first': start
}
})
pass
else:
log.error('{}: Failed while saving parts.'.format(group_name))
if server.connection:
server.connection.quit()
return False
if start == target_article:
if server.connection:
server.connection.quit()
return True
else:
end = start - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
else:
log.error('{}: Group doesn\'t exist in db.'.format(group_name))
if server.connection:
server.connection.quit()
return False
def backfill(group_name, date=None):
log.info('group: {}: backfilling group'.format(group_name))
server = Server()
_, count, first, last, _ = server.group(group_name)
if date:
target_article = server.day_to_post(group_name, server.days_old(date))
else:
target_article = server.day_to_post(group_name, config.scan.get('backfill_days', 10))
group = db.groups.find_one({'name': group_name})
if group:
# if the group hasn't been updated before, quit
if not group['first']:
log.error('group: {}: run a normal update prior to backfilling'.format(group_name))
if server.connection:
server.connection.quit()
return False
# if the first article we have is lower than the target
if target_article >= group['first']:
log.info('group: {}: Nothing to do, we already have the target post.'.format(group_name))
if server.connection:
server.connection.quit()
return True
# or if the target is below the server's first
if target_article < first:
target_article = first
total = group['first'] - target_article
end = group['first'] - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
retries = 0
while True:
messages = server.scan(group_name, start, end)
if messages:
if parts.save_all(messages):
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'first': start
}
})
retries = 0
else:
log.error('group: {}: failed while saving parts'.format(group_name))
if server.connection:
server.connection.quit()
return False
else:
log.error('group: {}: problem updating group - trying again'.format(group_name))
retries += 1
# keep trying the same block 3 times, then skip
if retries <= 3:
continue
if start == target_article:
if server.connection:
server.connection.quit()
return True
else:
end = start - 1
start = end - MESSAGE_LIMIT + 1
if target_article > start:
start = target_article
else:
log.error('group: {}: group doesn\'t exist in db.'.format(group_name))
if server.connection:
server.connection.quit()
return False
def update(group_name):
log.info('{}: Updating group...'.format(group_name))
server = Server()
_, count, first, last, _ = server.group(group_name)
group = db.groups.find_one({'name': group_name})
if group:
# if the group has been scanned before
if group['last']:
# pick up where we left off
start = group['last'] + 1
# if our last article is newer than the server's, something's wrong
if last < group['last']:
log.error('{}: Server\'s last article {:d} is lower than the local {:d}'.format(group_name, last,
group['last']))
if server.connection:
try:
server.connection.quit()
except:
pass
return False
else:
# otherwise, start from x days old
start = server.day_to_post(group_name, config.site['new_group_scan_days'])
if not start:
log.error('{}: Couldn\'t determine a start point for group.'.format(group_name))
if server.connection:
try:
server.connection.quit()
except:
pass
return False
else:
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'first': start
}
})
# either way, we're going upwards so end is the last available
end = last
# if total > 0, we have new parts
total = end - start + 1
start_date = server.post_date(group_name, start)
end_date = server.post_date(group_name, end)
total_date = end_date - start_date
log.debug('{}: Start: {:d} ({}) End: {:d} ({}) Total: {:d} ({} days, {} hours, {} minutes)'
.format(
group_name, start, start_date,
end, end_date,
total, total_date.days, total_date.seconds // 3600, (total_date.seconds // 60) % 60
)
)
if total > 0:
if not group['last']:
log.info('{}: Starting new group with {:d} days and {:d} new parts.'
.format(group_name, config.site['new_group_scan_days'], total))
else:
log.info('{}: Group has {:d} new parts.'.format(group_name, total))
retries = 0
# until we're finished, loop
while True:
# break the load into segments
if total > MESSAGE_LIMIT:
if start + MESSAGE_LIMIT > last:
end = last
else:
end = start + MESSAGE_LIMIT - 1
messages = server.scan(group_name, start, end)
if messages:
if parts.save_all(messages):
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'last': end
}
})
else:
log.error('{}: Failed while saving parts.'.format(group_name))
if server.connection:
try:
server.connection.quit()
except:
pass
return False
else:
log.error('Problem updating group - trying again...')
retries += 1
# keep trying the same block 3 times, then skip
if retries <= 3:
continue
if end == last:
if server.connection:
try:
server.connection.quit()
except:
pass
return True
else:
start = end + 1
log.info('{}: {:d} messages to go for this group.'.format(group_name, last - end))
else:
log.info('{}: No new records for group.'.format(group_name))
if server.connection:
server.connection.quit()
return True
else:
log.error('{}: No such group exists in the db.'.format(group_name))
if server.connection:
server.connection.quit()
return False
def update(group_name):
log.info('group: {}: updating group'.format(group_name))
server = Server()
_, count, first, last, _ = server.group(group_name)
group = db.groups.find_one({'name': group_name})
if group:
# if the group has been scanned before
if group['last']:
# pick up where we left off
start = group['last'] + 1
# if our last article is newer than the server's, something's wrong
if last < group['last']:
log.error('group: {}: last article {:d} on server is older than the local {:d}'.format(group_name, last,
group['last']))
if server.connection:
try:
server.connection.quit()
except:
pass
return False
else:
# otherwise, start from x days old
start = server.day_to_post(group_name, config.scan.get('new_group_scan_days', 5))
if not start:
log.error('group: {}: couldn\'t determine a start point for group'.format(group_name))
if server.connection:
try:
server.connection.quit()
except:
pass
return False
else:
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'first': start
}
})
# either way, we're going upwards so end is the last available
end = last
# if total > 0, we have new parts
total = end - start + 1
start_date = server.post_date(group_name, start)
end_date = server.post_date(group_name, end)
if start_date and end_date:
total_date = end_date - start_date
log.info('group: {}: pulling {} - {} ({}d, {}h, {}m)'.format(
group_name,
start, end,
total_date.days,
total_date.seconds // 3600,
(total_date.seconds // 60) % 60
))
else:
log.info('group: {}: pulling {} - {}'.format(group_name, start, end))
if total > 0:
if not group['last']:
log.info('group: {}: starting new group with {:d} days and {:d} new parts'
.format(group_name, config.scan.get('new_group_scan_days', 5), total))
else:
log.info('group: {}: group has {:d} new parts.'.format(group_name, total))
retries = 0
# until we're finished, loop
while True:
# break the load into segments
if total > MESSAGE_LIMIT:
if start + MESSAGE_LIMIT > last:
end = last
else:
end = start + MESSAGE_LIMIT - 1
messages = server.scan(group_name, start, end)
if messages:
if parts.save_all(messages):
db.groups.update({
'_id': group['_id']
},
{
'$set': {
'last': end
}
})
retries = 0
else:
log.error('group: {}: failed while saving parts'.format(group_name))
if server.connection:
try:
server.connection.quit()
except:
pass
return False
if end == last:
if server.connection:
try:
server.connection.quit()
except:
pass
return True
else:
start = end + 1
else:
log.info('group: {}: no new messages'.format(group_name))
if server.connection:
server.connection.quit()
return True
else:
log.error('group: {}: no group in db'.format(group_name))
if server.connection:
server.connection.quit()
return False
