def scan_missing_segments(group_name):
"""Scan for previously missed segments."""
log.info('missing: checking for missed segments')
with db_session() as db:
# recheck for anything to delete
expired = db.query(Miss).filter(
Miss.attempts >= config.scan.get('miss_retry_limit')).filter(
Miss.group_name == group_name).delete()
db.commit()
if expired:
log.info('missing: deleted {} expired misses'.format(expired))
# get missing articles for this group
missing_messages = [
r for r, in db.query(Miss.message).filter(
Miss.group_name == group_name).all()
]
if missing_messages:
# mash it into ranges
missing_ranges = intspan(missing_messages).ranges()
server = Server()
server.connect()
status, parts, messages, missed = server.scan(
group_name, message_ranges=missing_ranges)
# if we got some missing parts, save them
if parts:
pynab.parts.save_all(parts)
# even if they got blacklisted, delete the ones we got from the misses
if messages:
db.query(Miss).filter(Miss.message.in_(messages)).filter(
Miss.group_name == group_name).delete(False)
db.commit()
if missed:
# clear up those we didn't get
save_missing_segments(group_name, missed)
if server.connection:
try:
server.connection.quit()
except:
pass
def scan_missing_segments(group_name):
"""Scan for previously missed segments."""
log.info('missing: checking for missed segments')
with db_session() as db:
# recheck for anything to delete
expired = db.query(Miss).filter(Miss.attempts >= config.scan.get('miss_retry_limit')).filter(
Miss.group_name == group_name).delete()
db.commit()
if expired:
log.info('missing: deleted {} expired misses'.format(expired))
# get missing articles for this group
missing_messages = [r for r, in db.query(Miss.message).filter(Miss.group_name == group_name).all()]
if missing_messages:
# mash it into ranges
missing_ranges = intspan(missing_messages).ranges()
server = Server()
server.connect()
status, parts, messages, missed = server.scan(group_name, message_ranges=missing_ranges)
# if we got some missing parts, save them
if parts:
pynab.parts.save_all(parts)
# even if they got blacklisted, delete the ones we got from the misses
if messages:
db.query(Miss).filter(Miss.message.in_(messages)).filter(Miss.group_name == group_name).delete(False)
db.commit()
if missed:
# clear up those we didn't get
save_missing_segments(group_name, missed)
if server.connection:
try:
server.connection.quit()
except:
pass
def test_uncompress(self):
server = Server()
server.connect(False)
server.scan('alt.binaries.teevee', 563011234, 563031234)
class TestPynab(unittest.TestCase):
def setUp(self):
self.server = None
def test_connect(self):
self.server = Server()
self.server.connect()
self.assertTrue(self.server)
def test_capabilities(self):
self.test_connect()
print(self.server.connection.getcapabilities())
def test_fetch_headers(self):
self.test_connect()
groups = ['alt.binaries.teevee', 'alt.binaries.e-book', 'alt.binaries.moovee']
for group in groups:
(_, _, first, last, _) = self.server.connection.group(group)
for x in range(0, 20000, 10000):
y = x + 10000 - 1
parts = self.server.scan(group, last - y, last - x)
pynab.parts.save_all(parts)
def test_process_binaries(self):
pynab.binaries.process()
def test_process_releases(self):
pynab.releases.process()
def test_all(self):
self.test_fetch_headers()
self.test_process_binaries()
self.test_process_releases()
def test_print_binaries(self):
pprint.pprint([b for b in db.binaries.find()])
def test_day_to_post(self):
self.test_connect()
self.server.day_to_post('alt.binaries.teevee', 5)
def test_group_update(self):
pynab.groups.update('alt.binaries.teevee')
def test_group_backfill(self):
pynab.groups.backfill('alt.binaries.teevee')
def test_tvrage_process(self):
pynab.tvrage.process(100)
def test_omdb_search(self):
print(pynab.imdb.search('South Park Bigger Longer Uncut', '1999'))
def test_omdb_get_details(self):
print(pynab.imdb.get_details('tt1285016'))
def test_nzb_get(self):
release = db.releases.find_one()
pprint.pprint(pynab.nzbs.get_nzb_dict(release['nzb']))
def test_rar_process(self):
pynab.rars.process(5)
def test_nfo_process(self):
pynab.nfos.process(5)
def test_compress(self):
server = Server()
server.connect()
server.scan('alt.binaries.teevee', 563011234, 563031234)
def test_uncompress(self):
server = Server()
server.connect(False)
server.scan('alt.binaries.teevee', 563011234, 563031234)
def tearDown(self):
try:
self.server.connection.quit()
except:
pass
class TestPynab(unittest.TestCase):
def setUp(self):
self.server = None
def test_connect(self):
self.server = Server()
self.server.connect()
self.assertTrue(self.server)
def test_capabilities(self):
self.test_connect()
print(self.server.connection.getcapabilities())
def test_fetch_headers(self):
self.test_connect()
groups = ['alt.binaries.teevee']
for group in groups:
(_, _, first, last, _) = self.server.connection.group(group)
for x in range(0, 40000, 20000):
y = x + 20000 - 1
parts = self.server.scan(group, last - y, last - x)
pynab.parts.save_all(parts)
def test_group_update(self):
import pynab.groups
pynab.groups.update('alt.binaries.teevee')
def test_request_process(self):
import pynab.requests
pynab.requests.process()
def test_update_pres(self):
from scripts.nzedb_pre_import import largeNzedbPre, nzedbPre
largeNzedbPre()
nzedbPre()
def test_process_binaries(self):
import pynab.binaries
pynab.binaries.process()
def test_process_releases(self):
import pynab.releases
pynab.releases.process()
def test_update_blacklist(self):
import pynab.util
pynab.util.update_blacklist()
def test_update_regex(self):
import pynab.util
pynab.util.update_regex()
def test_process_requests(self):
import pynab.requests
pynab.requests.process()
def test_quick_postproc(self):
import scripts.quick_postprocess
scripts.quick_postprocess.local_postprocess()
def test_process_ids(self):
import pynab.ids
pynab.ids.process('movie')
def test_remove_metablacks(self):
from pynab.db import MetaBlack
with db_session() as db:
db.query(MetaBlack).delete()
db.commit()
def test_search_releases(self):
from sqlalchemy_searchable import search
from pynab.db import Release
with db_session() as db:
q = db.query(Release)
q = search(q, 'engaged e06')
print(q.first().search_name)
def test_nzb_parse(self):
import pynab.nzbs
from pynab.db import NZB
with db_session() as db:
nzb = db.query(NZB).filter(NZB.id==1).one()
import pprint
pprint.pprint(pynab.nzbs.get_nzb_details(nzb))
def test_scrape_nzbsu(self):
import requests
import time
from bs4 import BeautifulSoup
url = 'https://api.nzb.su/api?apikey=4d901407e99ae6c942416585c8a44673'
ua = {'User-agent': 'CouchPotato 3.0.1'}
results = []
for category in [5020,5030,5040,5050,5060,5070,5080,2010,2020,2030,2040,2050,2060,2070,4010,4020,4030,1010,1020,1030,1050,1080,1090,1100,4050,3010,3020,3030,3040,3050,7010,7020,7030,6010,6020,6030,6040,6050,6060,6070,8010]:
data = requests.get(url + '&t=search&cat={}&o=json'.format(category), headers=ua).json()
if 'item' in data['channel']:
results.extend(data['channel']['item'])
with open('dog_releases.csv', 'w', encoding='utf-8') as f:
f.write('"r","name","name","category_id","name","name"\r\n')
# turn results into useful data
for i, result in enumerate(results):
try:
resp = requests.get(url + '&t=details&id={}'.format(result['attr'][3]['@attributes']['value']), headers=ua)
soup = BeautifulSoup(resp.text)
group = soup.find(attrs={'name':'group'})['value']
f.write('"{}","{}","{}","{}","{}","{}"\r\n'.format(i, result['title'], group, result['attr'][1]['@attributes']['value'], *result['category'].split(' > ')))
time.sleep(5)
except:
continue
def test_categorise(self):
import nltk
import regex
import csv
import random
import pprint
#def determine_category(name, group_name=''):
def load_data(filename):
with open(filename, encoding='utf-8') as f:
f.readline()
csvfile = csv.reader(f, delimiter=',', quotechar='"')
data = []
for line in csvfile:
features = extract_features(line[1])
features['group'] = line[2]
features['name'] = line[1]
data.append((features, line[3]))
random.shuffle(data)
return data
train_data = load_data('tagged_releases_train.csv')
test_data = load_data('tagged_releases_test.csv')
nzbsu_data = load_data('tagged_releases_test_nzbsu.csv')
train_set = train_data
test_set = test_data
nzbsu_set = nzbsu_data
classifier = nltk.NaiveBayesClassifier.train(train_set)
from pickle import dump
with open('release_categoriser.pkl', 'wb') as out:
dump(classifier, out, -1)
errors = []
for features, tag in nzbsu_set:
guess = classifier.classify(features)
if guess[:2] != tag[:2]:
errors.append((tag, guess, features))
for tag, guess, features in errors:
print('correct={} guess={} name={}'.format(tag, guess, features['name'].encode('utf-8')))
print(classifier.show_most_informative_features())
print('test: {}'.format(nltk.classify.accuracy(classifier, test_set)))
print('test: {}'.format(nltk.classify.accuracy(classifier, nzbsu_set)))
def test_load_and_categorise(self):
from pynab.db import db_session, Release, Group, windowed_query
from pickle import load
with open('release_categoriser.pkl', 'rb') as cat_file:
categoriser = load(cat_file)
with db_session() as db:
errors = []
i = 0
query = db.query(Release).join(Group)
count = query.count()
for result in windowed_query(query, Release.id, 500):
features = extract_features(result.name)
features['group'] = result.group.name
features['name'] = result.name
guess = categoriser.classify(features)
if guess[:2] != str(result.category_id)[:2]:
errors.append((result.category_id, guess, features))
i += 1
if i % 500 == 0:
print('{} - {:.3f}%'.format((i/count)*100, (1 - (len(errors) / i)) * 100))
for tag, guess, features in errors:
print('correct={} guess={} name={}'.format(tag, guess, features['name'].encode('utf-8')))
print('accuracy={}'.format(1 - (len(errors)/i)))
def tearDown(self):
try:
self.server.connection.quit()
except:
pass
class TestPynab(unittest.TestCase):
def setUp(self):
self.server = None
def test_connect(self):
self.server = Server()
self.server.connect()
self.assertTrue(self.server)
def test_capabilities(self):
self.test_connect()
print(self.server.connection.getcapabilities())
def test_fetch_headers(self):
self.test_connect()
groups = ['alt.binaries.teevee']
for group in groups:
(_, _, first, last, _) = self.server.connection.group(group)
for x in range(0, 40000, 20000):
y = x + 20000 - 1
parts = self.server.scan(group, last - y, last - x)
pynab.parts.save_all(parts)
def test_group_update(self):
import pynab.groups
pynab.groups.update('alt.binaries.teevee')
def test_process_binaries(self):
import pynab.binaries
pynab.binaries.process()
def test_process_releases(self):
import pynab.releases
pynab.releases.process()
def test_update_blacklist(self):
import pynab.util
pynab.util.update_blacklist()
def test_update_regex(self):
import pynab.util
pynab.util.update_regex()
def test_search_releases(self):
from sqlalchemy_searchable import search
from pynab.db import Release
with db_session() as db:
q = db.query(Release)
q = search(q, 'engaged e06')
print(q.first().search_name)
def test_nzb_parse(self):
import pynab.nzbs
from pynab.db import NZB
with db_session() as db:
nzb = db.query(NZB).filter(NZB.id==1).one()
import pprint
pprint.pprint(pynab.nzbs.get_nzb_details(nzb))
def tearDown(self):
try:
self.server.connection.quit()
except:
pass
