def process(limit=None, category=0):
"""Process releases for NFO parts and download them."""
with Server() as server:
with db_session() as db:
# noinspection PyComparisonWithNone,PyComparisonWithNone
query = db.query(Release).join(Group).join(NZB).filter(
Release.nfo == None).filter(Release.nfo_metablack_id == None)
if category:
query = query.filter(Release.category_id == int(category))
if limit:
releases = query.order_by(Release.posted.desc()).limit(limit)
else:
releases = query.order_by(Release.posted.desc()).all()
for release in releases:
found = False
nzb = pynab.nzbs.get_nzb_details(release.nzb)
if nzb:
nfos = []
for nfo in nzb['nfos']:
for part in nfo['segments']:
if int(part['size']) > NFO_MAX_FILESIZE:
continue
nfos.append(part)
for nfo in nfos:
try:
article = server.get(release.group.name, [
nfo['message_id'],
])
except Exception as e:
# if usenet's not accessible, don't block it forever
log.error('nfo: unable to get nfo: {}'.format(e))
continue
if article:
data = gzip.compress(article.encode('utf-8'))
nfo = NFO(data=data)
db.add(nfo)
release.nfo = nfo
release.nfo_metablack_id = None
db.add(release)
log.debug('nfo: [{}] - nfo added'.format(
release.search_name))
found = True
break
if not found:
log.debug(
'nfo: [{}] - [{}] - no nfos in release'.format(
release.id, release.search_name))
mb = MetaBlack(nfo=release, status='IMPOSSIBLE')
db.add(mb)
db.commit()
def process(limit=None, category=0):
"""Process releases for SFV parts and download them."""
with Server() as server:
with db_session() as db:
# noinspection PyComparisonWithNone,PyComparisonWithNone
query = db.query(Release).join(Group).join(NZB).filter(
Release.sfv == None).filter(Release.sfv_metablack_id == None)
if category:
query = query.filter(Release.category_id == int(category))
if limit:
releases = query.order_by(Release.posted.desc()).limit(limit)
else:
releases = query.order_by(Release.posted.desc()).all()
for release in releases:
found = False
nzb = pynab.nzbs.get_nzb_details(release.nzb)
if nzb:
sfvs = []
for sfv in nzb['sfvs']:
for part in sfv['segments']:
if int(part['size']) > SFV_MAX_FILESIZE:
continue
sfvs.append(part)
for sfv in sfvs:
try:
article = server.get(release.group.name, [
sfv['message_id'],
])
except:
article = None
if article:
data = gzip.compress(article.encode('utf-8'))
sfv = SFV(data=data)
db.add(sfv)
release.sfv = sfv
release.sfv_metablack_id = None
db.add(release)
log.info('sfv: [{}] - sfv added'.format(
release.search_name))
found = True
break
if not found:
log.debug('sfv: [{}] - no sfvs in release'.format(
release.search_name))
mb = MetaBlack(sfv=release, status='IMPOSSIBLE')
db.add(mb)
db.commit()
def process(limit=None, category=0):
"""Processes release rarfiles to check for passwords and filecounts."""
with Server() as server:
with db_session() as db:
# noinspection PyComparisonWithNone
query = db.query(Release).join(Group).join(NZB).filter(~Release.files.any()). \
filter(Release.passworded == 'UNKNOWN').filter(Release.rar_metablack_id == None)
if category:
query = query.filter(Release.category_id == int(category))
if limit:
releases = query.order_by(Release.posted.desc()).limit(limit)
else:
releases = query.order_by(Release.posted.desc()).all()
for release in releases:
log.debug('rar: processing {}'.format(release.search_name))
nzb = pynab.nzbs.get_nzb_details(release.nzb)
if nzb and nzb['rars']:
try:
passworded, info = check_release_files(
server, release.group.name, nzb)
except Exception as e:
# if usenet isn't accessible, we don't want to blacklist it
log.error('rar: file info failed: {}'.format(e))
continue
if info:
log.info('rar: file info add [{}]'.format(
release.search_name))
release.passworded = passworded
size = 0
for file in info:
f = File(name=file['name'][:512],
size=file['size'])
f.release = release
size += file['size']
db.add(f)
if size != 0:
release.size = size
release.rar_metablack_id = None
db.add(release)
db.commit()
continue
log.debug('rar: [{}] - file info: no readable rars in release'.
format(release.search_name))
mb = MetaBlack(rar=release, status='IMPOSSIBLE')
db.add(mb)
db.commit()
def scan_missing_segments(group_name):
"""Scan for previously missed segments."""
log.info('missing: checking for missed segments')
with db_session() as db:
# recheck for anything to delete
expired = db.query(Miss).filter(
Miss.attempts >= config.scan.get('miss_retry_limit')).filter(
Miss.group_name == group_name).delete()
db.commit()
if expired:
log.info('missing: deleted {} expired misses'.format(expired))
# get missing articles for this group
missing_messages = [
r for r, in db.query(Miss.message).filter(
Miss.group_name == group_name).all()
]
if missing_messages:
# mash it into ranges
missing_ranges = intspan(missing_messages).ranges()
server = Server()
server.connect()
status, parts, messages, missed = server.scan(
group_name, message_ranges=missing_ranges)
# if we got some missing parts, save them
if parts:
pynab.parts.save_all(parts)
# even if they got blacklisted, delete the ones we got from the misses
if messages:
db.query(Miss).filter(Miss.message.in_(messages)).filter(
Miss.group_name == group_name).delete(False)
db.commit()
if missed:
# clear up those we didn't get
save_missing_segments(group_name, missed)
if server.connection:
try:
server.connection.quit()
except:
pass
def test_connect(self):
self.server = Server()
self.server.connect()
self.assertTrue(self.server)
def scan(group_name, direction='forward', date=None, target=None, limit=None):
log.info('group: {}: scanning group'.format(group_name))
with Server() as server:
_, count, first, last, _ = server.group(group_name)
if count:
with db_session() as db:
group = db.query(Group).filter(
Group.name == group_name).first()
if group:
# sort out missing first/lasts
if not group.first and not group.last:
group.first = last
group.last = last
direction = 'backward'
elif not group.first:
group.first = group.last
elif not group.last:
group.last = group.first
# check that our firsts and lasts are valid
if group.first < first:
log.error(
'group: {}: first article was older than first on server'
.format(group_name))
return True
elif group.last > last:
log.error(
'group: {}: last article was newer than last on server'
.format(group_name))
return True
db.merge(group)
# sort out a target
start = 0
mult = 0
if direction == 'forward':
start = group.last
target = last
mult = 1
elif direction == 'backward':
start = group.first
if not target:
target = server.day_to_post(
group_name,
server.days_old(date) if date else
config.scan.get('backfill_days', 10))
mult = -1
if not target:
log.info(
'group: {}: unable to continue'.format(group_name))
return True
if group.first <= target <= group.last:
log.info(
'group: {}: nothing to do, already have target'.
format(group_name))
return True
if first > target or last < target:
log.error(
'group: {}: server doesn\'t carry target article'.
format(group_name))
return True
iterations = 0
num = config.scan.get('message_scan_limit') * mult
for i in range(start, target, num):
# set the beginning and ends of the scan to their respective values
begin = i + mult
end = i + (mult *
config.scan.get('message_scan_limit'))
# check if the target is before our end
if abs(begin) <= abs(target) <= abs(end):
# we don't want to overscan
end = target
# at this point, we care about order
# flip them if one is bigger
begin, end = (begin, end) if begin < end else (end,
begin)
status, parts, messages, missed = server.scan(
group_name, first=begin, last=end)
try:
if direction == 'forward':
group.last = max(messages)
elif direction == 'backward':
group.first = min(messages)
except:
log.error(
'group: {}: problem updating group ({}-{})'.
format(group_name, start, end))
return False
# don't save misses if we're backfilling, there are too many
if status and missed and config.scan.get(
'retry_missed') and direction == 'forward':
save_missing_segments(group_name, missed)
if status and parts:
if pynab.parts.save_all(parts):
db.merge(group)
db.commit()
else:
log.error(
'group: {}: problem saving parts to db, restarting scan'
.format(group_name))
return False
to_go = abs(target - end)
log.info(
'group: {}: {:.0f} iterations ({} messages) to go'.
format(
group_name,
to_go / config.scan.get('message_scan_limit'),
to_go))
parts.clear()
del messages[:]
del missed[:]
iterations += 1
if limit and iterations >= 3: #* config.scan.get('message_scan_limit') >= limit:
log.info(
'group: {}: scan limit reached, ending early (will continue later)'
.format(group_name))
return False
log.info('group: {}: scan completed'.format(group_name))
return True
