def process_categories(cat_tuples):
"""Create torrent category if needed. Returns list of entities to be saved"""
cat_key = dao.category_key_from_tuples(cat_tuples)
cat = dao.get_from_key(cat_key)
if not cat:
enqueue_map_rebuild_if_needed()
return make_categories(cat_tuples)
return []
def import_torrent3(torrent_data, scraper):
tid = int(torrent_data.pop('tid'))
torrent_data.update(scraper.get_torrent_data(tid)) # TODO handle 'torrent deleted' and other errors here
torrent_data = prepare_torrent(torrent_data)
cat_key = dao.category_key_from_tuples(torrent_data['categories'])
torrent = dao.make_torrent(cat_key, torrent_data)
to_write = [torrent]
new_cats = process_categories(cat_key, torrent_data['categories'])
dao.write_multi(to_write)
def make_categories(cat_tuples):
"""Create and return entities for category and all its parent categories
If category already exists, this function returns empty list"""
tuples_copy = list(cat_tuples)
rv = []
while tuples_copy:
key = dao.category_key_from_tuples(tuples_copy)
cat = key.get()
if cat:
break
_, _, cat_title = tuples_copy.pop()
cat = dao.make_category(key, cat_title)
rv.append(cat)
return rv
def import_torrent(payload):
"""Run torrent import task for torrent, specified by torrent_data"""
torrent_dict = taskmaster.unpack_payload(payload)
tid = torrent_dict['id']
wc = webclient.RutrackerWebClient()
with dao.account_context() as account:
html = wc.get_torrent_page(account, tid)
p = parsing.Parser()
try:
torrent_data, category_tuples = p.parse_torrent_page(html)
except parsing.SkipTorrent as e:
logging.info('Skipping torrent %d: %s', tid, str(e))
return
torrent_dict.update(torrent_data)
to_write = process_categories(category_tuples)
cat_key = dao.category_key_from_tuples(category_tuples)
torrent = dao.make_torrent(cat_key, torrent_dict)
to_write.append(torrent)
dao.write_multi(to_write)
return []
elif cat and not cat.dirty:
cat.dirty = True
return [cat]
else:
return make_categories(cat_key, cat_tuple_list)
make_categories(cat_tuples):
tuples_copy = list(cat_tuples)
rv = []
while tuples_copy:
key = dao.category_key_from_tuples(tuples_copy)
cat = key.get()
if cat:
break
_, _, cat_title = tuples_copy.pop()
cat = dao.make_category(key, cat_title)
rv.append(cat)
return rv
def prepare_torrent(torrent_data): # TODO move this
"""Prepare torrent field values"""
return {
'id': int(torrent_data['tid']),
'title': torrent_data['title'],
'dt': datetime.datetime.utcfromtimestamp(int(torrent_data['timestamp'])),
