def update_last_update(self, dt=None):
# aggiorna sia variabile di cache che db
if dt and isinstance(dt, datetime.datetime):
now = dt.strftime('%d/%m/%Y %H:%M')
else:
now = u.now()
self._last_update = now
return self._execute(sql.UPDATE_GENERAL_SETTING, (now, 'last_update'),
rowcount=True)
def get_torrents(self, with_current_datetime=True):
if not self._torrents:
return []
if with_current_datetime:
now = u.now(stringify=False)
return [(t.id, t.title, t.description, t.magnet, now)
for t in self._torrents]
else:
return [(t.id, t.title, t.description, t.magnet)
for t in self._torrents]
def save_json_file(self):
if self._torrents: # non salvare il file se la lista è rimasta vuota
file_name = u.now(stringify='tntvillage_releases_%Y%m%d_%H%M.json')
file_path = os.path.join(self._json_base_dir, file_name)
json_data = [t._asdict() for t in self._torrents]
with open(file_path, 'w+') as f:
json.dump(json_data, f, indent=4)
return file_path
else:
return False
def iter_all(self,
sleep=1,
reset_previous_list=True,
limit=0,
yield_with_current_datetime=True):
"""Fetch all the torrents from all the pages.
:param sleep: time to wait after every request
:param reset_previous_list: empty the internal objects list
:param limit: only fetch this number of pages
:param yield_with_current_datetime: returns the torrent tuple with the current datetime too
:return: False if the first request fails, json path if save_json_file is True, else True
"""
logger.info('fetching all pages (iterable)')
fetched = self._fetch_total_pages(skip_request_if_present=True)
if not fetched:
raise ValueError('cannot fetch total number of pages')
time.sleep(sleep)
logger.info(
'starting loop to fetch all pages (reset_previous_list=%s, limit=%d)',
reset_previous_list, limit)
if reset_previous_list:
self._torrents = list()
for i in range(
1, self._total_pages + 1 if limit in (None, 0) else limit + 1):
logger.info('fetching page %d (sleep=%d)...', i, sleep)
result = self._fetch_page_by_index(i)
trs = result.soup.find_all('tr')
logger.info('fetched %d torrents from page %d', len(trs), i)
torrents_to_yield = list()
for tr in trs:
torrent = self._parse_torrent_data(tr)
if torrent:
self._torrents.append(torrent)
torrents_to_yield.append(torrent)
if torrents_to_yield:
if yield_with_current_datetime:
now = u.now(stringify=False)
yield [(t.id, t.title, t.description, t.magnet,
t.torrent_download_url, t.leech, t.seed, now)
for t in torrents_to_yield]
else:
yield [(t.id, t.title, t.description, t.magnet,
t.torrent_download_url, t.leech, t.seed)
for t in torrents_to_yield]
else:
# se il parsing della pagina non ha dato risultati, restituire l'html
yield result.html
import logging
import sqlite3
import datetime
import simplesqlitewrap as ssw
from bot import u
import bot.sql as sql
logger = logging.getLogger(__name__)
DEFAULT_SETTINGS = [('last_update', u.now()), ('json_file_id', None),
('searches_count', 0)]
class Database(ssw.Database):
def __init__(self, filename, **kwargs):
ssw.Database.__init__(
self,
filename,
connection_args={'detect_types': sqlite3.PARSE_DECLTYPES},
**kwargs)
self._init_tables()
self._last_update = self._get_general_setting(
'last_update') # cache della data/ora dell'ultimo update
self._json_file_id = self._get_general_setting('json_file_id')
self._total_entries = 0
self.update_total_enrties()