def load_sqlite_backup(backup_date: str, test, **kwargs):
if test:
logger.debug("load_sqlite_backup task")
return
with db_factory.get_instance() as db:
assert isinstance(db, CSqliteExtDatabase)
backup_filename = os.path.join(config.get('sqlite', 'backup_dir'),
_BACKUP_FORMAT % (backup_date))
db_backup = CSqliteExtDatabase(backup_filename)
db_backup.backup(db)
def sqlite_backup(test, **kwargs):
if test:
logger.debug("sqlite_backup task")
return
with db_factory.get_instance() as db:
assert isinstance(db, CSqliteExtDatabase)
filename = os.path.join(
config.get('sqlite', 'backup_dir'), _BACKUP_FORMAT %
(datetime.datetime.now().strftime(_BACKUP_DATE_FORMAT)))
db.backup_to_file(filename)
def execute(self, args=sys.argv[1:]):
parsed_args = self.parser.parse_args(args)
parsed_args_dict = vars(parsed_args)
setup_logging(parsed_args_dict.get('verbose', 0))
config_dir = parsed_args_dict.get('config_dir', None)
mode = parsed_args_dict.get('mode', None)
if config_dir or mode:
cfg.populate_config(config=config,
user_dir_config=config_dir,
mode=mode,
reload_=True)
indexers_config.clear()
indexers_config.update(cfg.read_indexers_file(config))
trackers_config.clear()
trackers_config.update(cfg.read_trackers_file(config))
db_factory.defaut_instance = config.get('db', 'database')
super().execute(args)
def read_from_url(url: str, headers: dict = {}) -> bytes:
"""Binary response content from torrent file url
Args:
url (str): url of torrent file
headers (dict, optional): key value pair of headers. Defaults to None.
Returns:
bytes: binary response content
"""
if 'user-agent' not in headers:
headers['user-agent'] = config.get('requests', 'user_agent')
try:
with requests.get(url, headers=headers) as req:
req.raise_for_status()
except requests.exceptions.InvalidSchema:
with requests.get(url, headers=headers, allow_redirects=False) as req:
if req.headers.get('Location') and req.headers.get(
'Location').startswith('magnet:'):
return req.headers.get('Location')
return req.content
"""
DEPRECATED : use of Jackett + Torznab specification
"""
from mediastrends import config
from mediastrends.torrent.Tracker import HttpTracker
ygg_tracker = HttpTracker(scheme=config.get('ygg', 'scheme'),
netloc=config.get('ygg', 'netloc'),
path=config.get('ygg', 'path'),
name='ygg')
class StatsScraper():
_HEADERS = {}
_HEADERS['user-agent'] = config.get('requests', 'user_agent')
_BATCH_SIZE = config.getint('requests', 'batch_size')
_RETRIES = config.getint('retry', 'tries')
_DELAY = config.getint('retry', 'delay')
def __init__(self, tracker: Tracker):
self.tracker = tracker
self._torrents_lookup = {}
self._info_hashes = []
self._parsed_content = {}
@property
def tracker(self):
return self._tracker
@tracker.setter
def tracker(self, tracker: Tracker):
if not (isinstance(tracker, HttpTracker)
or isinstance(tracker, UdpTracker)):
raise TypeError(
'Tracker object must be instance of HttpTracker or UdpTracker')
self._tracker = tracker
@property
def torrents(self):
return self._torrents
@torrents.setter
def torrents(self, torrents: list):
self._torrents_lookup = {t.info_hash: t for t in torrents}
self._parsed_content = {}
@property
def stats_collection(self):
return StatsCollection([
Stats(torrent=self._torrents_lookup[info_hash],
tracker=self._tracker,
seeders=c.get('complete'),
leechers=c.get('incomplete'),
completed=c.get('downloaded'))
for info_hash, c in self._parsed_content.items()
])
@retry((requests.exceptions.RequestException, OSError),
tries=_RETRIES,
delay=_DELAY,
jitter=(3, 10),
logger=logger)
def run(self, info_hashes: list):
content_infos = self._tracker.scrape(info_hashes)
self._parsed_content.update(content_infos)
def run_by_batch(self):
full_infos_hashes_list = list(self._torrents_lookup.keys())
logger.debug("Run by batch of %s" % self._BATCH_SIZE)
for info_hashes in tools.batch(full_infos_hashes_list,
self._BATCH_SIZE):
try:
self.run(info_hashes)
except (requests.exceptions.RequestException, OSError) as err:
logger.warning(err)
continue
def build_header(self, action: str = None):
self._headers['user-agent'] = config.get('requests', 'user_agent')
class Page(ABC):
_HEADERS = {}
_HEADERS['user-agent'] = config.get('requests', 'user_agent')
_RETRIES = config.getint('retry', 'tries')
_DELAY = config.getint('retry', 'delay')
def __init__(self, url, soup=None):
self.url = url
self._soup = soup
self._name = None
self._pub_date = None
self._seeders = None
self._leechers = None
self._completed = None
self._size = None
self._info_hash = None
self._valid_date = None
@property
def url(self):
return self._url
@url.setter
def url(self, url):
self._url = tools.quote_url(url)
return self
@property
def valid_date(self, valid_date):
return self.valid_date
@valid_date.setter
def valid_date(self, valid_date: datetime.datetime):
if not isinstance(valid_date, datetime.datetime):
raise ValueError("valid_date (%s) should be datetime object",
valid_date)
self._valid_date = valid_date.replace(microsecond=0)
return self
@property
def soup(self):
if not self._soup:
logger.debug("Scraping page: %s", self._url)
# self.soup = tools.parsed_html_content(self._url, headers = self._HEADERS)
self.soup = retry_call(tools.parsed_html_content,
fkwargs={
"url": self._url,
"headers": self._HEADERS
},
tries=self._RETRIES,
delay=self._DELAY,
jitter=(3, 10),
logger=logger)
return self._soup
@soup.setter
def soup(self, soup):
self._soup = soup
self.valid_date = datetime.datetime.now()
return self
@property
def info_hash(self):
raise NotImplementedError
@property
def pub_date(self):
raise NotImplementedError
@property
def name(self):
raise NotImplementedError
@property
def size(self):
raise NotImplementedError
@property
def seeders(self):
raise NotImplementedError
@property
def leechers(self):
raise NotImplementedError
@property
def completed(self):
raise NotImplementedError
@property
def category(self):
raise NotImplementedError