def download(self, response: Response, destination: Path) -> bool:
self.logger.debug({
'Method': response.request.method,
'Url': response.request.url,
})
with destination.open('wb') as w:
if not w.writable():
raise FsError(
'Destination not writable. Please, check permissions')
content = response.content
errors = []
if ~response.headers['Content-Type'].find('text/'):
errors.append('Content not binary type.')
if len(content) <= 0:
errors.append('Response has zero length.')
cli.syslog('%s Url: %s\nHistory: %s' %
(errors[-1], response.url, '\n'.join(
map(lambda _url: _url.url, response.history))))
if len(errors) > 0:
error_file = str(crc32(str(destination).encode()))
with Path(error_file).open('wb') as _w:
_w.write(content)
logger().warning(
'Non-byte response. See {}'.format(error_file))
logger().warning('\n'.join(errors))
w.close()
destination.unlink() # remove, if error
else:
w.write(content)
w.close()
return True
return False
def check_url(url: str) -> str:
"""
checks url for correctness
:param url:
:return:
"""
check = urlparse(url) # type: ParseResult
if check.scheme == '':
logger().warning('Url scheme has missing (%s). Use default scheme' %
url)
check.scheme = 'http'
if check.netloc == '':
logger().critical('Url netloc has missing (%s)')
raise InvalidUrlException(url)
return check.geturl()
def get_provider(url, args: ArgsListHelper) -> Generator:
caught = False
for _ in providers: # type: Provider
provider = _.new(args, url)
if provider.supported_urls() is None:
logger().warning('SUPPORTED_URLS is None for provider %s.%s',
provider.__class__.__module__,
provider.__class__.__name__)
continue
if provider.match():
caught = True
yield provider
if not caught:
raise ProviderNotFoundException.create(url)
def __init__(self, arguments: ArgsListHelper, url: str):
self.arguments = arguments
self.http = Http(self, url)
self._url = url
self.logger = logger()
self.store = store
self.print = print
"""
def __init__(self, provider=None, base_url: str = None):
from manga_py.libs.provider import Provider
if not isinstance(provider, Provider):
raise AttributeError('provider is not Provider type')
self.provider = provider # type: Provider
self.http_store = http_store # type: http_store
self.store = store # type: store
self.base_url = base_url
self.logger = logger()
self.http_store.init(base_url)
def run(self):
log = logger()
with ThreadPoolExecutor(max_workers=self._max_threads
) as executor: # type: ProcessPoolExecutor
future = {
executor.submit(self._target, file, self._each_file_callback):
file
for file in self._items
}
for idx in as_completed(future): # type: Future
url = future[idx]
try:
data = idx.result(timeout=30)
except Exception as exc:
log.warning('Error downloading')
else:
print('%r page is %d bytes' % (url, len(data)))
(self._final or self.__default_final)()
def __init__(self):
super().__init__()
atexit.register(self.exit)
self.log = logger(self.args.log_to_file)
if self.args.debug:
self.log.setLevel(DEBUG)