def __main__(self, keep_alive=False):
for domain in self.domains:
Thread(target=self.automator, args=(domain, keep_alive), name=domain.__name__, daemon=True).start()
try:
Sleep(15, True).join()
while True:
if False in [name in [thread.name for thread in enumerateT()]
for name in [domain.__name__ for domain in self.domains.difference(self.stopped)]]:
raise UnexpectedFewThreadsError
if self.domains == self.stopped:
raise SystemExit
sleep_ = Sleep(15, True)
self.links_writer()
sleep_.join()
except KeyboardInterrupt:
print("%sKeyboardInterrupt caught. Shutting down..." % currenttime())
self.links_writer()
raise SystemExit
except SystemExit:
self.links_writer()
raise
except UnexpectedFewThreadsError:
print("%sAn unexpected deletion of threads detected. Shutting down..." % currenttime())
self.links_writer()
raise SystemExit
finally:
string = "\nStatistics:\n"
for domain in self.domains:
string += "\n"
string += "%s:\nSaved: %s\nDownloaded: %s\n" % (domain.__name__,
self.statistics[domain.__name__]["saved"],
self.statistics[domain.__name__]["downloaded"])
print(string)
def links_writer(self):
try:
open(config_file, "rb")
except FileNotFoundError:
print("%sPlease stop deleting %s while the program is running." % (currenttime(), config_file))
dump(self.links, open("%s.bak" % config_file, "wb"))
dump(self.links, open("%s" % config_file, "wb"))
remove("%s.bak" % config_file)
print("%s%s overwritten." % (currenttime(), config_file))
def links_writer(self):
try:
open(config_file, "rb")
except FileNotFoundError:
print("%sPlease stop deleting %s while the program is running." %
(currenttime(), config_file))
dump(self.links, open("%s.bak" % config_file, "wb"))
dump(self.links, open("%s" % config_file, "wb"))
remove("%s.bak" % config_file)
print("%s%s overwritten." % (currenttime(), config_file))
def automator(self, domain, keep_alive=False):
token = False
try:
cutoff = self.links.data[domain.__name__]
except KeyError:
cutoff = None
while True:
if domain.__name__ not in self.links.getlinks:
self.links.getlinks[domain.__name__] = True
if not self.links.getlinks[domain.__name__]:
self.image_downloader(domain.__name__)
self.links.getlinks[domain.__name__] = True
if token and not keep_alive:
print("%sEnding %s downloader." % (currenttime(), domain.__name__))
break
token = False
if cutoff:
if domain.__name__ == "_4chan":
domain(cutoff=cutoff, disable_imt=True, disable_odt=True)
else:
domain(cutoff=cutoff)
else:
domain()
self.links.getlinks[domain.__name__] = False
token = True
self.stopped.add(domain)
def automator(self, domain, keep_alive=False):
token = False
try:
cutoff = self.links.data[domain.__name__]
except KeyError:
cutoff = None
while True:
if domain.__name__ not in self.links.getlinks:
self.links.getlinks[domain.__name__] = True
if not self.links.getlinks[domain.__name__]:
self.image_downloader(domain.__name__)
self.links.getlinks[domain.__name__] = True
if token and not keep_alive:
print("%sEnding %s downloader." %
(currenttime(), domain.__name__))
break
token = False
if cutoff:
if domain.__name__ == "_4chan":
domain(cutoff=cutoff, disable_imt=True, disable_odt=True)
else:
domain(cutoff=cutoff)
else:
domain()
self.links.getlinks[domain.__name__] = False
token = True
self.stopped.add(domain)
def __main__(self, keep_alive=False):
for domain in self.domains:
Thread(target=self.automator,
args=(domain, keep_alive),
name=domain.__name__,
daemon=True).start()
try:
Sleep(15, True).join()
while True:
if False in [
name in [thread.name for thread in enumerateT()]
for name in [
domain.__name__
for domain in self.domains.difference(self.stopped)
]
]:
raise UnexpectedFewThreadsError
if self.domains == self.stopped:
raise SystemExit
sleep_ = Sleep(15, True)
self.links_writer()
sleep_.join()
except KeyboardInterrupt:
print("%sKeyboardInterrupt caught. Shutting down..." %
currenttime())
self.links_writer()
raise SystemExit
except SystemExit:
self.links_writer()
raise
except UnexpectedFewThreadsError:
print(
"%sAn unexpected deletion of threads detected. Shutting down..."
% currenttime())
self.links_writer()
raise SystemExit
finally:
string = "\nStatistics:\n"
for domain in self.domains:
string += "\n"
string += "%s:\nSaved: %s\nDownloaded: %s\n" % (
domain.__name__, self.statistics[domain.__name__]["saved"],
self.statistics[domain.__name__]["downloaded"])
print(string)
def __init__(self, keep_alive=False):
self.domains = {self._4chan, self.konachan, }
self.stopped = set()
if not exists(config_file):
print('%s"links.json" not detected. Creating file...' % currenttime())
self.links = Config()
dump(self.links, open(config_file, "wb"))
else:
self.links = load(open(config_file, "rb"))
self.statistics = {domain.__name__: {"downloaded": 0, "saved": 0} for domain in self.domains}
self.__main__(keep_alive)
def __init__(self, keep_alive=False):
self.domains = {
self._4chan,
self.konachan,
}
self.stopped = set()
if not exists(config_file):
print('%s"links.json" not detected. Creating file...' %
currenttime())
self.links = Config()
dump(self.links, open(config_file, "wb"))
else:
self.links = load(open(config_file, "rb"))
self.statistics = {
domain.__name__: {
"downloaded": 0,
"saved": 0
}
for domain in self.domains
}
self.__main__(keep_alive)
def __init__(self, url, str_data=False, cooldown=1, error_limit=4, timeout=(4, 2), clip=54,
print_=True, params={}, bulk=False, **kwargs):
class CustomException(BaseException):
pass
link = url_make(url, params)
sc_counter = 0
lowerlimit = 0
self.content = b''
raised = False
if print_:
print("%sDownloading %s..." % (currenttime(), link[:(lambda: clip if clip else None)()]))
while True:
sleep_ = Sleep(cooldown, True)
try:
if bulk:
if "headers" in kwargs:
headers = kwargs["headers"]
del(kwargs["headers"])
if not match(r"bytes=\d*-", headers["range"]):
raise BaseException("Range header does not conform to 'bytes=[lowerlimit]-[upperlimit]"
" format.")
if "range" in headers:
print("%sOverriding range upper limit." % currenttime())
lowerlimit = match(r"(?<=bytes=)\d*(?=-)", headers["range"])
else:
headers = {"range": "bytes=0-"}
headers["range"] = sub(r"(?<=bytes=)\d*(?=-)", str(lowerlimit + len(self.content)),
headers["range"])
self.request = get(url, timeout=timeout, stream=True, params=params, headers=headers, **kwargs)
if "Content-Range" not in self.request.headers:
raise CustomException
for byte in self.request.iter_content(chunk_size=1024):
if byte:
self.content += byte
if str_data:
self.text = str(self.content)[2:-1]
else:
self.request = get(url, timeout=timeout, stream=True, params=params, **kwargs)
self.content = self.request.content
if str_data:
self.text = self.request.text
self.request.raise_for_status()
self.status_code = self.request.status_code
self.url = self.request.url
self.headers = self.request.headers
break
except (ChunkedEncodingError, ConnectionError, Timeout) as err:
raised, errmsg = True, err
except HTTPError as err:
raised, errmsg = True, err
if int(self.request.status_code/100) == 4:
if self.request.status_code == 429:
pass
else:
raise
else:
sc_counter += 1
if sc_counter >= error_limit:
raise
except CustomException:
bulk = False
print("%sServer does not accept custom ranges. Restarting request." % currenttime())
finally:
if raised:
print("%sError: %s" % (currenttime(), errmsg))
sleep_.join()