def download(self, media, api): return_bool = True if not overwrite_files and media.downloaded: return count = 0 sessions = [x for x in api.sessions if media.link in x.links] if not sessions: return session = sessions[0] while count < 11: links = [media.link] def choose_link(session, links): for link in links: r = api.json_request(link, session, "HEAD", stream=True, json_format=False) if not isinstance(r, requests.Response): continue header = r.headers content_length = header.get('content-length') if not content_length: continue content_length = int(content_length) return [link, content_length] result = choose_link(session, links) if not result: count += 1 continue link = result[0] content_length = result[1] media.size = content_length date_object = media.created_at download_path = os.path.join( media.directory, media.filename) timestamp = date_object.timestamp() if not overwrite_files: if main_helper.check_for_dupe_file(download_path, content_length): main_helper.format_image(download_path, timestamp) return_bool = False media.downloaded = True break r = api.json_request( link, session, stream=True, json_format=False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue downloader = main_helper.downloader(r, download_path, count) if not downloader: count += 1 continue main_helper.format_image(download_path, timestamp) link_string = f"Link: {link}" path_string = f"Link: {download_path}" print(link_string) print(path_string) media.downloaded = True break return return_bool
def download(media, session, directory, username): count = 0 while count < 11: links = media["links"] def choose_link(session, links): for link in links: r = json_request(session, link, "HEAD", True, False) if not r: continue header = r.headers content_length = int(header["content-length"]) if not content_length: continue return [link, content_length] result = choose_link(session, links) if not result: continue link = result[0] content_length = result[1] date_object = datetime.strptime(media["postedAt"], "%d-%m-%Y %H:%M:%S") og_filename = media["filename"] media["ext"] = os.path.splitext(og_filename)[1] media["ext"] = media["ext"].replace(".", "") download_path = media["directory"] + media["filename"] timestamp = date_object.timestamp() if not overwrite_files: if check_for_dupe_file(download_path, content_length): return r = json_request(session, link, "GET", True, False) if not r: count += 1 continue delete = False try: with open(download_path, 'wb') as f: delete = True for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) except (ConnectionResetError) as e: if delete: os.unlink(download_path) log_error.exception(e) count += 1 continue except Exception as e: if delete: os.unlink(download_path) log_error.exception(str(e) + "\n Tries: " + str(count)) count += 1 continue format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) return True
def download(medias): return_bool = True for media in medias: count = 0 session = media["session"] while count < 11: links = media["links"] def choose_link(session, links): for link in links: r = main_helper.json_request(session, link, "HEAD", stream=True, json_format=False) if not isinstance(r, requests.Response): continue header = r.headers content_length = header.get('content-length') if not content_length: continue content_length = int(content_length) return [link, content_length] result = choose_link(session, links) if not result: count += 1 continue link = result[0] content_length = result[1] date_object = datetime.strptime(media["postedAt"], "%d-%m-%Y %H:%M:%S") download_path = os.path.join(media["directory"], media["filename"]) timestamp = date_object.timestamp() if not overwrite_files: if main_helper.check_for_dupe_file(download_path, content_length): main_helper.format_image(download_path, timestamp) return_bool = False break r = main_helper.json_request(session, link, stream=True, json_format=False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue downloader = main_helper.downloader(r, download_path, count) if not downloader: count += 1 continue main_helper.format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) break return return_bool
def download(self, post: format_content.post_item, api): return_bool = True for media in post.medias: if not overwrite_files and media.downloaded: continue count = 0 session = media.session if not session: continue while count < 11: links = media.links def choose_link(session, links): for link in links: r = api.json_request(link, session, "HEAD", stream=True, json_format=False) if not isinstance(r, requests.Response): continue header = r.headers content_length = header.get('content-length') if not content_length: continue content_length = int(content_length) return [link, content_length] result = choose_link(session, links) if not result: count += 1 continue link = result[0] content_length = result[1] media.size = content_length date_object = datetime.strptime( post.postedAt, "%d-%m-%Y %H:%M:%S") download_path = os.path.join( media.directory, media.filename) timestamp = date_object.timestamp() if not overwrite_files: if main_helper.check_for_dupe_file(download_path, content_length): main_helper.format_image(download_path, timestamp) return_bool = False media.downloaded = True break r = api.json_request( link, session, stream=True, json_format=False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue downloader = main_helper.downloader(r, download_path, count) if not downloader: count += 1 continue main_helper.format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) media.downloaded = True break return return_bool
def download(thread, session, directory): thread_directory = thread["directory"] metadata_directory = os.path.join( thread_directory, "Metadata") os.makedirs(metadata_directory, exist_ok=True) metadata_filepath = os.path.join(metadata_directory, "Posts.json") with open(os.path.join(metadata_filepath), 'w') as outfile: json.dump(thread, outfile) return_bool = True medias = thread["posts"] for media in medias: count = 0 while count < 11: if "download_path" not in media: count += 1 continue ext = media["ext"].replace(".", "") filename = str(media["tim"])+"."+ext link = "http://i.4cdn.org/" + board_name + "/" + filename r = main_helper.json_request( session, link, "HEAD", True, False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue header = r.headers content_length = header.get('content-length') content_length = int(content_length) download_path = media["download_path"] timestamp = media["time"] if not overwrite_files: if main_helper.check_for_dupe_file(download_path, content_length): return_bool = False break r = main_helper.json_request(session, link, "GET", True, False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue delete = False try: with open(download_path, 'wb') as f: delete = True for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) except (ConnectionResetError) as e: if delete: os.unlink(download_path) main_helper.log_error.exception(e) count += 1 continue except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: count += 1 continue except Exception as e: if delete: os.unlink(download_path) main_helper.log_error.exception( str(e) + "\n Tries: "+str(count)) count += 1 continue main_helper.format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) break return return_bool
def download(medias, session, directory, username): return_bool = True for media in medias: count = 0 session = media["session"] while count < 11: links = media["links"] def choose_link(session, links): for link in links: r = json_request(session, link, "HEAD", stream=True, json_format=False) if not r: continue header = r.headers content_length = int(header["content-length"]) if not content_length: continue return [link, content_length] result = choose_link(session, links) if not result: count += 1 continue link = result[0] content_length = result[1] date_object = datetime.strptime(media["postedAt"], "%d-%m-%Y %H:%M:%S") download_path = os.path.join(media["directory"], media["filename"]) timestamp = date_object.timestamp() if not overwrite_files: if check_for_dupe_file(download_path, content_length): format_image(download_path, timestamp) return_bool = False break r = json_request(session, link, stream=True, json_format=False) if not r: return_bool = False count += 1 continue delete = False try: with open(download_path, 'wb') as f: delete = True for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) except (ConnectionResetError) as e: if delete: os.unlink(download_path) count += 1 continue except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: count += 1 continue except Exception as e: if delete: os.unlink(download_path) log_error.exception(str(e) + "\n Tries: " + str(count)) count += 1 continue format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) break return return_bool
def download(thread, session, directory): os.makedirs(directory, exist_ok=True) return_bool = True posts = thread["posts"] directory = thread["directory"] metadata_directory = os.path.join( directory, "Metadata") os.makedirs(metadata_directory, exist_ok=True) metadata_filepath = os.path.join(metadata_directory, "Posts.json") with open(os.path.join(metadata_filepath), 'w') as outfile: json.dump(thread, outfile) for post in posts: for media in post["files"]: count = 0 while count < 11: if "download_path" not in media: continue link = "https://bbw-chan.nl" + media["path"] r = main_helper.json_request( session, link, "HEAD", True, False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue header = r.headers content_length = header.get('content-length') content_length = int(content_length) download_path = media["download_path"] timestamp = post["creation"] if not overwrite_files: if main_helper.check_for_dupe_file(download_path, content_length): return_bool = False break r = main_helper.json_request( session, link, "GET", True, False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue os.makedirs(directory, exist_ok=True) delete = False try: with open(download_path, 'wb') as f: delete = True for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) except (ConnectionResetError) as e: if delete: os.unlink(download_path) count += 1 continue except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: count += 1 continue except Exception as e: if delete: os.unlink(download_path) count += 1 continue main_helper.format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) break return return_bool
def download(media, session): return_bool = True count = 0 while count < 11: links = media["links"] def choose_link(session, links): for link in links: r = main_helper.json_request(session, link, "HEAD", True, False) if not isinstance(r, requests.Response): continue header = r.headers content_length = int(header["content-length"]) if not content_length: continue return [link, content_length] result = choose_link(session, links) if not result: count += 1 continue link = result[0] content_length = result[1] date_object = datetime.strptime(media["postedAt"], "%d-%m-%Y %H:%M:%S") download_path = media["download_path"] timestamp = date_object.timestamp() if not overwrite_files: if main_helper.check_for_dupe_file(download_path, content_length): main_helper.format_image(download_path, timestamp) return_bool = False count += 1 break r = main_helper.json_request(session, link, "GET", True, False) if not isinstance(r, requests.Response): return_bool = False count += 1 continue delete = False try: with open(download_path, 'wb') as f: delete = True for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) except (ConnectionResetError) as e: if delete: os.unlink(download_path) count += 1 continue except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: count += 1 continue except Exception as e: if delete: os.unlink(download_path) count += 1 continue main_helper.format_image(download_path, timestamp) log_download.info("Link: {}".format(link)) log_download.info("Path: {}".format(download_path)) break return return_bool