def post_downloader(insta, post, debug: bool, final_dest: str, dump_metadata: bool, dump_comments: bool, dump_likes: bool): def on_item_start(_, __, item): bar.set_description_str("[" + "⤓ Downloading".center(15) + "]") bar.set_postfix_str("({}) {}".format(post.shortcode, item.typename)) def on_item_finish(_, __, ___, file_path): nonlocal downs, existings if file_path is not None: downs += 1 else: existings += 1 bar.set_description_str("[" + "♻︎ Existing".center(15) + "]") time.sleep(0.1) bar.update(1) def on_item_error(post, i, __, e): bar.set_description_str(Fore.RED + "[" + "✘ Failed".center(15) + "]") if not debug: tqdm.write(Fore.RED + "✖︎ [{} ({})] {}: {}".format(post.shortcode, i, e.__class__.__name__, str(e))) try: downs = existings = 0 bar = progress_bar(total=post.media_count, hide=debug) post.download(final_dest, on_item_start=on_item_start, on_item_finish=on_item_finish, on_item_error=on_item_error) if any((dump_metadata, dump_comments, dump_likes)): bar.set_description_str("[" + "Ⓓ Dumping".center(15) + "]") data = {} # TODO: show dump progress using 'tqdm.write' with carriage return (\r) if dump_metadata: data.update(post.as_dict(extra=True)) data.update({"media_items": [d.as_dict() for d in post.media_items()]}) if dump_comments: comments = post.comments() comments = [comment._asdict() for comment in comments[1]] data.update({"comments": list(comments)}) if dump_likes: likes = post.likes() likes = [like.username for like in likes] data.update({"likes": list(likes)}) data_dumper(final_dest, post.shortcode, data) bar.set_description_str(Fore.GREEN + "[" + "✔︎ Finished".center(15) + "]") except Exception as e: exc_type, exc_value, tb = sys.exc_info() insta.logger.error("{}: {}".format(exc_type.__name__, exc_value)) insta.logger.debug("".join(traceback.format_tb(tb))) if not debug: tqdm.write(Fore.RED + "✖︎ {}: {}".format(exc_type.__name__, str(e))) bar.set_description_str(Fore.RED + "[" + "✘ Failed".center(15) + "]") except KeyboardInterrupt: bar.set_description_str(Fore.MAGENTA + "[" + "⌧ Interrupted".center(15) + "]") raise finally: bar.close() return downs, existings
def on_item_finish(_, __, item, file_path): nonlocal downs, existings if file_path is not None: downs += 1 else: existings += 1 bar.set_description_str("[" + "♻︎ Existing".center(15) + "]") time.sleep(0.1) if dump_metadata: bar.set_description_str("[" + "Ⓓ Dumping".center(15) + "]") data_dumper(final_dest, to_datetime(item.created_time), item.as_dict(extra=True)) bar.update(1)
def on_post_finish(post): # TODO: show dump progress using 'tqdm.write' with carriage return (\r) if any((dump_metadata, dump_comments, dump_likes)): bar.set_description_str("[" + "Ⓓ Dumping".center(15) + "]") data = {} if dump_metadata: data.update(post.as_dict(extra=True)) data.update({"media_items": [d.as_dict() for d in post.media_items()]}) if dump_comments: comments = post.comments() comments = [comment._asdict() for comment in comments[1]] data.update({"comments": list(comments)}) if dump_likes: likes = post.likes() likes = [like.username for like in likes] data.update({"likes": list(likes)}) data_dumper(final_dest, post.shortcode, data) bar.update(1) nonlocal subbar if subbar is not None: subbar.close() subbar = None
def story_downloader(story, debug: bool, final_dest: str, dump_metadata: bool): def on_item_start(_, __, item): bar.set_description_str("[" + "⤓ Downloading".center(15) + "]") bar.set_postfix_str(item.typename) def on_item_finish(_, __, item, file_path): nonlocal downs, existings if file_path is not None: downs += 1 else: existings += 1 bar.set_description_str("[" + "♻︎ Existing".center(15) + "]") time.sleep(0.1) if dump_metadata: bar.set_description_str("[" + "Ⓓ Dumping".center(15) + "]") data_dumper(final_dest, to_datetime(item.created_time), item.as_dict(extra=True)) bar.update(1) def on_item_error(_, __, ___, e): bar.set_description_str(Fore.RED + "[" + "✘ Failed".center(15) + "]") if not debug: tqdm.write(Fore.RED + "✖︎ {}: {}".format(e.__class__.__name__, str(e))) try: downs = existings = 0 bar = progress_bar(story.reel_count, hide=debug) story.download(final_dest, on_item_start=on_item_start, on_item_finish=on_item_finish, on_item_error=on_item_error) if dump_metadata: data_dumper(final_dest, "story", story.as_dict(extra=True)) bar.set_description_str(Fore.GREEN + "[" + "✔︎ Finished".center(15) + "]") except KeyboardInterrupt: bar.set_description_str(Fore.MAGENTA + "[" + "⌧ Interrupted".center(15) + "]") raise finally: bar.close() return downs, existings
def dump_handler(**args): # Try to load object guest = False insta = load_obj() if insta is None: guest = True insta = Instagram() # Validate target parser = args.get("parser") targets = args.get("target") names = [] for target in targets: if len(target) <= 1: parser.error("invalid target parsed: '{}'".format(target)) if target[0] not in ("@", ":"): parser.error("invalid identifier of target: '{}'".format(target)) if target[0] != targets[0][0]: parser.error("all targets must be the same type") names.append(target[1:]) identifier = targets[0][0] # Gather options outfile = args.get("outfile") limit = args.get("limit") profiles_filter_str = args.get("profiles_filter") comments_filter_str = args.get("comments_filter") preload = args.get("preload") # Gather dump types followers = args.get("followers") followings = args.get("followings") comments = args.get("comments") likes = args.get("likes") # Validate dump types if identifier == "@" and any((comments, likes)): parser.error( "target '@' not allowed with arguments: -comments, -likes") elif identifier == ":" and any((followers, followings)): parser.error( "target ':' not allowed with arguments: -followers, -followings") struct_getter = None # Make entries Job = namedtuple("Job", "name handler") entries = [] for name in names: jobs = [] if identifier == "@": if not any((followers, followings)): jobs.append( Job("information", lambda profile: profile.as_dict())) else: if followers: jobs.append( Job("followers", lambda profile: profile.followers())) if followings: jobs.append( Job("followings", lambda profile: profile.followings())) if struct_getter is None: struct_getter = insta.profile elif identifier == ":": if not any((comments, likes)): jobs.append(Job("information", lambda post: post.as_dict())) else: if comments: jobs.append(Job("comments", lambda post: post.comments())) if likes: jobs.append(Job("likes", lambda post: post.likes())) if struct_getter is None: struct_getter = insta.post entries.append((name, jobs)) # Start dumping entries of jobs if guest: warn_print("You are not logged in currently (Anonymous/Guest).") print(Style.BRIGHT + Fore.GREEN + "❖ [Dump] {} entries ({} jobs)\n".format( len(entries), sum([len(jobs) for _, jobs in entries]))) for i, (name, jobs) in enumerate(entries, start=1): struct = None print(Style.BRIGHT + "{0}+ (Entry {1}/{2}) {3} {4}".format( Fore.BLUE, i, len(entries), struct_getter.__name__.title(), Fore.WHITE + name)) with error_catcher(do_exit=False): struct = struct_getter(name) struct._obtain_full_data() if not bool(struct): continue for j, job in enumerate(jobs, start=1): # retrieve items group = None results = None print("{0}► (Job {1}/{2}) Retrieving {3}".format( Fore.CYAN, j, len(jobs), job.name.title())) with error_catcher(do_exit=False): results = job.handler(struct) if not bool(results): warn_print("No results are returned.") continue try: if job.name == "information": if any((limit, profiles_filter_str, comments_filter_str, preload)): warn_print( "Disallow: -l/--limit, -PF/--profiles-filter, -CF/--comments-filter, --preload" ) elif job.name in ("likes", "followers", "followings"): if comments_filter_str: warn_print("Disallow: -CF/--comments-filter") print(Style.BRIGHT + "~ Total:", results.length) group = results group.limit(limit).preload(preload).ignore_errors(True) if profiles_filter_str: filterfunc = filterstr_to_filterfunc( profiles_filter_str) group.filter(filterfunc) results = (result.username for result in group) if not outfile else ( result.as_dict(extra=True) for result in group) elif job.name == "comments": if any((profiles_filter_str, preload)): warn_print( "Disallow: -PF/--profiles-filter, --preload") print(Style.BRIGHT + "~ Total:", results[0]) if comments_filter_str: filterfunc = filterstr_to_filterfunc( comments_filter_str) with error_catcher(do_exit=False): results = [ result._asdict() for result in results[1] if filterfunc(result) ] if limit is not None: results = results[:limit] else: raise ValueError("unable to resolve dump type") if outfile: outfile = os.path.abspath(outfile) dest, file = os.path.split(outfile) filename, _ = os.path.splitext(file) data_dumper(dest, filename, list(results)) print(Style.BRIGHT + Fore.GREEN + "⇟ Data Dumped => " + Fore.WHITE + os.path.join(dest, filename + ".json")) else: pretty_print(results) finally: if group is not None and job.name in ("likes", "followers", "followings"): errors = group.collect_errors() if errors: print(Fore.RED + " [{} Errors Collected During Posts Retrieving]" .format(len(errors))) for error in errors: print(Fore.RED + "> {} -> {}: {}".format( Fore.WHITE + Style.BRIGHT + error.name + Style.RESET_ALL + Fore.RED, Style.BRIGHT + error.exc_type.__name__, error.exc_value)) print()
def highlights_downloader(insta, highlights: list, debug: bool, final_dest: str, limit: int, dump_metadata: bool): def on_item_start(_, __, item): if subbar is not None: subbar.set_postfix_str(item.typename) def on_item_finish(_, __, item, file_path): nonlocal downs, existings if file_path is not None: downs += 1 else: existings += 1 bar.set_description_str("[" + "♻︎ Existing".center(15) + "]") subbar.set_description_str("[" + "↻ Verifying".center(15) + "]") time.sleep(0.1) if dump_metadata: bar.set_description_str("[" + "Ⓓ Dumping".center(15) + "]") subbar.set_description_str("[" + "↻ Verifying".center(15) + "]") data_dumper(subdir, to_datetime(item.created_time), item.as_dict(extra=True)) subbar.update(1) def on_item_error(_, i, ___, e): # # Interrupt the download process of the current highlight once it encounters an error # raise e from e # Ignore error and move on to the download process of the next reel item of the current highlight if subbar is not None: subbar.set_description_str(Fore.RED + "[" + "✘ Failed".center(15) + "]") else: bar.set_description_str(Fore.RED + "[" + "✘ Failed".center(15) + "]") if not debug: tqdm.write(Fore.RED + "✖︎ [{} ({})] {}: {}".format(highlight.title[:12], i, e.__class__.__name__, str(e))) try: highlights = highlights[:len(highlights) if limit is None else limit] downs = existings = 0 bar = progress_bar(total=len(highlights), hide=debug) for highlight in highlights[:len(highlights) if limit is None else limit]: subdir = os.path.join(final_dest, clean_filename(highlight.title)) subbar = progress_bar(total=highlight.reel_count, sub=True, hide=debug or highlight.reel_count == 1) bar.set_description_str("[" + "⤓ Downloading".center(15) + "]") bar.set_postfix_str("(" + highlight.title[:12] + ("..." if len(highlight.title) > 12 else "") + ")") if not os.path.isdir(subdir): os.mkdir(subdir) highlight.download(subdir, on_item_start=on_item_start, on_item_finish=on_item_finish, on_item_error=on_item_error) if dump_metadata: bar.set_description_str("[" + "Ⓓ Dumping".center(15) + "]") data_dumper(subdir, "highlight", highlight.as_dict(extra=True)) bar.update(1) bar.set_description_str(Fore.GREEN + "[" + "✔︎ Finished".center(15) + "]") except Exception as e: exc_type, exc_value, tb = sys.exc_info() insta.logger.error("{}: {}".format(exc_type.__name__, exc_value)) insta.logger.debug("".join(traceback.format_tb(tb))) if not debug: tqdm.write(Fore.RED + "✖︎ {}: {}".format(exc_type.__name__, str(e))) bar.set_description_str(Fore.RED + "[" + "✘ Failed".center(15) + "]") except KeyboardInterrupt: bar.set_description_str(Fore.MAGENTA + "[" + "⌧ Interrupted".center(15) + "]") raise finally: bar.close() return downs, existings