class MastoCrosspostUtils: def __init__(self, clientcred_key, access_token_key, instance_url): self.mastodon_api = Mastodon( client_id=clientcred_key, access_token=access_token_key, api_base_url=instance_url ) self.me = self.mastodon_api.account_verify_credentials() def scrape_toots(self, mstdn_acct_id, since=None): """ Get toots from an account since given toot id and filter them """ toots = self.mastodon_api.account_statuses( mstdn_acct_id, since_id=since, exclude_replies=True) filtered_toots = [] if len(toots): filtered_toots = list(filter(lambda x: x['reblog'] is None and x['poll'] is None and x['visibility'] in [ "public", "unlisted", "private"], toots[::-1])) return filtered_toots def get_following(self): return self.mastodon_api.account_following(self.me.id)
account = mastodon.account(j) accounts.append(account) with open(pathTo + 'accounts/accounts' + str(j) + '.json', 'w') as accountsFile: json.dump(account, accountsFile, default=datetimeconvert) if account['url'][:len(url)] == url: if len(account) > 0: if account["followers_count"] > 0: followersTemp = getLists(mastodon.account_followers(j)) if account["followers_count"] != len(followersTemp[1]): error.append(['followers', j]) followers.append(followersTemp) if account["following_count"] > 0: followingTemp = getLists(mastodon.account_following(j)) if account["following_count"] != len(followingTemp[1]): error.append(['followiong', j]) following.append(followingTemp) with open(pathTo + 'followers/followers' + str(j) + '.json', 'w') as followersFile: json.dump(followers, followersFile, default=datetimeconvert) with open(pathTo + 'following/following' + str(j) + '.json', 'w') as followingFile: json.dump(following, followingFile, default=datetimeconvert) j += 1 except Exception, e:
class FediGroupBot: def __init__(self, base_url, access_token, accept_dms, accept_retoots, save_file): self.base_url = base_url self.accept_dms = accept_dms self.accept_retoots = accept_retoots self.save_file = save_file self.masto = Mastodon(debug_requests=False, access_token=access_token, api_base_url=base_url) self.id = self.masto.account_verify_credentials().id self.username = self.masto.account_verify_credentials().username self.group_members = [] self.group_admins = [] def run(self): last_seen_id = 0 if os.path.exists(self.save_file): with open(self.save_file, 'r') as fd: try: last_seen_id = int(fd.readline()) except ValueError: pass while True: notifications = self.masto.notifications(since_id=last_seen_id) if notifications: self.__update_group_members() for notification in sorted(notifications, key=lambda x: x.id): if notification.id > last_seen_id: last_seen_id = notification.id self.__do_action(notification) with open(self.save_file, 'w') as fd: fd.write(str(last_seen_id)) time.sleep(2) def __update_group_members(self): self.group_members, max_id = [], None followers = self.masto.account_followers(self.id, limit=sys.maxsize, max_id=max_id) # NOQA: E501 while len(followers) > 1: self.group_members += [member.acct for member in followers] max_id = followers[-1].id followers = self.masto.account_followers( self.id, limit=sys.maxsize, max_id=max_id) # NOQA: E501 self.group_admins, max_id = [], None following = self.masto.account_following(self.id, limit=sys.maxsize, max_id=max_id) # NOQA: E501 while len(following) > 1: self.group_admins += [member.acct for member in following] max_id = followers[-1].id following = self.masto.account_following( self.id, limit=sys.maxsize, max_id=max_id) # NOQA: E501 def __do_action(self, notification): if notification.type != "mention": return if notification.status.in_reply_to_id is not None: return if self.accept_retoots and notification.status.visibility == "public" \ and notification.status.account.acct in self.group_members: self.masto.status_reblog(notification.status.id) if self.accept_dms and notification.status.visibility == "direct" \ and notification.status.account.acct in self.group_admins: new_status = re.sub("<br />", "\n", notification.status.content) new_status = re.sub("</p><p>", "\n\n", new_status) new_status = re.sub("<.*?>", "", new_status) if new_status.startswith("@" + self.username): new_status = re.sub("@" + self.username, "", new_status) new_status = html.unescape(new_status) media_ids = [] for media in notification.status.media_attachments: response = requests.get(media.url) mime_type = response.headers['Content-Type'] media_data = response.content media_ids.append( self.masto.media_post(media_data, mime_type, description=media.description)) self.masto.status_post( new_status, media_ids=media_ids, sensitive=notification.status.sensitive, visibility="public", spoiler_text=notification.status.spoiler_text)
link.insert_after(link["href"]) link.decompose() toot = soup.get_text() toot = toot.rstrip("\n") #remove trailing newline toot = toot.replace("@", "@\u202B") #put a zws between @ and username to avoid mentioning return(toot) client = Mastodon( client_id=cfg['client']['id'], client_secret = cfg['client']['secret'], access_token=cfg['secret'], api_base_url=cfg['site']) me = client.account_verify_credentials() following = client.account_following(me.id) db = sqlite3.connect("toots.db") db.text_factory=str c = db.cursor() c.execute("CREATE TABLE IF NOT EXISTS `toots` (id INT NOT NULL UNIQUE PRIMARY KEY, userid INT NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL) WITHOUT ROWID") db.commit() def handleCtrlC(signal, frame): print("\nPREMATURE EVACUATION - Saving chunks") db.commit() sys.exit(1) signal.signal(signal.SIGINT, handleCtrlC) def get_toots_legacy(client, id):
if 'runcount' not in runparams: runparams['runcount'] = 1 else: runparams['runcount']+=1 if 'my_id' not in runparams: my_id = mastodon.account_search(runparams['botname'])[0]['id'] runparams['my_id'] = my_id else: my_id = runparams['my_id'] if DEBUG: print('Found my id %i' % my_id) my_followed = mastodon.account_following(my_id) my_followed_list=[my_id] total_followed=0 for user in my_followed: if 'id' in user: my_followed_list.append(user['id']) total_followed+=1 if 'list_seen' not in runparams: runparams['list_seen'] = my_followed_list if DEBUG: print('I am currently already following %i persons' % total_followed) toots = mastodon.timeline_public(since_id=runparams['since_id'],limit=40) if DEBUG:
class MastodonEbooks: def __init__(self, options={}): self.api_base_url = options.get("api_base_url", "https://botsin.space") self.app_name = options.get("app_name", "ebooks") if path.exists("clientcred.secret") and path.exists("usercred.secret"): self.client = Mastodon(client_id="clientcred.secret", access_token="usercred.secret", api_base_url=self.api_base_url) if not path.exists("clientcred.secret"): print("No clientcred.secret, registering application") Mastodon.create_app(self.app_name, api_base_url=self.api_base_url, to_file="clientcred.secret") if not path.exists("usercred.secret"): print("No usercred.secret, registering application") self.email = input("Email: ") self.password = getpass("Password: "******"clientcred.secret", api_base_url=self.api_base_url) self.client.log_in(self.email, self.password, to_file="usercred.secret") def setup(self): me = self.client.account_verify_credentials() following = self.client.account_following(me.id) with open("corpus.txt", "w+", encoding="utf-8") as fp: for f in following: print("Downloading toots for user @{}".format(f.username)) for t in self._get_toots(f.id): fp.write(t + "\n") def gen_toot(self): with open("corpus.txt", encoding="utf-8") as fp: model = markovify.NewlineText(fp.read()) sentence = None # you will make that damn sentence while sentence is None or len(sentence) > 500: sentence = model.make_sentence(tries=100000) toot = sentence.replace("\0", "\n") return toot def post_toot(self, toot): self.client.status_post(toot, spoiler_text="markov toot") def _parse_toot(self, toot): if toot.spoiler_text != "": return if toot.reblog is not None: return if toot.visibility not in ["public", "unlisted"]: return soup = BeautifulSoup(toot.content, "html.parser") # pull the mentions out # for mention in soup.select("span.h-card"): # mention.unwrap() # for mention in soup.select("a.u-url.mention"): # mention.unwrap() # we will destroy the mentions until we're ready to use them # someday turbocat, you will talk to your sibilings for mention in soup.select("span.h-card"): mention.decompose() # make all linebreaks actual linebreaks for lb in soup.select("br"): lb.insert_after("\n") lb.decompose() # make each p element its own line because sometimes they decide not to be for p in soup.select("p"): p.insert_after("\n") p.unwrap() # keep hashtags in the toots for ht in soup.select("a.hashtag"): ht.unwrap() # unwrap all links (i like the bots posting links) for link in soup.select("a"): link.insert_after(link["href"]) link.decompose() text = map(lambda a: a.strip(), soup.get_text().strip().split("\n")) # next up: store this and patch markovify to take it # return {"text": text, "mentions": mentions, "links": links} # it's 4am though so we're not doing that now, but i still want the parser updates return "\0".join(list(text)) def _get_toots(self, id): i = 0 toots = self.client.account_statuses(id) while toots is not None: for toot in toots: t = self._parse_toot(toot) if t != None: yield t toots = self.client.fetch_next(toots) i += 1 if i % 10 == 0: print(i)
def main(): colorama_init() parser = ArgumentParser(description=__doc__, formatter_class=RawDescriptionHelpFormatter) parser.add_argument('--min-activity', dest='min_activity', type=parse_time_ago, default="1y", help=("Remove followings inactive for a given period" " (m for months, y for years, d for days) " "(default: %(default)s)")) parser.add_argument('--target-count', dest='target_count', type=int, help=("Target some following count (will try to stop" " when you have that many followings left)")) parser.add_argument('--unfollow', action='store_true', help="Actually unfollow") parser.add_argument('--followers', action='store_true', help="Instead of removing people you follow, remove " "people who follow YOU") parser.add_argument('--unmutuals', action='store_true', help="Remove people who follow you but that you " "don't follow") parser.add_argument('-v', '--verbose', action='store_true', help="Display more things") args = parser.parse_args() session = requests.Session() mastodon = Mastodon(access_token=settings.ACCESS_TOKEN, api_base_url=settings.API_BASE, ratelimit_method='pace') current_user = mastodon.account_verify_credentials() uid = current_user['id'] if args.unmutuals: args.followers = True if args.followers: followings_count = current_user['followers_count'] else: followings_count = current_user['following_count'] local_count = followings_count goal_msg = "" if args.target_count: goal_msg = "(goal: n>={})".format(args.target_count) now = datetime.now(tz=timezone.utc) def clog(c, s): tqdm.write(c + s + Fore.WHITE + Style.NORMAL) cprint(Fore.GREEN, "Current user: @{} (#{})".format(current_user['username'], uid)) cprint( Fore.GREEN, "{}: {} {}".format("Followers" if args.followers else "Followings", followings_count, goal_msg)) if args.unfollow: cprint(Fore.RED, "Action: unfollow") else: cprint(Fore.YELLOW, "Action: none") followings = None if args.followers: followings = mastodon.account_followers(uid) else: followings = mastodon.account_following(uid) followings = mastodon.fetch_remaining(followings) bar = tqdm(list(followings)) for f in bar: fid = f.get('id') acct = f.get('acct') fullhandle = "@{}".format(acct) if '@' in acct: inst = acct.split('@', 1)[1].lower() else: inst = None if args.target_count is not None and local_count <= args.target_count: clog(Fore.RED + Style.BRIGHT, "{} followings left; stopping".format(local_count)) break title_printed = False def title(): nonlocal title_printed if title_printed: return title_printed = True clog(Fore.WHITE + Style.BRIGHT, "Account: {} (#{})".format(f.get('acct'), fid)) if args.verbose: title() try: bar.set_description(fullhandle.ljust(30, ' ')) act = False if args.unmutuals and inst not in settings.SKIP_INSTANCES: try: relations = mastodon.account_relationships(fid) is_mutual = relations[0]["following"] or relations[0][ "requested"] if not is_mutual: clog(Fore.YELLOW, "- Unmutual ({})".format(relations)) act = True except (UserGone, Error, requests.RequestException) as e: act = False clog(Fore.YELLOW, "- Exception ({})".format(e)) elif args.min_activity and inst not in settings.SKIP_INSTANCES: try: last_toot = get_last_toot(mastodon, fid) if last_toot < now - args.min_activity: # force a cache miss to be Sure last_toot = get_last_toot(mastodon, fid, force=True) if last_toot < now - args.min_activity: act = True msg = "(!)" title() clog(Fore.WHITE, "- Last toot: {} {}".format(last_toot, msg)) else: msg = "(pass)" if args.verbose: clog(Fore.WHITE, "- Last toot: {} {}".format(last_toot, msg)) except UserGone as e: moved = f.get('moved') if moved: # TODO: follow new account and unfollow old act = False title() clog( Fore.YELLOW, "- User moved ({}) [NOT IMPLEMENTED]".format( moved)) else: act = True title() clog(Fore.YELLOW, "- User gone ({})".format(e)) except (Error, requests.RequestException) as e: if inst and inst in settings.ASSUME_DEAD_INSTANCES: act = True title() clog(Fore.YELLOW, "- Instance gone ({})".format(e)) else: raise if act: local_count -= 1 if args.unfollow: if args.followers: clog(Fore.GREEN + Style.BRIGHT, "- Removing follower {}".format(fullhandle)) mastodon.account_block(fid) mastodon.account_unblock(fid) else: clog(Fore.GREEN + Style.BRIGHT, "- Unfollowing {}".format(fullhandle)) mastodon.account_unfollow(fid) else: clog(Fore.GREEN + Style.BRIGHT, "- (not) unfollowing {}".format(fullhandle)) clog(Fore.WHITE, ("- {}/{} followings left".format( local_count, followings_count))) except Exception as e: title() clog(Fore.RED, "- Error: {}".format(str(e)))
def main(instance, config, profile): configpath = os.path.expanduser(config) if os.path.isfile(configpath) and not os.access(configpath, os.W_OK): # warn the user before they're asked for input cprint( "Config file does not appear to be writable: {}".format( configpath), fg('red')) config = parse_config(configpath) # make sure profile name is legal profile = re.sub(r'\s+', '', profile) # disallow whitespace profile = profile.lower() # force to lowercase if profile == '' or profile in RESERVED: cprint("Invalid profile name: {}".format(profile), fg('red')) sys.exit(1) if not config.has_section(profile): config.add_section(profile) instance, client_id, client_secret, token = \ get_or_input_profile(config, profile, instance) if not token: cprint("Could not log you in. Please try again later.", fg('red')) sys.exit(1) mastodon = Mastodon(client_id=client_id, client_secret=client_secret, access_token=token, api_base_url="https://" + instance) # update config before writing if "token" not in config[profile]: config[profile] = { 'instance': instance, 'client_id': client_id, 'client_secret': client_secret, 'token': token } save_config(configpath, config) say_error = lambda a, b: cprint( "Invalid command. Use 'help' for a list of commands.", fg('white') + bg('red')) print("You are connected to ", end="") cprint(instance, fg('green') + attr('bold')) print("Enter a command. Use 'help' for a list of commands.") print("\n") user = mastodon.account_verify_credentials() prompt = "[@{} ({})]: ".format(str(user['username']), profile) # Completion setup stuff for i in mastodon.account_following(user['id'], limit=80): bisect.insort(completion_list, '@' + i['acct']) readline.set_completer(complete) readline.parse_and_bind("tab: complete") readline.set_completer_delims(' ') while True: command = input(prompt).split(' ', 1) rest = "" try: rest = command[1] except IndexError: pass command = command[0] cmd_func = commands.get(command, say_error) cmd_func(mastodon, rest)
bot_client_secret_fn = os.path.join('data', 'instances', domain, 'bot_client.secret') user_secret_file = os.path.join( 'data', 'accounts', common.md5('{0},{1}'.format(domain, username)), 'user.secret') api_base_url = 'https://{0}'.format(domain) mastodon = Mastodon(client_id=bot_client_secret_fn, access_token=user_secret_file, api_base_url=api_base_url) user_id = data.get_id(domain) follower_list = [] max_id = None while (True): _follower_list = mastodon.account_following(user_id, max_id=max_id) if len(_follower_list) <= 0: break follower_list += _follower_list if '_pagination_next' not in _follower_list[-1]: break max_id = _follower_list[-1]['_pagination_next'] follower_list = [i['acct'] for i in follower_list] follower_list = [ '@{0}'.format(i) if '@' in i else '@{0}@{1}'.format(i, domain) for i in follower_list ] all_follower_list = config['instance_data_list'] all_follower_list = [i['domain'] for i in all_follower_list] all_follower_list = [data.get_username(i) for i in all_follower_list]