class MastoCrosspostUtils:
def __init__(self, clientcred_key, access_token_key, instance_url):
self.mastodon_api = Mastodon(
client_id=clientcred_key,
access_token=access_token_key,
api_base_url=instance_url
)
self.me = self.mastodon_api.account_verify_credentials()
def scrape_toots(self, mstdn_acct_id, since=None):
""" Get toots from an account since given toot id and filter them
"""
toots = self.mastodon_api.account_statuses(
mstdn_acct_id, since_id=since, exclude_replies=True)
filtered_toots = []
if len(toots):
filtered_toots = list(filter(lambda x:
x['reblog'] is None and
x['poll'] is None and
x['visibility'] in [
"public", "unlisted", "private"],
toots[::-1]))
return filtered_toots
def get_following(self):
return self.mastodon_api.account_following(self.me.id)
account = mastodon.account(j)
accounts.append(account)
with open(pathTo + 'accounts/accounts' + str(j) + '.json',
'w') as accountsFile:
json.dump(account, accountsFile, default=datetimeconvert)
if account['url'][:len(url)] == url:
if len(account) > 0:
if account["followers_count"] > 0:
followersTemp = getLists(mastodon.account_followers(j))
if account["followers_count"] != len(followersTemp[1]):
error.append(['followers', j])
followers.append(followersTemp)
if account["following_count"] > 0:
followingTemp = getLists(mastodon.account_following(j))
if account["following_count"] != len(followingTemp[1]):
error.append(['followiong', j])
following.append(followingTemp)
with open(pathTo + 'followers/followers' + str(j) + '.json',
'w') as followersFile:
json.dump(followers,
followersFile,
default=datetimeconvert)
with open(pathTo + 'following/following' + str(j) + '.json',
'w') as followingFile:
json.dump(following,
followingFile,
default=datetimeconvert)
j += 1
except Exception, e:
class FediGroupBot:
def __init__(self, base_url, access_token, accept_dms, accept_retoots,
save_file):
self.base_url = base_url
self.accept_dms = accept_dms
self.accept_retoots = accept_retoots
self.save_file = save_file
self.masto = Mastodon(debug_requests=False,
access_token=access_token,
api_base_url=base_url)
self.id = self.masto.account_verify_credentials().id
self.username = self.masto.account_verify_credentials().username
self.group_members = []
self.group_admins = []
def run(self):
last_seen_id = 0
if os.path.exists(self.save_file):
with open(self.save_file, 'r') as fd:
try:
last_seen_id = int(fd.readline())
except ValueError:
pass
while True:
notifications = self.masto.notifications(since_id=last_seen_id)
if notifications:
self.__update_group_members()
for notification in sorted(notifications, key=lambda x: x.id):
if notification.id > last_seen_id:
last_seen_id = notification.id
self.__do_action(notification)
with open(self.save_file, 'w') as fd:
fd.write(str(last_seen_id))
time.sleep(2)
def __update_group_members(self):
self.group_members, max_id = [], None
followers = self.masto.account_followers(self.id,
limit=sys.maxsize,
max_id=max_id) # NOQA: E501
while len(followers) > 1:
self.group_members += [member.acct for member in followers]
max_id = followers[-1].id
followers = self.masto.account_followers(
self.id, limit=sys.maxsize, max_id=max_id) # NOQA: E501
self.group_admins, max_id = [], None
following = self.masto.account_following(self.id,
limit=sys.maxsize,
max_id=max_id) # NOQA: E501
while len(following) > 1:
self.group_admins += [member.acct for member in following]
max_id = followers[-1].id
following = self.masto.account_following(
self.id, limit=sys.maxsize, max_id=max_id) # NOQA: E501
def __do_action(self, notification):
if notification.type != "mention":
return
if notification.status.in_reply_to_id is not None:
return
if self.accept_retoots and notification.status.visibility == "public" \
and notification.status.account.acct in self.group_members:
self.masto.status_reblog(notification.status.id)
if self.accept_dms and notification.status.visibility == "direct" \
and notification.status.account.acct in self.group_admins:
new_status = re.sub("<br />", "\n", notification.status.content)
new_status = re.sub("</p><p>", "\n\n", new_status)
new_status = re.sub("<.*?>", "", new_status)
if new_status.startswith("@" + self.username):
new_status = re.sub("@" + self.username, "", new_status)
new_status = html.unescape(new_status)
media_ids = []
for media in notification.status.media_attachments:
response = requests.get(media.url)
mime_type = response.headers['Content-Type']
media_data = response.content
media_ids.append(
self.masto.media_post(media_data,
mime_type,
description=media.description))
self.masto.status_post(
new_status,
media_ids=media_ids,
sensitive=notification.status.sensitive,
visibility="public",
spoiler_text=notification.status.spoiler_text)
link.insert_after(link["href"])
link.decompose()
toot = soup.get_text()
toot = toot.rstrip("\n") #remove trailing newline
toot = toot.replace("@", "@\u202B") #put a zws between @ and username to avoid mentioning
return(toot)
client = Mastodon(
client_id=cfg['client']['id'],
client_secret = cfg['client']['secret'],
access_token=cfg['secret'],
api_base_url=cfg['site'])
me = client.account_verify_credentials()
following = client.account_following(me.id)
db = sqlite3.connect("toots.db")
db.text_factory=str
c = db.cursor()
c.execute("CREATE TABLE IF NOT EXISTS `toots` (id INT NOT NULL UNIQUE PRIMARY KEY, userid INT NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL) WITHOUT ROWID")
db.commit()
def handleCtrlC(signal, frame):
print("\nPREMATURE EVACUATION - Saving chunks")
db.commit()
sys.exit(1)
signal.signal(signal.SIGINT, handleCtrlC)
def get_toots_legacy(client, id):
if 'runcount' not in runparams:
runparams['runcount'] = 1
else:
runparams['runcount']+=1
if 'my_id' not in runparams:
my_id = mastodon.account_search(runparams['botname'])[0]['id']
runparams['my_id'] = my_id
else:
my_id = runparams['my_id']
if DEBUG:
print('Found my id %i' % my_id)
my_followed = mastodon.account_following(my_id)
my_followed_list=[my_id]
total_followed=0
for user in my_followed:
if 'id' in user:
my_followed_list.append(user['id'])
total_followed+=1
if 'list_seen' not in runparams:
runparams['list_seen'] = my_followed_list
if DEBUG:
print('I am currently already following %i persons' % total_followed)
toots = mastodon.timeline_public(since_id=runparams['since_id'],limit=40)
if DEBUG:
class MastodonEbooks:
def __init__(self, options={}):
self.api_base_url = options.get("api_base_url", "https://botsin.space")
self.app_name = options.get("app_name", "ebooks")
if path.exists("clientcred.secret") and path.exists("usercred.secret"):
self.client = Mastodon(client_id="clientcred.secret",
access_token="usercred.secret",
api_base_url=self.api_base_url)
if not path.exists("clientcred.secret"):
print("No clientcred.secret, registering application")
Mastodon.create_app(self.app_name,
api_base_url=self.api_base_url,
to_file="clientcred.secret")
if not path.exists("usercred.secret"):
print("No usercred.secret, registering application")
self.email = input("Email: ")
self.password = getpass("Password: "******"clientcred.secret",
api_base_url=self.api_base_url)
self.client.log_in(self.email,
self.password,
to_file="usercred.secret")
def setup(self):
me = self.client.account_verify_credentials()
following = self.client.account_following(me.id)
with open("corpus.txt", "w+", encoding="utf-8") as fp:
for f in following:
print("Downloading toots for user @{}".format(f.username))
for t in self._get_toots(f.id):
fp.write(t + "\n")
def gen_toot(self):
with open("corpus.txt", encoding="utf-8") as fp:
model = markovify.NewlineText(fp.read())
sentence = None
# you will make that damn sentence
while sentence is None or len(sentence) > 500:
sentence = model.make_sentence(tries=100000)
toot = sentence.replace("\0", "\n")
return toot
def post_toot(self, toot):
self.client.status_post(toot, spoiler_text="markov toot")
def _parse_toot(self, toot):
if toot.spoiler_text != "": return
if toot.reblog is not None: return
if toot.visibility not in ["public", "unlisted"]: return
soup = BeautifulSoup(toot.content, "html.parser")
# pull the mentions out
# for mention in soup.select("span.h-card"):
# mention.unwrap()
# for mention in soup.select("a.u-url.mention"):
# mention.unwrap()
# we will destroy the mentions until we're ready to use them
# someday turbocat, you will talk to your sibilings
for mention in soup.select("span.h-card"):
mention.decompose()
# make all linebreaks actual linebreaks
for lb in soup.select("br"):
lb.insert_after("\n")
lb.decompose()
# make each p element its own line because sometimes they decide not to be
for p in soup.select("p"):
p.insert_after("\n")
p.unwrap()
# keep hashtags in the toots
for ht in soup.select("a.hashtag"):
ht.unwrap()
# unwrap all links (i like the bots posting links)
for link in soup.select("a"):
link.insert_after(link["href"])
link.decompose()
text = map(lambda a: a.strip(), soup.get_text().strip().split("\n"))
# next up: store this and patch markovify to take it
# return {"text": text, "mentions": mentions, "links": links}
# it's 4am though so we're not doing that now, but i still want the parser updates
return "\0".join(list(text))
def _get_toots(self, id):
i = 0
toots = self.client.account_statuses(id)
while toots is not None:
for toot in toots:
t = self._parse_toot(toot)
if t != None:
yield t
toots = self.client.fetch_next(toots)
i += 1
if i % 10 == 0:
print(i)
def main():
colorama_init()
parser = ArgumentParser(description=__doc__,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('--min-activity',
dest='min_activity',
type=parse_time_ago,
default="1y",
help=("Remove followings inactive for a given period"
" (m for months, y for years, d for days) "
"(default: %(default)s)"))
parser.add_argument('--target-count',
dest='target_count',
type=int,
help=("Target some following count (will try to stop"
" when you have that many followings left)"))
parser.add_argument('--unfollow',
action='store_true',
help="Actually unfollow")
parser.add_argument('--followers',
action='store_true',
help="Instead of removing people you follow, remove "
"people who follow YOU")
parser.add_argument('--unmutuals',
action='store_true',
help="Remove people who follow you but that you "
"don't follow")
parser.add_argument('-v',
'--verbose',
action='store_true',
help="Display more things")
args = parser.parse_args()
session = requests.Session()
mastodon = Mastodon(access_token=settings.ACCESS_TOKEN,
api_base_url=settings.API_BASE,
ratelimit_method='pace')
current_user = mastodon.account_verify_credentials()
uid = current_user['id']
if args.unmutuals:
args.followers = True
if args.followers:
followings_count = current_user['followers_count']
else:
followings_count = current_user['following_count']
local_count = followings_count
goal_msg = ""
if args.target_count:
goal_msg = "(goal: n>={})".format(args.target_count)
now = datetime.now(tz=timezone.utc)
def clog(c, s):
tqdm.write(c + s + Fore.WHITE + Style.NORMAL)
cprint(Fore.GREEN,
"Current user: @{} (#{})".format(current_user['username'], uid))
cprint(
Fore.GREEN,
"{}: {} {}".format("Followers" if args.followers else "Followings",
followings_count, goal_msg))
if args.unfollow:
cprint(Fore.RED, "Action: unfollow")
else:
cprint(Fore.YELLOW, "Action: none")
followings = None
if args.followers:
followings = mastodon.account_followers(uid)
else:
followings = mastodon.account_following(uid)
followings = mastodon.fetch_remaining(followings)
bar = tqdm(list(followings))
for f in bar:
fid = f.get('id')
acct = f.get('acct')
fullhandle = "@{}".format(acct)
if '@' in acct:
inst = acct.split('@', 1)[1].lower()
else:
inst = None
if args.target_count is not None and local_count <= args.target_count:
clog(Fore.RED + Style.BRIGHT,
"{} followings left; stopping".format(local_count))
break
title_printed = False
def title():
nonlocal title_printed
if title_printed:
return
title_printed = True
clog(Fore.WHITE + Style.BRIGHT,
"Account: {} (#{})".format(f.get('acct'), fid))
if args.verbose:
title()
try:
bar.set_description(fullhandle.ljust(30, ' '))
act = False
if args.unmutuals and inst not in settings.SKIP_INSTANCES:
try:
relations = mastodon.account_relationships(fid)
is_mutual = relations[0]["following"] or relations[0][
"requested"]
if not is_mutual:
clog(Fore.YELLOW, "- Unmutual ({})".format(relations))
act = True
except (UserGone, Error, requests.RequestException) as e:
act = False
clog(Fore.YELLOW, "- Exception ({})".format(e))
elif args.min_activity and inst not in settings.SKIP_INSTANCES:
try:
last_toot = get_last_toot(mastodon, fid)
if last_toot < now - args.min_activity:
# force a cache miss to be Sure
last_toot = get_last_toot(mastodon, fid, force=True)
if last_toot < now - args.min_activity:
act = True
msg = "(!)"
title()
clog(Fore.WHITE,
"- Last toot: {} {}".format(last_toot, msg))
else:
msg = "(pass)"
if args.verbose:
clog(Fore.WHITE,
"- Last toot: {} {}".format(last_toot, msg))
except UserGone as e:
moved = f.get('moved')
if moved:
# TODO: follow new account and unfollow old
act = False
title()
clog(
Fore.YELLOW,
"- User moved ({}) [NOT IMPLEMENTED]".format(
moved))
else:
act = True
title()
clog(Fore.YELLOW, "- User gone ({})".format(e))
except (Error, requests.RequestException) as e:
if inst and inst in settings.ASSUME_DEAD_INSTANCES:
act = True
title()
clog(Fore.YELLOW, "- Instance gone ({})".format(e))
else:
raise
if act:
local_count -= 1
if args.unfollow:
if args.followers:
clog(Fore.GREEN + Style.BRIGHT,
"- Removing follower {}".format(fullhandle))
mastodon.account_block(fid)
mastodon.account_unblock(fid)
else:
clog(Fore.GREEN + Style.BRIGHT,
"- Unfollowing {}".format(fullhandle))
mastodon.account_unfollow(fid)
else:
clog(Fore.GREEN + Style.BRIGHT,
"- (not) unfollowing {}".format(fullhandle))
clog(Fore.WHITE, ("- {}/{} followings left".format(
local_count, followings_count)))
except Exception as e:
title()
clog(Fore.RED, "- Error: {}".format(str(e)))
def main(instance, config, profile):
configpath = os.path.expanduser(config)
if os.path.isfile(configpath) and not os.access(configpath, os.W_OK):
# warn the user before they're asked for input
cprint(
"Config file does not appear to be writable: {}".format(
configpath), fg('red'))
config = parse_config(configpath)
# make sure profile name is legal
profile = re.sub(r'\s+', '', profile) # disallow whitespace
profile = profile.lower() # force to lowercase
if profile == '' or profile in RESERVED:
cprint("Invalid profile name: {}".format(profile), fg('red'))
sys.exit(1)
if not config.has_section(profile):
config.add_section(profile)
instance, client_id, client_secret, token = \
get_or_input_profile(config, profile, instance)
if not token:
cprint("Could not log you in. Please try again later.", fg('red'))
sys.exit(1)
mastodon = Mastodon(client_id=client_id,
client_secret=client_secret,
access_token=token,
api_base_url="https://" + instance)
# update config before writing
if "token" not in config[profile]:
config[profile] = {
'instance': instance,
'client_id': client_id,
'client_secret': client_secret,
'token': token
}
save_config(configpath, config)
say_error = lambda a, b: cprint(
"Invalid command. Use 'help' for a list of commands.",
fg('white') + bg('red'))
print("You are connected to ", end="")
cprint(instance, fg('green') + attr('bold'))
print("Enter a command. Use 'help' for a list of commands.")
print("\n")
user = mastodon.account_verify_credentials()
prompt = "[@{} ({})]: ".format(str(user['username']), profile)
# Completion setup stuff
for i in mastodon.account_following(user['id'], limit=80):
bisect.insort(completion_list, '@' + i['acct'])
readline.set_completer(complete)
readline.parse_and_bind("tab: complete")
readline.set_completer_delims(' ')
while True:
command = input(prompt).split(' ', 1)
rest = ""
try:
rest = command[1]
except IndexError:
pass
command = command[0]
cmd_func = commands.get(command, say_error)
cmd_func(mastodon, rest)
bot_client_secret_fn = os.path.join('data', 'instances', domain,
'bot_client.secret')
user_secret_file = os.path.join(
'data', 'accounts', common.md5('{0},{1}'.format(domain, username)),
'user.secret')
api_base_url = 'https://{0}'.format(domain)
mastodon = Mastodon(client_id=bot_client_secret_fn,
access_token=user_secret_file,
api_base_url=api_base_url)
user_id = data.get_id(domain)
follower_list = []
max_id = None
while (True):
_follower_list = mastodon.account_following(user_id, max_id=max_id)
if len(_follower_list) <= 0:
break
follower_list += _follower_list
if '_pagination_next' not in _follower_list[-1]:
break
max_id = _follower_list[-1]['_pagination_next']
follower_list = [i['acct'] for i in follower_list]
follower_list = [
'@{0}'.format(i) if '@' in i else '@{0}@{1}'.format(i, domain)
for i in follower_list
]
all_follower_list = config['instance_data_list']
all_follower_list = [i['domain'] for i in all_follower_list]
all_follower_list = [data.get_username(i) for i in all_follower_list]
