def to_dataframe(user:str = 'secrets/topstbot.user.secret',
url:str = 'https://botsin.space',
since: str = '2020-03-01T00:00:00.000Z',
tagged:List[str] = ['de', 'day']) -> pd.DataFrame:
"""
Fetch all posts that are tagged with [tagged].
"""
mastodon = Mastodon(access_token=user, api_base_url=url)
my_id = mastodon.me()['id']
since_date = pd.to_datetime(since)
statuses = mastodon.account_statuses(my_id)
while pd.to_datetime(statuses[-1]['created_at']) > since_date:
statuses += mastodon.account_statuses(my_id, max_id=statuses[-1]['id'])
headers = ['created_at', 'content', 'url']
columns = {h: [] for h in headers}
for status in statuses:
if len(set([s.name for s in status['tags']]).intersection(set(tagged)))!=len(tagged):
continue
columns[headers[0]].append(status[headers[0]])
columns[headers[1]].append(status[headers[1]])
columns[headers[2]].append(status[headers[2]])
result = pd.DataFrame(columns, columns=headers)
result[headers[0]] = pd.to_datetime(result[headers[0]])
return result.set_index(headers[0])
def mining(id, return_type="list", switch=None):
print(return_type + " is selected!")
Mastodon = login(switch)
#timelineからlastestなmax_idを取得
tl = Mastodon.timeline_local(limit=1)
initial_max_id = tl[0]['id']
toot = Mastodon.account_statuses(id, initial_max_id, None, 40)
while True:
last_max_id = toot[len(toot) - 1]['id']
#続きのtootを取得
last_toot = Mastodon.account_statuses(id, last_max_id, None, 40)
toot.extend(last_toot)
# final_max_lenge = len(toot)-1
final_max_lenge = len(last_toot) - 1
# account = Mastodon.account(id)
# count = account['statuses_count']
toot_count = toot[0]['account']['statuses_count']
print(str(len(toot)) + '/' + str(toot_count))
if final_max_lenge < 39:
break
if return_type == "json":
filename = str(id)
jsoner(toot, filename)
else:
return toot
class Gensokyo:
def __init__(self):
self.token = Mastodon(client_id="clientcred.secret",
access_token="usercred.secret",
api_base_url="https://gensokyo.town")
def get_account_toot(self):
"""
アカウントの過去のtootを全て取得し、絵文字と使用回数のリストを作る
"""
toots = []
extend_toots = toots.extend
temp_toots = self.token.account_statuses(id=113, limit=40)
last_year = timezone("Asia/Tokyo").localize(
dt.datetime((dt.date.today() - relativedelta(years=1)).year, 1, 2,
0, 0, 0, 0))
while temp_toots[0]["created_at"].astimezone(
timezone("Asia/Tokyo")) > last_year:
for toot in temp_toots:
if "トレンド" not in toot[
"content"] and toot["created_at"].astimezone(
timezone("Asia/Tokyo")) > last_year:
#toot = {content: "<p>内容</p>"}
print(toot["content"][3:10])
extend_toots(toot["content"].split("<br />"))
time.sleep(3)
temp_toots = self.token.account_statuses(
id=113, max_id=temp_toots[-1]["id"] - 1, limit=40)
return (toots)
def post_rank(self, emoji_rank):
temp_rank = 0
temp_num = 0
toot = "{}年に使用された絵文字の使用回数ランキングです。\n".format(dt.date.today().year)
#ランキング作る
for i, emoji in emoji_rank.iterrows():
if emoji["num"] == temp_num:
temp = "{}位: {} ({}回)\n".format(temp_rank, emoji["emoji"],
emoji["num"])
else:
temp = "{}位: {} ({}回)\n".format(i + 1, emoji["emoji"],
emoji["num"])
temp_num = emoji["num"]
temp_rank = i + 1
if len(toot) + len(temp) >= 500:
self.token.status_post(status=toot, visibility="unlisted")
toot = ""
time.sleep(1)
toot += temp
self.token.status_post(status=toot, visibility="unlisted")
def clean(config: CleanTootsConfig, delete: bool, headless: bool):
"""
Delete Toots based on rules in config file.
Without the `--delete` flag, toots will only be displayed.
"""
if not _config_has_sections(config):
return
h = html2text.HTML2Text()
h.ignore_links = True
h.ignore_emphasis = True
h.ignore_images = True
h.ignore_tables = True
for section in config.sections():
section = config[section]
user_secret_file = config.file(section.get("user_secret_file"))
mastodon = Mastodon(access_token=user_secret_file)
user = mastodon.me()
page = mastodon.account_statuses(user["id"])
would_delete = []
protected = []
while page:
for toot in page:
protection_reason = _toot_protection_reason(toot, section)
if protection_reason:
protected.append({"toot": toot, "reason": protection_reason})
else:
would_delete.append(toot)
page = mastodon.fetch_next(page)
_delete_or_log(delete, h, headless, mastodon, protected, would_delete)
def toot_on_mastodon(configs: dict, post_text: str, image_filenames: [str],
reply_to_latest_post: bool = False):
api = Mastodon(configs['mastodon']['client_id'],
configs['mastodon']['client_secret'],
configs['mastodon']['access_token'],
configs['mastodon']['instance_url'])
post_media = []
for filename in image_filenames:
with open(filename, 'rb') as f:
post_media.append(api.media_post(f.read(), 'image/png'))
latest_status_id = None
success_message = "tooted on Mastodon"
if reply_to_latest_post:
account_id = api.account_verify_credentials()['id']
latest_status = api.account_statuses(account_id, limit=1)[0]
latest_status_id = latest_status['id']
success_message = ("replied to %s" %
shorten_text(latest_status['content']))
api.status_post(post_text, in_reply_to_id=latest_status_id,
media_ids=post_media)
print(success_message)
class Main(object):
'''Main class'''
def __init__(self):
'''Constructor of the Main class'''
# parse the command line
rtargs = CliParse()
self.args = rtargs.arguments
# read the configuration file
cfgparse = ConfParse(self.args.pathtoconf)
self.cfgvalues = cfgparse.confvalues
self.twp = TootWasPosted(self.cfgvalues)
# activate the mastodon api
self.api = Mastodon(client_id=self.cfgvalues['clientcred'],
access_token=self.cfgvalues['usercred'],
api_base_url=self.cfgvalues['instanceurl'])
self.main()
def main(self):
'''Main of the Main class'''
for user in self.cfgvalues['userstoboost']:
lasttoots = self.api.account_statuses(
self.api.account_search(user, limit=1)[0]['id'])
lasttoots.reverse()
if self.args.limit:
lasttoots = lasttoots[(len(lasttoots) - self.args.limit):]
tootstosend = []
# test if the last 20 toots were posted
for lasttoot in lasttoots:
if not self.twp.wasposted(lasttoot['id']):
Validate(self.cfgvalues, self.args, self.api, lasttoot)
sys.exit(0)
class MastoCrosspostUtils:
def __init__(self, clientcred_key, access_token_key, instance_url):
self.mastodon_api = Mastodon(
client_id=clientcred_key,
access_token=access_token_key,
api_base_url=instance_url
)
self.me = self.mastodon_api.account_verify_credentials()
def scrape_toots(self, mstdn_acct_id, since=None):
""" Get toots from an account since given toot id and filter them
"""
toots = self.mastodon_api.account_statuses(
mstdn_acct_id, since_id=since, exclude_replies=True)
filtered_toots = []
if len(toots):
filtered_toots = list(filter(lambda x:
x['reblog'] is None and
x['poll'] is None and
x['visibility'] in [
"public", "unlisted", "private"],
toots[::-1]))
return filtered_toots
def get_following(self):
return self.mastodon_api.account_following(self.me.id)
def get_statuses(*,
client=None,
account=None,
username=None,
count=None,
limit=40):
"""
Get the statuses for an account, by account dict or username
Args:
client (mastodon.Mastodon, optional): Mastodon.py client to use.
If not provided, will initialize a new, non-authenticated client.
Strongly recommended *against* passing, as non-authed clients have higher rate limits.
account (dict, optional): Account dict. Precisely one of this or `username` is required.
username (str, optional): Username. Precisely one of this or `account` is required.
count (int, optional): Number of statuses to fetch. If omitted, retrieves all.
limit (int, optional): Number of statuses to fetch per request. Defaults to 40.
This is usually limited to 40 serverside.
Returns:
list: Statuses retrieved
"""
assert (account is None) != (username is None), \
f'''
get_statuses requires precisely one of `account`, `username`
received `account`={account}, `username`={username}
'''
if client is None:
client = Mastodon(
api_base_url=config('MASTODON_API'),
ratelimit_method='throw',
)
if account is None:
account = get_account(client, username)
BOT_LOG.info(f'Getting statuses for account id {account["id"]}...')
full_statuses = []
max_id = None
while True:
BOT_LOG.info(
f'Getting statuses for account id {account["id"]} with max_id={max_id}. Current count: {len(full_statuses)}'
)
statuses = client.account_statuses(id=account['id'],
max_id=max_id,
limit=limit)
if len(statuses) > 0:
full_statuses += statuses
max_id = statuses[-1]['id']
else:
break
if count is not None and count <= len(full_statuses):
break
BOT_LOG.info(f'Got {len(full_statuses)} statuses.')
if count is None:
return (full_statuses)
return (full_statuses[:count])
def yield_statuses(mastodon: Mastodon,
account_id: int,
*,
since_id: int,
limit: int = 200) -> Generator[Status, None, None]:
statuses = mastodon.account_statuses(account_id,
limit=MAX_TOOTS,
since_id=since_id)
yield from statuses
while statuses:
statuses = mastodon.fetch_next(statuses)
if statuses:
yield from statuses
def getToots(id, lim, max, vis=["public"]):
text = ""
mstdn = Mastodon(client_id=session['client_id'],
client_secret=session['client_secret'],
access_token=session['access_token'],
api_base_url=session['uri'])
ltl = mstdn.account_statuses(id, limit=lim, max_id=max)
for row in ltl:
if row["reblog"] == None:
if row["visibility"] in vis:
text += reform(row["content"]) + "\n"
toot_id = row["id"]
return (text, toot_id)
def main():
''' メインルーチン '''
# Mastodon初期化
mastodon = Mastodon(client_id=CID_FILE,
access_token=TOKEN_FILE,
api_base_url=URL)
# 自分の最新トゥート1件を取得する
user_dict = mastodon.account_verify_credentials()
user_toots = mastodon.account_statuses(user_dict['id'], limit=1)
# トゥートのアプリ名があれば表示する
if user_toots[0]['reblog'] is None:
print(check_appname(user_toots[0])) # 通常のトゥートの場合
else:
print(check_appname(user_toots[0]['reblog'])) # ブーストされたトゥートの場合
def recent_artworks(count=7) -> list[tuple[str, str]]:
"""
Return a list of (post_url, thumbnail_url) tuples for recent public media
posts on donphan, with the most popular (most faves) closer to the middle
"""
CACHE_KEY = "www.codl.fr:4:artworks:{}".format(count)
r = get_redis()
cached = r.get(CACHE_KEY)
if not cached:
access_token = app.config.get("DONPHAN_ACCESS_TOKEN")
if not access_token:
raise NoMastodonAccess()
session = requests.Session()
session.headers.update({"user-agent": "www.codl.fr"})
m = Mastodon(
access_token=access_token,
api_base_url="https://donphan.social/",
session=session,
)
me = m.me()
statuses = m.account_statuses(me["id"],
only_media=True,
exclude_replies=True,
limit=40)
artworks = list()
for status in filter(
lambda a: not a["sensitive"] and a["visibility"] == "public",
sorted(statuses,
key=lambda a: a["favourites_count"],
reverse=True),
):
artwork = (status["url"],
status["media_attachments"][0]["preview_url"])
artworks.append(artwork)
artworks = list(reversed(artworks))
if len(artworks) > count:
break
r.set(CACHE_KEY, pickle.dumps(artworks), ex=3600)
else:
artworks = pickle.loads(cached)
return artworks
def main():
''' メインルーチン '''
# Mastodon初期化
mastodon = Mastodon(client_id=CID_FILE,
access_token=TOKEN_FILE,
api_base_url=URL)
# 対象アカウントのユーザーIDを取得する。
user_list = mastodon.account_search(USERNAME, limit=1)
user_id = user_list[0]['id']
# 対象アカウントの最新トゥート10件を取得する
user_toots = mastodon.account_statuses(user_id, limit=1)
# トゥートのアプリ名があれば表示する
if user_toots[0]['reblog'] is None:
print(check_appname(user_toots[0])) # 通常のトゥートの場合
else:
print(check_appname(user_toots[0]['reblog'])) # ブーストされたトゥートの場合
def POST(msg):
if (len(msg) > 500):
msg = msg[:500]
th.status_post(msg, visibility='public')
pp = pprint.PrettyPrinter(indent=4)
# Set up Mastodon
token = open('token.secret', 'r').read().strip('\n')
print(token)
th = Mastodon(access_token=token, api_base_url='https://' + DOMAIN)
last = th.account_statuses(th.me().id, limit=1)
#pp.pprint(last)
start = last[0].reblog.id
print(start)
while True:
print('conn')
r = th.timeline(timeline='local', min_id=start, limit=40)
print(len(r))
if (len(r) == 0):
break
start = r[0].id
ids = [st.id for st in r if st.favourites_count > THRESHOLD]
#sts = [st for st in r if st.favourites_count > THRESHOLD ]
consumer_secret=c.TWITTER_CONSUMER_SECRET,
access_token_key=bridge.twitter_oauth_token,
access_token_secret=bridge.twitter_oauth_secret,
tweet_mode='extended' # Allow tweets longer than 140 raw characters
)
#
# Fetch from Mastodon
#
new_toots: List[Any] = []
l.debug(f"-- {bridge.id}: {bridge.mastodon_user}@{mastodonhost.hostname} --")
try:
new_toots = mast_api.account_statuses(
bridge.mastodon_account_id,
since_id=bridge.mastodon_last_id
)
except MastodonAPIError as e:
l.error(e)
if any(x in repr(e) for x in ['revoked', 'invalid', 'not found', 'Forbidden', 'Unauthorized']):
l.warning(f"Disabling bridge for user {bridge.mastodon_user}@{mastodonhost.hostname}")
bridge.enabled = False
continue
except MastodonNetworkError as e:
l.error(f"Error with user {bridge.mastodon_user}@{mastodonhost.hostname}: {e}")
mastodonhost.defer()
session.commit()
'bot_client.secret')
user_secret_file = os.path.join(
'data', 'accounts', common.md5('{0},{1}'.format(domain, username)),
'user.secret')
api_base_url = 'https://{0}'.format(domain)
mastodon = Mastodon(client_id=bot_client_secret_fn,
access_token=user_secret_file,
api_base_url=api_base_url)
user_id = data.get_id(domain)
status_list = []
max_id = None
while (True):
_status_list = mastodon.account_statuses(id=user_id, max_id=max_id)
# print('TSKYQWIY len(_status_list)={0}'.format(len(_status_list)))
if len(_status_list) <= 0:
break
status_list += _status_list
if '_pagination_next' not in _status_list[-1]:
break
max_id = _status_list[-1]['_pagination_next']['max_id']
# print('MLKDQIUA len(status_list)={0}'.format(len(status_list)))
status_list = filter(
lambda i: i['created_at'].timestamp() < timestamp - config[
'remove_toot_timeout'], status_list)
status_list = list(status_list)
for status in status_list:
# print('PVHSPADZ delete {0}'.format(status['id']))
html = response.read()
data = json.loads(html)
index += 15
if (len(data['data']) < 15):
cont = False
for account in data['data']:
if (account['host'] == host):
if account['name'] not in result['followed']:
dicti = mastodon.follows(account['name'] + "@" + host)
result['followed'][account['name']] = dicti.id
# Retoot
if (rt):
for account in result['followed']:
#print(result['followed'][account])
for status in reversed(
mastodon.account_statuses(result['followed'][account])):
if ("comment" not in status.uri
and status.id not in result['retoot']):
print "rebloging " + status.content
rb = mastodon.status_reblog(status.id)
result['retoot'].append(status.id)
if (unrt):
mastodon.status_unreblog(rb.reblog.id)
file = open("peertube_to_masto.json", "w")
file.write(json.dumps(result))
file.close()
pprint(result)
class MastodonPublisher:
"""
Ease the publishing of content to Mastodon
"""
MAX_LEN_TOOT = 500
def __init__(self,
config: Configuration,
secrets_file: str = 'mastodon.secret') -> None:
self.logger = config.bot.logger
self.media_only = config.media.media_only
self.nsfw_marked = config.reddit.nsfw_marked
self.mastodon_config = config.mastodon_config
self.post_recorder = config.bot.post_recorder
self.num_non_promo_posts = 0
self.promo = config.promo
api_base_url = 'https://' + self.mastodon_config.domain
# Log into Mastodon if enabled in settings
if not os.path.exists(secrets_file):
# If the secret file doesn't exist,
# it means the setup process hasn't happened yet
self.logger.warning(
'Mastodon API keys not found. (See wiki for help).')
user_name = input(
"[ .. ] Enter email address for Mastodon account: ")
password = input("[ .. ] Enter password for Mastodon account: ")
config.bot.logger.info('Generating login key for Mastodon...')
try:
Mastodon.create_app(
'Tootbot',
website='https://gitlab.com/marvin8/tootbot',
api_base_url=api_base_url,
to_file=secrets_file)
self.mastodon = Mastodon(client_id=secrets_file,
api_base_url='https://' +
self.mastodon_config.domain)
self.mastodon.log_in(user_name, password, to_file=secrets_file)
# Make sure authentication is working
self.userinfo = self.mastodon.account_verify_credentials()
mastodon_username = self.userinfo['username']
config.bot.logger.info(
'Successfully authenticated on %s as @%s',
self.mastodon_config.domain, mastodon_username)
config.bot.logger.info(
'Mastodon login information now stored in %s file',
secrets_file)
except MastodonError as mastodon_error:
config.bot.logger.error(
'Error while logging into Mastodon: %s', mastodon_error)
config.bot.logger.error(
'Tootbot cannot continue, now shutting down')
sys.exit(1)
else:
try:
self.mastodon = Mastodon(access_token=secrets_file,
api_base_url=api_base_url)
# Make sure authentication is working
self.userinfo = self.mastodon.account_verify_credentials()
mastodon_username = self.userinfo['username']
config.bot.logger.info(
'Successfully authenticated on %s as @%s',
self.mastodon_config.domain, mastodon_username)
except MastodonError as mastodon_error:
config.bot.logger.error(
'Error while logging into Mastodon: %s', mastodon_error)
config.bot.logger.error(
'Tootbot cannot continue, now shutting down')
sys.exit(1)
def make_post(self, posts: dict, reddit_helper: RedditHelper,
media_helper: LinkedMediaHelper) -> None:
"""
Makes a post on mastodon from a selection of reddit submissions.
Arguments:
posts: A dictionary of subreddit specific hash tags and PRAW Submission objects
reddit_helper: Helper class to work with Reddit
media_helper: Helper class to retrieve media linked to from a reddit Submission.
"""
break_to_mainloop = False
for additional_hashtags, source_posts in posts.items():
if break_to_mainloop:
break
for post in source_posts:
# Grab post details from dictionary
post_id = source_posts[post].id
shared_url = source_posts[post].url
if not (self.post_recorder.duplicate_check(post_id)
or self.post_recorder.duplicate_check(shared_url)):
self.logger.debug('Processing reddit post: %s',
source_posts[post])
attachments = MediaAttachment(source_posts[post],
media_helper, self.logger)
number_attachments = len(attachments.media_paths)
self._remove_posted_earlier(attachments)
if number_attachments > 0 and len(
attachments.media_paths) == 0:
self.logger.info(
'Skipping %s because all attachments have already been posted',
post_id)
self.post_recorder.log_post(
post_id,
'Mastodon: Skipped because all images have already been posted',
'', '')
continue
self.logger.debug('Media posts only: %s', self.media_only)
# Make sure the post contains media,
# if MEDIA_POSTS_ONLY in config is set to True
if (self.media_only and len(attachments.media_paths) > 0) or \
(not self.media_only):
self.logger.debug('Going to post Toot.')
try:
promo_message = None
if self.num_non_promo_posts >= self.promo.every > 0:
promo_message = self.promo.message
self.num_non_promo_posts = -1
# Generate post caption
caption = reddit_helper.get_caption(
source_posts[post],
MastodonPublisher.MAX_LEN_TOOT,
add_hash_tags=additional_hashtags,
promo_message=promo_message)
# Upload media files if available
media_ids = None
if len(attachments.media_paths) > 0:
self.logger.info(
'Posting to Mastodon with media(s): %s',
caption)
media_ids = self._post_attachments(
attachments, post_id)
else:
self.logger.info(
'Posting to Mastodon without media: %s',
caption)
spoiler = None
if source_posts[post].over_18 and self.nsfw_marked:
spoiler = 'NSFW'
toot = self.mastodon.status_post(
status=caption,
media_ids=media_ids,
sensitive=self.mastodon_config.
media_always_sensitive,
spoiler_text=spoiler)
# Log the toot
self.post_recorder.log_post(
post_id, toot["url"], shared_url, '')
self.num_non_promo_posts += 1
self.mastodon_config.number_of_errors = 0
except MastodonError as mastodon_error:
self.logger.error('Error while posting toot: %s',
mastodon_error)
# Log the post anyways so we don't get into a loop of the same error
self.post_recorder.log_post(
post_id, 'Error while posting toot: %s' %
mastodon_error, '', '')
self.mastodon_config.number_of_errors += 1
else:
self.logger.warning(
'Skipping %s, non-media posts disabled or media file not found',
post_id)
# Log the post anyways
self.post_recorder.log_post(
post_id,
'Skipping, non-media posts disabled or media file not found',
'', '')
# Clean up media file
attachments.destroy()
# Return control to main loop
break_to_mainloop = True
break
self.logger.info('Skipping %s because it was already posted',
post_id)
def _post_attachments(self, attachments: MediaAttachment,
post_id: str) -> List[dict]:
"""
_post_attachments post any media in attachments.media_paths list
Arguments:
attachments: object with a list of paths to media to be posted on Mastodon
Returns:
media_ids: List of dicts returned by mastodon.media_post
"""
media_ids = []
for checksum, media_path in attachments.media_paths.items():
self.logger.info('Media %s with checksum: %s', media_path,
checksum)
media = self.mastodon.media_post(media_path)
# Log the media upload
self.post_recorder.log_post(post_id, '', media_path, checksum)
media_ids.append(media)
return media_ids
def _remove_posted_earlier(self, attachments: MediaAttachment) -> None:
"""
_remove_posted_earlier checks che checksum of all proposed attachments and removes any from
the list that have already been posted earlier.
Arguments:
attachments: object with list of paths to media files proposed to be posted on Mastodon
"""
# Build a list of checksums for files that have already been posted earlier
checksums = []
for checksum in attachments.media_paths:
self.logger.debug('Media attachment (path, checksum): %s, %s',
attachments.media_paths[checksum], checksum)
if attachments.media_paths[checksum] is None:
checksums.append(checksum)
# Check for duplicate of attachment sha256
elif self.post_recorder.duplicate_check(checksum):
self.logger.info(
'Media with checksum %s has already been posted', checksum)
checksums.append(checksum)
# Remove all empty or previously posted images
for checksum in checksums:
attachments.destroy_one_attachment(checksum)
def delete_toots(self, older_than_days: int) -> None:
"""
Deletes old toots that are older than "older_than_days" days old in batches of up to
whatever the limit is set to in the account_statuses call to mastodon. This limit should be
kept low enough to not trigger rate limiting by the mastodon server.
For example with the limit set to 10, this method will delete up to 10 old toots and then
return.
Arguments:
older_than_days (int): This value is used to determine the most recent toot that will
be considered for deletion.
"""
try:
toots = self.mastodon.account_statuses(self.userinfo['id'],
limit=10)
now = arrow.get(arrow.now().format('YYYY-MM-DD HH:mm:ss'),
'YYYY-MM-DD HH:mm:ss')
oldest_to_keep = now.shift(days=-older_than_days)
# List of toots is paginated. This while loop finds the first "page" of toots that
# contains toots old enough to need deleting
while True:
if len(toots) == 0:
break
last_toot_created_at = arrow.get(toots[-1]['created_at'])
if last_toot_created_at < oldest_to_keep:
break
max_id = toots[-1]['id']
self.logger.debug(
'Last toot in list %s from %s is not older than %s',
max_id, last_toot_created_at, oldest_to_keep)
toots = self.mastodon.account_statuses(self.userinfo['id'],
max_id=max_id,
limit=10)
# Actually deleting toots that are older than "older_than_days"
for toot in toots:
created_at = arrow.get(toot['created_at'])
if created_at < oldest_to_keep:
self.logger.info('Deleting toot %s from %s', toot['url'],
toot['created_at'])
self.mastodon.status_delete(toot['id'])
except MastodonError as mastodon_error:
self.logger.error('Encountered error while deleting_toots: %s ',
mastodon_error)
class MastodonEbooks:
def __init__(self, options={}):
self.api_base_url = options.get("api_base_url", "https://botsin.space")
self.app_name = options.get("app_name", "ebooks")
if path.exists("clientcred.secret") and path.exists("usercred.secret"):
self.client = Mastodon(client_id="clientcred.secret",
access_token="usercred.secret",
api_base_url=self.api_base_url)
if not path.exists("clientcred.secret"):
print("No clientcred.secret, registering application")
Mastodon.create_app(self.app_name,
api_base_url=self.api_base_url,
to_file="clientcred.secret")
if not path.exists("usercred.secret"):
print("No usercred.secret, registering application")
self.email = input("Email: ")
self.password = getpass("Password: "******"clientcred.secret",
api_base_url=self.api_base_url)
self.client.log_in(self.email,
self.password,
to_file="usercred.secret")
def setup(self):
me = self.client.account_verify_credentials()
following = self.client.account_following(me.id)
with open("corpus.txt", "w+", encoding="utf-8") as fp:
for f in following:
print("Downloading toots for user @{}".format(f.username))
for t in self._get_toots(f.id):
fp.write(t + "\n")
def gen_toot(self):
with open("corpus.txt", encoding="utf-8") as fp:
model = markovify.NewlineText(fp.read())
sentence = None
# you will make that damn sentence
while sentence is None or len(sentence) > 500:
sentence = model.make_sentence(tries=100000)
toot = sentence.replace("\0", "\n")
return toot
def post_toot(self, toot):
self.client.status_post(toot, spoiler_text="markov toot")
def _parse_toot(self, toot):
if toot.spoiler_text != "": return
if toot.reblog is not None: return
if toot.visibility not in ["public", "unlisted"]: return
soup = BeautifulSoup(toot.content, "html.parser")
# pull the mentions out
# for mention in soup.select("span.h-card"):
# mention.unwrap()
# for mention in soup.select("a.u-url.mention"):
# mention.unwrap()
# we will destroy the mentions until we're ready to use them
# someday turbocat, you will talk to your sibilings
for mention in soup.select("span.h-card"):
mention.decompose()
# make all linebreaks actual linebreaks
for lb in soup.select("br"):
lb.insert_after("\n")
lb.decompose()
# make each p element its own line because sometimes they decide not to be
for p in soup.select("p"):
p.insert_after("\n")
p.unwrap()
# keep hashtags in the toots
for ht in soup.select("a.hashtag"):
ht.unwrap()
# unwrap all links (i like the bots posting links)
for link in soup.select("a"):
link.insert_after(link["href"])
link.decompose()
text = map(lambda a: a.strip(), soup.get_text().strip().split("\n"))
# next up: store this and patch markovify to take it
# return {"text": text, "mentions": mentions, "links": links}
# it's 4am though so we're not doing that now, but i still want the parser updates
return "\0".join(list(text))
def _get_toots(self, id):
i = 0
toots = self.client.account_statuses(id)
while toots is not None:
for toot in toots:
t = self._parse_toot(toot)
if t != None:
yield t
toots = self.client.fetch_next(toots)
i += 1
if i % 10 == 0:
print(i)
from mastodon import Mastodon
url = sys.argv[1]
cid_file = 'client_id.txt'
token_file = 'access_token.txt'
username = '******'
mastodon = Mastodon(client_id=cid_file,
access_token=token_file,
api_base_url=url)
# 対象アカウントのユーザーIDを取得する。
user_list = mastodon.account_search(username, limit=1)
user_id = user_list[0]['id']
# 対象アカウントの最新トゥート10件を取得する
user_toots = mastodon.account_statuses(user_id, limit=10)
# トゥートを全て表示する
print(user_toots)
from mastodon import Mastodon
from datetime import datetime
from dateutil.tz import tzutc
import sys
import pickle
mastodon = Mastodon(access_token='user.secret',
api_base_url='https://machteburch.social')
stored = pickle.load(open('users.pickle', 'rb'))
print("Getting new toots... ", end='')
toots = []
for user, lsid in stored.items():
for rawtoot in reversed(mastodon.account_statuses(user, since_id=lsid)):
stored[user] = int(rawtoot['id'])
if rawtoot['reblog'] == None:
for tag in rawtoot['tags']:
if tag['name'] == 'mastoadmin': toots.append(rawtoot)
toots = sorted(toots, key=lambda toot: toot['created_at'].timestamp())
print('Found: ' + str(len(toots)))
print('Retooting... ', end='')
for toot in toots:
if len(sys.argv) > 1 and sys.argv[1] == 'test':
print(str(toot['id']) + ' by ' + toot['account']['username'])
else:
print(str(toot['id']) + ' by ' + toot['account']['username'])
class Twoot:
def __app_questions(self):
# defaults
d_name = 'twoot.py'
d_url = 'https://github.com/wtsnjp/twoot.py'
# ask questions
print('\n#1 First, decide about your application.')
name = input('Name (optional; empty for "{}"): '.format(d_name))
url = input('Website (optional; empty for "{}"): '.format(d_url))
# set config
if len(name) < 1:
name = d_name
if len(url) < 1:
url = d_url
return name, url
def __mastodon_questions(self, app_name, app_url):
# ask questions
print('\n#2 Tell me about your Mastodon account.')
inst = input('Instance (e.g., https://mastodon.social): ').rstrip('/')
mail = input('Login e-mail (never stored): ')
pw = getpass(prompt='Login password (never stored): ')
# register application
cl_id, cl_sc = Mastodon.create_app(app_name,
website=app_url,
api_base_url=inst)
# application certification & login
mastodon = Mastodon(client_id=cl_id,
client_secret=cl_sc,
api_base_url=inst)
access_token = mastodon.log_in(mail, pw)
# set config
self.config['mastodon'] = {
'instance': inst,
'access_token': access_token
}
return mastodon
def __twitter_questions(self):
# ask questions
print('\n#3 Tell me about your Twitter account.')
print(
'cf. You can get keys & tokens from https://developer.twitter.com/'
)
cs_key = input('API key: ')
cs_secret = input('API secret key: ')
ac_tok = input('Access token: ')
ac_sec = input('Access token secret: ')
# OAuth
auth = Twitter.OAuth(ac_tok, ac_sec, cs_key, cs_secret)
twitter = Twitter.Twitter(auth=auth)
# set config
self.config['twitter'] = {
'consumer_key': cs_key,
'consumer_secret': cs_secret,
'access_token': ac_tok,
'access_token_secret': ac_sec
}
return twitter
def __init__(self, profile='default', setup=False):
# files
twoot_dir = os.path.expanduser('~/.' + PROG_NAME)
if not os.path.isdir(twoot_dir):
os.mkdir(twoot_dir)
logger.debug('Selected profile: ' + profile)
self.config_file = twoot_dir + '/{}.json'.format(profile)
self.data_file = twoot_dir + '/{}.pickle'.format(profile)
# config
if setup or not os.path.isfile(self.config_file):
# setup mode
logger.debug('Selected mode: setup')
self.setup = True
# initialize
self.config = {'max_twoots': 1000}
# ask for config entries
print('Welcome to Twoot! Please answer a few questions.')
app_name, app_url = self.__app_questions()
self.mastodon = self.__mastodon_questions(app_name, app_url)
self.twitter = self.__twitter_questions()
print('\nAll configuration done. Thanks!')
# save config
logger.debug('Saving current config to {}'.format(
self.config_file))
with open(self.config_file, 'w') as f:
json.dump(self.config, f, indent=4, sort_keys=True)
else:
# normal mode
logger.debug('Selected mode: normal')
self.setup = False
# load config
logger.debug('Loading config file {}'.format(self.config_file))
with open(self.config_file) as f:
self.config = json.loads(f.read())
# setup Mastodon
ms = self.config['mastodon']
# Note: for HTTP debugging, set debug_requests=True
self.mastodon = Mastodon(access_token=ms['access_token'],
api_base_url=ms['instance'])
# setup Twitter
tw = self.config['twitter']
t_auth = Twitter.OAuth(tw['access_token'],
tw['access_token_secret'],
tw['consumer_key'], tw['consumer_secret'])
self.twitter = Twitter.Twitter(auth=t_auth)
self.twitter_upload = Twitter.Twitter(domain='upload.twitter.com',
auth=t_auth)
# data
self.twoots = []
if os.path.isfile(self.data_file):
logger.debug('Loading data file {}'.format(self.data_file))
with open(self.data_file, 'rb') as f:
self.data = pickle.load(f)
else:
logger.debug('No data file found; initialzing')
self.data = {'twoots': []}
# fetch self account information
if not self.data.get('mastodon_account', False):
ms_avc = self.mastodon.account_verify_credentials
try:
logger.debug(
'Fetching Mastodon account information (verify credentials)'
)
self.data['mastodon_account'] = ms_avc()
except Exception as e:
logger.exception(
'Failed to verify credentials for Mastodon: {}'.format(e))
logger.critical('Unable to continue; abort!')
raise
if not self.data.get('twitter_account', False):
tw_avc = self.twitter.account.verify_credentials
try:
logger.debug(
'Fetching Twitter account information (verify credentials)'
)
self.data['twitter_account'] = tw_avc()
except Exception as e:
logger.exception(
'Failed to verify credentials for Twitter: {}'.format(e))
logger.critical('Unable to continue; abort!')
raise
# save data anyway
with open(self.data_file, 'wb') as f:
pickle.dump(self.data, f)
# utility
self.html2text = html2text.HTML2Text()
self.html2text.body_width = 0
def __update_last_id(self, key, value):
"""Update the last id (last_toot or last_tweet) in the data file."""
# load the latest data
if os.path.isfile(self.data_file):
with open(self.data_file, 'rb') as f:
data = pickle.load(f)
else:
data = {'twoots': []}
# update the target
data[key] = value
with open(self.data_file, 'wb') as f:
pickle.dump(data, f)
def get_new_toots(self, dry_run=False, update=False):
"""Get new toots of the author.
Using account_statuses API, get the author's new toots, i.e., the toots
from the owner's account since the last toot id, and return the list of
toot dicts. If the last toot id cannot be found in the data, the id of
latest toot is recoreded and return an empty list.
Returns:
list: toot dicts
"""
res = []
# fetch necessary information
my_id = self.data['mastodon_account']['id']
last_id = self.data.get('last_toot', False)
# try to get toots
try:
# get toots for sync
if last_id:
logger.debug('Getting new toots for sync')
r = self.mastodon.account_statuses(my_id, since_id=last_id)
logger.debug('Number of new toots: {}'.format(len(r)))
res = r
# get toots only for updating last_toot
else:
logger.debug('Getting new toots only for fetching information')
r = self.mastodon.account_statuses(my_id)
# update the last toot ID
if len(r) > 0:
new_last_id = r[0]['id'] # r[0] is the latest
# update the data file immediately
if not dry_run or update:
logger.debug(
'Updating the last toot: {}'.format(new_last_id))
self.__update_last_id('last_toot', new_last_id)
except Exception as e:
logger.exception('Failed to get new toots: {}'.format(e))
return res
def get_new_tweets(self, dry_run=False, update=False):
"""Get new tweets of the author.
Using statuses/user_timeline API, get the author's new tweets, i.e.,
the tweets from the owner's account since the last tweet id, and return
the list of Tweet dicts. If the last tweet id cannot be found in the
data, the id of latest tweet is recoreded and return an empty list.
Returns:
list: toot dicts
"""
res = []
# fetch necessary information
my_id = self.data['twitter_account']['id']
last_id = self.data.get('last_tweet', False)
# try to get tweets
try:
# get tweets for sync
if last_id:
logger.debug('Getting new tweets for sync')
r = self.twitter.statuses.user_timeline(user_id=my_id,
since_id=last_id,
tweet_mode="extended")
logger.debug('Number of new tweets: {}'.format(len(r)))
res = r
# get tweets only for updating last_tweet
else:
logger.debug(
'Getting new tweets only for fetching information')
r = self.twitter.statuses.user_timeline(user_id=my_id,
tweet_mode="extended")
# update the last tweet ID
if len(r) > 0:
new_last_id = r[0]['id'] # r[0] is the latest
# update the data file immediately
if not dry_run or update:
logger.debug(
'Updating the last tweet: {}'.format(new_last_id))
self.__update_last_id('last_tweet', new_last_id)
except Exception as e:
logger.exception('Failed to get new tweets: {}'.format(e))
return res
def __store_twoot(self, toot_id, tweet_id):
"""Store a twoot (a pair of toot_id and tweet_id) in the data.
Insert the newest twoot to the HEAD of data['twoot'].
This is because it makes it easier to keep the number of stored twoots
less than max_twoots and also efficient in searching calculation.
"""
twoot = {'toot_id': toot_id, 'tweet_id': tweet_id}
logger.debug('Storing a twoot: {}'.format(twoot))
self.twoots.insert(0, twoot)
def __find_paired_toot(self, tweet_id):
"""Returns the id of paired toot of `tweet_id`.
Args:
tweet_id (int): Id of a tweet
Returns:
int: Id of the paired toot of `tweet_id`
"""
for t in self.twoots + self.data['twoots']:
if t['tweet_id'] == tweet_id:
toot_id = t['toot_id']
return toot_id
return None
def __find_paired_tweet(self, toot_id):
"""Returns the id of paired tweet of `toot_id`.
Args:
toot_id (int): Id of a toot
Returns:
int: Id of the paired tweet of `toot_id`
"""
for t in self.twoots + self.data['twoots']:
if t['toot_id'] == toot_id:
tweet_id = t['tweet_id']
return tweet_id
return None
def __html2text(self, html):
"""Convert html to text.
This process is essential for treating toots because the API of
Mastodon give us a toot in HTML format. This conversion is also useful
for tweets sometime because some specific letters (e.g., '<' and '>')
are encoded in character references of HTML even for the Twitter API.
Args:
html (str): a html text
Returns:
str: the plain text
"""
# prevent removing line break, indents, and char escapes
escapeable = [
('\n', '<br>'), # line break
(' ', ' '), # space
('\\', '\'), # backslash
('+', '+'), # plus
('-', '-'), # hyphen
('.', '.'), # period
]
for p in escapeable:
html = html.replace(p[0], p[1])
# basically, trust html2text
text = self.html2text.handle(html).strip()
# treat links and hashtags
text = re.sub(r'\[#(.*?)\]\(.*?\)', r'#\1', text)
text = re.sub(r'\[.*?\]\((.*?)\)', r'\1', text)
return text
def __pre_process(self, text, remove_words=[]):
"""Format a text nicely before posting.
This function do four things:
1. convert HTML to plain text
2. expand shorten links
3. remove given `remove_words` such as links of attached media
4. search usernames and escape by adding a dot (.) if any
5. delete tailing spaces
Args:
text (str): the text
remove_words (str): the list of words to remove
Returns:
str: the pre-processed text
"""
# process HTML tags/escapes
text = self.__html2text(text)
# expand links
links = [w for w in text.split() if urlparse(w.strip()).scheme]
for l in links:
# check the link
if not re.match(r'http(s|)://', l):
continue
# expand link with HTTP(S) HEAD request
try:
r = requests.head(l)
url = r.headers.get('location', l)
text = text.replace(l, url)
except Exception as e:
logger.exception('HTTP(S) HEAD request failed: {}'.format(e))
# remove specified words
for w in remove_words:
text = text.replace(w, '')
# prevent mentions
text = re.sub(r'([\s\n(]@)([_\w\d])', r'\1.\2', text)
# no tailing spaces
text = re.sub(r'[ \t]+\n', r'\n', text).strip()
return text
def __replace_rt_cite(self, text, tweet_id):
"""Replace the `rt_cite` place holder
Args:
text (str): the preprocessed text
tweet_id (int): Id of the original tweet
Returns:
str: the replaced text
"""
rt_cite = self.config.get('rt_cite', None)
if not rt_cite:
return text
rt_cite_re = '({})$'.format('|'.join(rt_cite))
if not re.search(rt_cite_re, text):
return text
my_id = self.data['twitter_account']['id']
try:
r = self.twitter.statuses.user_timeline(user_id=my_id,
count=1,
max_id=tweet_id - 1,
tweet_mode="extended")
except Exception as e:
logger.exception('Failed to get the previous tweet: {}'.format(e))
return text
rtd_tw = r[0].get('retweeted_status', None)
if rtd_tw is None:
logger.warn(
'Found a rt_cite place holder but cannot identify the RT')
return text
rtd_user, rtd_id = rtd_tw['user']['screen_name'], rtd_tw['id']
rtd_url = 'https://twitter.com/{}/status/{}'.format(rtd_user, rtd_id)
return re.sub(rt_cite_re, rtd_url, text)
def __download_image(self, url):
"""Download an image from `url`.
Args:
url (str): the image url
Returns:
raw binary data
str: content type
"""
r = requests.get(url)
if r.status_code != 200:
logger.warn('Failed to get an image from {}'.format(url))
return None
c_type = r.headers['content-type']
if 'image' not in c_type:
logger.warn('Data from {} is not an image'.format(url))
return None
return r.content, c_type
def __download_video(self, url):
"""Download a video from `url`.
Args:
url (str): the video url
Returns:
raw binary data
str: content type
"""
r = requests.get(url)
if r.status_code != 200:
logger.warn('Failed to get a video from {}'.format(url))
return None
c_type = r.headers['content-type']
if 'video' not in c_type:
logger.warn('Data from {} is not a video'.format(url))
return None
return r.content, c_type
def __post_media_to_mastodon(self, media):
"""Get actual data of `media` from Twitter and post it to Mastodon.
Args:
media: a Twitter media dict
Returns:
a Mastodon media dict
"""
media_type = media['type']
if media_type == 'photo':
img, mime_type = self.__download_image(media['media_url_https'])
try:
r = self.mastodon.media_post(img, mime_type=mime_type)
# NOTE: only under development
#logger.debug('Recieved media info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to post an image: {}'.format(e))
return None
elif media_type == 'animated_gif':
video_url = media['video_info']['variants'][0]['url']
video, mime_type = self.__download_video(video_url)
try:
r = self.mastodon.media_post(video, mime_type=mime_type)
# NOTE: only under development
#logger.debug('Recieved media info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to post a video: {}'.format(e))
return None
else:
logger.warn('Unknown media type found. Skipping.')
def __toot(self, text, in_reply_to_id=None, media_ids=None):
try:
r = self.mastodon.status_post(text,
in_reply_to_id=in_reply_to_id,
media_ids=media_ids)
# NOTE: only under development
#logger.debug('Recieved toot info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to create a toot: {}'.format(e))
return None
def __boost(self, target_id):
try:
r = self.mastodon.status_reblog(target_id)
# NOTE: only under development
#logger.debug('Recieved toot (BT) info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to create a toot (BT): {}'.format(e))
return None
def create_toot_from_tweet(self, tweet, dry_run=False):
"""Create a toot corresponding to the tweet.
Try to create a toot (or BT) if `tweet` satisfy:
1. normal tweet
2. so-called "self retweet" (create a corresponding BT)
3. so-called "self reply" (create a corresponding thread)
Otherwise, the tweet will be just skipped. In case `dry_run` is True,
the actual post will never executed but only the messages are output.
Args:
tweet: a tweet dict
dry_run (bool): the flag
"""
my_id = self.data['twitter_account']['id']
tweet_id = tweet['id']
synced_tweets = [
t['tweet_id'] for t in self.twoots + self.data['twoots']
]
def debug_skip(tw_id, reason):
logger.debug('Skipping a tweet (id: {}) because {}'.format(
tw_id, reason))
# skip if already forwarded
if tweet_id in synced_tweets:
debug_skip(tweet_id, 'it is already forwarded')
return
# reply case; a bit complecated
in_reply_to_tweet_id = None
in_reply_to_user_id = tweet.get('in_reply_to_user_id', None)
user_mentions = tweet.get('entities', {}).get('user_mentions', [])
if in_reply_to_user_id:
# skip reply for other users
if in_reply_to_user_id != my_id or len(user_mentions) > 1:
debug_skip(tweet_id, 'it is a reply for other users')
return
# reply to multiple users including oneself
if re.match(r'@[_\w\d]', tweet['full_text']):
debug_skip(tweet_id, 'it is a self reply but also to others')
return
# if self reply, store in_reply_to_tweet_id for creating a thread
logger.debug('The tweet (id: {}) is a self reply'.format(tweet_id))
in_reply_to_tweet_id = tweet['in_reply_to_status_id']
# RT case; more complecated
retweeted_tweet = tweet.get('retweeted_status', None)
if retweeted_tweet:
retweeted_tweet_id = retweeted_tweet['id']
# if self RT of a synced tweet, exec BT on the paired toot
if retweeted_tweet_id in synced_tweets:
target_toot_id = self.__find_paired_toot(retweeted_tweet_id)
logger.debug('Boost a toot (id: {})'.format(target_toot_id))
# execute BT
if not dry_run:
r = self.__boost(target_toot_id)
if r:
toot_id = r['id']
self.__store_twoot(toot_id, tweet_id)
# no more process for RT
return
# otherwise, just skip
else:
debug_skip(tweet_id, 'it is an RT')
return
# treat media
twitter_media = tweet.get('extended_entities', {}).get('media', [])
media_num = 0
# if dry run, don't upload
if dry_run:
media_num = len(twitter_media)
else:
mastodon_media = [
self.__post_media_to_mastodon(m) for m in twitter_media
]
media_ids = [m['id'] for m in mastodon_media if m is not None]
media_num = len(media_ids)
# treat text
media_urls = [m['expanded_url'] for m in twitter_media]
text = self.__pre_process(tweet['full_text'], remove_words=media_urls)
text = self.__replace_rt_cite(text, tweet['id'])
# try to create a toot
if media_num > 0:
logger.debug('Trying to toot: {} (with {} media)'.format(
repr(text), media_num))
else:
logger.debug('Trying to toot: {}'.format(repr(text)))
if not dry_run:
# NOTE: these branches are for calculation efficiency
# if the tweet is in a thread and in sync, copy as a thread
if in_reply_to_tweet_id in synced_tweets:
r = self.__toot(text,
in_reply_to_id=self.__find_paired_toot(
in_reply_to_tweet_id),
media_ids=media_ids)
# otherwise, just toot it
else:
r = self.__toot(text, media_ids=media_ids)
# store the twoot
if r:
toot_id = r['id']
self.__store_twoot(toot_id, tweet_id)
logger.info(
'Forwarded a tweet (id: {}) as a toot (id: {})'.format(
tweet_id, toot_id))
def __post_media_to_twitter(self, media):
"""Get actual data of `media` from Mastodon and post it to Twitter.
Args:
media: a Mastodon media dict
Returns:
a Twitter media dict
"""
media_type = media['type']
if media_type == 'image':
img, mime_type = self.__download_image(media['url'])
try:
r = self.twitter_upload.media.upload(media=img)
# NOTE: only under development
#logger.debug('Recieved media info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to post an image: {}'.format(e))
return None
elif media_type == 'gifv':
video, mime_type = self.__download_video(media['url'])
try:
# init
init_res = self.twitter_upload.media.upload(
command='INIT',
total_bytes=len(video),
media_type=mime_type)
media_id = init_res['media_id_string']
# append
append_res = self.twitter_upload.media.upload(
command='APPEND',
media_id=media_id,
media=video,
segment_index=0)
# finalize
r = self.twitter_upload.media.upload(command='FINALIZE',
media_id=media_id)
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to post an image: {}'.format(e))
return None
else:
logger.warn('Unknown media type found. Skipping.')
def __tweet(self, text, in_reply_to_id=None, media_ids=None):
try:
r = self.twitter.statuses.update(
status=text,
in_reply_to_status_id=in_reply_to_id,
media_ids=','.join(media_ids))
# NOTE: only under development
#logger.debug('Recieved tweet info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to create a tweet: {}'.format(e))
return None
def __retweet(self, target_id):
try:
r = self.twitter.statuses.retweet(_id=target_id)
# NOTE: only under development
#logger.debug('Recieved toot (BT) info: {}'.format(str(r)))
return r
# if failed, report it
except Exception as e:
logger.exception('Failed to create a tweet (RT): {}'.format(e))
return None
def create_tweet_from_toot(self, toot, dry_run=False):
"""Create a tweet corresponding to the toot.
Try to create a tweet (or RT) if `toot` satisfy:
1. normal toot
2. so-called "self boost" (create a corresponding RT)
3. so-called "self reply" (create a corresponding thread)
Otherwise, the toot will be just skipped. In case `dry_run` is True,
the actual post will never executed but only the messages are output.
Args:
toot: a toot dict
dry_run (bool): the flag
"""
my_id = self.data['mastodon_account']['id']
toot_id = toot['id']
synced_toots = [
t['toot_id'] for t in self.twoots + self.data['twoots']
]
def debug_skip(tt_id, reason):
logger.debug('Skipping a toot (id: {}) because {}'.format(
tt_id, reason))
# skip if already forwarded
if toot_id in synced_toots:
debug_skip(toot_id, 'it is already forwarded')
return
# reply case; a bit complecated
in_reply_to_toot_id = None
in_reply_to_account_id = toot['in_reply_to_account_id']
if in_reply_to_account_id:
# skip reply for other users
if in_reply_to_account_id != my_id:
debug_skip(toot_id, 'it is a reply for other users')
return
# if self reply, store in_reply_to_toot_id for creating a thread
logger.debug('The toot (id: {}) is a self reply'.format(toot_id))
in_reply_to_toot_id = toot['in_reply_to_id']
# BT case; more complecated
boosted_toot = toot.get('reblog', None)
if boosted_toot:
boosted_toot_id = boosted_toot['id']
# if self BT of a synced toot, exec RT on the paired tweet
if boosted_toot_id in synced_toots:
target_tweet_id = self.__find_paired_tweet(boosted_toot_id)
logger.debug(
'Retweet a tweet (id: {})'.format(target_tweet_id))
# execute RT
if not dry_run:
r = self.__retweet(target_tweet_id)
if r:
tweet_id = r['id']
self.__store_twoot(toot_id, tweet_id)
# no more process for BT
return
# otherwise, just skip
else:
debug_skip(toot_id, 'because it is a BT')
return
# treat media
mastodon_media = toot.get('media_attachments', [])
media_num = 0
# if dry run, don't upload
if dry_run:
media_num = len(mastodon_media)
else:
twitter_media = [
self.__post_media_to_twitter(m) for m in mastodon_media
]
media_ids = [
m['media_id_string'] for m in twitter_media if m is not None
]
media_num = len(media_ids)
# treat text
text = self.__pre_process(toot['content'])
# try to create a tweet
if media_num > 0:
logger.debug('Trying to tweet: {} (with {} media)'.format(
repr(text), media_num))
else:
logger.debug('Trying to tweet: {}'.format(repr(text)))
if not dry_run:
# NOTE: these branches are for calculation efficiency
# if the toot is in a thread and in sync, copy as a thread
if in_reply_to_toot_id in synced_toots:
r = self.__tweet(text,
in_reply_to_id=self.__find_paired_tweet(
in_reply_to_toot_id),
media_ids=media_ids)
# otherwise, just tweet it
else:
r = self.__tweet(text, media_ids=media_ids)
# store the twoot
if r:
tweet_id = r['id']
self.__store_twoot(toot_id, tweet_id)
logger.info(
'Forwarded a toot (id: {}) as a tweet (id: {})'.format(
toot_id, tweet_id))
def tweets2toots(self, tweets, dry_run=False):
# process from the oldest one
for t in reversed(tweets):
# NOTE: only under development
#logger.debug('Processing tweet info: {}'.format(t))
# create a toot if necessary
self.create_toot_from_tweet(t, dry_run)
def toots2tweets(self, toots, dry_run=False):
# process from the oldest one
for t in reversed(toots):
# NOTE: only under development
#logger.debug('Processing toot info: {}'.format(t))
# create a toot if necessary
self.create_tweet_from_toot(t, dry_run)
def __save_data(self):
"""Save up-to-dated data (twoots) to the data file."""
# load the latest data
with open(self.data_file, 'rb') as f:
data = pickle.load(f)
# concat the new twoots to data
data['twoots'] = self.twoots + data['twoots']
# keep the number of stored twoots less than max_twoots
data['twoots'] = data['twoots'][:self.config['max_twoots']]
# save data
with open(self.data_file, 'wb') as f:
pickle.dump(data, f)
def run(self, dry_run=False, update=False):
if dry_run:
if self.setup:
logger.warn(
'Option --dry-run (-n) has no effect for setup mode')
dry_run = False
else:
logger.debug('Dry running')
else:
logger.debug('Running')
# tweets -> toots
toots = self.get_new_toots(dry_run, update)
if not self.setup:
self.toots2tweets(toots, dry_run)
# toots -> tweets
tweets = self.get_new_tweets(dry_run, update)
if not self.setup:
self.tweets2toots(tweets, dry_run)
# update the entire data
if len(self.twoots) > 0:
logger.debug('Saving up-to-dated data to {}'.format(
self.data_file))
self.__save_data()
# show current status for debugging
logger.debug('Number of stored twoots: {}'.format(
len(self.data['twoots'])))
access_token_secret=config.twitter.access_token_secret)
creds = mastodon.account_verify_credentials()
mastodon_account_id = creds.id
twitter_user = twitter.VerifyCredentials()
twitter_id = twitter_user.id
if not os.path.exists('.sync_id'):
logger.error(
"There's no .sync_id file containing your latest synced Toot, creating one"
)
with open('.sync_id', 'w') as fh:
fh.write("0")
toots = mastodon.account_statuses(mastodon_account_id)
latest_toots = []
for toot in toots:
if (toot['visibility'] not in ['public', 'unlisted']
or # Skip private Toots
len(toot['mentions']) > 0 or # Skip replies
toot['reblog']): # Skip reblogs/boosts
continue
# Unescape HTML entities in content
# This means, that e.g. a Toot "Hi, I'm a bot" is escaped as
# "Hi, I'm a bot". Unescaping this text reconstructs the
# original string "I'm"
content = html.unescape(toot['content'])
# !pip3 install Mastodon.py
# このコードは割と雑な作りで、エラーハンドリング等をしていません。
# だいたい動くと思いますがもう少しマシな実装を使った方がいいかもしれません。
from mastodon import Mastodon
# ユーザー設定→開発でアプリを登録することで以下の値を得ることができる
mastodon = Mastodon(access_token='your_access_token',
client_id='your_client_id',
client_secret='your_client_secret',
api_base_url='https://instance.com')
account = mastodon.account_verify_credentials()
toots = mastodon.account_statuses(account)
for toot in toots:
from time import sleep
print(toot.id)
mastodon.status_delete(toot)
sleep(5) # インスタンスへの負荷に応じて調整してください
# print('deleted first 20 toots')
next_toots = mastodon.fetch_next(toots)
while len(next_toots) != 0:
for toot in next_toots:
from time import sleep
print(toot.id)
sleep(5) # インスタンスへの負荷に応じて調整してください
mastodon.status_delete(toot)
def check_toots(config, options, retry_count=0):
"""
The main function, uses the Mastodon API to check all toots in the user timeline, and delete any that do not meet any of the exclusion criteria from the config file.
"""
try:
if not options.quiet:
print(
"Fetching account details for @",
config["username"],
"@",
config["base_url"],
sep="",
)
if options.pace:
mastodon = Mastodon(
access_token=config["access_token"],
api_base_url="https://" + config["base_url"],
ratelimit_method="pace",
)
else:
mastodon = Mastodon(
access_token=config["access_token"],
api_base_url="https://" + config["base_url"],
ratelimit_method="wait",
)
user_id = mastodon.account_verify_credentials(
).id # verify user and get ID
account = mastodon.account(user_id) # get the account
timeline = mastodon.account_statuses(user_id,
limit=40) # initial batch
if not options.quiet:
print("Checking", str(account.statuses_count), "toots")
# check first batch
# check_batch() then recursively keeps looping until all toots have been checked
check_batch(config, options, mastodon, user_id, timeline)
except KeyboardInterrupt:
print("Operation aborted.")
except KeyError as val:
print("\n⚠️ error with in your config.yaml file!")
print("Please ensure there is a value for " + str(val) + "\n")
except MastodonAPIError as e:
if e.args[1] == 401:
print(
"\n🙅 User and/or access token does not exist or has been deleted (401)\n"
)
elif e.args[1] == 404:
print("\n🔭 Can't find that server (404)\n")
else:
print("\n😕 Server has returned an error (5xx)\n")
if options.verbose:
print(e, "\n")
except MastodonNetworkError as e:
if retry_count == 0:
print(
"\n📡 ephemetoot cannot connect to the server - are you online?"
)
if options.verbose:
print(e)
if retry_count < 4:
print("Waiting " + str(options.retry_mins) +
" minutes before trying again")
time.sleep(60 * options.retry_mins)
retry_count += 1
print("Attempt " + str(retry_count + 1))
check_toots(config, options, retry_count)
else:
print("Gave up waiting for network\n")
except Exception as e:
if options.verbose:
print("ERROR:", e)
else:
print("ERROR:", str(e.args[0]), "\n")
def backup(args):
"""
Backup toots, followers, etc from your Mastodon account
"""
(username, domain) = args.user.split("@")
url = 'https://' + domain
client_secret = domain + '.client.secret'
user_secret = domain + '.user.' + username + '.secret'
status_file = domain + '.user.' + username + '.json'
data = None
if os.path.isfile(status_file):
print("Loading existing backup")
with open(status_file, mode = 'r', encoding = 'utf-8') as fp:
data = json.load(fp)
if not os.path.isfile(client_secret):
print("Registering app")
Mastodon.create_app(
'mastodon-backup',
api_base_url = url,
to_file = client_secret)
if not os.path.isfile(user_secret):
print("Log in")
mastodon = Mastodon(
client_id = client_secret,
api_base_url = url)
url = mastodon.auth_request_url(
client_id = client_secret,
scopes=['read'])
print("Visit the following URL and authorize the app:")
print(url)
print("Then paste the access token here:")
token = sys.stdin.readline().rstrip()
mastodon.log_in(
username = username,
code = token,
to_file = user_secret,
scopes=['read'])
else:
mastodon = Mastodon(
client_id = client_secret,
access_token = user_secret,
api_base_url = url)
print("Get user info")
user = mastodon.account_verify_credentials()
def find_id(list, id):
"""Return the list item whose id attribute matches."""
return next((item for item in list if item["id"] == id), None)
def fetch_up_to(page, id):
statuses = []
# use a generator expression to find our last status
found = find_id(page, id)
# get the remaining pages
while len(page) > 0 and found is None:
statuses.extend(page)
sys.stdout.flush()
page = mastodon.fetch_next(page)
if page is None:
break
found = find_id(page, id)
page = page[0:page.index(found)]
statuses.extend(page)
print("Fetched a total of %d new toots" % len(statuses))
return statuses
if data is None or not "statuses" in data:
print("Get statuses (this may take a while)")
statuses = mastodon.account_statuses(user["id"])
statuses = mastodon.fetch_remaining(
first_page = statuses)
else:
id = data["statuses"][0]["id"]
print("Get new statuses")
statuses = fetch_up_to(mastodon.account_statuses(user["id"]), id)
statuses.extend(data["statuses"])
if data is None or not "favourites" in data:
print("Get favourites (this may take a while)")
favourites = mastodon.favourites()
favourites = mastodon.fetch_remaining(
first_page = favourites)
else:
id = data["favourites"][0]["id"]
print("Get new favourites")
favourites = fetch_up_to(mastodon.favourites(), id)
favourites.extend(data["favourites"])
data = { 'account': user,
'statuses': statuses,
'favourites': favourites }
print("Saving %d statuses and %d favourites" % (
len(statuses),
len(favourites)))
date_handler = lambda obj: (
obj.isoformat()
if isinstance(obj, (datetime.datetime, datetime.date))
else None)
with open(status_file, mode = 'w', encoding = 'utf-8') as fp:
data = json.dump(data, fp, indent = 2, default = date_handler)
# Log in and start up
mastodon_api = Mastodon(client_id="mtt_mastodon_client.secret",
access_token="mtt_mastodon_user.secret",
ratelimit_method="wait",
api_base_url=MASTODON_BASE_URL)
twitter_api = twitter.Api(
consumer_key=TWITTER_CONSUMER_KEY,
consumer_secret=TWITTER_CONSUMER_SECRET,
access_token_key=TWITTER_ACCESS_KEY,
access_token_secret=TWITTER_ACCESS_SECRET,
tweet_mode='extended' # Allow tweets longer than 140 raw characters
)
ma_account_id = mastodon_api.account_verify_credentials()["id"]
try:
since_toot_id = mastodon_api.account_statuses(ma_account_id)[0]["id"]
except:
since_toot_id = 0
print("Tweeting any toots after toot " + str(since_toot_id))
since_tweet_id = twitter_api.GetUserTimeline()[0].id
print("Tooting any tweets after tweet " + str(since_tweet_id))
# Set "last URL length update" time to 1970
last_url_len_update = 0
while True:
# Fetch twitter short URL length, if needed
if time.time() - last_url_len_update > 60 * 60 * 24:
twitter_api._config = None
url_length = max(twitter_api.GetShortUrlLength(False),
twitter_api.GetShortUrlLength(True)) + 1
parser = ArgumentParser()
parser.add_argument(
"--test", action="store_true", help="do a test run without deleting any toots"
)
options = parser.parse_args()
if options.test:
print("This is a test run...")
print("Fetching account details...")
mastodon = Mastodon(access_token=config.access_token, api_base_url=config.base_url)
cutoff_date = datetime.now(timezone.utc) - timedelta(days=config.days_to_keep)
user_id = mastodon.account_verify_credentials().id
timeline = mastodon.account_statuses(user_id, limit=40)
def checkToots(timeline, deleted_count=0):
for toot in timeline:
try:
if config.save_pinned and hasattr(toot, "pinned") and toot.pinned:
print("📌 skipping pinned toot - " + str(toot.id))
elif toot.id in config.toots_to_save:
print("💾 skipping saved toot - " + str(toot.id))
elif cutoff_date > toot.created_at:
if hasattr(toot, "reblog") and toot.reblog:
print(
"👎 unboosting toot "
+ str(toot.id)
+ " boosted "
from mastodon import Mastodon
url = sys.argv[1]
cid_file = 'client_id.txt'
token_file = 'access_token.txt'
mastodon = Mastodon(
client_id=cid_file,
access_token=token_file,
api_base_url=url
)
# 自分の最新トゥート1件を取得する
user_dict = mastodon.account_verify_credentials()
user_toots = mastodon.account_statuses(user_dict['id'], limit=1)
# トゥートの時間と内容を表示する
print(user_toots[0]['created_at'], user_toots[0]['content'])
USER_ID = 38871
def json_handler(obj):
if hasattr(obj, 'isoformat'):
return obj.isoformat()
else:
raise TypeError(obj)
mastodon = Mastodon(client_id='clientcred.secret',
access_token='usercred.secret',
api_base_url='https://cybre.space',
ratelimit_method='pace')
statuses = mastodon.account_statuses(USER_ID, limit=40)
stdout.write('[\n')
first = True
while True:
max_id = None
stderr.write('writing {} toots\n'.format(len(statuses)))
for toot in statuses:
if not first:
stdout.write(',\n')
first = False
dump(toot, stdout, default=json_handler, indent=1)
