def read_comments(self):
c_list = {}
comments = []
# comment_history = self.account.comment_history(limit=self.limit)
# query = Query(start_author=self.username, limit=self.limit)
# comment_history = Discussions_by_comments(query)
comment_history = self.get_comments()
days_done = False
for c in comment_history:
if c.permlink in c_list:
continue
if c.is_comment() and c.author == self.username:
sc = SteemComment(comment=c)
days_done = self.days is not None and not in_recent_n_days(c, self.days)
if days_done:
break
else:
if self.receiver is None or len(self.receiver) == 0 \
or c.parent_author == self.receiver:
c = sc.refresh()
sc.log()
c_list[c.permlink] = 1
comments.append(c)
print ('{} comments are fetched'.format(len(comments)))
return comments
def vote(self, post=None, url=None, weight=None, retries=VOTE_RETRIES):
c = SteemComment(comment=post, url=url)
if retries <= 0:
logger.error("Vote {} failed after retries for {} times".format(
c.get_url(), VOTE_RETRIES))
return False
while time.time() - self.last_vote_timestamp < MINIMUM_VOTE_INTERVAL:
wait_time = round(
MINIMUM_VOTE_INTERVAL +
random.random() * MINIMUM_VOTE_INTERVAL * 0.2, 2)
logger.info(
"Sleep {} seconds to avoid voting too frequently.".format(
wait_time))
time.sleep(wait_time)
if time.time() - self.last_vote_timestamp >= MINIMUM_VOTE_INTERVAL:
return self.vote(post, url, weight, retries - 1)
success = False
try:
weight = weight or self.weight(c)
success = self.voter.vote(c.get_comment(), weight=weight)
self.last_vote_timestamp = time.time()
except:
logger.error(
"Failed when voting {} with error: {} . {} retry times left.".
format(c.get_url(), traceback.format_exc(), retries - 1))
return self.vote(post, url, weight, retries - 1)
self.after_success(success)
return success
def read_posts(self):
posts = {}
votes = self.get_votes(self.account, up=self.upvote)
for v in votes:
# filter votes with 7 days
# dt = datetime.strptime(v['time'], "%Y-%m-%dT%H:%M:%S")
dt = datetime.strptime(v['last_update'], "%Y-%m-%dT%H:%M:%S")
dt = dt.replace(tzinfo=timezone.utc)
if not in_recent_days(dt, self.days):
continue
# get post data
# authorperm = v['authorperm']
authorperm = "@{}/{}".format(v['author'], v['permlink'])
if not authorperm in posts:
c = SteemComment(author_perm=authorperm)
if not self.within_nth_day(c, self.days):
continue
posts[authorperm] = {
"author": c.get_comment().author,
"permlink": v['permlink'],
"authorperm": authorperm,
"voters": [],
"percents": [],
"created": c.get_comment()['created']
}
# get voters and percentage
posts[authorperm]['voters'].append(v['voter'])
# posts[authorperm]['percents'].append(v['percent'])
posts[authorperm]['percents'].append(v['vote_percent'])
res = list(posts.values())
res.sort(key=lambda post: post['created'])
return res
def vote(self, post=None, url=None, weight=None, retries=VOTE_RETRIES):
c = SteemComment(comment=post, url=url)
if retries <= 0:
logger.error("Vote {} failed after retries for {} times".format(
c.get_url(), VOTE_RETRIES))
return False
while time.time() - self.last_vote_timestamp < MINIMUM_VOTE_INTERVAL:
self.sleep()
if time.time() - self.last_vote_timestamp >= MINIMUM_VOTE_INTERVAL:
return self.vote(post, url, weight, retries - 1)
success = False
try:
weight = weight or self.weight(c)
success = self.voter.vote(c.get_comment(), weight=weight)
self.last_vote_timestamp = time.time()
except:
logger.error(
"Failed when voting {} with error: {} . {} retry times left.".
format(c.get_url(), traceback.format_exc(), retries - 1))
return self.vote(post, url, weight, retries - 1)
self.after_success(success, c)
return success
def _measure_similarity(self, post):
target = SteemComment(comment=post)
logger.info("measure {}".format(post.title))
return SimilarityCheck(
{
"title": self.source.title(),
"body": self.source.get_text_body()
}, {
"title": target.title(),
"body": target.get_text_body()
}).compare()
def _downvote(self, post, weight):
c = SteemComment(comment=post)
if not c.is_downvoted_by(self.author):
post.downvote(weight=weight, voter=self.author)
logger.info(
"Downvoted to [{}] [{}] with weight [{:.2f}] successfully".
format(post.title, c.get_url(), weight))
return True
else:
logger.info(
"Skip downvote because I already downvoted this post [{}] [{}]"
.format(post.title, c.get_url()))
return False
def what_to_vote(self, ops):
# posted_with_steemcn = self.ops.is_app(APP) and not self.ops.is_comment()
is_post = not self.ops.is_comment()
if is_post:
logger.info("Find post {} published with [{}] app".format(
self.ops.get_url(), APP))
c = SteemComment(ops=ops)
if c.is_upvoted_by(self.by()):
logger.info("Post {} is already upvoted. Skip.".format(
c.get_url()))
return False
else:
return True
return False
def _write_content(self, post):
folder = self._get_content_folder()
c = SteemComment(comment=post)
# retrieve necessary data from steem
title = self._yaml_compatible(post.title, "''")
permlink = post["permlink"]
body = c.get_compatible_markdown()
position = self._get_position(body)
date_str = post.json()["created"]
date = date_str.replace('T', ' ')
tags = "\n".join(["- {}".format(tag) for tag in c.get_tags()])
category = 'cross-posting'
if len(c.get_tags()) > 0:
category = c.get_tags()[0]
thumbnail = self._yaml_compatible(c.get_pic_url(), "")
url = c.get_url()
# build content with template
template = get_message("blog", footer=True)
content = template.format(title=title, permlink=permlink,
position=position, date=date,
tags=tags, category=category,
thumbnail=thumbnail, body=body, url=url)
# write into MD files
filename = os.path.join(folder, "{}_{}.md".format(date_str.split('T')[0], permlink))
with open(filename, "w", encoding="utf-8") as f:
f.write(content)
logger.info("Download post [{}] into file {}".format(title, filename))
def reply(self, message_id=None, post=None, url=None):
""" reply to the' post """
c = SteemComment(comment=post, url=url)
receiver = c.get_comment().author
if not self._has_replied(receiver, message_id):
title = c.get_comment().title
message = self._get_reply_body(message_id, receiver)
self.writer.reply(c.get_comment(), message)
self._add_reply_record(receiver, message_id, c)
logger.info("Replied to @{}'s post [{}] with [{}] message".format(receiver, title, message_id))
return True
else:
logger.info("Skip reply account @{} with [{}] message, because we already reliped before".format(receiver, message_id))
return False
def context(self, ops):
if isinstance(ops, Comment):
self.ops = SteemComment(comment=ops)
else:
self.ops = SteemOperation(ops=ops)
logger.debug("watch operation: {}; tags: {}".format(
self.ops.get_url(), self.ops.get_tags()))
def read_posts_with_limit(self, limit, last_post=None):
posts = []
if last_post:
q = Query(limit=limit, tag=self.tag,
start_author=last_post.author,
start_permlink=last_post.permlink
# before_date=str(last_post['created'])
)
else:
q = Query(limit=limit, tag=self.tag)
blogs = Discussions_by_created(q)
logger.info ('reading posts: {}'.format(len(blogs)))
c_list = {}
days_done = False
for c in blogs:
if c.permlink in c_list:
continue
if not c.is_comment():
if self.keyword is None or self.keyword in c.title:
days_done = self.days is not None and not in_recent_n_days(c, self.days)
if days_done:
break
else:
SteemComment(comment=c).log()
c_list[c.permlink] = 1
posts.append(c)
return {
"posts": posts,
"days_done": days_done
}
def what_to_vote(self, ops):
posted_with_steemcn = self.ops.is_app(
APP) and not self.ops.is_comment()
if posted_with_steemcn:
logger.info("Find post {} published with [{}] app".format(
self.ops.get_url(), APP))
c = SteemComment(ops=ops)
beneficiary = c.get_beneficiaries(account=BENEFICIARY_ACCOUNT)
if beneficiary >= BENEFICIARY_THRESHOLD:
logger.info("Post {} has set {} beneficiary.".format(
self.ops.get_url(), beneficiary))
if c.is_upvoted_by(self.by()):
logger.info("Post {} is already upvoted. Skip.".format(
c.get_url()))
return False
else:
return True
return False
def _has_replied(self, receiver, message_id):
# has reply records in DB, or has replied by checking steem APIs
if self._has_reply_record(receiver, message_id):
return True
(replied, comment) = self._has_reply_comment(receiver, message_id)
if replied:
c = SteemComment(comment=comment.get_parent())
# cache the record into database since not found
self._add_reply_record(receiver, message_id, c, comment["created"])
return True
return False
def data(self):
data = super().data()
# exclude the posts in yesterday's summary
yesterday_posts = self.get_yesterday_posts()
yesterday_authorperms = [
SteemComment(url=url).get_author_perm() for url in yesterday_posts
]
data = [
d for d in data if not d['authorperm'] in yesterday_authorperms
]
return data
def read_posts(self, start=0):
c_list = {}
posts = []
if self.limit:
blogs = self.account.get_blog(start_entry_id=start, limit=self.limit)
else:
blogs = self.account.blog_history(start=start, limit=self.limit, reblogs=False)
days_done = False
for c in blogs:
if c.permlink in c_list:
continue
if not c.is_comment() and c.author == self.username:
sc = SteemComment(comment=c)
tags = sc.get_tags()
if self.tag is None or self.tag in tags:
if self.keyword is None or self.keyword in c.title:
days_done = self.days is not None and not in_recent_n_days(c, self.days)
if days_done:
break
else:
c = sc.refresh()
sc.log()
c_list[c.permlink] = 1
posts.append(c)
print ('{} posts are fetched'.format(len(posts)))
return posts
def read_posts_with_limit(self, limit, last_post=None):
posts = []
if last_post:
q = Query(limit=limit,
tag=self.tag,
start_author=last_post.author,
start_permlink=last_post.permlink
# before_date=str(last_post['created'])
)
else:
q = Query(limit=limit, tag=self.tag)
if self.mode == "comment":
blogs = Comment_discussions_by_payout(q)
else:
blogs = Discussions_by_created(q)
if isinstance(blogs, types.GeneratorType):
logger.info("reading posts...")
else:
logger.info('reading posts: {}'.format(len(blogs)))
c_list = {}
days_done = False
for c in blogs:
if c.permlink in c_list:
continue
if self.mode == "comment":
type_match = c.is_comment()
elif self.mode == "post":
type_match = not c.is_comment()
else: # self.mode == "all"
type_match = True
if type_match:
if self.keyword is None or self.keyword in c.title:
days_done = self.days is not None and not in_recent_n_days(
c, self.days)
if days_done:
break
else:
SteemComment(comment=c).log()
c_list[c.permlink] = 1
posts.append(c)
return {"posts": posts, "days_done": days_done}
def read_posts(self, start=0):
if not self.exists:
return []
c_list = {}
posts = []
if self.reblog:
blogs = self.account.history_reverse(only_ops=["custom_json"])
elif self.limit:
blogs = self.account.get_blog(start_entry_id=start,
limit=self.limit)
else:
blogs = self.account.blog_history(start=start,
limit=self.limit,
reblogs=self.reblog)
days_done = False
for c in blogs:
if self.reblog:
if c['id'] == "follow":
json_data = json.loads(c['json'])
if len(json_data) > 0 and json_data[0] == "reblog":
info = json_data[1]
authorperm = "@{}/{}".format(info["author"],
info["permlink"])
timestamp = c["timestamp"]
c = Comment(authorperm)
c["reblogged"] = timestamp
else:
continue
else:
continue
if c.authorperm in c_list:
continue
if not c.is_comment():
if ((not self.reblog) and c.author == self.username) or (
self.reblog and c.author != self.username):
sc = SteemComment(comment=c)
tags = sc.get_tags()
if self.tag is None or self.tag in tags:
if self.keyword is None or self.keyword in c.title:
days_done = self.days is not None and not in_recent_n_days(
c, self.days, self.reblog)
if days_done:
break
else:
c = sc.refresh()
sc.log()
c_list[c.authorperm] = 1
posts.append(c)
print('{} posts are fetched'.format(len(posts)))
return posts
def vote(self, post=None, url=None):
c = SteemComment(comment=post, url=url)
receiver = c.get_comment().author
title = c.get_comment().title
if not c.is_upvoted_by(self.author):
self.voter.upvote(c.get_comment(), weight=float(VOTE_WEIGHT))
logger.info("I have upvoted @{}'s post [{}] successfully".format(
receiver, title))
return True
else:
logger.info(
"Skip upvote @{} because I already upvoted his/her post [{}]".
format(receiver, title))
return False
def crawl(self):
post = SteemComment(url=TEAMCN_CEREMONY_POST).get_comment()
return [post]
def perform_vote():
if isinstance(ops, Comment):
c = SteemComment(comment=ops)
else:
c = SteemComment(ops=ops)
self.append_to_vote_queue(post=c.get_comment())
def print_post(ops):
c = SteemComment(ops=ops)
if not c.get_comment().is_comment():
c.log()
def get_post_payout(self, post=None, url=None):
c = SteemComment(comment=post, url=url)
return c.get_payout()