def save(self):
count = len(self.roster)
if count > 0:
filename = self._get_roster_data_file()
write_json_array_to_csv(self.roster, filename)
logger.info("Crawled {} names to add into roster {}".format(
count, filename))
def _diff_files(self):
os.chdir(SOURCE_REPO_FOLDER)
res = subprocess.run(['git', 'diff', 'HEAD', 'HEAD~1', '--name-only'], stdout=subprocess.PIPE).stdout.decode('utf-8')
os.chdir("..")
files = [f for f in res.split("\n") if len(f) > 0]
logger.info("{} different files:\n{}".format(len(files), res))
return files
def vote(self, post=None, url=None, weight=None, retries=VOTE_RETRIES):
c = SteemComment(comment=post, url=url)
if retries <= 0:
logger.error("Vote {} failed after retries for {} times".format(
c.get_url(), VOTE_RETRIES))
return False
while time.time() - self.last_vote_timestamp < MINIMUM_VOTE_INTERVAL:
wait_time = round(
MINIMUM_VOTE_INTERVAL +
random.random() * MINIMUM_VOTE_INTERVAL * 0.2, 2)
logger.info(
"Sleep {} seconds to avoid voting too frequently.".format(
wait_time))
time.sleep(wait_time)
if time.time() - self.last_vote_timestamp >= MINIMUM_VOTE_INTERVAL:
return self.vote(post, url, weight, retries - 1)
success = False
try:
weight = weight or self.weight(c)
success = self.voter.vote(c.get_comment(), weight=weight)
self.last_vote_timestamp = time.time()
except:
logger.error(
"Failed when voting {} with error: {} . {} retry times left.".
format(c.get_url(), traceback.format_exc(), retries - 1))
return self.vote(post, url, weight, retries - 1)
self.after_success(success)
return success
def sleep(self):
wait_time = round(
MINIMUM_VOTE_INTERVAL * 1.2 +
random.random() * MINIMUM_VOTE_INTERVAL * 0.3, 2)
logger.info("Sleep {} seconds to avoid voting too frequently.".format(
wait_time))
time.sleep(wait_time)
def run(self, callback=default_callback, lookback=0, start=-1, days=-1):
try:
if lookback > 0 or start > 0 or days > 0:
if self.last_streamed_block == 0:
if start > 0:
start_block = start
elif lookback > 0:
start_block = self.blockchain.get_current_block_num() - int(lookback) #200000
else:
start_date = days_ago(days)
start_block = self.blockchain.get_estimated_block_num(start_date)
else:
start_block = self.last_streamed_block + 1
stop_block = self.blockchain.get_current_block_num()
logger.info("Streaming for operations {} has started from block {} to {}".format(self.operations, start_block, stop_block))
for ops in self.blockchain.stream(opNames=self.operations,
start=start_block, stop=stop_block,
max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8):
try:
callback(ops)
except:
logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc()))
else:
logger.info("Streaming for operations {} has started from the latest blocks".format(self.operations))
for ops in self.blockchain.stream(opNames=self.operations,
max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8):
try:
callback(ops)
except:
logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc()))
except:
logger.error("Failed when streaming operations {} with error: {}".format(self.operations, traceback.format_exc()))
def read_posts_with_limit(self, limit, last_post=None):
posts = []
if last_post:
blogs = get_discussions("created",
token=self.token,
limit=limit,
start_author=last_post['author'],
start_permlink=last_post['permlink'])
else:
blogs = get_discussions("created", token=self.token, limit=limit)
logger.info('reading posts: {}'.format(len(blogs)))
c_list = {}
days_done = False
for ops in blogs:
c = SteemOperation(ops)
if ops['authorperm'] in c_list:
continue
tags = c.get_tags()
if self.tag is None or self.tag in tags:
if self.keyword is None or self.keyword in c.title():
days_done = self.days is not None and not in_recent_n_days(
ops, self.days)
if days_done:
break
else:
c.log(scot=True)
c_list[ops['authorperm']] = 1
posts.append(ops)
return {"posts": posts, "days_done": days_done}
def detect(self, strict=True):
try:
summary = {}
for i in range(0, TIMES):
results = detect_langs(self.text)
for res in results:
lang = res.lang
prob = float(res.prob) #float
if not lang in summary:
summary[lang] = prob
else:
summary[lang] += prob
languages = sorted(summary, key=summary.get, reverse=True)
language = languages[0]
logger.info("language detection: lang = {} ; summary = {}".format(
language, summary))
if strict:
return language
else:
return languages
except:
logger.info(
"failed when detecting language for text: {}\nError: {}".
format(self.text, traceback.format_exc()))
return None
def watch(self, ops):
author = ops['author']
def perform_vote():
if isinstance(ops, Comment):
c = SteemComment(comment=ops)
else:
c = SteemComment(ops=ops)
self.append_to_vote_queue(post=c.get_comment())
self.ctx(ops)
try:
if self.what_to_vote(ops) and self.who_to_vote(
author) and self.is_ready():
delay = self.when_to_vote(ops) # mins
if delay is not None and delay > 0:
secs = 60.0 * delay
logger.info("I'll vote after {} seconds".format(secs))
t = Timer(secs, perform_vote)
t.start()
else:
logger.info("I'll vote immediately")
perform_vote()
except:
logger.error(
"Failed when watching the comment [{}] with error: {} .".
format(ops, traceback.format_exc()))
def _write_content(self, post):
folder = self._get_content_folder()
c = SteemComment(comment=post)
# retrieve necessary data from steem
title = self._yaml_compatible(post.title, "''")
# permlink = post["permlink"]
permlink = self.permalink_filtering(post.title)
body = c.get_compatible_markdown()
position = self._get_position(body)
date_str = post.json()["created"]
date = date_str.replace('T', ' ')
tags = "\n".join(["- {}".format(tag) for tag in c.get_tags()])
category = "\n".join(["- {}".format(c) for c in c.get_category()])
thumbnail = self._yaml_compatible(c.get_pic_url(), "")
url = c.get_url()
# build content with template
template = get_message("blog", footer=True)
content = template.format(title=title, permlink=permlink,
position=position, date=date,
tags=tags, category=category,
thumbnail=thumbnail, body=body, url=url)
# write into MD files
filename = os.path.join(folder, "{}_{}.md".format(date_str.split('T')[0], permlink))
with open(filename, "w", encoding="utf-8") as f:
f.write(content)
logger.info("Download post [{}] into file {}".format(title, filename))
def read_posts_with_limit(self, limit, last_post=None):
posts = []
if last_post:
q = Query(limit=limit, tag=self.tag,
start_author=last_post.author,
start_permlink=last_post.permlink
# before_date=str(last_post['created'])
)
else:
q = Query(limit=limit, tag=self.tag)
blogs = Discussions_by_created(q)
logger.info ('reading posts: {}'.format(len(blogs)))
c_list = {}
days_done = False
for c in blogs:
if c.permlink in c_list:
continue
if not c.is_comment():
if self.keyword is None or self.keyword in c.title:
days_done = self.days is not None and not in_recent_n_days(c, self.days)
if days_done:
break
else:
SteemComment(comment=c).log()
c_list[c.permlink] = 1
posts.append(c)
return {
"posts": posts,
"days_done": days_done
}
def wait_for_vote():
while True:
while (len(self._vote_queue) > 0):
post = self._vote_queue.pop(0)
self.vote(post)
time.sleep(1)
logger.info("Vote Queue Stopped.")
def post(self, title, body, tags, self_vote=False):
self.steem.post(title,
body,
author=self.author,
tags=tags,
self_vote=self_vote)
logger.info("Authored the post [{}] successfully".format(title))
def list_all_posts(self, folder=None):
""" list all the posts in the blog folder """
folder = folder or self.blog_folder
files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
logger.info("{} posts in blog folder {}".format(len(files), folder))
return files
def setup_source_repo(self):
git_clone_cmd = "git clone --depth 1 --branch {} --single-branch https://{}github.com/{}.git {}".format(
SOURCE_BRANCH, self._get_github_pat(),
self._get_repo(prefix=False), SOURCE_REPO_FOLDER)
os.system(git_clone_cmd)
# on `source` branch after clone
logger.info(
"Cloned source repo into workspace: {}".format(SOURCE_REPO_FOLDER))
def _has_reply_comment(self, receiver, message_id):
comments = self._read_comments()
for c in comments:
# check the receiver and the message_id fingerprint
if c.parent_author == receiver and verify_message(message_id, c.body):
logger.info("I found I replied to @{} with message [{}] by searching comment history".format(receiver, message_id))
return (True, c)
return (False, None)
def download(self, href, config):
logger.warning(self.errored)
token = href.split('=')[1]
file_name = token + '-' + str(config['height'])
file_path = DOWNLOAD_PATH + "/" + file_name + ".mp4"
if path.exists(file_path):
logger.debug('Already exists')
return file_path
if href in self.errored:
logger.warning('Was errored before', href)
return None
else:
logger.debug('Wasn\'t errored')
try:
yt = pytube.YouTube(href)
video_filter = yt.streams\
.filter(subtype='mp4') \
.filter(progressive=False)
quality = 0
for video in video_filter.all():
resolution = video.resolution
logger.debug(f"get {video.url}")
if resolution is not None:
resolution = int(video.resolution.replace('p', ''))
if resolution <= config['height'] and resolution >= quality:
quality = resolution
video_filter = video_filter.filter(resolution=str(quality) + "p")
video = video_filter.first()
logger.info("Quality: " + str(quality) + "p")
if video is None:
self.errored[href] = True
self.__save_cache__()
return None
subtype = video.subtype
print(f"Downloading {DOWNLOAD_PATH}")
video.download(
DOWNLOAD_PATH,
filename=file_name
)
return file_path
except Exception as error:
logger.error('Error handled', error)
self.errored[href] = True
self.__save_cache__()
return None
def _read(self):
if self.article is None:
try:
self.article = self.g.extract(url=self.url)
except:
logger.info(
"failed when loading article content for {}\nError: {}".
format(self.url, traceback.format_exc()))
return self.article
def refresh(self):
c = self.get_comment()
try:
c.refresh()
except ContentDoesNotExistsException:
logger.info("Failed when refresh {} with title [{}], which is probably deleted.".format(self.get_url(), c.title))
return None
return c
def refresh(self):
c = self.get_comment()
try:
c.refresh()
except ContentDoesNotExistsException:
logger.info("failed when refresh {}".format(self.get_url()))
return None
return c
def publish(self, title, body, tags):
""" publish the post """
if not self._has_published(title):
self.writer.post(title, body, tags)
logger.info("I have published the post [{}] successfully".format(title))
return True
else:
logger.info("Skip this post [{}], because I already published the post with the same title".format(title))
return False
def how_to_vote(self, post):
self.voted_posts += 1
logger.info("voting {} / {} posts".format(self.voted_posts,
self.posts_num))
weight = self.voter.estimate_vote_pct_for_n_votes(
days=CURATION_CYCLE, n=self.posts_num) * VOTE_PERCENTAGE
if weight > UPVOTE_LIMIT:
weight = UPVOTE_LIMIT
return weight
def log(self, scot=False):
if scot:
logger.info(
"@%s | %s | %s | %s" % (self.author(), self.title(),
self.get_url(), self.ops['created']))
if 'type' in self.ops and self.ops['type'] == "comment":
logger.info(
"@%s | %s | %s | %s" % (self.author(), self.title(),
self.get_url(), self.ops['timestamp']))
def publish(self):
count = len(self.roster)
if count > 0:
filename = self._get_roster_page_file()
content = self._content()
with open(filename, "w", encoding="utf-8") as f:
f.write(content)
logger.info("Published {} names into the page {}".format(
count, filename))
def send_author_rewards(self, receiver, url, percentage=50, memo=""):
logger.info(
"Sending author reward to {} for post {} with percentage {}".
format(receiver, url, percentage))
payout = self.get_post_payout(url=url)
for token, reward in payout.items():
# send author reward to receivers
paid_author_reward = float(reward['author']) * percentage / 100
self.transfer(receiver, token, paid_author_reward, memo)
def run(self):
# crawl posts
posts = self.crawl()
if len(posts) > 0:
for post in posts:
self.parse(post)
else:
logger.info("No posts are fetched.")
return self._roster_dict
def set_smart_duration(self):
if not self.account:
return
if self._source_repo_exists() and self._blog_exists():
self.days = settings.get_env_var("DURATION") or 1.5
logger.info("The download duration has been set to {} days".format(self.days))
else:
self.days = None
logger.info("The download duration has been expanded to the entire lifetime of the account")
def __init__(self, account=None, tag=None, days=None, incremental=False):
SteemReader.__init__(self, account=account, tag=tag, days=days)
self.roster = []
self._roster_dict = {}
self.public_folder = "public"
self.incremental = incremental
if self.incremental:
logger.info("launch: incremental mode")
self.days = 1.5
self.fetch_history()
def verify_message(id, body):
try:
html = markdown(body)
soup = BeautifulSoup(html, "html.parser")
msg_div = soup.find("div", {"message_id": id})
if msg_div:
return True
except:
logger.info("Failed when verifying message, with error: {}".format(traceback.format_exc()))
return False
def __init__(self):
self.author = self.by()
self.bot = VoteBot(author=self.author,
mode=self.mode(),
config=self.config())
self.voter = self.bot.voter
self.ops = None
self.ctx = {}
for k, v in self.vp_limit().items():
logger.info("set VP limit of {} to {}%".format(k, v))
self.voter.set_vp_limit(k, v)
def how_to_vote(self, post):
self.voted_posts += 1
logger.info("voting {} / {} posts".format(self.voted_posts,
self.posts_num))
weight = self.voter.estimate_vote_pct_for_n_votes(days=VOTE_CYCLE,
n=self.posts_num)
if weight > 100:
weight = 100
return weight