def add_block_content_to_db(block):
block_number = int(block['previous'][:8], base=16) + 1
# add qualifying posts, and for edited posts and comments, schedule updates
try:
for transaction in block['transactions']:
for o in transaction['operations']:
# check for votes for existing video posts, and if so schedule Steem RPC update
if o[0] == 'vote':
try:
vote = o[1]
post = db.session.query(Post) \
.filter(and_(Post.author == vote['author'],
Post.permlink == vote['permlink'])).first()
if post:
post.pending_steem_info_update = True
post.steem_info_update_requested = datetime.now()
db.session.commit()
except Exception as e:
log('Monitor error when evaluating vote...')
log(str(e))
# check for existing post (or add skeleton record if not present) and schedule updates
if o[0] == 'comment':
try:
comment = o[1]
if comment['parent_author'] == '': # if top-level post
post = db.session.query(Post) \
.filter(and_(Post.author == comment['author'],
Post.permlink == comment['permlink'])).first()
if post:
post.pending_steem_info_update = True
post.steem_info_update_requested = datetime.now()
post.pending_video_info_update = True
post.video_info_update_requested = datetime.now()
db.session.commit()
else:
video_type, video_id, category = get_valid_video(comment)
if video_type and video_id and category:
post_skeleton = Post(author=comment['author'], permlink=comment['permlink'],
category=category, video_type=video_type, video_id=video_id,
block_number=block_number)
db.session.add(post_skeleton)
db.session.commit()
except Exception as e:
log('Monitor error when evaluating comment...')
log(str(e))
except Exception as e:
log('BIG ERROR:' + str(e))
time.sleep(30)
def updateRecentPostScores(self):
try:
q = '''
update posts set
trending_score =
(pow(pending_payout_value, 0.4) * 1000000) / pow(EXTRACT(EPOCH FROM current_timestamp - created) + 300, 0.2),
hot_score =
(sqrt(pending_payout_value - least(9.99, pending_payout_value)) * 1000000) / (EXTRACT(EPOCH FROM current_timestamp - created) + 60)
where EXTRACT(EPOCH FROM current_timestamp - created) > 600
and EXTRACT(EPOCH FROM current_timestamp - created) <= 86400
'''
db.engine.execute(text(q).execution_options(autocommit=True))
except Exception as e:
log('Failed to update recent scores...')
log(str(e))
def run(self):
last_updated_recent_post_scores = datetime.now() - timedelta(
seconds=240)
last_updated_older_post_scores = datetime.now() - timedelta(
seconds=3540)
while True:
time.sleep(0.5)
try:
# update recent post scores every 15 minutes
if (datetime.now() -
last_updated_recent_post_scores).seconds > 900:
log('##### Updating recent post scores...')
self.updateRecentPostScores()
last_updated_recent_post_scores = datetime.now()
log('Updated recent post scores!')
# update older post scores every two hours
if (datetime.now() -
last_updated_older_post_scores).seconds > 7200:
log('##### Updating older post scores...')
self.updateOlderPostScores()
last_updated_older_post_scores = datetime.now()
log('Updated older post scores!')
post = db.session.query(Post) \
.filter(Post.pending_video_info_update).order_by(Post.video_info_update_requested).first()
if post:
post = self.update_steem_info(post)
if post:
post = self.update_video_info(post)
else:
post = db.session.query(Post) \
.filter(Post.pending_steem_info_update).order_by(Post.steem_info_update_requested).first()
if post:
post = self.update_steem_info(post)
else:
time.sleep(1)
except Exception as e:
log('Error in post-info-updater--run: ' + str(e))
time.sleep(20)
def update_video_info(self, post):
log(post.author + '/' + post.permlink)
try:
if post.video_type == 'youtube':
video_id = post.video_id
video_api_key = app.config['YOUTUBE_API_KEY']
# url = 'https://www.googleapis.com/youtube/v3/videos?part=snippet%2CcontentDetails%2Cstatistics%2Cstatus%2Cplayer&id=' + video_id + '&key=' + video_api_key
url = 'https://www.googleapis.com/youtube/v3/videos?part=snippet%2CcontentDetails&id=' + video_id + '&key=' + video_api_key
try:
js = requests.get(url, timeout=2).json()
except Exception as e:
log(url)
log('Problem accessing YouTube Video info for: @' +
post.author + '/' + post.permlink + '!')
time.sleep(5)
return
items = js['items']
if len(items) == 1:
item = items[0]
post.video_thumbnail_image_url = item['snippet'][
'thumbnails']['medium']['url']
post.video_duration_seconds = seconds_from_youtube_duration(
item['contentDetails']['duration'])
post.video_provider_channel_id = item['snippet'][
'channelId']
video_published = datetime.strptime(
item['snippet']['publishedAt'][:-5],
'%Y-%m-%dT%H:%M:%S')
if post.created > video_published:
post.video_post_publish_delay_seconds = (
post.created - video_published).total_seconds()
else:
post.video_post_publish_delay_seconds = 0
# todo - decide which metadata to store in DB
# post.video_info = {'snippet': item['snippet'], 'contentDetails': item['contentDetails']}
elif post.video_type == 'dtube':
try:
url = 'https://steemit.com/dtube/@' + post.author + '/' + post.permlink + '.json'
try:
js = requests.get(url, timeout=2).json()['post']
except Exception as e:
log('Problem accessing DTube Video info for: @' +
post.author + '/' + post.permlink + '!')
time.sleep(5)
return
metadata = js.get('json_metadata', '[]')
post.video_thumbnail_image_url = 'https://ipfs.io/ipfs/' + metadata[
'video']['info']['snaphash']
post.video_duration_seconds = int(
metadata['video']['info']['duration'])
post.video_provider_channel_id = ''
post.video_post_publish_delay_seconds = 0
# todo - decide which metadata to store in DB
# post.video_info = metadata
except Exception as e:
# todo - fix regex so invalid dtubes don't reach here, then remove
log('Problem updating updating dtube video info: ' +
str(e))
db.session.delete(post)
db.session.commit()
log('Assumed Invalid, and Deleted post! : @' +
post.author + '/' + post.permlink)
return
elif post.video_type == 'dlive':
try:
url = 'https://steemit.com/dlive/@' + post.author + '/' + post.permlink + '.json'
try:
js = requests.get(url, timeout=2).json()['post']
except Exception as e:
log(url)
log('Problem accessing DLive Video info for: @' +
post.author + '/' + post.permlink + '!')
time.sleep(5)
return
metadata = js.get('json_metadata', '[]')
post.video_thumbnail_image_url = metadata.get(
'thumbnail', '')
post.video_duration_seconds = -1
post.video_provider_channel_id = ''
post.video_post_publish_delay_seconds = 0
# todo - decide which metadata to store in DB
# post.video_info = metadata
except Exception as e:
# todo - fix intake filter regex so invalid dlives don't reach here, then remove
log('Problem updating updating dlive video info: ' +
str(e))
db.session.delete(post)
db.session.commit()
log('Assumed Invalid, and Deleted post! : @' +
post.author + '/' + post.permlink)
return
# todo - implement support
elif post.video_type == 'vimeo':
pass
post.pending_video_info_update = False
post.video_info_update_requested = None
db.session.commit()
except Exception as e:
log('Updating video info failed for: @' + post.author + '@' +
post.permlink + '!')
log(str(e))
db.session.delete(post)
db.session.commit()
log('Assumed Invalid, and Deleted post!')
return post
def update_steem_info(self, post):
try:
# trap http type errors and retry fetch
content = {}
while not content:
try:
content = steem.get_content(post.author, post.permlink)
except Exception as e:
log('Problem getting Steem info from API for: @' +
post.author + '/' + post.permlink + '!')
log(str(e))
time.sleep(5)
post.created = datetime.strptime(content['created'],
'%Y-%m-%dT%H:%M:%S')
post.category = content['category']
js = content.get('json_metadata', '[]')
metadata = json.loads(js)
tags = metadata.get('tags', [])
post.tags = ' '.join(tags)
post.is_nsfw = True if post.tags.lower().find(
'nsfw') >= 0 else False
post.title = content['title']
post.has_declined_payout = False if float(
content['max_accepted_payout'].split(' ')[0]) > 0 else True
post.pending_payout_value = float(
content['pending_payout_value'].split(' ')[0])
post.total_payout_value = float(
content['total_payout_value'].split(' ')[0])
post.has_paidout = True if post.total_payout_value > 0 else False
post.steem_json = content # todo - decide what of this should be stored
post.steem_thumbnail_image_url = ''
new_type, new_video_id, new_category = get_valid_video(content)
# if valid on update, use new values, otherwise assume old values remain
# this check is applied so dtube posts, edited in steemit are still retained
if new_type and new_video_id and new_category:
post.video_type, post.video_id, post.category = new_type, new_video_id, new_category
post.description = markdown_to_safe_html(content['body'])
# update experimental votes information
log('Starting vote add...')
post.votes_sparkline_data = get_sparkline_data_from_content(
content)
log(str(post.votes_sparkline_data))
post.voters_list = ' '.join(get_voters_list_from_content(content))
log(post.voters_list)
log('Done vote add!')
post.pending_steem_info_update = False
post.steem_info_update_requested = None
db.session.commit()
log('Committed vote add!')
return post
except Exception as e:
log('Problem updating Steem info for: @' + post.author + '/' +
post.permlink + '!')
log(str(e))
db.session.delete(post)
db.session.commit()
log('Assumed Invalid, and Deleted post!'
) # todo - decide whether there's a better approach to this
if (datetime.now() -
last_updated_older_post_scores).seconds > 7200:
log('##### Updating older post scores...')
self.updateOlderPostScores()
last_updated_older_post_scores = datetime.now()
log('Updated older post scores!')
post = db.session.query(Post) \
.filter(Post.pending_video_info_update).order_by(Post.video_info_update_requested).first()
if post:
post = self.update_steem_info(post)
if post:
post = self.update_video_info(post)
else:
post = db.session.query(Post) \
.filter(Post.pending_steem_info_update).order_by(Post.steem_info_update_requested).first()
if post:
post = self.update_steem_info(post)
else:
time.sleep(1)
except Exception as e:
log('Error in post-info-updater--run: ' + str(e))
time.sleep(20)
time.sleep(10)
log('Started Post Info Updater')
# start thread for updating post info
thread_1 = PostUpdateThread(db, app)
thread_1.start()
def run(self):
while True:
try:
b = Blockchain(Steem(nodes=app.config['STEEM_NODES']))
log('Using Steem API node(s): ' + str(app.config['STEEM_NODES']))
log('Blockchain head is ' + str(steem.head_block_number))
# start from max block present in table
post = db.session.query(Post).order_by(Post.id.desc()).first()
if post:
self.start_block = post.block_number
log('Starting streaming (in catch up) from block: ' + str(self.start_block))
else:
self.start_block = self.start_block
log('Starting streaming from (specified) block: ' + str(self.start_block))
for blockcount, block in enumerate(b.stream_from(start_block=self.start_block, full_blocks=True)):
block_number = self.start_block + blockcount
if (self.start_block + blockcount) % 20 == 0:
log('Read Block:' + str(block_number))
try:
add_block_content_to_db(block)
except Exception as e:
log('ERROR: Problem adding block to database.')
log(str(e))
time.sleep(10)
break
except Exception as e:
log('ERROR: Problem collecting raw blocks from ' + str(app.config['STEEM_NODES']))
log(str(e))
time.sleep(10)
if (self.start_block + blockcount) % 20 == 0:
log('Read Block:' + str(block_number))
try:
add_block_content_to_db(block)
except Exception as e:
log('ERROR: Problem adding block to database.')
log(str(e))
time.sleep(10)
break
except Exception as e:
log('ERROR: Problem collecting raw blocks from ' + str(app.config['STEEM_NODES']))
log(str(e))
time.sleep(10)
time.sleep(5)
log('Started Blockchain Monitor')
if app.config['RECREATE_DATABASE']:
log('Dropping/Recreating Database...')
try:
db.drop_all()
db.create_all()
log('Successfully dropped/recreated. Using new database...')
time.sleep(1)
except Exception as e:
log('Error Dropping/Recreating Database: ' + str(e))
else:
log('Using existing database...')
# add data from raw JSON block files
#log('Populating DB from Files...')
#populate_db_from_raw_blocks_json_files('/home/app/raw_blocks/', datetime(2017, 6, 1))