def run(self):
while True:
try:
b = Blockchain(Steem(nodes=app.config['STEEM_NODES']))
log('Using Steem API node(s): ' + str(app.config['STEEM_NODES']))
log('Blockchain head is ' + str(steem.head_block_number))
# start from max block present in table
post = db.session.query(Post).order_by(Post.id.desc()).first()
if post:
self.start_block = post.block_number
log('Starting streaming (in catch up) from block: ' + str(self.start_block))
else:
self.start_block = self.start_block
log('Starting streaming from (specified) block: ' + str(self.start_block))
for blockcount, block in enumerate(b.stream_from(start_block=self.start_block, full_blocks=True)):
block_number = self.start_block + blockcount
if (self.start_block + blockcount) % 20 == 0:
log('Read Block:' + str(block_number))
try:
add_block_content_to_db(block)
except Exception as e:
log('ERROR: Problem adding block to database.')
log(str(e))
time.sleep(10)
break
except Exception as e:
log('ERROR: Problem collecting raw blocks from ' + str(app.config['STEEM_NODES']))
log(str(e))
time.sleep(10)
def run(filename):
b = Blockchain()
# automatically resume from where we left off
# previous + last + 1
start_block = get_previous_block_num(get_last_line(filename)) + 2
upstream_blk = b.get_current_block()
checkin_count = 0
print("Our local block: {}".format(start_block))
print("The Upstream block: {}".format(b.get_current_block_num()))
print("Progress: {}%\n".format(
progress(start_block, b.get_current_block_num())))
with open(filename, 'a+') as file:
checkin_ts = time.time()
for block in b.stream_from(start_block=start_block, full_blocks=True):
if checkin_ts + 10 < time.time():
pdone = progress(int(block['block_id'][:8], base=16),
b.get_current_block_num())
dl_rate = checkin_count / 10
print("Rate: {}blk/sec, Progress: {}%".format(dl_rate, pdone))
checkin_ts = time.time()
checkin_count = 0
file.write(json.dumps(block, sort_keys=True) + '\n')
checkin_count += 1
def get_transactionid(username):
time.sleep(6)
found = False
count = 0
while found == False and count < 5:
count = count +1
print ("Attempt to get an ID number: " +str(count))
b = Blockchain()
end_block = steem.head_block_number
start_block = end_block -100
for block in b.stream_from(start_block=start_block, end_block=end_block, full_blocks=True):
transactions = block['transactions']
for transaction in transactions:
if (transaction['operations'][0][0] == "custom_json"):
if (transaction['operations'][0][1]['required_posting_auths'][0]) == user:
print (transaction)
if (transaction['operations'][0][1]['required_posting_auths'][0]) == user and (transaction['operations'][0][1]['id'])=="sm_find_match":
print (transaction['transaction_id'])
trxid = (transaction['transaction_id'])
#check if this is a valid id
r = requests.get('https://steemmonsters.com/battle/status?id=' +str(trxid))
data = r.json()
try:
status = data['status']
print ("ID Status:" +str(status))
if int(status) == 1:
found = True
return (trxid)
except:
found = False
return
def run():
blockchain = Blockchain()
head_block = blockchain.get_current_block_num()
start_block = head_block - (int(sys.argv[1]))
stream = blockchain.stream_from(start_block=start_block)
block_count = int(sys.argv[1])
current_block = start_block
stats = {}
operations = 0
counter = 0
print("Starting from block {} for {} blocks\n".format(
start_block, block_count))
for post in stream:
if post['block'] != current_block:
counter += 1
print("Block {}/{} {:.2f}%".format(counter, block_count,
counter / block_count * 100))
current_block = post['block']
elif post['block'] == start_block + block_count:
break
operation = post['op'][0]
if operation not in stats:
stats[operation] = 1
else:
stats[operation] += 1
operations += 1
print("operations {}\n".format(operations))
for operation in stats:
print(operation, stats[operation])
def run(filename):
b = Blockchain()
# automatically resume from where we left off
# previous + last + 1
start_block = get_previous_block_num(get_last_line(filename)) + 2
with open(filename, 'a+') as file:
for block in b.stream_from(start_block=start_block, full_blocks=True):
file.write(json.dumps(block, sort_keys=True) + '\n')
class Steem_node():
def __init__(self, block, block_count):
self.block = block
self.end_block = block_count + self.block - 1
self.tag = 'transfer'
self.nodes = [
'https://api.steemit.com', 'https://rpc.buildteam.io',
'https://rpc.steemviz.com'
]
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def process_transaction(self, index, block, operation):
date = block['timestamp']
to = operation['to']
user = operation['from']
amount = operation['amount']
memo = operation['memo']
db.insert_selection(self.block, index, date, to, user, amount, memo)
def run(self):
run = 1
# run until end_block
while run == 1:
try:
# stream full blocks starting at start_block
stream = self.b.stream_from(start_block=self.block,
end_block=self.end_block,
full_blocks=True)
# process each block indiviudally
for block in stream:
print('\nBlock: ', self.block)
# keep track of transaction index inside block
index = 0
# go over each transaction indivually, process if tag is
# met and update index
for transaction in block['transactions']:
if transaction['operations'][0][0] == self.tag:
self.process_transaction(
index, block, transaction['operations'][0][1])
index += 1
# Check if current block equals the end_block, break if so
# else update the current block
if self.block == self.end_block:
run = 0
else:
self.block += 1
except Exception as e:
print('Error:', e)
continue
class Steem_node():
def __init__(self, block, block_count, tag):
self.block = block
self.end_block = block_count + self.block - 1
self.tag = tag
self.nodes = [
'https://rpc.buildteam.io', 'https://api.steemit.com',
'https://rpc.steemviz.com'
]
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def process_operation(self, operation, tag, timestamp):
if tag == 'transfer':
transfer = operations.Transfer(operation, self.block, timestamp)
transfer.print_operation()
elif tag == 'transfer_to_vesting':
transfer_to_vesting = operations.Transfer_to_vesting(
operation, self.block, timestamp)
transfer_to_vesting.print_operation()
elif tag == 'withdraw_vesting':
withdraw_vesting = operations.Withdraw_vesting(
operation, self.block, timestamp)
withdraw_vesting.print_operation()
elif tag == 'convert':
convert = operations.Convert(operation, self.block, timestamp)
convert.print_operation()
def run(self):
run = 1
while run == 1:
try:
stream = self.b.stream_from(start_block=self.block,
end_block=self.end_block,
full_blocks=True)
for block in stream:
print('Block:', self.block)
transaction_index = 0
for transaction in block['transactions']:
if transaction['operations'][0][0] == self.tag:
self.process_operation(
transaction['operations'][0][1], self.tag,
block['timestamp'])
transaction_index += 1
if self.block == self.end_block:
run = 0
else:
self.block += 1
except Exception as e:
print(repr(e))
continue
async def ops_iter(blockchain:Blockchain=None, start_block:int=None):
ops_func = blockchain.stream_from(
start_block=start_block, batch_operations=False)
loop_elapsed = 0
while True:
logger.debug('new op', interval=loop_elapsed)
loop_start = time.perf_counter()
ops = await execute_sync(next, ops_func)
yield ops
loop_elapsed = time.perf_counter() - loop_start
def listen_steemd():
b = Blockchain()
h = b.stream_from(
start_block=db_last_block() + 1,
full_blocks=True,
)
for block in h:
num = int(block['previous'][:8], base=16) + 1
print("[LIVE] Got block {} at {} with {} txs".format(
num, block['timestamp'], len(block['transactions'])))
process_blocks([block])
class Steem_node():
def __init__(self, block, account):
self.block = block
self.account = account
self.nodes = [
'https://rpc.steemviz.com', 'https://rpc.buildteam.io',
'https://api.steemit.com'
]
self.steemPostingKey = os.environ.get('steemPostingKey')
self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes)
self.b = Blockchain(self.steem)
self.queue = classes.Queue(self.steem, self.account)
print('\nConnected to: {}'.format(self.nodes[0]))
def process_timestamp(self, block):
timestamp = block['timestamp']
datetime_object = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S')
if datetime_object.second == 54:
account = Account(self.account)
print(f'\nVoting power: {account.voting_power()}%\n')
if account.voting_power() == 100:
self.queue.run_voting_round()
def process_transaction(self, transaction):
if transaction['operations'][0][0] == 'transfer':
operation = transaction['operations'][0][1]
self.process_transfer(operation)
def process_transfer(self, operation):
user = operation['from']
amount = operation['amount']
memo = operation['memo']
bid = classes.Bid(user, amount, memo)
if operation['to'] == self.account:
self.queue.add_bid(bid)
def run(self):
while True:
try:
stream = self.b.stream_from(start_block=self.block,
full_blocks=True)
for block in stream:
print('Block:', self.block)
self.process_timestamp(block)
for transaction in block['transactions']:
self.process_transaction(transaction)
self.block += 1
except Exception as e:
print(repr(e))
continue
def scrape_blockchain(mongo):
s = Steem()
# see how far behind we are
missing = list(range(last_block_num(mongo), s.last_irreversible_block_num))
# if we are far behind blockchain head
# split work in chunks of 100
if len(missing) > 100:
for batch in partition_all(100, missing):
results = s.get_blocks(batch)
insert_blocks(mongo, results)
# otherwise continue as normal
blockchain = Blockchain(mode="irreversible")
hist = blockchain.stream_from(start_block=last_block_num(mongo), full_blocks=True)
insert_blocks(mongo, hist)
def run():
conn = dbconn.get_connection()
lastId = dbutil.get_last_blockId(conn)
b = Blockchain()
for block in b.stream_from(start_block=lastId + 1, full_blocks=True):
print(block['block_id'])
btxs = list(map(lambda x: x['operations'][0], block['transactions']))
for tx in btxs:
if tx[0] == 'comment' and tx[1]['parent_author'] == '':
try:
meta = json.loads(tx[1]['json_metadata'])
if tx[1]['body'].startswith("@@ ", ) == False:
article = Article()
article.author = tx[1]['author']
article.subject = tx[1]['title']
article.content = tx[1]['body']
article.tag = meta['tags']
html = mistune.markdown(article.content)
article.text = BeautifulSoup(
html, "html.parser").get_text().strip()
article.preview = article.text[0:100]
article.hashId = base64.b64encode(
md5(article.preview.encode(
"UTF-8")).digest())[0:32]
article.created = datetime.datetime.strptime(
block['timestamp'], '%Y-%m-%dT%H:%M:%S')
article.url = "https://steemkr.com/@" + article.author + "/" + tx[
1]['permlink']
if "image" in meta and len(meta['image']) > 0:
article.img = meta['image'][0]
try:
if detect(article.text) == "ko":
print(meta['tags'])
dbutil.insert_article(conn, article)
except:
pass
except:
pass
dbutil.insert_blockId(conn, block_num_from_hash(block['block_id']),
block['timestamp'])
class Steem_node():
def __init__(self, block, block_count, operation):
self.block = block
self.end_block = block_count + self.block - 1
self.operation = operation
self.nodes = ['https://rpc.buildteam.io', 'https://api.steemit.com',
'https://rpc.steemviz.com']
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def run(self):
run = 1
while run == 1:
try:
stream = self.b.stream_from(start_block=self.block,
end_block=self.end_block,
full_blocks=True)
for block in stream:
print('\nBlock: ', self.block)
transaction_index = 0
for transaction in block['transactions']:
if transaction['operations'][0][0] == self.operation:
print(transaction_index,
transaction['operations'][0][1])
transaction_index += 1
if self.block == self.end_block:
run = 0
else:
self.block += 1
except Exception as e:
continue
class Upvotebot():
def __init__(self):
self.node_index = 0
self.nodes = [
'https://api.steemit.com', 'https://rpc.buildteam.io',
'https://rpc.steemviz.com'
]
self.steemPostingKey = os.environ.get('steemPostingKey')
self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes)
self.b = Blockchain(self.steem, mode='head')
self.block = self.b.get_current_block_num()
self.upvote_list = []
self.trail_list = []
self.day = None
self.hour = None
self.timestamp = None
self.db = database.Database()
# Voting is done via Steemconnect with the steemconnect plugin from
# @emre: https://github.com/emre/steemconnect-python-client
# Set client_id and client_sercret
# Sign up at: https://v2.steemconnect.com/dashboard
#
# To perform a vote the access_token is fetched from the DB and checked
# to not be expired, if so it is renewed and stored inside the DB.
# Votes are then removed from the queue and their log is updated.
# When a vote fails it gets added to the error database for manual review.
def vote(self, voter, author, permlink, weight, type):
result = self.db.get_user_auth(voter)
access_token, refresh_token, expire_on = result[0]
dt = datetime.now()
c = Client(
client_id=CLIENT_ID,
client_secret=CLIENT_SECRET,
access_token=access_token,
)
try:
# Verify access_token
if dt > expire_on:
access_token = self.refresh_token(refresh_token)
result = c.refresh_access_token(
refresh_token,
"login,vote" # scopes
)
access_token = result['access_token']
refresh_token = result['refresh_token']
expires_in = result['expires_in']
self.db.update_authentication_tokens(
voter,
access_token,
refresh_token,
expires_in,
self.timestamp,
)
print('Updated access token\n')
# Perform vote
if not self.is_already_voted(voter, author, permlink):
vote = Vote(voter, author, permlink, weight)
result = c.broadcast([vote.to_operation_structure()])
# Log vote
if 'error' in result:
message = result['error_description']
self.db.add_to_error_log(
voter,
author,
permlink,
weight,
type,
message,
self.timestamp,
)
else:
message = 'Succes'
if type == 'vote':
self.db.update_log(
voter,
permlink,
message,
'vote_log',
)
elif type == 'trail':
self.db.update_log(
voter,
permlink,
message,
'trail_log',
)
print(f"Voter: {voter}\nAuthor: {author}\n" +
f"Permlink: {permlink}\nWeight: {weight}\n" +
"Upvote succes\n")
except Exception as error:
print(voter, author, permlink, weight, error)
self.db.add_to_error_log(
voter,
author,
permlink,
weight,
type,
error,
self.timestamp,
)
# Trails with a delay of 0 are immediatly voted upon. Else the expire_on
# time of the vote is calculated and added to the queue. The weight of
# vote is relative between the weight of the vote casted by the voter that
# is being trailed and the trailer who has set a custom relative weight.
def add_trail_to_queue(self, trail, author, permlink, weight):
for voter in self.db.get_trail_voters(trail):
voter, voter_weight, limit, delay = voter
vote_log = self.db.get_trail_log(voter, trail, self.timestamp)
upvote_weight = weight / 100 * voter_weight / 100
print(f"\nTrail: {trail}\nVoter: {voter}\nAuthor: {author}\n" +
f"Weight: {upvote_weight}\nLimit: {limit}\nDelay: {delay}\n")
if delay != 0:
self.db.add_trail_to_queue(
trail,
author,
voter,
upvote_weight,
limit,
delay,
permlink,
self.timestamp,
)
elif len(vote_log) < limit:
self.db.add_to_trail_log(
author,
voter,
permlink,
upvote_weight,
self.timestamp,
)
self.vote(voter, author, permlink, weight, type)
# The time for when a vote is to be made is calculated from the current
# block time + the delay. If the delay is 0 an immediate vote is performed.
# Votes get added to the queue and log
def add_to_queue(self, author, permlink):
for vote in self.db.get_votes(author):
voter, weight, limit, delay = vote
vote_log = self.db.get_vote_log(voter, author, self.timestamp)
print(f"\nVoter: {voter}\nWeight: {weight}\nLimit: " +
f"{limit}\nDelay: {delay}")
if delay != 0:
self.db.add_to_queue(
author,
voter,
weight,
limit,
delay,
permlink,
self.timestamp,
)
elif len(vote_log) < limit:
self.db.add_to_vote_log(
author,
voter,
permlink,
weight,
self.timestamp,
)
self.vote(voter, author, permlink, weight, type)
# Go thourgh all the active votes and look if the voter has already
# voted on this post.
def is_already_voted(self, voter, author, permlink):
try:
identifier = f"@{author}/{permlink}"
post = Post(identifier, self.steem)
active_votes = post['active_votes']
for vote in active_votes:
if vote['voter'] == voter:
return 1
return 0
except Exception as error:
return 0
# Posts are verified to be the initial post and not an update
def is_new_post(self, author, permlink):
identifier = f"@{author}/{permlink}"
post = Post(identifier, self.steem)
created = post['created']
last_update = post['last_update']
if created != last_update:
return 0
else:
return 1
# Comment is checked to be of the post type and not a comment to a post.
# Posts are then added to queue
def process_comment(self, comment):
parent_author = comment['parent_author']
author = comment['author']
if parent_author == '' and author in self.upvote_list:
permlink = comment['permlink']
if self.is_new_post(author, permlink):
print(f'\n\n{self.timestamp} Block: {self.block}')
print(f"New post\nAuthor: {author}")
self.add_to_queue(author, permlink)
# Votes are checked for their voter and if the voter is being trailed
def process_vote(self, vote):
trail = vote['voter']
if trail in self.trail_list:
author = vote['author']
permlink = vote['permlink']
weight = vote['weight']
print(f"\n\nTrail: {trail}\nAuthor: {author}\nPermlink: " +
f"{permlink}\nWeight: {weight}\n")
self.add_trail_to_queue(trail, author, permlink, weight)
# Each block the queue is fetched and checked for votes that have reached
# their maturity time.
def process_queue(self):
dt = datetime.strptime(self.timestamp, '%Y-%m-%dT%H:%M:%S')
result = self.db.get_queue()
if len(result) > 0:
for vote in result:
id, voter, author, permlink, weight, type, expire_on = vote
if dt > expire_on:
self.vote(voter, author, permlink, weight, type)
self.db.remove_from_queue(id)
# Stream block indefinitely, process each block for comments
# Stream is set to head mode
def run(self):
stream = self.b.stream_from(
start_block=self.block,
full_blocks=True,
)
while True:
try:
for block in stream:
# Set block time and process queue
self.timestamp = block['timestamp']
print(f'{self.timestamp} Block: {self.block}', end='\r')
self.process_queue()
# Fetch author to be upvoted list
self.upvote_list = []
for author in self.db.get_authors():
author = re.sub(
'[^A-Za-z0-9^-]+',
'',
author[0].lower(),
)
self.upvote_list.append(author)
# Fetch trails to be tracked
self.trail_list = []
for trail in self.db.get_trails():
trail = re.sub(
'[^A-Za-z0-9^-]+',
'',
trail[0].lower(),
)
self.trail_list.append(trail)
# Look at each single transaxtion
for transaction in block['transactions']:
transaction_type = transaction['operations'][0][0]
if transaction_type == 'comment':
comment = transaction['operations'][0][1]
self.process_comment(comment)
elif transaction_type == 'vote':
vote = transaction['operations'][0][1]
self.process_vote(vote)
self.block += 1
except Exception as e:
print(e)
# common exception for api.steemit.com
# when the head block is reached
s = ("TypeError('an integer is"
" required (got type NoneType)',)")
if repr(e) == s:
time.sleep(1)
else:
stream = self.b.stream_from(
start_block=self.block,
full_blocks=True,
)
continue
class Steem_node():
def __init__(self, acount, number):
self.tag = 'vote'
self.account = account
self.number = number
self.nodes = [
'https://api.steemit.com', 'https://rpc.buildteam.io',
'https://rpc.steemviz.com'
]
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
self.timestamp = None
self.block = self.b.get_current_block_num()
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def send_sms(self, memo):
print(twilio_api.send_sms(self.number, memo))
# Convert rshares to dollar value
# Author: emrebeyler
# Source: https://goo.gl/dV7yVa
def convert_rshares(self, rshares):
reward_fund = self.steem.get_reward_fund()
reward_balance, recent_claims = (reward_fund["reward_balance"],
reward_fund["recent_claims"])
base_price = self.steem.get_current_median_history_price()["base"]
fund_per_share = Amount(reward_balance).amount / float(recent_claims)
payout = float(rshares) * fund_per_share * Amount(base_price).amount
return payout
def process_vote(self, vote):
print(vote)
voter = vote['voter']
author = vote['author']
permlink = vote['permlink']
weight = vote['weight']
# Verify that utopian-io is the voter and that is votes on account
if voter == 'utopian-io' and author == account:
print(f"\nVoter: {voter}\nAuthor: {author}\nPermlink: {permlink}" +
f"\nWeight: {weight}\n{self.timestamp}\n")
# Construct post from the permlink to retrieve vote rshares
post = Post(f'@{self.account}/{permlink}', self.steem)
# Look for utopian in active_votes
active_votes = post['active_votes']
for vote in active_votes:
if vote['voter'] == 'utopian-io':
payout = self.convert_rshares(vote['rshares'])
# Construct sms
url = f"http://steemit.com/@{self.account}/{permlink}"
body = (f"Upvote from Utopian-io for ${payout:.2f}\n" +
f"\n{url}\n\nBlock: {self.block}" + f"\n{self.timestamp}")
self.send_sms(body)
def run(self):
print(self.block)
while True:
try:
# stream full blocks
stream = self.b.stream_from(start_block=self.block,
full_blocks=True)
# process each block indiviudally
for block in stream:
self.timestamp = block['timestamp']
print(f'{self.timestamp} Block: {self.block}', end='\r')
# go over each transaction indivually, process if tag is
# met and update index
for transaction in block['transactions']:
if transaction['operations'][0][0] == self.tag:
transfer = transaction['operations'][0][1]
self.process_vote(transfer)
self.block += 1
except Exception as e:
# common exception for api.steemit.com
# when the head block is reached
s = ("TypeError('an integer is"
" required (got type NoneType)',)")
if repr(e) == s:
time.sleep(3)
class FunfunPosting(object):
def __init__(self):
self.steem = Steem()
self.blockchain = Blockchain()
self.account = config.ACCOUNT['user']
connObj = DBActions()
self.conn = connObj.getConnection()
def _getLastBlockId(self):
cursor = self.conn.cursor()
query = """
SELECT max(last_block) as latest FROM last_block
"""
cursor.execute(query)
ret = cursor.fetchone()
if ret is None or len(ret) < 1:
return 1
return ret['latest']
def _updateLastBlockId(self, block_no):
cursor = self.conn.cursor()
query = """
UPDATE last_block SET last_block = %s WHERE id = 1
"""
try:
cursor.execute(query, (block_no, ))
except:
traceback.print_exc()
self.conn.rollback()
return False
return True
def _getUserId(self, user_text_id):
cursor = self.conn.cursor()
query = """
SELECT id FROM users
WHERE steemid = %s
LIMIT 1
"""
cursor.execute(query, (user_text_id, ))
ret = cursor.fetchone()
if ret is None or len(ret) < 1:
return -1
return ret['id']
def findUser(self, user_text_id):
cursor = self.conn.cursor()
query = """
SELECT count(*) as cnt FROM users
WHERE steemid = %s
"""
cursor.execute(query, (user_text_id, ))
ret = cursor.fetchone()
if ret is None or len(ret) < 1:
return False
return True if ret['cnt'] == 1 else False
def addUser(self, user_text_id):
cursor = self.conn.cursor()
query = """
INSERT INTO users
(steemid, added_at)
VALUES
(%s, CURRENT_TIMESTAMP)
"""
try:
cursor.execute(query, (user_text_id, ))
except:
traceback.print_exc()
self.conn.rollback()
return False
self.conn.commit()
return True
def deleteUser(self, user_text_id):
cursor = self.conn.cursor()
query = """
DELETE FROM users
WHERE steemid = %s
"""
try:
cursor.execute(query, (user_text_id, ))
except:
traceback.print_exc()
self.conn.rollback()
return False
self.conn.commit()
return True
def getUserLists(self):
cursor = self.conn.cursor()
query = """
SELECT steemid FROM users
"""
cursor.execute(query)
ret = cursor.fetchall()
if ret is None or len(ret) < 1:
return []
result = ['@{}'.format(item['steemid']) for item in ret]
return result
def getThemeLists(self):
cursor = self.conn.cursor()
query = """
SELECT * FROM theme
"""
cursor.execute(query)
ret = cursor.fetchall()
if ret is None or len(ret) < 1:
return []
result = [{
"id": item['id'],
"theme_name": item['theme_name'],
"added_time": item['added_time']
} for item in ret]
return result
def addTheme(self, theme_name):
cursor = self.conn.cursor()
query = """
INSERT INTO theme
(theme_name, added_time)
VALUES
(%s, CURRENT_TIMESTAMP)
"""
try:
cursor.execute(query, (theme_name, ))
except:
traceback.print_exc()
self.conn.rollback()
return False
self.conn.commit()
return True
def modifyTheme(self, theme_id, theme_name):
cursor = self.conn.cursor()
query = """
UPDATE theme
SET theme_name = %s
WHERE id = %s
"""
try:
cursor.execute(query, (
theme_name,
theme_id,
))
except:
traceback.print_exc()
self.conn.rollback()
return False
self.conn.commit()
return True
def deleteTheme(self, theme_name):
cursor = self.conn.cursor()
query = """
DELETE FROM theme
WHERE theme_name = %s
"""
try:
cursor.execute(query, (theme_name, ))
except:
traceback.print_exc()
self.conn.rollback()
return False
self.conn.commit()
return True
def _findThemeId(self, theme_name):
cursor = self.conn.cursor()
query = """
SELECT id FROM theme
WHERE theme_name = %s
LIMIT 1
"""
cursor.execute(query, (theme_name, ))
ret = cursor.fetchone()
if ret is None or len(ret) < 1:
return -1
return ret['id']
def _insertPostInfo(self,
user_id,
theme_id,
user_text_id,
permlink,
tags,
written_at,
is_regular=False):
cursor = self.conn.cursor()
query = """
INSERT INTO posts
(user_id, theme_id, user_text_id, permlink, tags, written_at, is_voted, is_regular)
VALUES
(%s, %s, %s, %s, %s, %s, 0, %s)
"""
try:
cursor.execute(query, (
user_id,
theme_id,
user_text_id,
permlink,
tags,
written_at,
is_regular,
))
except pymysql.err.IntegrityError:
print("Duplicated or revised post")
return False, 1
except:
traceback.print_exc()
self.conn.rollback()
return False, 2
self.conn.commit()
return True, 0
def _voting(self,
user_text_id,
permlink,
is_regular=False,
is_voted=False):
if is_voted == True:
return True
cursor = self.conn.cursor()
query = """
UPDATE posts
SET is_voted = 1
WHERE user_text_id = %s
AND permlink = %s
"""
try:
cursor.execute(query, (
user_text_id,
permlink,
))
except:
traceback.print_exc()
self.conn.rollback()
return False
## VOTE
post = "@{}/{}".format(user_text_id, permlink)
try:
"""
{
"voter": "badfatcat",
"author": "claudene",
"permlink": "update-the-new-coins-in-the-philippines-the-p10-pesos-and-p5-pesos-2018-20171231t62751604z",
"weight": 10000,
"_id": "1af544c59ea75a816ed756578105d9befa442224",
"type": "vote",
"timestamp": "2018-01-15 14:32:33",
"block_num": 19001685,
"trx_id": "1d0490ac6aea885b373a5dc0f1bebb22d9f9fd96"
}
"""
rate = 50
if is_regular == True:
rate = 100
ret = self.steem.vote(post, rate, "funfund")
except steembase.exceptions.RPCError:
print("Already voted!")
except:
traceback.print_exc()
self.conn.rollback()
return False
query = """
UPDATE posts
SET is_voted = 2, voted_at = CURRENT_TIMESTAMP
WHERE user_text_id = %s
AND permlink = %s
"""
try:
cursor.execute(query, (
user_text_id,
permlink,
))
except:
traceback.print_exc()
self.conn.rollback()
return False
self.conn.commit()
time.sleep(21)
return True
def filterPosts(self, post_arr):
"""
{
'signatures':[
'1f2e21012c8ce7f64275b2144bddbd114c8d918dd9ba031274496bf7144c968af1443f2a99efe47ad8b609357ef157e14cc692a5e8b8d52da83d84aa5f3d25b795'
],
'expiration':'2018-06-27T16:50:33',
'transaction_num':24,
'extensions':[
],
'operations':[
[
'comment',
{
'body':'[](https://dlive.io/livestream/gregoryarts/cf6aaf79-7a28-11e8-9907-0242ac110003)\n\nMinecraft 1.12.2\n\nnotifications thanks :) https://zygmunt.pro/apps/dcr/ \n\nhttps://tipanddonation.com/gregoryarts \n\nMój sprzęt \n\nProcesor AMD FX-8320 8 x 4,5 Ghz \nPłyta główna MSI 970 Gaming (MS-7693) \nPamięć ram HyperX Savage 8GB 2400MHz DDR3 CL11 (2x4GB) \nGrafika Gigabyte GeForce GTX 1050 Ti OC 4GB GDDR5 \nZasilacz Radix ECO III 650W Obudowa z okienkiem ;) (projekt własny - filmy z jej tworzeniem dostępne na kanale d-tube)\n\nMy live stream is at [DLive](https://dlive.io/livestream/gregoryarts/cf6aaf79-7a28-11e8-9907-0242ac110003)',
'json_metadata':'{"tags":["dlive","dlive-broadcast","game","minecraft","gamplay","mods","polish"],"app":"dlive/0.1","format":"markdown","language":"en","thumbnail":"https://images.dlive.io/cf1db880-7a28-11e8-9118-0242ac110002"}',
'title':'[PL] Minecraft - granie na modach #001',
'author':'gregoryarts',
'parent_author':'',
'permlink':'cf6aaf79-7a28-11e8-9907-0242ac110003',
'parent_permlink':'dlive'
}
]
],
'ref_block_num':35368,
'transaction_id':'b02bd95c0a07850ceb606771e09ea733679dbf25',
'block_num':23693882,
'ref_block_prefix':4217244095
}
"""
filtered_result = []
for item in post_arr:
try:
operations = item.get('operations')
if operations == None:
print("An important info is null")
continue
content_obj = operations[0][1]
parent_author = content_obj.get('parent_author')
try:
json_metadata = json.loads(content_obj['json_metadata'])
tags = json_metadata.get('tags', [])
except:
continue
author = content_obj.get('author')
original_theme = content_obj.get('title')
permlink = content_obj.get('permlink')
written_at = item.get('expiration')
if author == None or original_theme == None or permlink == None or parent_author == None:
print("@{}/{}: An important info is null".format(
author, permlink))
continue
except:
traceback.print_exc()
print(content_obj)
print("Error in TX!!")
sys.exit(1)
# Tag check
if "kr-funfun" not in tags:
print("@{}/{}: No 'kr-funfun' tag in the post".format(
author, permlink))
continue
# User check
user_id = self._getUserId(author)
if user_id == -1:
print("@{}/{}: User {} is not member of kr-funfun".format(
author, permlink, author))
continue
# Theme check
is_regular = False
extract_theme_obj = re.search(r':(.*?)\]', original_theme)
theme_id = -1
if extract_theme_obj != None:
theme = extract_theme_obj.group(1).strip()
theme_id = self._findThemeId(theme)
if theme_id == -1:
is_regular = False
else:
is_regular = True
string_tags = ','.join(tags)
is_ok, code = self._insertPostInfo(user_id, theme_id, author,
permlink, string_tags,
written_at, is_regular)
is_voted = False
if is_ok == False and code >= 2:
self.conn.rollback()
print("Something wrong in writing funfun posts")
sys.exit(1)
elif is_ok == False and code == 1:
self.conn.rollback()
is_voted = True
else:
is_voted = False
filtered_result.append({
"user_text_id": author,
"permlink": permlink,
"is_regular": is_regular,
"is_voted": is_voted
})
self.conn.commit()
return filtered_result
def main(self):
last_block_id = self._getLastBlockId()
# Block streaming..
for block in self.blockchain.stream_from(start_block=last_block_id,
full_blocks=True):
for tx in block['transactions']:
ret = []
tx_type = tx['operations'][0][0]
title = tx['operations'][0][1].get('title')
if tx_type == 'comment' and title != '' and title != None:
ret.append(tx)
last_block_id = tx['block_num']
filtered_posts = self.filterPosts(ret)
if len(filtered_posts) > 0:
print("Gathered {} kr-funfun posts!".format(
len(filtered_posts)))
for item in filtered_posts:
print("Voting @{}/{}..".format(item['user_text_id'],
item['permlink']))
is_ok = self._voting(item['user_text_id'],
item['permlink'], item['is_regular'],
item['is_voted'])
if is_ok == False:
print("Something wrong in @{}/{}".format(
item['user_text_id'], item['permlink']))
self.conn.rollback()
sys.exit(1)
self.conn.commit()
self._updateLastBlockId(last_block_id)
self.conn.commit()
'interval',
seconds=15,
id='process_vote_queue')
scheduler.add_job(process_rewards_pools,
'interval',
minutes=10,
id='process_rewards_pools')
scheduler.start()
quick = False
# last_save_height_time = time.time()
last_save_block_processed = last_block_processed
while True:
for block in b.stream_from(start_block=last_block_processed,
batch_operations=False,
full_blocks=True):
if (len(block) > 0):
block_num = block_num_from_hash(block['block_id'])
timestamp = block['timestamp']
# If behind by more than X (for initial indexes), set quick to true to prevent unneeded past operations
remaining_blocks = props[
'last_irreversible_block_num'] - block_num
if remaining_blocks > quick_value:
quick = True
dt = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S')
l('----------------------------------')
l('#{} - {} - {} ops ({} remaining|quick:{})'.format(
block_num, dt, len(block['transactions']),
remaining_blocks, quick))
for idx, tx in enumerate(block['transactions']):
class YoBlockchainFollower(YoBaseService):
service_name = 'blockchain_follower'
def __init__(self, **kwargs):
super().__init__(**kwargs)
# init blockchain
steemd_url = self.yo_app.config.steemd_url
self.steemd = steem.steemd.Steemd(nodes=[steemd_url])
self.blockchain = Blockchain(steemd_instance=self.steemd)
self.start_block = self.yo_app.config.steemd_start_block
self.last_block_num_handled = 0
# init ops handlers
self.op_map = defaultdict(list)
self.op_map.update({
'vote': [self.handle_vote],
'account_update': [self.handle_account_update],
'transfer': [self.handle_send, self.handle_receive],
'custom_json': [self.handle_follow, self.handle_resteem],
'withdraw_vesting': [self.handle_power_down],
'comment': [self.handle_mention, self.handle_comment]
})
start_block = self.get_start_block()
self.ops_func = self.blockchain.stream_from(
start_block=start_block, batch_operations=True)
self.ops = lambda: self.execute_sync(next, self.ops_func)
def get_start_block(self):
start_block = None
try:
if isinstance(self.start_block, int) and self.start_block < 0:
start_block = self.blockchain.get_current_block_num() - self.start_block
except Exception:
self.log.exception('service error')
start_block = None
self.log.debug('get_start_block', start_block=start_block)
return start_block
async def store_notification(self, notifications):
return await self.db.create_notifications(notifications)
# pylint gets confused about these for some reason
# pylint: disable=no-member
async def handle_vote(self, op):
vote_info = op['op'][1]
self.log.info(
'handle_vote',
permlink=vote_info['permlink'],
author=vote_info['author'],
voter=vote_info['voter'],
weight=vote_info['weight'])
return [
dict(
trx_id=op['trx_id'],
from_username=vote_info['voter'],
to_username=vote_info['author'],
json_data=json.dumps(vote_info),
notify_type=VOTE,
priority_level=Priority.LOW.value)
]
async def handle_follow(self, op):
if op['op'][1]['id'] != 'follow':
self.log.debug('handle_follow noop')
return []
op_data = op['op'][1]
follow_data = json.loads(op_data['json'])
if follow_data[0] != 'follow':
return []
follower = follow_data[1]['follower']
following = follow_data[1]['following']
if len(op_data['required_posting_auths']) != 1:
self.log.error('inavlid follow op, got %d posting auths, expected 1',
op_data['required_posting_auths'])
return []
if op_data['required_posting_auths'][0] != follower:
self.log.error('invalid follow op, follower must be signer')
return []
self.log.debug('handle_follow', follower=follower, following=following)
return [
dict(
trx_id=op['trx_id'],
from_username=follower,
to_username=following,
json_data=json.dumps(follow_data[1]),
notify_type=FOLLOW,
priority_level=Priority.LOW.value)
]
async def handle_account_update(self, op):
op_data = op['op'][1]
self.log.debug('handle_account_update', account=op_data['account'])
return [
dict(
trx_id=op['trx_id'],
to_username=op_data['account'],
json_data=json.dumps(op_data),
notify_type=ACCOUNT_UPDATE,
priority_level=Priority.LOW.value)
]
async def handle_send(self, op):
op_data = op['op'][1]
send_data = {
'amount': op_data['amount'],
'from': op_data['from'],
'memo': op_data['memo'],
'to': op_data['to'],
}
self.log.debug(
'handle_send',
_from=send_data['from'],
amount=send_data['amount'],
to=send_data['to'])
return [
dict(
trx_id=op['trx_id'],
to_username=send_data['from'],
json_data=json.dumps(send_data),
notify_type=SEND_STEEM,
priority_level=Priority.LOW.value)
]
async def handle_receive(self, op):
op_data = op['op'][1]
receive_data = {
'amount': op_data['amount'],
'from': op_data['from'],
'memo': op_data['memo'],
'to': op_data['to'],
}
self.log.debug(
'handle_receive',
to=receive_data['to'],
amount=receive_data['amount'],
_from=receive_data['from'])
return [
dict(
trx_id=op['trx_id'],
to_username=receive_data['to'],
from_username=receive_data['from'],
json_data=json.dumps(receive_data),
notify_type=RECEIVE_STEEM,
priority_level=Priority.LOW.value)
]
async def handle_power_down(self, op):
op_data = op['op'][1]
self.log.debug(
'handle_power_down',
account=op_data['account'],
amount=op_data['vesting_shares'])
return [
dict(
trx_id=op['trx_id'],
to_username=op_data['account'],
json_data=json.dumps(op_data),
notify_type=POWER_DOWN,
priority_level=Priority.LOW.value)
]
async def handle_mention(self, op):
comment_data = op['op'][1]
haystack = comment_data['body'] + '\n'
data = {
'author': comment_data['author'],
'permlink': comment_data['permlink'],
}
notifications = []
for match in set(re.findall(MENTION_PATTERN, haystack)):
self.log.debug('handle_mention', author=data['author'], mentioned=match)
notifications.append(
dict(
trx_id=op['trx_id'],
to_username=match,
from_username=data['author'],
json_data=json.dumps(data),
notify_type=MENTION,
priority_level=Priority.LOW.value))
return notifications
async def handle_comment(self, op):
self.log.debug('handle_comment', op=['op'][0])
op_data = op['op'][1]
if op_data['parent_author'] == '':
# top level post
return []
parent_id = '@' + op_data['parent_author'] + '/' + op_data['parent_permlink']
parent = steem.post.Post(parent_id)
note_type = COMMENT_REPLY if parent.is_comment() else POST_REPLY
self.log.debug(
'handle_comment',
note_type=note_type,
author=op_data['author'],
parent_id=parent_id)
return [
dict(
trx_id=op['trx_id'],
to_username=op_data['parent_author'],
from_username=op_data['author'],
json_data=json.dumps(op_data),
notify_type=note_type,
priority_level=Priority.LOW.value)
]
async def handle_resteem(self, op):
op_data = op['op'][1]
resteem_data = json.loads(op_data['json'])
if resteem_data[0] != 'reblog':
self.log.debug('handle_resteem noop')
return []
account = resteem_data[1]['account']
author = resteem_data[1]['author']
permlink = resteem_data[1]['permlink']
if len(op_data['required_posting_auths']) != 1:
self.log.error('inavlid resteem op, got %d posting auths, expected 1',
op_data['required_posting_auths'])
return []
if op_data['required_posting_auths'][0] != account:
self.log.error('invalid resteem op, account must be signer')
return []
self.log.debug(
'handle_resteem', account=account, author=author, permlink=permlink)
return [
dict(
trx_id=op['trx_id'],
from_username=account,
to_username=author,
json_data=json.dumps(resteem_data[1]),
notify_type=RESTEEM,
priority_level=Priority.LOW.value)
]
# pylint: enable=no-member
async def notify(self, blockchain_op):
""" Handle notification for a particular op
"""
op_type = blockchain_op['op'][0]
futures = [handler(blockchain_op) for handler in self.op_map[op_type]]
if futures:
self.log.debug(
'operation triggering handlers', op_type=op_type, handlers=futures)
return await asyncio.gather(*futures)
else:
self.log.debug('skipping operation', op_type=op_type)
return []
async def ops_iter(self):
start_block = self.get_start_block()
ops_func = self.blockchain.stream_from(
start_block=start_block, batch_operations=True)
while True:
ops = await self.execute_sync(next, ops_func)
yield ops
async def main_task(self):
self.log.debug('main task executed')
async for ops in self.ops_iter():
block_num = ops[0]['block']
self.log.debug('main task', op_in_block=len(ops), block_num=block_num)
unstored_notifications = await asyncio.gather(
*[self.notify(op) for op in ops])
combined_notifications = list(
itertools.chain(*itertools.chain(*unstored_notifications)))
logger.debug(
'main_task',
block_num=block_num,
op_count=len(ops),
unstored_count=sum(map(len, unstored_notifications)),
combined_notifications=len(combined_notifications))
resp = await self.store_notification(combined_notifications)
if resp:
self.last_block_num_handled = block_num
class Upvotebot():
def __init__(self, account):
self.block = None
self.node_index = 0
self.list = [
'https://api.steemit.com', 'https://rpc.buildteam.io',
'https://rpc.steemviz.com'
]
self.nodes = [self.list[self.node_index]]
self.steemPostingKey = os.environ.get('steemPostingKey')
self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes)
self.b = Blockchain(self.steem)
self.account = account
self.upvote_list = json.load(open('upvote_list.json'))
self.day = None
self.hour = None
self.queue = classes.Queue(account, self.steem)
print('Running photobot \nConnected to: {}'.format(self.nodes[0]))
# set block times
def set_times(self, datetime_object):
self.day = datetime_object.day
self.hour = datetime_object.hour
print('Block times set')
# process block time
def time_handler(self, datetime_object):
if not self.day and not self.hour:
self.set_times(datetime_object)
day = datetime_object.day
hour = datetime_object.hour
# clear memory for a new day
if day is not self.day:
print("\nNew day: cleared memory\n")
self.queue.reset()
self.day = day
# reload the upvote_list every hour
if hour is not self.hour:
print("\nNew hour: reloaded upvote list\n")
self.upvote_list = json.load(open('upvote_list.json'))
self.hour = hour
# check if the operation is a post and not a comment, also check
# if the author is in the upvote list.
def process_operation(self, operation, timestamp):
parent_author = operation['operations'][0][1]['parent_author']
if parent_author == '':
author = operation['operations'][0][1]['author']
permlink = operation['operations'][0][1]['permlink']
identifier = '@{}/{}'.format(author, permlink)
if author in self.upvote_list:
self.queue.add_to_queue(identifier, timestamp)
# in case one of the nodes fail switch over to the next node
def reconfigure_node(self):
self.node_index = (self.node_index + 1) % len(self.list)
self.nodes = [self.list[self.node_index]]
self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes)
self.b = Blockchain(self.steem)
self.queue.steem = self.steem
print('New node: {}\n'.format(self.nodes))
# Stream block indefinitely, process each transaction in each block
def run(self, block_num, upvote_age):
# filter command line arguments
if block_num == 'head':
self.block = self.blockchain.get_current_block_num()
else:
self.block = int(block_num)
# set and convert upvote age to seconds
self.queue.upvote_age = upvote_age * 60
while True:
try:
stream = self.b.stream_from(start_block=self.block,
full_blocks=True)
for block in stream:
# process the timestamp
timestamp = block['timestamp']
print(self.block, timestamp)
self.queue.process_queue(timestamp)
datetime_object = datetime.strptime(
timestamp, '%Y-%m-%dT%H:%M:%S')
self.time_handler(datetime_object)
# go over each transaction and process comments
for operation in block['transactions']:
operation_type = operation['operations'][0][0]
if operation_type == 'comment':
self.process_operation(operation, timestamp)
self.block += 1
except Exception as e:
# common exception for api.steemit.com
# when the head block is reached
s = ("TypeError('an integer is"
" required (got type NoneType)',)")
if repr(e) == s:
time.sleep(3)
else:
print(self.block, repr(e))
self.reconfigure_node()
continue
process_platform_history()
process_global_props()
process_rewards_pools()
rebuild_forums_cache()
rebuild_bots_cache()
scheduler = BackgroundScheduler()
scheduler.add_job(process_global_props, 'interval', seconds=9, id='process_global_props')
scheduler.add_job(rebuild_forums_cache, 'interval', minutes=1, id='rebuild_forums_cache')
scheduler.add_job(rebuild_bots_cache, 'interval', minutes=1, id='rebuild_bots_cache')
scheduler.add_job(process_vote_queue, 'interval', seconds=15, id='process_vote_queue')
scheduler.add_job(process_rewards_pools, 'interval', minutes=10, id='process_rewards_pools')
scheduler.start()
quick = False
for block in b.stream_from(start_block=last_block_processed, full_blocks=True):
if(len(block['transactions']) > 0):
block_num = block_num_from_hash(block['block_id'])
timestamp = block['timestamp']
# If behind by more than X (for initial indexes), set quick to true to prevent unneeded past operations
remaining_blocks = props['last_irreversible_block_num'] - block_num
if remaining_blocks > quick_value:
quick = True
dt = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S')
l('----------------------------------')
l('#{} - {} - {} ops ({} remaining|quick:{})'.format(block_num,
dt, len(block['transactions']), remaining_blocks, quick))
for idx, tx in enumerate(block['transactions']):
txid = block['transaction_ids'][idx]
# Is this a group of ops for the forum?
if len(tx['operations']) > 1:
