def __init__(self, **kwargs):
super().__init__(**kwargs)
# init blockchain
steemd_url = self.yo_app.config.steemd_url
self.steemd = steem.steemd.Steemd(nodes=[steemd_url])
self.blockchain = Blockchain(steemd_instance=self.steemd)
self.start_block = self.yo_app.config.steemd_start_block
self.last_block_num_handled = 0
# init ops handlers
self.op_map = defaultdict(list)
self.op_map.update({
'vote': [self.handle_vote],
'account_update': [self.handle_account_update],
'transfer': [self.handle_send, self.handle_receive],
'custom_json': [self.handle_follow, self.handle_resteem],
'withdraw_vesting': [self.handle_power_down],
'comment': [self.handle_mention, self.handle_comment]
})
start_block = self.get_start_block()
self.ops_func = self.blockchain.stream_from(
start_block=start_block, batch_operations=True)
self.ops = lambda: self.execute_sync(next, self.ops_func)
def run(self):
while True:
try:
b = Blockchain(Steem(nodes=app.config['STEEM_NODES']))
log('Using Steem API node(s): ' + str(app.config['STEEM_NODES']))
log('Blockchain head is ' + str(steem.head_block_number))
# start from max block present in table
post = db.session.query(Post).order_by(Post.id.desc()).first()
if post:
self.start_block = post.block_number
log('Starting streaming (in catch up) from block: ' + str(self.start_block))
else:
self.start_block = self.start_block
log('Starting streaming from (specified) block: ' + str(self.start_block))
for blockcount, block in enumerate(b.stream_from(start_block=self.start_block, full_blocks=True)):
block_number = self.start_block + blockcount
if (self.start_block + blockcount) % 20 == 0:
log('Read Block:' + str(block_number))
try:
add_block_content_to_db(block)
except Exception as e:
log('ERROR: Problem adding block to database.')
log(str(e))
time.sleep(10)
break
except Exception as e:
log('ERROR: Problem collecting raw blocks from ' + str(app.config['STEEM_NODES']))
log(str(e))
time.sleep(10)
def scrape_operations(mongo):
"""Fetch all operations (including virtual) from last known block forward."""
indexer = Indexer(mongo)
last_block = indexer.get_checkpoint('operations')
log.info('\n> Fetching operations, starting with block %d...' % last_block)
blockchain = Blockchain(mode="irreversible")
history = blockchain.history(
start_block=last_block,
)
for operation in history:
# insert operation
with suppress(DuplicateKeyError):
transform = compose(strip_dot_from_keys, json_expand, typify)
mongo.Operations.insert_one(transform(operation))
# if this is a new block, checkpoint it, and schedule batch processing
if operation['block_num'] != last_block:
last_block = operation['block_num']
indexer.set_checkpoint('operations', last_block - 1)
if last_block % 10 == 0:
log.info("Checkpoint: %s (%s)" % (
last_block,
blockchain.steem.hostname
))
def stream():
REGEX = "(?<=^|(?<=[^a-zA-Z0-9-_\.]))!([A-Za-z]+[A-Za-z0-9]+)" # REGEX code for '!' marker
blockchain = Blockchain() # blockchain instance
stream = map(
Post, blockchain.stream(filter_by=['comment'])) # stream comments only
print('\033[37mSearching...')
for post in stream:
curator = post['author']
p_parent = post['parent_permlink']
a_parent = post['parent_author']
f = regex.findall(REGEX, post['body']) # REGEX searches for !'flag'
# filters
if post.is_comment():
if curator in curators:
if flag in f:
if p_parent in log:
continue
else:
# start action thread
acaoThread = threading.Thread(target=action,
args=(
a_parent,
p_parent,
curator,
))
acaoThread.start()
def validate_operations(mongo):
""" Scan latest N blocks in the database and validate its operations. """
blockchain = Blockchain(mode="irreversible")
highest_block = mongo.Operations.find_one({}, sort=[('block_num', -1)
])['block_num']
lowest_block = max(1, highest_block - 250_000)
for block_num in range(highest_block, lowest_block, -1):
if block_num % 100 == 0:
log.info('Validating block #%s' % block_num)
block = list(
blockchain.stream(start_block=block_num, end_block=block_num))
# remove all invalid or changed operations
conditions = {
'block_num': block_num,
'_id': {
'$nin': [x['_id'] for x in block]
}
}
mongo.Operations.delete_many(conditions)
# insert any missing operations
for op in block:
with suppress(DuplicateKeyError):
transform = compose(strip_dot_from_keys, json_expand, typify)
mongo.Operations.insert_one(transform(op))
# re-process comments (does not re-add deleted posts)
for comment in (x for x in block if x['type'] == 'comment'):
upsert_comment(mongo,
'%s/%s' % (comment['author'], comment['permlink']))
def get_transactionid(username):
time.sleep(6)
found = False
count = 0
while found == False and count < 5:
count = count +1
print ("Attempt to get an ID number: " +str(count))
b = Blockchain()
end_block = steem.head_block_number
start_block = end_block -100
for block in b.stream_from(start_block=start_block, end_block=end_block, full_blocks=True):
transactions = block['transactions']
for transaction in transactions:
if (transaction['operations'][0][0] == "custom_json"):
if (transaction['operations'][0][1]['required_posting_auths'][0]) == user:
print (transaction)
if (transaction['operations'][0][1]['required_posting_auths'][0]) == user and (transaction['operations'][0][1]['id'])=="sm_find_match":
print (transaction['transaction_id'])
trxid = (transaction['transaction_id'])
#check if this is a valid id
r = requests.get('https://steemmonsters.com/battle/status?id=' +str(trxid))
data = r.json()
try:
status = data['status']
print ("ID Status:" +str(status))
if int(status) == 1:
found = True
return (trxid)
except:
found = False
return
def run():
username = os.environ.get("STEEM_NAME")
blockchain = Blockchain()
stream = map(Post, blockchain.stream(filter_by=["comment"]))
upvote_list = json.load(open('upvote_list.json'))
wif = upvote_list["voters"][username]["wif"]
steem = Steem(wif=wif)
upvoted = {}
date = int(time.strftime("%d"))
hour = int(time.strftime("%-H"))
print("Entering blockchain stream!")
while True:
try:
for post in stream:
if int(time.strftime("%d")) is not date:
upvoted = {}
date = int(time.strftime("%d"))
if int(time.strftime("%-H")) is not hour:
upvote_list = json.load(open('upvote_list.json'))
hour = int(time.strftime("%-H"))
if valid_post(post, upvote_list, upvoted):
try:
author = post["author"]
schedule_upvote(upvote_list, username, author)
print("Upvoted {}".format(author))
upvoted[author] += 1
except Exception as error:
print(repr(error))
continue
except Exception as error:
#print(repr(error))
continue
def run():
blockchain = Blockchain()
head_block = blockchain.get_current_block_num()
start_block = head_block - (int(sys.argv[1]))
stream = blockchain.stream_from(start_block=start_block)
block_count = int(sys.argv[1])
current_block = start_block
stats = {}
operations = 0
counter = 0
print("Starting from block {} for {} blocks\n".format(
start_block, block_count))
for post in stream:
if post['block'] != current_block:
counter += 1
print("Block {}/{} {:.2f}%".format(counter, block_count,
counter / block_count * 100))
current_block = post['block']
elif post['block'] == start_block + block_count:
break
operation = post['op'][0]
if operation not in stats:
stats[operation] = 1
else:
stats[operation] += 1
operations += 1
print("operations {}\n".format(operations))
for operation in stats:
print(operation, stats[operation])
def main():
accounts = []
votes_db = pickledb.load('steem_account_votes.db', False)
comments_db = pickledb.load('steem_account_comments.db', False)
transfers_db = pickledb.load('steem_account_trasnfers.db', False)
vesting_transfers_db = pickledb.load('steem_account_vesting_transfers.db', False)
result_db = pickledb.load('result.db', False)
steemd = Steem(STEEM_NODES)
chain = Blockchain(steemd_instance = steemd, mode = 'head')
logging.getLogger().setLevel(20)
databases = {'account_create': accounts, 'vote': votes_db, 'comment': comments_db, 'transfer': transfers_db, 'transfer_to_vesting': vesting_transfers_db}
fill(databases, chain, round((1497970800 - 1451606400) / 3), chain.info()['head_block_number'])
votes_db.dump()
comments_db.dump()
transfers_db.dump()
vesting_transfers_db.dump()
analyze(databases, result_db)
result_db.dump()
def run(user_json):
username = "******"
wif = os.environ.get("UNLOCK")
steem = Steem(wif=wif)
blockchain = Blockchain()
stream = map(Post, blockchain.stream(filter_by=["comment"]))
print("Checking posts on the blockchain!")
while True:
try:
for post in stream:
title = post["title"]
author = post["author"]
print(author)
if author in user_json:
# if valid_post(post, user_json):
try:
title = post["title"]
author = post["author"]
print("Upvoting post {} by {}!".format(title, author))
post.upvote(weight=user_json[author]["upvote_weight"],
voter=username)
except Exception as error:
print(repr(error))
continue
except Exception as error:
print(repr(error))
continue
def reconfigure_node(self):
self.node_index = (self.node_index + 1) % len(self.list)
self.nodes = [self.list[self.node_index]]
self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes)
self.b = Blockchain(self.steem)
self.queue.steem = self.steem
print('New node: {}\n'.format(self.nodes))
def run(filename):
b = Blockchain()
# automatically resume from where we left off
# previous + last + 1
start_block = get_previous_block_num(get_last_line(filename)) + 2
with open(filename, 'a+') as file:
for block in b.stream_from(start_block=start_block, full_blocks=True):
file.write(json.dumps(block, sort_keys=True) + '\n')
def run():
account = "sttest1"
account = Account(account)
wif = os.environ.get("UNLOCK")
steem = Steem(wif=wif)
blockchain = Blockchain()
stream = map(Post, blockchain.stream(filter_by=["comment"]))
upvote_list = json.load(open('upvote_list.json'))
upvoted = {}
upvote_queue = {}
date = int(time.strftime("%d"))
hour = int(time.strftime("%-H"))
print("Entering blockchain stream!")
while True:
try:
for post in stream:
time_string = str(post.time_elapsed())
post_age = time_string.split(":")
try:
# Omit posts older than 10 min
if int(post_age[0]) > 0 or int(post_age[1]) > 10:
break
except Exception as e:
#print (repr(e))
break
# start upvoting cycle at 100% voting power
if account.voting_power() == 100:
start_upvote_cycle(upvote_queue, account, steem)
# check for new date
if int(time.strftime("%d")) is not date:
upvoted = {}
date = int(time.strftime("%d"))
#check for new hour
if int(time.strftime("%-H")) is not hour:
upvote_list = json.load(open('upvote_list.json'))
hour = int(time.strftime("%-H"))
# verify post and add to queue
if valid_post(post, upvote_list, upvoted):
try:
author = post["author"]
print ("\nAdding to queue{}\nPermlink: {}".format(author, post['identifier']))
if post['identifier'] not in upvote_queue:
upvote_queue[post['identifier']] = upvote_list[author]["upvote_weight"]
print ("Upvoted {} for {}%".format(author, upvote_list[author]["upvote_weight"]))
upvoted[author] += 1
except Exception as error:
print(repr(error))
continue
except Exception as error:
#print(repr(error))
continue
def __init__(self, authorsLiked, who, mainWin): self.authorsLiked = authorsLiked self.tagsLiked = ['polish'] self.who = who self.b = Blockchain() self.stream = self.b.stream() self.mainWin = mainWin
def __init__(self, block, block_count, operation):
self.block = block
self.end_block = block_count + self.block - 1
self.operation = operation
self.nodes = ['https://rpc.buildteam.io', 'https://api.steemit.com',
'https://rpc.steemviz.com']
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def run():
blockchain = Blockchain()
id = blockchain.get_current_block_num()
global current_id
if id != current_id:
block = blockchain.get_current_block()
thread = threading.Thread(target=parse_next_block, args=(block, ))
print("Start new thread", thread.name)
thread.start()
current_id = id
threading.Timer(1, run).start()
def listen_steemd():
b = Blockchain()
h = b.stream_from(
start_block=db_last_block() + 1,
full_blocks=True,
)
for block in h:
num = int(block['previous'][:8], base=16) + 1
print("[LIVE] Got block {} at {} with {} txs".format(
num, block['timestamp'], len(block['transactions'])))
process_blocks([block])
def run(tags):
log.info("Follow mode activated", tags=tags)
if tags is None or len(tags) < 1:
raise ValueError("You must specify at least one tag")
log.debug("initializing...")
steem = Steem(keys=[cred.key])
account = Account(cred.id, steem)
chain = Blockchain(steem)
log.debug("ready", steem=steem, account=account, blockchain=chain)
log.info("Gathering our following list...")
following = account.get_following()
pending = []
log.info("Following list retrieved", count=len(following))
log.info("Watching for new posts...")
while True:
stream = map(Post, chain.stream(filter_by=['comment']))
try:
for post in stream:
count = len(pending)
if count > 0:
copy = list(pending)
for i in range(count):
if have_bandwidth(steem, account):
user = copy[i]
log.info("following user", user=user)
steem.follow(user, account=cred.id)
del pending[0]
else:
log.warn("Waiting for more bandwidth before following another user")
break
if post.is_main_post():
log.debug("found a top-level post", author=post.author, tags=post.tags)
if post.author != cred.id:
for tag in tags:
if tag in post.tags:
if post.author not in following:
pending.append(post.author)
following.append(post.author)
break
except PostDoesNotExist as e:
log.debug("Post has vanished", exception=e)
except RPCError as e:
log.error("RPC problem while streaming posts", exception=e)
def __init__(self, block, account):
self.block = block
self.account = account
self.nodes = [
'https://rpc.steemviz.com', 'https://rpc.buildteam.io',
'https://api.steemit.com'
]
self.steemPostingKey = os.environ.get('steemPostingKey')
self.steem = Steem(wif=self.steemPostingKey, nodes=self.nodes)
self.b = Blockchain(self.steem)
self.queue = classes.Queue(self.steem, self.account)
print('\nConnected to: {}'.format(self.nodes[0]))
def __init__(self, acount, number):
self.tag = 'vote'
self.account = account
self.number = number
self.nodes = [
'https://api.steemit.com', 'https://rpc.buildteam.io',
'https://rpc.steemviz.com'
]
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
self.timestamp = None
self.block = self.b.get_current_block_num()
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def stream_blockchain():
blockchain = Blockchain()
stream = map(Post, blockchain.stream(filter_by=['comment']))
while True:
try:
for post in stream:
tags = post["tags"]
if post.is_main_post() and "utopian-io" in tags:
author = post["author"]
title = post["title"]
print("{} posted {}".format(author, title))
except Exception as error:
print(repr(error))
continue
def stream(self, opNames, *args, **kwargs):
warnings.warn(
"The stream() call has been moved to `steem.blockchain.Blockchain.stream()`",
DeprecationWarning)
from steem.blockchain import Blockchain
return Blockchain(mode=kwargs.get("mode", "irreversible")).stream(
opNames, *args, **kwargs)
def scrape_blockchain(mongo):
s = Steem()
# see how far behind we are
missing = list(range(last_block_num(mongo), s.last_irreversible_block_num))
# if we are far behind blockchain head
# split work in chunks of 100
if len(missing) > 100:
for batch in partition_all(100, missing):
results = s.get_blocks(batch)
insert_blocks(mongo, results)
# otherwise continue as normal
blockchain = Blockchain(mode="irreversible")
hist = blockchain.stream_from(start_block=last_block_num(mongo), full_blocks=True)
insert_blocks(mongo, hist)
def get_block_headers_between(start_datetime, end_datetime, steem):
""" Returns block headers between two dates"""
latest_block_num = Blockchain(steem).get_current_block_num()
end_offset_num, _ = find_nearest_block_num(end_datetime, steem, latest_block_num)
return get_block_headers_between_offset_start(start_datetime, end_datetime,
steem=steem,
end_offset_num=end_offset_num)
def main():
global steem
global tnt_server
global steem_space
global followers_space
global chain
print('starting datafeed.py..')
ws_connection = os.environ['STEEMIT_WEBSOCKET_CONNECTION']
print('Connecting to ', ws_connection)
sys.stdout.flush()
steem = Steem(ws_connection)
chain = Blockchain(steem_instance=steem, mode='head')
while True:
try:
print('Connecting to tarantool (datastore:3301)..')
sys.stdout.flush()
tnt_server = tarantool.connect('datastore', 3301)
steem_space = tnt_server.space('steem')
followers_space = tnt_server.space('followers')
except Exception as e:
print('Cannot connect to tarantool server', file=sys.stderr)
print(str(e), file=sys.stderr)
sys.stderr.flush()
time.sleep(10)
continue
else:
while True:
run()
print('[run] exited, continue..')
def main():
global tnt_server
global steem_space
global followers_space
global chain
print('starting datafeed.py..')
sys.stdout.flush()
chain = Blockchain(mode='head')
while True:
try:
print('Connecting to tarantool (datastore:3301)..')
sys.stdout.flush()
tnt_server = tarantool.connect('datastore', 3301)
steem_space = tnt_server.space('steem')
followers_space = tnt_server.space('followers')
except Exception as e:
print('Cannot connect to tarantool server', file=sys.stderr)
print(str(e), file=sys.stderr)
sys.stderr.flush()
time.sleep(10)
continue
else:
while True:
run()
print('[run] exited, continue..')
def list_accounts(self, start=None, step=1000, limit=None, **kwargs):
warnings.warn(
"The list_accounts() call has been moved to `steem.blockchain.Blockchain.get_all_accounts()`",
DeprecationWarning)
from steem.blockchain import Blockchain
return Blockchain(
mode=kwargs.get("mode", "irreversible")).get_all_accounts(
start=start, steps=step, **kwargs)
def run():
# upvote posts with 30% weight
upvote_pct = 30
whoami = 'makerhacks'
# stream comments as they are published on the blockchain
# turn them into convenient Post objects while we're at it
b = Blockchain()
stream = map(Post, b.stream(filter_by=['comment']))
for post in stream:
if post.json_metadata:
mentions = post.json_metadata.get('users', [])
# if post mentions more than 10 people its likely spam
if mentions and len(mentions) < 10:
post.upvote(weight=upvote_pct, voter=whoami)
def run():
steem = Steem()
blockchain = Blockchain()
stream = blockchain.stream(filter_by=["transfer"])
username = "******"
while True:
try:
for transfer in stream:
if transfer["to"] == username:
url, permlink = transfer["memo"].split("@")
if "https://steemit.com/" in url:
steem.vote(f"@{permlink}", 100)
except Exception as error:
print(repr(error))
continue
class Steem_node():
def __init__(self, block, block_count):
self.block = block
self.end_block = block_count + self.block - 1
self.tag = 'transfer'
self.nodes = [
'https://api.steemit.com', 'https://rpc.buildteam.io',
'https://rpc.steemviz.com'
]
self.steem = Steem(nodes=self.nodes)
self.b = Blockchain(self.steem)
print('Booted\nConnected to: {}'.format(self.nodes[0]))
def process_transaction(self, index, block, operation):
date = block['timestamp']
to = operation['to']
user = operation['from']
amount = operation['amount']
memo = operation['memo']
db.insert_selection(self.block, index, date, to, user, amount, memo)
def run(self):
run = 1
# run until end_block
while run == 1:
try:
# stream full blocks starting at start_block
stream = self.b.stream_from(start_block=self.block,
end_block=self.end_block,
full_blocks=True)
# process each block indiviudally
for block in stream:
print('\nBlock: ', self.block)
# keep track of transaction index inside block
index = 0
# go over each transaction indivually, process if tag is
# met and update index
for transaction in block['transactions']:
if transaction['operations'][0][0] == self.tag:
self.process_transaction(
index, block, transaction['operations'][0][1])
index += 1
# Check if current block equals the end_block, break if so
# else update the current block
if self.block == self.end_block:
run = 0
else:
self.block += 1
except Exception as e:
print('Error:', e)
continue
from steem.blockchain import Blockchain
# parse the whole chain
for event in Blockchain().replay():
print("Event: %s" % event['op_type'])
print("Time: %s" % event['timestamp'])
print("Body: %s\n" % event['op'])
# parse only payments from specific datetime until now
b = Blockchain()
history = b.replay(
start_block=b.get_block_from_time("2016-09-01T00:00:00"),
end_block=b.get_current_block(),
filter_by=['transfer']
)
for payment in history:
print("@%s sent %s to @%s" % (payment['from'], payment['amount'], payment['to']))
# Output:
# @victoriart sent 1.000 SBD to @null
# @dude sent 5.095 STEEM to @bittrex
# @devil sent 5.107 STEEM to @poloniex
# @pinoytravel sent 0.010 SBD to @null
# @aladdin sent 5.013 STEEM to @poloniex
# @mrwang sent 31.211 STEEM to @blocktrades
# @kodi sent 0.030 SBD to @steembingo
