def test_stream_threading(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
ops_stream_no_threading = []
opNames = ["transfer", "vote"]
block_num_list2 = []
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, threading=False):
ops_stream_no_threading.append(op)
if op["block_num"] not in block_num_list2:
block_num_list2.append(op["block_num"])
for n in range(5):
ops_stream = []
block_num_list = []
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, threading=True, thread_num=8):
ops_stream.append(op)
if op["block_num"] not in block_num_list:
block_num_list.append(op["block_num"])
self.assertEqual(ops_stream[0]["block_num"], ops_stream_no_threading[0]["block_num"])
self.assertEqual(ops_stream[-1]["block_num"], ops_stream_no_threading[-1]["block_num"])
self.assertEqual(len(ops_stream_no_threading), len(ops_stream))
self.assertEqual(len(block_num_list), len(block_num_list2))
for i in range(len(block_num_list)):
self.assertEqual(block_num_list[i], block_num_list2[i])
class Crawler:
def __init__(self, name, blockchain):
self.__name = name
if blockchain == "hive":
self._blockchain = Blockchain(
blockchain_instance=blockchains.HIVE_INSTANCE)
pass
elif blockchain == "steem":
self._blockchain = Blockchain(
blockchain_instance=blockchains.STEEM_INSTANCE)
else:
raise NotImplementedError
def _stream(self, opNames=[]):
if hasattr(self, "start") and hasattr(self, "stop"):
# get starting block id
start_block_id = self._blockchain.get_estimated_block_num(
self.start, accurate=True)
stop_block_id = self._blockchain.get_estimated_block_num(
self.stop, accurate=True)
# looping trough generator
for op_json in self._blockchain.stream(opNames=opNames,
start=start_block_id,
stop=stop_block_id):
yield op_json
else:
print(
"[DEBUG] Timeframe was not set in criteria, direct streaming used"
)
# looping trough generator
for op_json in self._blockchain.stream(opNames=opNames):
yield op_json
def set_timeframe(self, start, stop):
if isinstance(start, datetime.datetime):
self.start = start
else:
raise TypeError(
"start argument must be an instance of datetime.datetime")
if isinstance(stop, datetime.datetime):
self.stop = stop
else:
raise TypeError(
"stop argument must be an instance of datetime.datetime")
if start >= stop:
raise RuntimeError("stop({}) < start({})".format(stop, start))
def read_missing_block_data(self, start_block=None):
# Go trough all transfer ops
cnt = 0
blockchain = Blockchain()
current_block = blockchain.get_current_block_num()
if start_block is not None:
trx_num = start_block["trx_num"]
block_num = start_block["block_num"]
start_block = block_num
# print("account %s - %d" % (account["name"], start_block))
else:
trx_num = 0
block_num = 0
start_block = current_block - self.block_history
data = []
for op in blockchain.stream(start=start_block,
stop=current_block,
only_ops=self.only_ops,
only_virtual_ops=self.only_virtual_ops):
if op["block_num"] < start_block:
# last_block = op["block"]
continue
elif op["block_num"] == start_block:
if op["trx_num"] <= trx_num:
continue
data.append(op)
cnt += 1
return data
def test_stream_batch(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False):
ops_stream.append(op)
self.assertTrue(ops_stream[0]["block_num"] >= self.start)
self.assertTrue(ops_stream[-1]["block_num"] <= self.stop)
op_stat = b.ops_statistics(start=self.start, stop=self.stop)
self.assertEqual(op_stat["vote"] + op_stat["transfer"], len(ops_stream))
ops_blocks = []
for op in b.blocks(start=self.start, stop=self.stop, max_batch_size=self.max_batch_size, threading=False):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if isinstance(op, dict) and "type" in op and "value" in op:
op_type = op["type"]
if len(op_type) > 10 and op_type[len(op_type) - 10:] == "_operation":
op_type = op_type[:-10]
if op_type in opNames:
op_stat4[op_type] += 1
elif op[0] in opNames:
op_stat4[op[0]] += 1
self.assertTrue(block.identifier >= self.start)
self.assertTrue(block.identifier <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
def profiling(name_list):
stm = Steem()
set_shared_steem_instance(stm)
del stm
print("start")
for name in name_list:
print("account: %s" % (name))
acc = Account(name)
max_index = acc.virtual_op_count()
print(max_index)
stopTime = datetime(2018, 4, 22, 0, 0, 0)
hist_elem = None
for h in acc.history_reverse(stop=stopTime):
hist_elem = h
print(hist_elem)
print("blockchain")
blockchain_object = Blockchain()
current_num = blockchain_object.get_current_block_num()
startBlockNumber = current_num - 20
endBlockNumber = current_num
block_elem = None
for o in blockchain_object.stream(start=startBlockNumber,
stop=endBlockNumber):
print("block %d" % (o["block_num"]))
block_elem = o
print(block_elem)
def test_stream_threading(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
op_stat = b.ops_statistics(start=self.start, stop=self.stop)
ops_blocks = []
for op in b.blocks(start=self.start,
stop=self.stop,
threading=True,
thread_num=8):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if op[0] in opNames:
op_stat4[op[0]] += 1
self.assertTrue(block.identifier >= self.start)
self.assertTrue(block.identifier <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
def test_stream2(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
stop_block = b.get_current_block_num()
start_block = stop_block - 10
ops_stream = []
for op in b.stream(start=start_block, stop=stop_block):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
def listen_blockchain_ops(opNames: list):
"""Listens to Steem blockchain and yields specified operations.
:param opNames: List of operations to yield
:type opNames: list
"""
bc = Blockchain(mode="head")
block_num = bc.get_current_block_num()
for op in bc.stream(opNames=opNames, start=block_num, threading=True):
yield op
def stream_blocks(self, start=None):
blockchain = Blockchain()
current_block = blockchain.get_current_block_num()
ops = []
for op in blockchain.stream(start=start,
stop=current_block,
only_ops=self.only_ops,
only_virtual_ops=self.only_virtual_ops):
ops.append(op)
return ops
def test_stream(self, node_param):
if node_param == "non_appbase":
bts = self.bts
else:
bts = self.appbase
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=self.start, stop=self.stop):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
op_stat = b.ops_statistics(start=self.start, stop=self.stop)
op_stat2 = {"transfer": 0, "vote": 0}
for op in ops_stream:
self.assertIn(op["type"], opNames)
op_stat2[op["type"]] += 1
self.assertTrue(op["block_num"] >= self.start)
self.assertTrue(op["block_num"] <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat2["transfer"])
self.assertEqual(op_stat["vote"], op_stat2["vote"])
ops_ops = []
for op in b.ops(start=self.start, stop=self.stop):
ops_ops.append(op)
self.assertTrue(len(ops_ops) > 0)
op_stat3 = {"transfer": 0, "vote": 0}
for op in ops_ops:
if op["op"][0] in opNames:
op_stat3[op["op"][0]] += 1
self.assertTrue(op["block_num"] >= self.start)
self.assertTrue(op["block_num"] <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat3["transfer"])
self.assertEqual(op_stat["vote"], op_stat3["vote"])
ops_blocks = []
for op in b.blocks(start=self.start, stop=self.stop):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if op[0] in opNames:
op_stat4[op[0]] += 1
self.assertTrue(block.identifier >= self.start)
self.assertTrue(block.identifier <= self.stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
ops_blocks = []
for op in b.blocks():
ops_blocks.append(op)
break
self.assertTrue(len(ops_blocks) == 1)
class SteemStream:
def __init__(self, operations=[]):
self.blockchain = Blockchain(mode="head")
self.operations = operations
self.last_streamed_block = 0
self.max_batch_size = 50
self.threading = False
def run(self, callback=default_callback, lookback=0, start=-1, days=-1):
try:
if lookback > 0 or start > 0 or days > 0:
if self.last_streamed_block == 0:
if start > 0:
start_block = start
elif lookback > 0:
start_block = self.blockchain.get_current_block_num() - int(lookback) #200000
else:
start_date = days_ago(days)
start_block = self.blockchain.get_estimated_block_num(start_date)
else:
start_block = self.last_streamed_block + 1
stop_block = self.blockchain.get_current_block_num()
logger.info("Streaming for operations {} has started from block {} to {}".format(self.operations, start_block, stop_block))
for ops in self.blockchain.stream(opNames=self.operations,
start=start_block, stop=stop_block,
max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8):
try:
callback(ops)
except:
logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc()))
else:
logger.info("Streaming for operations {} has started from the latest blocks".format(self.operations))
for ops in self.blockchain.stream(opNames=self.operations,
max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8):
try:
callback(ops)
except:
logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc()))
except:
logger.error("Failed when streaming operations {} with error: {}".format(self.operations, traceback.format_exc()))
def test_stream2(self, node_param):
if node_param == "normal":
bts = self.bts
else:
bts = self.testnet
b = Blockchain(steem_instance=bts)
stop_block = b.get_current_block_num()
start_block = stop_block - 10
ops_stream = []
for op in b.stream(start=start_block, stop=stop_block):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
def stream_votes(stm, threading, thread_num):
b = Blockchain(steem_instance=stm)
opcount = 0
start_time = time.time()
for op in b.stream(start=23483000, stop=23483200, threading=threading, thread_num=thread_num,
opNames=['vote']):
sys.stdout.write("\r%s" % op['block_num'])
opcount += 1
now = time.time()
total_duration = now - start_time
print(" votes: %d, time %.2f" % (opcount, total_duration))
return opcount, total_duration
def test_stream_batch2(self):
bts = self.bts
b = Blockchain(steem_instance=bts)
ops_stream = []
start_block = 25097000
stop_block = 25097100
opNames = ["account_create", "custom_json"]
for op in b.stream(start=int(start_block), stop=int(stop_block), opNames=opNames, max_batch_size=50, threading=False, thread_num=8):
ops_stream.append(op)
self.assertTrue(ops_stream[0]["block_num"] >= start_block)
self.assertTrue(ops_stream[-1]["block_num"] <= stop_block)
op_stat = b.ops_statistics(start=start_block, stop=stop_block)
self.assertEqual(op_stat["account_create"] + op_stat["custom_json"], len(ops_stream))
async def stream():
try:
stm = Steem(node=[
"https://api.steemit.com",
"https://steemd.minnowsupportproject.org", "https://anyx.io"
])
chain = Blockchain(stm, "head")
for tx in chain.stream(['custom_json']):
if tx['id'] == 'sm_sell_cards':
user_perm_posting = ""
user_perm_active = ""
try:
user_perm_posting = tx['required_posting_auths'][0]
except:
pass
try:
user_perm_active = tx['required_auths'][0]
except:
pass
json_data = json.loads(tx['json'])
loop.create_task(
process(json_data, user_perm_posting, user_perm_active))
elif tx['id'] == 'sm_update_price':
market_id_list = json.loads(tx['json'])['ids']
card_uid_dict = []
for market_id in market_id_list:
time.sleep(2)
try:
response_json = requests.get(
f"https://steemmonsters.com/market/status?id={market_id}"
).json()
card_uid = response_json['cards'][0]['uid']
card_uid_dict.append(card_uid)
except:
pass
user_perm_posting = ""
user_perm_active = ""
try:
user_perm_posting = tx['required_posting_auths'][0]
except:
pass
try:
user_perm_active = tx['required_auths'][0]
except:
pass
loop.create_task(
process(card_uid_dict, user_perm_posting,
user_perm_active))
await asyncio.sleep(0)
except:
print(traceback.format_exc())
def anyx():
SV = os.environ.get('SV')
di = {
'16': .55,
'15': .03,
'5': .42,
'49': .45,
'27': .42,
'38': .42,
'70': .50,
'71': .45,
'72': .55,
'73': .45,
'74': .4,
}
dic = {'16': .7, '5': .50, '49': .7, '27': .56, '38': .68}
stm = Steem(node="https://anyx.io", keys=SV)
acc = Account('sourovafrin', steem_instance=stm)
bo = Blockchain(stm, 'head')
print("started anyx")
for detail in bo.stream(['custom_json']):
try:
if detail['id'] == 'sm_sell_cards':
for i in ast.literal_eval(detail['json']):
lin = requests.get(
"https://steemmonsters.com/cards/find?ids=" +
i['cards'][0]).json()
for ii in lin:
id = ii['market_id']
idd = str(ii['card_detail_id'])
price = float(ii['buy_price'])
if int(ii['edition']) == 1 and price <= di[idd]:
acc.transfer(
'svirus', 2, 'SBD',
"sm_market_purchase:" + id + ":sourovafrin")
elif int(ii['edition']) == 0 and price <= dic[idd]:
acc.transfer(
'svirus', 2, 'SBD',
"sm_market_purchase:" + id + ":sourovafrin")
except Exception as e:
print("Error found anyx: {}".format(e))
def main(conn, logger, debug=False):
# Main Loop
s = Steem(node=nodes)
theChain = Blockchain(steem_instance=s, mode="head")
Start_block_num = read_blocknum_fromfile("virtual_ops_block.txt")
if not Start_block_num:
Start_block_num = theChain.get_current_block_num()
# print('Start For loop')
try:
for block in theChain.stream(only_virtual_ops=True,
start=Start_block_num):
print("Try add to DB")
addblock_to_db(block, conn, logger)
except Exception as e:
return
def main(conn, logger, debug=False):
s = Steem(node=nodes)
theChain = Blockchain(steem_instance=s, mode="head")
# print('Start For loop')
Start_block_num = read_blocknum_fromfile("ops_block.txt")
if not Start_block_num:
Start_block_num = theChain.get_current_block_num()
try:
for block in theChain.stream(start=Start_block_num):
# print("Run Again")
# print(f"{block['block_num']}")
yay = addblock_to_db(block, conn, logger)
# print("Got out ok")
except Exception as e:
logger.error(f"Error in Stream: {e}")
return
def test_stream(self):
bts = self.bts
start = self.start
stop = self.stop
b = Blockchain(steem_instance=bts)
ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames, start=start, stop=stop):
ops_stream.append(op)
self.assertTrue(len(ops_stream) > 0)
ops_raw_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames,
raw_ops=True,
start=start,
stop=stop):
ops_raw_stream.append(op)
self.assertTrue(len(ops_raw_stream) > 0)
only_ops_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames,
start=start,
stop=stop,
only_ops=True):
only_ops_stream.append(op)
self.assertTrue(len(only_ops_stream) > 0)
only_ops_raw_stream = []
opNames = ["transfer", "vote"]
for op in b.stream(opNames=opNames,
raw_ops=True,
start=start,
stop=stop,
only_ops=True):
only_ops_raw_stream.append(op)
self.assertTrue(len(only_ops_raw_stream) > 0)
op_stat = b.ops_statistics(start=start, stop=stop)
op_stat2 = {"transfer": 0, "vote": 0}
for op in ops_stream:
self.assertIn(op["type"], opNames)
op_stat2[op["type"]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat2["transfer"])
self.assertEqual(op_stat["vote"], op_stat2["vote"])
op_stat3 = {"transfer": 0, "vote": 0}
for op in ops_raw_stream:
self.assertIn(op["op"][0], opNames)
op_stat3[op["op"][0]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat3["transfer"])
self.assertEqual(op_stat["vote"], op_stat3["vote"])
op_stat5 = {"transfer": 0, "vote": 0}
for op in only_ops_stream:
self.assertIn(op["type"], opNames)
op_stat5[op["type"]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat5["transfer"])
self.assertEqual(op_stat["vote"], op_stat5["vote"])
op_stat6 = {"transfer": 0, "vote": 0}
for op in only_ops_raw_stream:
self.assertIn(op["op"][0], opNames)
op_stat6[op["op"][0]] += 1
self.assertTrue(op["block_num"] >= start)
self.assertTrue(op["block_num"] <= stop)
self.assertEqual(op_stat["transfer"], op_stat6["transfer"])
self.assertEqual(op_stat["vote"], op_stat6["vote"])
ops_blocks = []
for op in b.blocks(start=start, stop=stop):
ops_blocks.append(op)
op_stat4 = {"transfer": 0, "vote": 0}
self.assertTrue(len(ops_blocks) > 0)
for block in ops_blocks:
for tran in block["transactions"]:
for op in tran['operations']:
if isinstance(op, list) and op[0] in opNames:
op_stat4[op[0]] += 1
elif isinstance(op, dict):
op_type = op["type"]
if len(op_type) > 10 and op_type[len(op_type) -
10:] == "_operation":
op_type = op_type[:-10]
if op_type in opNames:
op_stat4[op_type] += 1
self.assertTrue(block.identifier >= start)
self.assertTrue(block.identifier <= stop)
self.assertEqual(op_stat["transfer"], op_stat4["transfer"])
self.assertEqual(op_stat["vote"], op_stat4["vote"])
ops_blocks = []
for op in b.blocks():
ops_blocks.append(op)
break
self.assertTrue(len(ops_blocks) == 1)
class Distribubot:
def __init__(self, config, data_file, hived_instance):
self.config = config
self.data_file = data_file
self.hive = hived_instance
self.token_config = {}
# add log stats
self.log_data = {
"start_time": 0,
"last_block_num": None,
"new_commands": 0,
"stop_block_num": 0,
"stop_block_num": 0,
"time_for_blocks": 0
}
config_cnt = 0
necessary_fields = [
"token_account", "symbol", "min_token_in_wallet",
"comment_command", "token_memo", "reply", "sucess_reply_body",
"fail_reply_body", "no_token_left_body", "user_can_specify_amount",
"usage_upvote_percentage", "no_token_left_for_today",
"token_in_wallet_for_each_outgoing_token",
"maximum_amount_per_comment", "count_only_staked_token"
]
self.token_config = check_config(self.config["config"],
necessary_fields, self.hive)
self.blockchain = Blockchain(mode='head',
blockchain_instance=self.hive)
def run(self, start_block, stop_block):
self.hive.wallet.unlock(self.config["wallet_password"])
self.blockchain = Blockchain(mode='head',
blockchain_instance=self.hive)
current_block = self.blockchain.get_current_block_num()
if stop_block is None or stop_block > current_block:
stop_block = current_block
if start_block is None:
start_block = current_block
last_block_num = current_block - 1
else:
last_block_num = start_block - 1
self.log_data["start_block_num"] = start_block
for op in self.blockchain.stream(start=start_block,
stop=stop_block,
opNames=["comment"]):
self.log_data = print_block_log(self.log_data, op,
self.config["print_log_at_block"])
last_block_num = op["block_num"]
if op["type"] == "comment":
token = None
for key in self.token_config:
if op["body"].find(
self.token_config[key]["comment_command"]) >= 0:
token = key
if token is None:
continue
if op["author"] == self.token_config[token]["token_account"]:
continue
cnt = 0
c_comment = None
c_parent = None
authorperm = construct_authorperm(op)
use_tags_api = True
while c_comment is None and cnt < 10:
cnt += 1
try:
c_comment = Comment(authorperm,
use_tags_api=use_tags_api,
blockchain_instance=self.hive)
c_comment.refresh()
except:
if cnt > 5:
use_tags_api = False
nodelist = NodeList()
nodelist.update_nodes()
self.hive = Hive(node=nodelist.get_hive_nodes(),
num_retries=5,
call_num_retries=3,
timeout=15)
time.sleep(3)
if cnt == 10 or c_comment is None:
logger.warn("Could not read %s/%s" %
(op["author"], op["permlink"]))
continue
if 'depth' in c_comment:
if c_comment['depth'] == 0:
continue
else:
if c_comment["parent_author"] == '':
continue
if abs((c_comment["created"] -
op['timestamp']).total_seconds()) > 9.0:
logger.warn("Skip %s, as edited" % c_comment["authorperm"])
continue
already_voted = False
if self.token_config[token]["upvote_token_receiver"]:
parent_identifier = construct_authorperm(
c_comment["parent_author"],
c_comment["parent_permlink"])
c_parent = Comment(parent_identifier,
blockchain_instance=self.hive)
for v in c_parent.get_votes(raw_data=True):
if self.token_config[token]["token_account"] == v[
"voter"]:
already_voted = True
else:
for v in c_comment.get_votes(raw_data=True):
if self.token_config[token]["token_account"] == v[
"voter"]:
already_voted = True
if already_voted:
continue
already_replied = None
cnt = 0
if self.token_config[token]["usage_upvote_percentage"] == 0:
while already_replied is None and cnt < 5:
cnt += 1
try:
already_replied = False
for r in c_comment.get_all_replies():
if r["author"] == self.token_config[token][
"token_account"]:
already_replied = True
except:
already_replied = None
self.hive.rpc.next()
if already_replied is None:
already_replied = False
for r in c_comment.get_all_replies():
if r["author"] == self.token_config[token][
"token_account"]:
already_replied = True
if already_replied:
continue
muting_acc = Account(self.token_config[token]["token_account"],
blockchain_instance=self.hive)
blocked_accounts = muting_acc.get_mutings()
if c_comment["author"] in blocked_accounts:
logger.info("%s is blocked" % c_comment["author"])
continue
# Load bot token balance
bot_wallet = Wallet(self.token_config[token]["token_account"],
blockchain_instance=self.hive)
symbol = bot_wallet.get_token(
self.token_config[token]["symbol"])
# parse amount when user_can_specify_amount is true
amount = self.token_config[token]["default_amount"]
if self.token_config[token]["user_can_specify_amount"]:
start_index = c_comment["body"].find(
self.token_config[token]["comment_command"])
stop_index = c_comment["body"][start_index:].find("\n")
if stop_index >= 0:
command = c_comment["body"][start_index +
1:start_index + stop_index]
else:
command = c_comment["body"][start_index + 1:]
command_args = command.replace(' ', ' ').split(" ")[1:]
if len(command_args) > 0:
try:
amount = float(command_args[0])
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
logger.warn("%s - %s - %s" %
(str(exc_type), str(fname),
str(exc_tb.tb_lineno)))
logger.info("Could not parse amount")
if self.token_config[token][
"maximum_amount_per_comment"] and amount > self.token_config[
token]["maximum_amount_per_comment"]:
amount = self.token_config[token][
"maximum_amount_per_comment"]
if not self.config["no_broadcast"] and self.hive.wallet.locked(
):
self.hive.wallet.unlock(self.config["wallet_password"])
self.log_data["new_commands"] += 1
wallet = Wallet(c_comment["author"],
blockchain_instance=self.hive)
token_in_wallet = wallet.get_token(
self.token_config[token]["symbol"])
balance = 0
if token_in_wallet is not None:
logger.info(token_in_wallet)
if self.token_config[token]["count_only_staked_token"]:
balance = 0
else:
balance = float(token_in_wallet["balance"])
if "stake" in token_in_wallet:
balance += float(token_in_wallet['stake'])
if 'delegationsIn' in token_in_wallet and float(
token_in_wallet['delegationsIn']) > 0:
balance += float(token_in_wallet['delegationsIn'])
if 'pendingUnstake' in token_in_wallet and float(
token_in_wallet['pendingUnstake']) > 0:
balance += float(token_in_wallet['pendingUnstake'])
if balance > self.token_config[token][
"min_token_in_wallet"]:
if self.token_config[token][
"token_in_wallet_for_each_outgoing_token"] > 0:
max_token_to_give = int(
balance / self.token_config[token]
["token_in_wallet_for_each_outgoing_token"])
else:
max_token_to_give = self.token_config[token][
"maximum_amount_per_comment"]
else:
max_token_to_give = 0
else:
max_token_to_give = 0
logger.info("token to give for %s: %f" %
(c_comment["author"], max_token_to_give))
db_data = read_data(self.data_file)
if "accounts" in db_data and c_comment["author"] in db_data[
"accounts"] and token in db_data["accounts"][
c_comment["author"]]:
if db_data["accounts"][c_comment["author"]][token][
"last_date"] == date.today(
) and self.token_config[token][
"token_in_wallet_for_each_outgoing_token"] > 0:
max_token_to_give = max_token_to_give - db_data[
"accounts"][c_comment["author"]][token]["amount"]
if amount > max_token_to_give:
amount = max_token_to_give
if amount > self.token_config[token][
"maximum_amount_per_comment"]:
amount = self.token_config[token][
"maximum_amount_per_comment"]
if token_in_wallet is None or balance < self.token_config[
token]["min_token_in_wallet"]:
reply_body = self.token_config[token]["fail_reply_body"]
elif max_token_to_give < 1:
reply_body = self.token_config[token][
"no_token_left_for_today"]
elif c_comment["parent_author"] == c_comment["author"]:
reply_body = "You cannot sent token to yourself."
elif float(symbol["balance"]) < amount:
reply_body = self.token_config[token]["no_token_left_body"]
else:
if "{}" in self.token_config[token]["sucess_reply_body"]:
reply_body = (self.token_config[token]
["sucess_reply_body"]).format(
c_comment["parent_author"])
else:
reply_body = self.token_config[token][
"sucess_reply_body"]
if "{}" in self.token_config[token]["token_memo"]:
token_memo = (
self.token_config[token]["token_memo"]).format(
c_comment["author"])
else:
token_memo = self.token_config[token]["token_memo"]
sendwallet = Wallet(
self.token_config[token]["token_account"],
blockchain_instance=self.hive)
try:
logger.info(
"Sending %.2f %s to %s" %
(amount, self.token_config[token]["symbol"],
c_comment["parent_author"]))
sendwallet.transfer(c_comment["parent_author"], amount,
self.token_config[token]["symbol"],
token_memo)
if "accounts" in db_data:
accounts = db_data["accounts"]
else:
accounts = {}
if c_comment["author"] not in accounts:
accounts[c_comment["author"]] = {}
accounts[c_comment["author"]][token] = {
"last_date": date.today(),
"n_comments": 1,
"amount": amount
}
elif token not in accounts[c_comment["author"]]:
accounts[c_comment["author"]][token] = {
"last_date": date.today(),
"n_comments": 1,
"amount": amount
}
else:
if accounts[c_comment["author"]][token][
"last_date"] < date.today():
accounts[c_comment["author"]][token] = {
"last_date": date.today(),
"n_comments": 1,
"amount": amount
}
else:
accounts[c_comment["author"]][token][
"n_comments"] += 1
accounts[c_comment["author"]][token][
"amount"] += amount
store_data(self.data_file, "accounts", accounts)
logger.info(
"%s - %s" %
(c_comment["author"],
str(accounts[c_comment["author"]][token])))
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
logger.warn(
"%s - %s - %s" %
(str(exc_type), str(fname), str(exc_tb.tb_lineno)))
logger.warn("Could not send %s token" %
self.token_config[token]["symbol"])
continue
reply_identifier = construct_authorperm(
c_comment["parent_author"], c_comment["parent_permlink"])
if self.config["no_broadcast"]:
logger.info("%s" % reply_body)
else:
try:
self.hive.post(
"",
reply_body,
author=self.token_config[token]["token_account"],
reply_identifier=reply_identifier)
if self.token_config[token][
"usage_upvote_percentage"] <= 0:
time.sleep(5)
self.hive.post(
"",
"Command accepted!",
author=self.token_config[token]
["token_account"],
reply_identifier=c_comment["authorperm"])
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
logger.warn(
"%s - %s - %s" %
(str(exc_type), str(fname), str(exc_tb.tb_lineno)))
logger.warn("Could not reply to post")
continue
if self.token_config[token]["usage_upvote_percentage"] > 0:
time.sleep(5)
upvote_percentge = self.token_config[token][
"usage_upvote_percentage"]
if self.token_config[token]["scale_upvote_weight"]:
upvote_percentge = upvote_percentge * amount / self.token_config[
token]["maximum_amount_per_comment"]
print("Upvote with %.2f %%" % upvote_percentge)
if self.token_config[token]["upvote_token_receiver"]:
if c_parent is None:
c_parent = Comment(
parent_identifier,
blockchain_instance=self.hive)
try:
c_parent.upvote(upvote_percentge,
voter=self.token_config[token]
["token_account"])
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
logger.warn("%s - %s - %s" %
(str(exc_type), str(fname),
str(exc_tb.tb_lineno)))
logger.warn("Could not upvote comment")
else:
try:
c_comment.upvote(upvote_percentge,
voter=self.token_config[token]
["token_account"])
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
logger.warn("%s - %s - %s" %
(str(exc_type), str(fname),
str(exc_tb.tb_lineno)))
logger.warn("Could not upvote comment")
time.sleep(4)
return last_block_num
from beem.account import Account
from beem.blockchain import Blockchain
from beem.comment import Comment
from beem.utils import construct_authorperm
from voteleader import db, voter, wif
nodes_urls = [
"https://api.hive.blog",
"https://api.deathwing.me",
"https://hived.privex.io",
]
hive = Hive(node=nodes_urls, keys=wif)
blockchain = Blockchain(blockchain_instance=hive)
stream = blockchain.stream(opNames=["comment"],
raw_ops=False,
threading=True,
thread_num=4)
def tally(author):
today = datetime.now(timezone.utc)
one_week = today - timedelta(days=7)
acc = Account(author, blockchain_instance=hive)
blog = acc.get_blog()
return 1 + sum(post["created"] >= one_week for post in blog)
def monitor():
table = db.load_table("leaderboard")
vote_table = db["vote_history"]
print("[Monitor Starting up...]")
import sys
from datetime import timedelta
import time
import io
from beem.blockchain import Blockchain
from beem.utils import parse_time
import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
class DemoBot(object):
def vote(self, vote_event):
w = vote_event["weight"]
if w > 0:
print("Vote by", vote_event["voter"], "for", vote_event["author"])
else:
if w < 0:
print("Downvote by", vote_event["voter"], "for",
vote_event["author"])
else:
print("(Down)vote by", vote_event["voter"], "for",
vote_event["author"], "CANCELED")
if __name__ == "__main__":
tb = DemoBot()
blockchain = Blockchain()
for vote in blockchain.stream(opNames=["vote"]):
tb.vote(vote)
latest_update_block = b.get_estimated_block_num(latest_update)
else:
latest_update_block = start_block
print("latest update %s - %d to %d" %
(str(latest_update), latest_update_block, stop_block))
start_block = max([latest_update_block, start_block]) + 1
if stop_block > start_block + 6000:
stop_block = start_block + 6000
cnt = 0
updated_accounts = []
posts_dict = {}
changed_member_data = []
for ops in b.stream(start=start_block,
stop=stop_block,
opNames=["comment"],
max_batch_size=max_batch_size,
threading=threading,
thread_num=8):
#print(ops)
timestamp = ops["timestamp"]
# timestamp = timestamp.replace(tzinfo=None)
# continue
if ops["author"] not in member_accounts:
continue
if ops["block_num"] <= latest_update_block:
continue
if ops["block_num"] - last_block_print > 50:
last_block_print = ops["block_num"]
print("blocks left %d - post found: %d" %
(ops["block_num"] - stop_block, len(posts_dict)))
authorperm = construct_authorperm(ops)
flagged_posts = []
if last_streamed_block == 0:
start_block = b.get_current_block_num() - int(201600)
else:
start_block = last_streamed_block + 1
stop_block = b.get_current_block_num()
last_block_print = start_block
cnt = 0
updated_accounts = []
posts_dict = {}
changed_member_data = []
ops = None
for ops in b.stream(start=start_block,
stop=stop_block,
opNames=["comment", "transfer", "vote", "custom_json"],
max_batch_size=max_batch_size,
threading=threading,
thread_num=8):
#print(ops)
timestamp = ops["timestamp"]
# timestamp = timestamp.replace(tzinfo=None)
# continue
last_streamed_block = ops["block_num"]
if ops["type"] == "transfer" and ops["to"] == "rewarding":
authorperm = ops["memo"].split(",")[0]
command = ",".join(ops["memo"].split(",")[1:])
commandsTrx.add({
"authorperm": authorperm,
"command": command,
"account": ops["from"],
"valid": True,
def s_load_binary(file_obj):
'''load contents from file_obj, returning a generator that yields one
element at a time'''
for line in file_obj:
elt = loads(unhexlify(line[:-1]))
yield elt
if __name__ == "__main__":
blockchain = Blockchain()
threading = True
thread_num = 8
cur_block = blockchain.get_current_block()
stop = cur_block.identifier
startdate = cur_block.time() - timedelta(seconds=3600)
start = blockchain.get_estimated_block_num(startdate, accurate=True)
outf = gzip.open('blocks1.pkl', 'w')
blocks = 0
for block in blockchain.stream(opNames=[], start=start, stop=stop, threading=threading, thread_num=thread_num):
s_dump_binary(block, outf)
blocks = blocks + 1
if blocks % 200 == 0:
print(blocks, "blocks streamed")
outf.close()
for block in s_load_binary(gzip.open('blocks1.pkl')):
print(block)
next_account_id = accountTrxStorage.get_latest_index() + 1
if end_block_round > end_block:
end_block_round = end_block
data_available = False
db_data = []
db_account = []
db_follow = []
accounts = accountTrxStorage.get_accounts()
last_block_num = None
new_accounts_found = 0
new_follow_found = 0
last_trx_id = '0' * 40
op_num = 0
print("start_streaming")
for op in b.stream(start=int(start_block), stop=int(end_block_round), opNames=["account_create", "account_create_with_delegation", "create_claimed_account", "pow", "pow2", "custom_json"], max_batch_size=max_batch_size, threading=threading, thread_num=8):
block_num = op["block_num"]
# print(block_num)
if last_block_num is None:
new_accounts_found = 0
new_follow_found = 0
start_time = time.time()
last_block_num = block_num
if op["trx_id"] == last_trx_id:
op_num += 1
else:
op_num = 0
if "trx_num" in op:
trx_num = op["trx_num"]
else:
trx_num = 0
transaction.update(field_updates=fields, reference=creator_doc_ref)
transaction = db.transaction()
pref = global_preferences.get().to_dict()
last_block_num_synced = pref['last_block_num_synced'] if pref else 26256743 - 1 # HF20 -1
last_block_num = last_block_num_synced + 1
print("Let's sync with blockchain...")
for op in blockchain.stream(
# opNames=[
# 'claim_account',
# 'create_claimed_account',
# 'account_create',
# ],
# start=26256743, # first block of HF20
# start=26265398, # first block after HF20 with `create_claimed_account` op
# start=26289186, # first block after HF20 with `account_create` op
start=last_block_num_synced + 1,
max_batch_size=50,
):
op["timestamp"] = op["timestamp"].__str__()
if last_block_num < op['block_num']: # that means that all ops from last_block are already synced
last_block_num_synced = op['block_num'] - 1
current_block_num = blockchain.get_current_block_num()
transaction.set(global_preferences, {
"last_block_num_synced": last_block_num_synced,
"current_block_num": current_block_num,
"blocks_behind": current_block_num - op['block_num'],
"last_trx_id_synced": op["trx_id"],
import dataset
from beem import Steem
from beem.blockchain import Blockchain
# Ask for user ID
hive_id = input("Hive User ID: ")
# hive_id = yournamehere # if you want to hard code it.
hive = Steem(node='https://anyx.io')
db = dataset.connect('sqlite:///mydatabase.db')
# System Variables
blockchain = Blockchain(steem_instance=hive)
stream = blockchain.stream(opNames=['transfer'],
raw_ops=False,
threading=True,
thread_num=4)
table = db[hive_id]
# parse json data to SQL insert
def update_db(post):
try:
table.insert(dict(post))
db.commit()
except Exception as e:
print(f'[Error: {e} moving on]')
db.rollback()
def monitor():
from __future__ import print_function
import sys
from datetime import timedelta
import time
import io
from beem.blockchain import Blockchain
from beem.utils import parse_time
import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
class DemoBot(object):
def comment(self, comment_event):
print('Comment by {} on post {} by {}:'.format(
comment_event['author'], comment_event['parent_permlink'],
comment_event['parent_author']))
print(comment_event['body'])
print()
if __name__ == "__main__":
tb = DemoBot()
blockchain = Blockchain()
for vote in blockchain.stream(opNames=["comment"]):
tb.comment(vote)
except exceptions.ContentDoesNotExistsException:
print("Could not find Comment: %s" % (authorperm))
al = list()
if not vote_event["voter"] in self.looked_up:
al.append(vote_event["voter"])
self.looked_up.add(vote_event["voter"])
if not vote_event["author"] in self.looked_up:
al.append(vote_event["author"])
self.looked_up.add(vote_event["author"])
if len(al) > 0:
lookup_accounts(al)
if __name__ == "__main__":
wtw = WatchingTheWatchers()
tb = WatchingTheWatchersBot(wtw)
blockchain = Blockchain()
threading = True
thread_num = 16
cur_block = blockchain.get_current_block()
stop = cur_block.identifier
startdate = cur_block.time() - timedelta(days=1)
start = blockchain.get_estimated_block_num(startdate, accurate=True)
for vote in blockchain.stream(opNames=["vote"],
start=start,
stop=stop,
threading=threading,
thread_num=thread_num):
tb.vote(vote)
wtw.report()
