def search(self, search_str, start=None, stop=None, use_block_num=True):
""" Returns ops in the given range"""
ops = []
if start is not None:
start = addTzInfo(start)
if stop is not None:
stop = addTzInfo(stop)
for op in self:
if use_block_num and start is not None and isinstance(start, int):
if op["block"] < start:
continue
elif not use_block_num and start is not None and isinstance(
start, int):
if op["index"] < start:
continue
elif start is not None and isinstance(start,
(datetime, date, time)):
if start > formatTimeString(op["timestamp"]):
continue
if use_block_num and stop is not None and isinstance(stop, int):
if op["block"] > stop:
continue
elif not use_block_num and stop is not None and isinstance(
stop, int):
if op["index"] > stop:
continue
elif stop is not None and isinstance(stop, (datetime, date, time)):
if stop < formatTimeString(op["timestamp"]):
continue
op_string = json.dumps(list(op.values()))
if re.search(search_str, op_string):
ops.append(op)
return ops
def acc_to_dict(account):
output = account.copy()
if 'json_metadata' in output:
output["json_metadata"] = json.dumps(output["json_metadata"],
separators=[',', ':'])
parse_times = [
"last_owner_update", "last_account_update", "created",
"last_owner_proved", "last_active_proved", "last_account_recovery",
"last_vote_time", "sbd_seconds_last_update",
"sbd_last_interest_payment", "savings_sbd_seconds_last_update",
"savings_sbd_last_interest_payment", "next_vesting_withdrawal",
"last_market_bandwidth_update", "last_post", "last_root_post",
"last_bandwidth_update"
]
for p in parse_times:
if p in output:
date = output.get(p, datetime(1970, 1, 1, 0, 0))
if isinstance(date, (datetime, date)):
output[p] = formatTimeString(date)
else:
output[p] = date
amounts = [
"balance", "savings_balance", "sbd_balance", "savings_sbd_balance",
"reward_sbd_balance", "reward_steem_balance", "reward_vesting_balance",
"reward_vesting_steem", "vesting_shares", "delegated_vesting_shares",
"received_vesting_shares"
]
for p in amounts:
if p in output and isinstance(output[p], Amount):
output[p] = output[p].json()
return json.loads(str(json.dumps(output)))
def add_mngt_shares(self, last_op, mgnt_shares, op_count):
index = last_op["index"]
timestamp = last_op["timestamp"]
sponsee = {}
memo = ""
latest_share = self.trxStorage.get_lastest_share_type("Mgmt")
if latest_share is not None:
start_index = latest_share["index"] + 1
else:
start_index = op_count / 100 * 3
for account in mgnt_shares:
shares = mgnt_shares[account]
sponsor = account
data = {
"index": start_index,
"source": "mgmt",
"memo": "",
"account": account,
"sponsor": sponsor,
"sponsee": sponsee,
"shares": shares,
"vests": float(0),
"timestamp": formatTimeString(timestamp),
"status": "Valid",
"share_type": "Mgmt"
}
start_index += 1
self.trxStorage.add(data)
def test_formatDateTimetoTimeStamp(self):
t = "1970-01-01T00:00:00"
t = formatTimeString(t)
timestamp = formatToTimeStamp(t)
self.assertEqual(timestamp, 0)
t2 = "2018-07-10T10:08:39"
timestamp = formatToTimeStamp(t2)
self.assertEqual(timestamp, 1531217319)
t3 = datetime(2018, 7, 10, 10, 8, 39)
timestamp = formatToTimeStamp(t3)
self.assertEqual(timestamp, 1531217319)
def update_delegations(self):
delegations = self.delegation_acc.get_vesting_delegations(
start_account='', limit=1000, account=None)
for d in delegations:
if d["delegatee"] in self.accounts:
self.accounts[
d["delegatee"]]["delegated_hp"] = self.hive.vests_to_hp(
float(
Amount(d["vesting_shares"],
blockchain_instance=self.hive)))
self.accounts[
d["delegatee"]]["delegation_timestamp"] = formatTimeString(
d["min_delegation_time"]).replace(tzinfo=None)
def get_ops(self,
start=None,
stop=None,
use_block_num=True,
only_ops=[],
exclude_ops=[]):
""" Returns ops in the given range"""
if start is not None:
start = addTzInfo(start)
if stop is not None:
stop = addTzInfo(stop)
for op in self:
if use_block_num and start is not None and isinstance(start, int):
if op["block"] < start:
continue
elif not use_block_num and start is not None and isinstance(
start, int):
if op["index"] < start:
continue
elif start is not None and isinstance(start,
(datetime, date, time)):
if start > formatTimeString(op["timestamp"]):
continue
if use_block_num and stop is not None and isinstance(stop, int):
if op["block"] > stop:
continue
elif not use_block_num and stop is not None and isinstance(
stop, int):
if op["index"] > stop:
continue
elif stop is not None and isinstance(stop, (datetime, date, time)):
if stop < formatTimeString(op["timestamp"]):
continue
if exclude_ops and op["type"] in exclude_ops:
continue
if not only_ops or op["type"] in only_ops:
yield op
def store_blocks(self, ops):
filename = self.get_filename()
if self.db_type == "shelve":
with shelve.open(filename) as db:
db['ops'] = ops
elif self.db_type == "pickle":
db = {"ops": ops}
save(filename, db)
elif self.db_type == "deepdish":
db = {"ops": ops}
dd.io.save(filename, db, compression=None)
elif self.db_type == "sqlite":
with dataset.connect('sqlite:///%s?check_same_thread=False' %
(filename)) as db:
table = db["ops"]
for op in ops:
json_params = [
"required_auths", "required_posting_auths",
"extensions", "proposal_ids", "active", "posting",
"props", "owner", "new_owner_authority",
"recent_owner_authority"
]
amount_params = [
"reward_steem", "reward_sbd", "reward_vests", "fee",
"max_accepted_payout", "amount", "sbd_amount",
"steem_amount", "daily_pay", "min_to_receive",
"amount_to_sell"
]
datetime_params = [
"start_date", "end_date", "escrow_expiration",
"expiration", "ratification_deadline"
]
price_params = ["exchange_rate"]
for key in op:
if key in json_params:
op[key] = json.dumps(op[key])
elif key in amount_params:
op[key] = str(Amount(op[key]))
elif key in datetime_params:
op[key] = formatTimeString(op[key])
elif key in price_params:
op[key] = str(Price(op[key]))
if "id" in op:
json_id = op.pop("id")
op["json_id"] = json_id
table.insert(op)
def parse_transfer_out_op(self, op):
amount = Amount(op["amount"], steem_instance=self.steem)
index = op["index"]
account = op["from"]
timestamp = op["timestamp"]
encrypted = False
processed_memo = ascii(op["memo"]).replace('\n', '').replace('\\n', '').replace('\\', '')
if len(processed_memo) > 2 and (processed_memo[0] == '#' or processed_memo[1] == '#' or processed_memo[2] == '#') and account == "steembasicincome":
if processed_memo[1] == '#':
processed_memo = processed_memo[1:-1]
elif processed_memo[2] == '#':
processed_memo = processed_memo[2:-2]
memo = Memo(account, op["to"], steem_instance=self.steem)
processed_memo = ascii(memo.decrypt(processed_memo)).replace('\n', '')
encrypted = True
if amount.amount < 1:
data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": None, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp}
self.transactionOutStorage.add(data)
return
if amount.symbol == "SBD":
# self.trxStorage.get_account(op["to"], share_type="SBD")
shares = -int(amount.amount)
if "http" in op["memo"] or "STEEM" not in op["memo"]:
data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": None, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp}
self.transactionOutStorage.add(data)
return
trx = self.trxStorage.get_SBD_transfer(op["to"], shares, formatTimeString(op["timestamp"]))
sponsee = json.dumps({})
if trx:
sponsee = trx["sponsee"]
self.new_transfer_record(op["index"], processed_memo, op["to"], op["to"], sponsee, shares, op["timestamp"], share_type="Refund")
# self.new_transfer_record(op["index"], op["to"], "", shares, op["timestamp"], share_type="Refund")
data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": sponsee, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp}
self.transactionOutStorage.add(data)
return
else:
data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": None, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp}
self.transactionOutStorage.add(data)
return
def new_delegation_record(self,
index,
account,
vests,
timestamp,
status="Valid",
share_type="Delegation"):
data = {
"index": index,
"source": self.account["name"],
"memo": "",
"account": account,
"sponsor": account,
"sponsee": json.dumps({}),
"shares": 0,
"vests": float(vests),
"timestamp": formatTimeString(timestamp),
"status": status,
"share_type": share_type
}
self.trxStorage.add(data)
def new_transfer_record(self,
index,
memo,
account,
sponsor,
sponsee,
shares,
timestamp,
status="Valid",
share_type="Standard"):
data = {
"index": index,
"source": self.account["name"],
"memo": memo,
"account": account,
"sponsor": sponsor,
"sponsee": sponsee,
"shares": shares,
"vests": float(0),
"timestamp": formatTimeString(timestamp),
"status": status,
"share_type": share_type
}
self.trxStorage.add(data)
transactionStorage = TransactionMemoDB(db2)
conf_setup = confStorage.get()
last_cycle = conf_setup["last_cycle"]
share_cycle_min = conf_setup["share_cycle_min"]
sp_share_ratio = conf_setup["sp_share_ratio"]
rshares_per_cycle = conf_setup["rshares_per_cycle"]
upvote_multiplier = conf_setup["upvote_multiplier"]
last_paid_post = conf_setup["last_paid_post"]
last_paid_comment = conf_setup["last_paid_comment"]
print("last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60))
if last_cycle is None:
last_cycle = datetime.utcnow() - timedelta(seconds = 60 * 145)
confStorage.update({"last_cycle": last_cycle})
elif True: # doing same maintanence
data = trxStorage.get_all_data()
data = sorted(data, key=lambda x: (datetime.utcnow() - x["timestamp"]).total_seconds(), reverse=True)
# data = sorted(data, key=lambda x: (datetime.utcnow() - x["timestamp"]).total_seconds(), reverse=True)
key_list = []
key = keyStorage.get("steembasicincome", "memo")
if key is not None:
key_list.append(key["wif"])
#print(key_list)
nodes = NodeList()
try:
nodes.update_nodes()
def test_formatTimeString(self):
t = "2018-07-10T10:08:39"
t = formatTimeString(t)
t2 = addTzInfo(datetime(2018, 7, 10, 10, 8, 39))
self.assertEqual(t, t2)
sponsee_dict, "Valid")
except:
print("error: %s" % processed_memo)
if False: #check all trx datasets
print("check trx dataset")
for op in data:
if op["status"] == "Valid":
try:
share_type = op["share_type"]
sponsor = op["sponsor"]
sponsee = json.loads(op["sponsee"])
shares = op["shares"]
share_age = 0
if isinstance(op["timestamp"], str):
timestamp = formatTimeString(op["timestamp"])
else:
timestamp = op["timestamp"]
except:
print("error at: %s" % str(op))
if False: #reset last cycle
print("reset last cycle")
confStorage.update({
"last_cycle":
last_cycle - timedelta(seconds=60 * share_cycle_min)
})
if False: # reset management shares
print("Reset all mgmt trx data")
trxStorage.delete_all("mgmt")
shares_sum = 0
total_mgnt_shares_sum = 0
try:
if settings is None:
settings = accountsTrx.get(pending_vote["voter"])
if settings is not None and "upvote_comment" in settings and settings["upvote_comment"] is not None:
json_metadata = {'app': 'rewarding/%s' % (rewarding_version)}
reply_body = settings["upvote_comment"]
reply_body = reply_body.replace("{{name}}", "@%s" % c["author"] ).replace("{{voter}}", "@%s" % pending_vote["voter"])
c.reply(reply_body, author=pending_vote["voter"], meta=json_metadata)
except:
print("Could not leave comment!")
voteLogTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "author": c["author"],
"timestamp": datetime.utcnow(), "vote_weight": vote_weight, "vote_delay_min": pending_vote["vote_delay_min"],
"voted_after_min": age_min, "vp": vp, "vote_when_vp_reached": pending_vote["vote_when_vp_reached"],
"trail_vote": pending_vote["trail_vote"], "main_post": pending_vote["main_post"],
"voter_to_follow": pending_vote["voter_to_follow"]})
expiration = formatTimeString(reply_message["expiration"]).replace(tzinfo=None)
delete_pending_votes.append({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"]})
else:
expiration = datetime.utcnow()
broadcastVoteTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"],
"weight": vote_weight, "vote_delay_min": pending_vote["vote_delay_min"], "min_vp": pending_vote["min_vp"],
"vote_when_vp_reached": pending_vote["vote_when_vp_reached"], "main_post": pending_vote["main_post"],
"author": c["author"], "voted_after_min": 0, "created": datetime.utcnow(), "vp": settings["vp"], "down_vp": settings["down_vp"],
"maximum_vote_delay_min": pending_vote["maximum_vote_delay_min"], "comment_timestamp": pending_vote["comment_timestamp"],
"trail_vote": pending_vote["trail_vote"], "voter_to_follow": pending_vote["voter_to_follow"], "leave_comment": pending_vote["leave_comment"],
"vote_timestamp": pending_vote["comment_timestamp"] + timedelta(seconds=pending_vote["vote_delay_min"]/60),
"max_votes_per_day": pending_vote["max_votes_per_day"], "max_votes_per_week": pending_vote["max_votes_per_week"]})
delete_pending_votes.append({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"]})
try:
if settings is None:
settings = accountsTrx.get(voter_acc["name"])
if settings is not None and "upvote_comment" in settings and settings["upvote_comment"] is not None:
json_metadata = {'app': 'rewarding/%s' % (rewarding_version)}
reply_body = settings["upvote_comment"]
reply_body = reply_body.replace("{{name}}", "@%s" % c["author"] ).replace("{{voter}}", "@%s" % voter_acc["name"])
c.reply(reply_body, author=voter_acc["name"], meta=json_metadata)
except:
print("Could not leave comment!")
voteLogTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "author": c["author"],
"timestamp": datetime.utcnow(), "vote_weight": vote_weight, "vote_delay_min": pending_vote["vote_delay_min"],
"voted_after_min": age_min, "vp": voter_acc.vp, "vote_when_vp_reached": pending_vote["vote_when_vp_reached"],
"trail_vote": pending_vote["trail_vote"], "main_post": pending_vote["main_post"],
"voter_to_follow": pending_vote["voter_to_follow"]})
broadcastVoteTrx.add({"expiration": formatTimeString(reply_message["expiration"]).replace(tzinfo=None), "authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"],
"weight": vote_weight})
delete_pending_votes.append({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"]})
continue
else:
broadcastVoteTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"],
"weight": vote_weight, "valid": True})
for pending_vote in delete_pending_votes:
pendingVotesTrx.delete(pending_vote["authorperm"], pending_vote["voter"], pending_vote["vote_when_vp_reached"])
delete_pending_votes = []
print("time vote %.2f s - %d votes" % (time.time() - start_prep_time, vote_count))
votes_above_vp = 0
votes_below_vp = 0
for pending_vote in pendingVotesTrx.get_command_list_vp_reached():
backed_symbol_amount = 0
index = 0
hard_fork_reached = False
year_reached = False
next_year_reached = False
data = init()
print("duplicate indices %d" % len(duplicate_indices))
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if _id in duplicate_indices:
continue
block = ops["block"]
timestamp = ops["timestamp"].replace("T", " ")
date = formatTimeString(ops["timestamp"]).strftime(date_format)
trx_id = ops["trx_id"]
if trx_id == "0000000000000000000000000000000000000000":
trx_id = "virtual_id_" + ops["_id"]
if limit_to_year and not year_reached and timestamp[:4] == str(
current_year):
if has_fork and hard_fork_reached:
year_reached = True
elif has_fork:
year_reached = False
else:
year_reached = True
if year_reached:
if symbol_amount > 0:
amount = Amount(symbol_amount,
def update_account_hist(self):
if self.hist_account is None:
return
votes = []
daily_curation = 0
daily_author_SP = 0
daily_author_SBD = 0
daily_author_STEEM = 0
self.append_account_hist()
new_op_found = False
start_block = self.account_hist_info["start_block"]
if start_block == 0:
first_call = True
else:
first_call = False
trx_ids = self.account_hist_info["trx_ids"]
for op in self.account_history:
if op["block"] < start_block:
# last_block = op["block"]
continue
elif op["block"] == start_block:
if op["trx_id"] in trx_ids:
continue
else:
trx_ids.append(op["trx_id"])
else:
trx_ids = [op["trx_id"]]
start_block = op["block"]
new_op_found = True
op_timedelta = formatTimedelta(
addTzInfo(datetime.utcnow()) -
formatTimeString(op["timestamp"]))
op_local_time = formatTimeString(op["timestamp"]).astimezone(
tz.tzlocal())
# print("Read %d" % op["index"])
if op["type"] == "vote":
if op["voter"] == self.hist_account["name"]:
continue
if op["weight"] >= 0:
self.lastUpvotesListWidget.insertItem(
0, "%s - %s (%.2f %%) upvote %s" %
(op_timedelta, op["voter"], op["weight"] / 100,
op["permlink"]))
hist_item = "%s - %s - %s (%.2f %%) upvote %s" % (
op_local_time, op["type"], op["voter"],
op["weight"] / 100, op["permlink"])
tray_item = "%s - %s (%.2f %%) upvote %s" % (
op["type"], op["voter"], op["weight"] / 100,
op["permlink"])
else:
hist_item = "%s - %s - %s (%.2f %%) downvote %s" % (
op_local_time, op["type"], op["voter"],
op["weight"] / 100, op["permlink"])
tray_item = "%s - %s (%.2f %%) downvote %s" % (
op["type"], op["voter"], op["weight"] / 100,
op["permlink"])
self.accountHistListWidget.insertItem(0, hist_item)
elif op["type"] == "curation_reward":
curation_reward = self.stm.vests_to_sp(
Amount(op["reward"], steem_instance=self.stm))
self.lastCurationListWidget.insertItem(
0, "%s - %.3f HP for %s" %
(op_timedelta, curation_reward,
construct_authorperm(op["comment_author"],
op["comment_permlink"])))
hist_item = "%s - %s - %.3f HP for %s" % (
op_local_time, op["type"], curation_reward,
construct_authorperm(op["comment_author"],
op["comment_permlink"]))
tray_item = "%s - %.3f HP for %s" % (
op["type"], curation_reward,
construct_authorperm(op["comment_author"],
op["comment_permlink"]))
self.accountHistListWidget.insertItem(0, hist_item)
elif op["type"] == "author_reward":
sbd_payout = (Amount(op["sbd_payout"],
steem_instance=self.stm))
steem_payout = (Amount(op["steem_payout"],
steem_instance=self.stm))
sp_payout = self.stm.vests_to_sp(
Amount(op["vesting_payout"], steem_instance=self.stm))
self.lastAuthorListWidget.insertItem(
0, "%s - %s %s %.3f HP for %s" %
(op_timedelta, str(sbd_payout), str(steem_payout),
sp_payout, op["permlink"]))
hist_item = "%s - %s - %s %s %.3f SP for %s" % (
op_local_time, op["type"], str(sbd_payout),
str(steem_payout), sp_payout, op["permlink"])
tray_item = "%s - %s %s %.3f SP for %s" % (
op["type"], str(sbd_payout), str(steem_payout), sp_payout,
op["permlink"])
self.accountHistListWidget.insertItem(0, hist_item)
elif op["type"] == "custom_json":
hist_item = "%s - %s - %s" % (op_local_time, op["type"],
op["id"])
tray_item = "%s - %s" % (op["type"], op["id"])
self.accountHistListWidget.insertItem(0, hist_item)
elif op["type"] == "transfer":
hist_item = "%s - %s - %s from %s" % (
op_local_time, op["type"],
str(Amount(op["amount"],
steem_instance=self.stm)), op["from"])
tray_item = "%s - %s from %s" % (
op["type"],
str(Amount(op["amount"],
steem_instance=self.stm)), op["from"])
self.accountHistListWidget.insertItem(0, hist_item)
elif op["type"] == "comment":
comment_type = "post"
if op["parent_author"] != "":
hist_item = "%s - comment on %s - %s from %s" % (
op_local_time,
construct_authorperm(
op["parent_author"],
op["parent_permlink"]), op["title"], op["author"])
tray_item = "comment from %s: %s on %s" % (
op["author"], op["body"][:100], op["title"])
else:
hist_item = "%s - post - %s from %s" % (
op_local_time, op["title"], op["author"])
tray_item = "post from %s: %s" % (op["author"],
op["title"])
self.accountHistListWidget.insertItem(0, hist_item)
else:
hist_item = "%s - %s" % (op_local_time, op["type"])
tray_item = "%s" % (op["type"])
self.accountHistListWidget.insertItem(0, hist_item)
if self.accountHistNotificationCheckBox.isChecked(
) and not first_call:
self.tray.showMessage(self.hist_account["name"], tray_item)
if new_op_found:
self.account_hist_info["start_block"] = start_block
self.account_hist_info["trx_ids"] = trx_ids
for op in self.account_history:
if op["type"] == "vote":
if op["voter"] == self.hist_account["name"]:
continue
votes.append(op)
elif op["type"] == "curation_reward":
curation_reward = self.stm.vests_to_sp(
Amount(op["reward"], steem_instance=self.stm))
daily_curation += curation_reward
elif op["type"] == "author_reward":
sbd_payout = (Amount(op["sbd_payout"],
steem_instance=self.stm))
steem_payout = (Amount(op["steem_payout"],
steem_instance=self.stm))
sp_payout = self.stm.vests_to_sp(
Amount(op["vesting_payout"], steem_instance=self.stm))
daily_author_SP += sp_payout
daily_author_STEEM += float(steem_payout)
daily_author_SBD += float(sbd_payout)
reward_text = "Curation reward (last 24 h): %.3f HP\n" % daily_curation
reward_text += "Author reward (last 24 h):\n"
reward_text += "%.3f HP - %.3f HIVE - %.3f HBD" % (
daily_author_SP, (daily_author_STEEM), (daily_author_SBD))
self.text2.setText(reward_text)
f["trust_score_norm"] = 0
if "trust_score_n" not in f:
f["trust_score_n"] = 0
if "created_at" not in f:
f["created_at"] = None
accounts[f["name"]] = f
else:
accounts = account2TrxStorage.get_accounts()
print("read all accounts")
if use_mongodb:
latest_created_at = account2TrxStorage.get_latest_timestamp()
else:
latest_created_at = followsTrxStorage.get_latest_created_at()
print(latest_created_at)
if latest_created_at is not None and isinstance(latest_created_at, str):
latest_created_at = formatTimeString(latest_created_at)
d = 0.85
trust_seed = {}
trust_score = {}
trust_score_old = {}
sybil = {}
sybil_list = []
N = len(accounts)
trust_sum = 0
trust_seed_sum = 0
reverse_id = {}
date_now = datetime.utcnow()
last_activity_days = {}
def test_history(self):
account = self.account
zero_element = 0
h_all_raw = []
for h in account.history_reverse(raw_output=True):
h_all_raw.append(h)
index = h_all_raw[0][0]
for op in h_all_raw:
self.assertEqual(op[0], index)
index -= 1
# h_all_raw = h_all_raw[zero_element:]
zero_element = h_all_raw[-1][0]
h_list = []
for h in account.history(stop=10,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
# self.assertEqual(h_list[0][0], zero_element)
self.assertEqual(h_list[-1][0], 10)
self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-11 + zero_element][1]['block'])
h_list = []
for h in account.history(start=1,
stop=9,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.history(start=start,
stop=stop,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
h_list = []
for h in account.history_reverse(start=10,
stop=0,
use_block_num=False,
batch_size=10,
raw_output=False):
h_list.append(h)
# zero_element = h_list[-1]['index']
self.assertEqual(h_list[0]['index'], 10)
# self.assertEqual(h_list[-1]['index'], zero_element)
self.assertEqual(h_list[0]['block'],
h_all_raw[-11 + zero_element][1]['block'])
self.assertEqual(h_list[-1]['block'], h_all_raw[-1][1]['block'])
h_list = []
for h in account.history_reverse(start=9,
stop=1,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 9)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.history_reverse(start=start,
stop=stop,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
# self.assertEqual(h_list[0][0], 8)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
order=1,
raw_output=True):
h_list.append(h)
# self.assertEqual(h_list[0][0], zero_element)
self.assertEqual(h_list[-1][0], 10)
self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-11 + zero_element][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
start=1,
stop=9,
order=1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
start=start,
stop=stop,
order=1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
order=-1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 10)
# self.assertEqual(h_list[-1][0], zero_element)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-11 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'], h_all_raw[-1][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
start=9,
stop=1,
order=-1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 9)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.get_account_history(10,
10,
start=start,
stop=stop,
order=-1,
raw_output=True):
h_list.append(h)
for i in range(len(h_list)):
self.assertEqual(h_list[i][0], 9 - i)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
upvote_multiplier = conf_setup["upvote_multiplier"]
last_paid_post = conf_setup["last_paid_post"]
last_paid_comment = conf_setup["last_paid_comment"]
last_delegation_check = conf_setup["last_delegation_check"]
minimum_vote_threshold = conf_setup["minimum_vote_threshold"]
upvote_multiplier_adjusted = conf_setup["upvote_multiplier_adjusted"]
accountTrx = {}
for account in accounts:
if account == "steembasicincome":
accountTrx["sbi"] = AccountTrx(db, "sbi")
else:
accountTrx[account] = AccountTrx(db, account)
print("sbi_update_member_db: last_cycle: %s - %.2f min" %
(formatTimeString(last_cycle),
(datetime.utcnow() - last_cycle).total_seconds() / 60))
print("last_paid_post: %s - last_paid_comment: %s" %
(formatTimeString(last_paid_post),
formatTimeString(last_paid_comment)))
if last_cycle is None:
last_cycle = datetime.utcnow() - timedelta(seconds=60 * 145)
confStorage.update({"last_cycle": last_cycle})
elif (datetime.utcnow() -
last_cycle).total_seconds() > 60 * share_cycle_min:
new_cycle = (datetime.utcnow() -
last_cycle).total_seconds() > 60 * share_cycle_min
current_cycle = last_cycle + timedelta(seconds=60 * share_cycle_min)
print("Update member database, new cycle: %s" % str(new_cycle))
def test_history(self, node_param):
if node_param == "non_appbase":
account = self.account
zero_element = 0
else:
account = self.account_appbase
zero_element = 1 # Bug in steem
h_all_raw = []
for h in account.history_reverse(raw_output=True):
h_all_raw.append(h)
# h_all_raw = h_all_raw[zero_element:]
zero_element = h_all_raw[-1][0]
h_list = []
for h in account.history(stop=10,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
# self.assertEqual(h_list[0][0], zero_element)
self.assertEqual(h_list[-1][0], 10)
self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-11 + zero_element][1]['block'])
h_list = []
for h in account.history(start=1,
stop=9,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.history(start=start,
stop=stop,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
h_list = []
for h in account.history_reverse(start=10,
stop=0,
use_block_num=False,
batch_size=10,
raw_output=False):
h_list.append(h)
# zero_element = h_list[-1]['index']
self.assertEqual(h_list[0]['index'], 10)
# self.assertEqual(h_list[-1]['index'], zero_element)
self.assertEqual(h_list[0]['block'],
h_all_raw[-11 + zero_element][1]['block'])
self.assertEqual(h_list[-1]['block'], h_all_raw[-1][1]['block'])
h_list = []
for h in account.history_reverse(start=9,
stop=1,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 9)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.history_reverse(start=start,
stop=stop,
use_block_num=False,
batch_size=10,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 9)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
order=1,
raw_output=True):
h_list.append(h)
# self.assertEqual(h_list[0][0], zero_element)
self.assertEqual(h_list[-1][0], 10)
self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-11 + zero_element][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
start=1,
stop=9,
order=1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
start=start,
stop=stop,
order=1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 1)
self.assertEqual(h_list[-1][0], 9)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
order=-1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 10)
# self.assertEqual(h_list[-1][0], zero_element)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-11 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'], h_all_raw[-1][1]['block'])
h_list = []
for h in account.get_account_history(10,
10,
use_block_num=False,
start=9,
stop=1,
order=-1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 9)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
start = formatTimeString(h_list[0][1]["timestamp"])
stop = formatTimeString(h_list[-1][1]["timestamp"])
h_list = []
for h in account.get_account_history(10,
10,
start=start,
stop=stop,
order=-1,
raw_output=True):
h_list.append(h)
self.assertEqual(h_list[0][0], 9)
self.assertEqual(h_list[-1][0], 1)
self.assertEqual(h_list[0][1]['block'],
h_all_raw[-10 + zero_element][1]['block'])
self.assertEqual(h_list[-1][1]['block'],
h_all_raw[-2 + zero_element][1]['block'])
databaseConnector2 = config_data["databaseConnector2"]
start_prep_time = time.time()
# sqlDataBaseFile = os.path.join(path, database)
# databaseConnector = "sqlite:///" + sqlDataBaseFile
db = dataset.connect(databaseConnector)
db2 = dataset.connect(databaseConnector2)
accountStorage = AccountsDB(db2)
accounts = accountStorage.get()
other_accounts = accountStorage.get_transfer()
confStorage = ConfigurationDB(db2)
conf_setup = confStorage.get()
last_cycle = conf_setup["last_cycle"]
share_cycle_min = conf_setup["share_cycle_min"]
print("sbi_store_ops_db: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60))
if last_cycle is not None and (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min:
# Update current node list from @fullnodeupdate
nodes = NodeList()
nodes.update_nodes()
# nodes.update_nodes(weights={"hist": 1})
stm = Steem(node=nodes.get_nodes())
# print(str(stm))
print("Fetch new account history ops.")
blockchain = Blockchain(steem_instance=stm)
accountTrx[account] = AccountTrx(db, account)
# Create keyStorage
trxStorage = TrxDB(db2)
memberStorage = MemberDB(db2)
keyStorage = KeysDB(db2)
transactionStorage = TransactionMemoDB(db2)
transactionOutStorage = TransactionOutDB(db2)
confStorage = ConfigurationDB(db2)
conf_setup = confStorage.get()
last_cycle = conf_setup["last_cycle"]
share_cycle_min = conf_setup["share_cycle_min"]
print("sbi_transfer: last_cycle: %s - %.2f min" %
(formatTimeString(last_cycle),
(datetime.utcnow() - last_cycle).total_seconds() / 60))
if last_cycle is not None and (datetime.utcnow() - last_cycle
).total_seconds() > 60 * share_cycle_min:
key_list = []
print("Parse new transfers.")
key = keyStorage.get("steembasicincome", "memo")
if key is not None:
key_list.append(key["wif"])
#print(key_list)
nodes = NodeList()
try:
nodes.update_nodes()
except:
last_paid_comment = conf_setup["last_paid_comment"]
last_delegation_check = conf_setup["last_delegation_check"]
minimum_vote_threshold = conf_setup["minimum_vote_threshold"]
upvote_multiplier_adjusted = conf_setup["upvote_multiplier_adjusted"]
comment_vote_divider = conf_setup["comment_vote_divider"]
accountTrx = {}
for account in accounts:
if account == "steembasicincome":
accountTrx["sbi"] = AccountTrx(db, "sbi")
else:
accountTrx[account] = AccountTrx(db, account)
print("sbi_update_member_db: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60))
print("last_paid_post: %s - last_paid_comment: %s" % (formatTimeString(last_paid_post), formatTimeString(last_paid_comment)))
if last_cycle is None:
last_cycle = datetime.utcnow() - timedelta(seconds = 60 * 145)
confStorage.update({"last_cycle": last_cycle})
elif (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min:
new_cycle = (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min
current_cycle = last_cycle + timedelta(seconds=60 * share_cycle_min)
print("Update member database, new cycle: %s" % str(new_cycle))
# memberStorage.wipe(True)
member_accounts = memberStorage.get_all_accounts()
data = trxStorage.get_all_data()
n_records = 0
member_data = {}
for op in data:
if op["status"] == "Valid":
share_type = op["share_type"]
if share_type in [
"RemovedDelegation", "Delegation", "DelegationLeased",
"Mgmt", "MgmtTransfer"
]:
continue
sponsor = op["sponsor"]
sponsee = json.loads(op["sponsee"])
shares = op["shares"]
share_age = 0
if isinstance(op["timestamp"], str):
timestamp = formatTimeString(op["timestamp"])
else:
timestamp = op["timestamp"]
if shares == 0:
continue
if sponsor not in member_data:
member = Member(sponsor, shares, timestamp)
member.append_share_age(timestamp, shares)
member_data[sponsor] = member
else:
member_data[sponsor]["latest_enrollment"] = timestamp
member_data[sponsor]["shares"] += shares
member_data[sponsor].append_share_age(timestamp, shares)
if len(sponsee) == 0:
continue
for s in sponsee:
account_name = data["new_account_name"]
if not fetch_account_ids or last_account_id < 965379:
next_account_id = accounts_name_ids[account_name]
else:
acc = Account(account_name, full=False)
api_calls += 1
next_account_id = acc["id"]
if next_account_id == last_account_id:
print("same account id")
continue
if next_account_id - last_account_id != 1:
raise ValueError("Missing data acc %s id %d to %d - block id %d" % (account_name, last_account_id, next_account_id, block_num))
timestamp = data["timestamp"]
if isinstance(timestamp, str):
timestamp = formatTimeString(timestamp)
account2_data = {"id":
next_account_id, "name": account_name, "created_at":
timestamp, "block_num":
data["block_num"], "followers": 0,
"following": 0,
# "active_at": formatTimeString(data["timestamp"]),
"cached_at": timestamp}
accounts[account_name] = next_account_id
db_account.append(account2_data)
last_account_id = next_account_id
if cnt % follow_db_batch_size == 0 and cnt > 0:
print(data["timestamp"])
print("api_calls %d" % api_calls)
start_time = time.time()
conf_setup = confStorage.get()
last_cycle = conf_setup["last_cycle"]
share_cycle_min = conf_setup["share_cycle_min"]
sp_share_ratio = conf_setup["sp_share_ratio"]
rshares_per_cycle = conf_setup["rshares_per_cycle"]
upvote_multiplier = conf_setup["upvote_multiplier"]
last_paid_post = conf_setup["last_paid_post"]
last_paid_comment = conf_setup["last_paid_comment"]
minimum_vote_threshold = conf_setup["minimum_vote_threshold"]
comment_vote_divider = conf_setup["comment_vote_divider"]
comment_vote_timeout_h = conf_setup["comment_vote_timeout_h"]
print("last_cycle: %s - %.2f min" %
(formatTimeString(last_cycle),
(datetime.utcnow() - last_cycle).total_seconds() / 60))
if True:
last_cycle = datetime.utcnow() - timedelta(seconds=60 * 145)
confStorage.update({"last_cycle": last_cycle})
print("update member database")
# memberStorage.wipe(True)
member_accounts = memberStorage.get_all_accounts()
# Update current node list from @fullnodeupdate
nodes = NodeList()
nodes.update_nodes()
stm = Steem(node=nodes.get_nodes())
# stm = Steem()
member_data = {}
n_records = 0
accountTrx[account] = AccountTrx(db, account)
# Create keyStorage
trxStorage = TrxDB(db2)
memberStorage = MemberDB(db2)
keyStorage = KeysDB(db2)
transactionStorage = TransactionMemoDB(db2)
transactionOutStorage = TransactionOutDB(db2)
confStorage = ConfigurationDB(db2)
conf_setup = confStorage.get()
last_cycle = conf_setup["last_cycle"]
share_cycle_min = conf_setup["share_cycle_min"]
print("sbi_transfer: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60))
if last_cycle is not None and (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min:
key_list = []
print("Parse new transfers.")
key = keyStorage.get("steembasicincome", "memo")
if key is not None:
key_list.append(key["wif"])
#print(key_list)
nodes = NodeList()
try:
nodes.update_nodes()
except:
print("could not update nodes")
stm = Steem(keys=key_list, node=nodes.get_nodes(hive=hive_blockchain))
upvote_multiplier = conf_setup["upvote_multiplier"]
last_paid_post = conf_setup["last_paid_post"]
last_paid_comment = conf_setup["last_paid_comment"]
last_delegation_check = conf_setup["last_delegation_check"]
minimum_vote_threshold = conf_setup["minimum_vote_threshold"]
upvote_multiplier_adjusted = conf_setup["upvote_multiplier_adjusted"]
accountTrx = {}
for account in accounts:
if account == "steembasicincome":
accountTrx["sbi"] = AccountTrx(db, "sbi")
else:
accountTrx[account] = AccountTrx(db, account)
print("sbi_update_curation_rshares: last_cycle: %s - %.2f min" %
(formatTimeString(last_cycle),
(datetime.utcnow() - last_cycle).total_seconds() / 60))
print("last_paid_post: %s - last_paid_comment: %s" %
(formatTimeString(last_paid_post),
formatTimeString(last_paid_comment)))
if (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min:
new_cycle = (datetime.utcnow() -
last_cycle).total_seconds() > 60 * share_cycle_min
current_cycle = last_cycle + timedelta(seconds=60 * share_cycle_min)
print("Update member database, new cycle: %s" % str(new_cycle))
# memberStorage.wipe(True)
member_accounts = memberStorage.get_all_accounts()
op_in_trx = op["op_in_trx"]
else:
op_in_trx += 1
if virtual_op > 0:
op_in_trx = 0
if trx_in_block > 255:
trx_in_block = 0
d = {
"block": op["block"],
"op_acc_index": start_index,
"op_acc_name": account["name"],
"trx_in_block": trx_in_block,
"op_in_trx": op_in_trx,
"virtual_op": virtual_op,
"timestamp": formatTimeString(op["timestamp"]),
"type": op["type"],
"op_dict": json.dumps(op)
}
#op_in_trx += 1
start_index += 1
last_block = op["block"]
last_trx = trx_in_block
data.append(d)
if cnt % 1000 == 0:
print(op["timestamp"])
accountTrx[account_name].add_batch(data)
data = []
cnt += 1
if len(data) > 0:
print(op["timestamp"])
