def search(self, search_str, start=None, stop=None, use_block_num=True): """ Returns ops in the given range""" ops = [] if start is not None: start = addTzInfo(start) if stop is not None: stop = addTzInfo(stop) for op in self: if use_block_num and start is not None and isinstance(start, int): if op["block"] < start: continue elif not use_block_num and start is not None and isinstance( start, int): if op["index"] < start: continue elif start is not None and isinstance(start, (datetime, date, time)): if start > formatTimeString(op["timestamp"]): continue if use_block_num and stop is not None and isinstance(stop, int): if op["block"] > stop: continue elif not use_block_num and stop is not None and isinstance( stop, int): if op["index"] > stop: continue elif stop is not None and isinstance(stop, (datetime, date, time)): if stop < formatTimeString(op["timestamp"]): continue op_string = json.dumps(list(op.values())) if re.search(search_str, op_string): ops.append(op) return ops
def acc_to_dict(account): output = account.copy() if 'json_metadata' in output: output["json_metadata"] = json.dumps(output["json_metadata"], separators=[',', ':']) parse_times = [ "last_owner_update", "last_account_update", "created", "last_owner_proved", "last_active_proved", "last_account_recovery", "last_vote_time", "sbd_seconds_last_update", "sbd_last_interest_payment", "savings_sbd_seconds_last_update", "savings_sbd_last_interest_payment", "next_vesting_withdrawal", "last_market_bandwidth_update", "last_post", "last_root_post", "last_bandwidth_update" ] for p in parse_times: if p in output: date = output.get(p, datetime(1970, 1, 1, 0, 0)) if isinstance(date, (datetime, date)): output[p] = formatTimeString(date) else: output[p] = date amounts = [ "balance", "savings_balance", "sbd_balance", "savings_sbd_balance", "reward_sbd_balance", "reward_steem_balance", "reward_vesting_balance", "reward_vesting_steem", "vesting_shares", "delegated_vesting_shares", "received_vesting_shares" ] for p in amounts: if p in output and isinstance(output[p], Amount): output[p] = output[p].json() return json.loads(str(json.dumps(output)))
def add_mngt_shares(self, last_op, mgnt_shares, op_count): index = last_op["index"] timestamp = last_op["timestamp"] sponsee = {} memo = "" latest_share = self.trxStorage.get_lastest_share_type("Mgmt") if latest_share is not None: start_index = latest_share["index"] + 1 else: start_index = op_count / 100 * 3 for account in mgnt_shares: shares = mgnt_shares[account] sponsor = account data = { "index": start_index, "source": "mgmt", "memo": "", "account": account, "sponsor": sponsor, "sponsee": sponsee, "shares": shares, "vests": float(0), "timestamp": formatTimeString(timestamp), "status": "Valid", "share_type": "Mgmt" } start_index += 1 self.trxStorage.add(data)
def test_formatDateTimetoTimeStamp(self): t = "1970-01-01T00:00:00" t = formatTimeString(t) timestamp = formatToTimeStamp(t) self.assertEqual(timestamp, 0) t2 = "2018-07-10T10:08:39" timestamp = formatToTimeStamp(t2) self.assertEqual(timestamp, 1531217319) t3 = datetime(2018, 7, 10, 10, 8, 39) timestamp = formatToTimeStamp(t3) self.assertEqual(timestamp, 1531217319)
def update_delegations(self): delegations = self.delegation_acc.get_vesting_delegations( start_account='', limit=1000, account=None) for d in delegations: if d["delegatee"] in self.accounts: self.accounts[ d["delegatee"]]["delegated_hp"] = self.hive.vests_to_hp( float( Amount(d["vesting_shares"], blockchain_instance=self.hive))) self.accounts[ d["delegatee"]]["delegation_timestamp"] = formatTimeString( d["min_delegation_time"]).replace(tzinfo=None)
def get_ops(self, start=None, stop=None, use_block_num=True, only_ops=[], exclude_ops=[]): """ Returns ops in the given range""" if start is not None: start = addTzInfo(start) if stop is not None: stop = addTzInfo(stop) for op in self: if use_block_num and start is not None and isinstance(start, int): if op["block"] < start: continue elif not use_block_num and start is not None and isinstance( start, int): if op["index"] < start: continue elif start is not None and isinstance(start, (datetime, date, time)): if start > formatTimeString(op["timestamp"]): continue if use_block_num and stop is not None and isinstance(stop, int): if op["block"] > stop: continue elif not use_block_num and stop is not None and isinstance( stop, int): if op["index"] > stop: continue elif stop is not None and isinstance(stop, (datetime, date, time)): if stop < formatTimeString(op["timestamp"]): continue if exclude_ops and op["type"] in exclude_ops: continue if not only_ops or op["type"] in only_ops: yield op
def store_blocks(self, ops): filename = self.get_filename() if self.db_type == "shelve": with shelve.open(filename) as db: db['ops'] = ops elif self.db_type == "pickle": db = {"ops": ops} save(filename, db) elif self.db_type == "deepdish": db = {"ops": ops} dd.io.save(filename, db, compression=None) elif self.db_type == "sqlite": with dataset.connect('sqlite:///%s?check_same_thread=False' % (filename)) as db: table = db["ops"] for op in ops: json_params = [ "required_auths", "required_posting_auths", "extensions", "proposal_ids", "active", "posting", "props", "owner", "new_owner_authority", "recent_owner_authority" ] amount_params = [ "reward_steem", "reward_sbd", "reward_vests", "fee", "max_accepted_payout", "amount", "sbd_amount", "steem_amount", "daily_pay", "min_to_receive", "amount_to_sell" ] datetime_params = [ "start_date", "end_date", "escrow_expiration", "expiration", "ratification_deadline" ] price_params = ["exchange_rate"] for key in op: if key in json_params: op[key] = json.dumps(op[key]) elif key in amount_params: op[key] = str(Amount(op[key])) elif key in datetime_params: op[key] = formatTimeString(op[key]) elif key in price_params: op[key] = str(Price(op[key])) if "id" in op: json_id = op.pop("id") op["json_id"] = json_id table.insert(op)
def parse_transfer_out_op(self, op): amount = Amount(op["amount"], steem_instance=self.steem) index = op["index"] account = op["from"] timestamp = op["timestamp"] encrypted = False processed_memo = ascii(op["memo"]).replace('\n', '').replace('\\n', '').replace('\\', '') if len(processed_memo) > 2 and (processed_memo[0] == '#' or processed_memo[1] == '#' or processed_memo[2] == '#') and account == "steembasicincome": if processed_memo[1] == '#': processed_memo = processed_memo[1:-1] elif processed_memo[2] == '#': processed_memo = processed_memo[2:-2] memo = Memo(account, op["to"], steem_instance=self.steem) processed_memo = ascii(memo.decrypt(processed_memo)).replace('\n', '') encrypted = True if amount.amount < 1: data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": None, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp} self.transactionOutStorage.add(data) return if amount.symbol == "SBD": # self.trxStorage.get_account(op["to"], share_type="SBD") shares = -int(amount.amount) if "http" in op["memo"] or "STEEM" not in op["memo"]: data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": None, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp} self.transactionOutStorage.add(data) return trx = self.trxStorage.get_SBD_transfer(op["to"], shares, formatTimeString(op["timestamp"])) sponsee = json.dumps({}) if trx: sponsee = trx["sponsee"] self.new_transfer_record(op["index"], processed_memo, op["to"], op["to"], sponsee, shares, op["timestamp"], share_type="Refund") # self.new_transfer_record(op["index"], op["to"], "", shares, op["timestamp"], share_type="Refund") data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": sponsee, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp} self.transactionOutStorage.add(data) return else: data = {"index": index, "sender": account, "to": op["to"], "memo": processed_memo, "encrypted": encrypted, "referenced_accounts": None, "amount": amount.amount, "amount_symbol": amount.symbol, "timestamp": timestamp} self.transactionOutStorage.add(data) return
def new_delegation_record(self, index, account, vests, timestamp, status="Valid", share_type="Delegation"): data = { "index": index, "source": self.account["name"], "memo": "", "account": account, "sponsor": account, "sponsee": json.dumps({}), "shares": 0, "vests": float(vests), "timestamp": formatTimeString(timestamp), "status": status, "share_type": share_type } self.trxStorage.add(data)
def new_transfer_record(self, index, memo, account, sponsor, sponsee, shares, timestamp, status="Valid", share_type="Standard"): data = { "index": index, "source": self.account["name"], "memo": memo, "account": account, "sponsor": sponsor, "sponsee": sponsee, "shares": shares, "vests": float(0), "timestamp": formatTimeString(timestamp), "status": status, "share_type": share_type } self.trxStorage.add(data)
transactionStorage = TransactionMemoDB(db2) conf_setup = confStorage.get() last_cycle = conf_setup["last_cycle"] share_cycle_min = conf_setup["share_cycle_min"] sp_share_ratio = conf_setup["sp_share_ratio"] rshares_per_cycle = conf_setup["rshares_per_cycle"] upvote_multiplier = conf_setup["upvote_multiplier"] last_paid_post = conf_setup["last_paid_post"] last_paid_comment = conf_setup["last_paid_comment"] print("last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) if last_cycle is None: last_cycle = datetime.utcnow() - timedelta(seconds = 60 * 145) confStorage.update({"last_cycle": last_cycle}) elif True: # doing same maintanence data = trxStorage.get_all_data() data = sorted(data, key=lambda x: (datetime.utcnow() - x["timestamp"]).total_seconds(), reverse=True) # data = sorted(data, key=lambda x: (datetime.utcnow() - x["timestamp"]).total_seconds(), reverse=True) key_list = [] key = keyStorage.get("steembasicincome", "memo") if key is not None: key_list.append(key["wif"]) #print(key_list) nodes = NodeList() try: nodes.update_nodes()
def test_formatTimeString(self): t = "2018-07-10T10:08:39" t = formatTimeString(t) t2 = addTzInfo(datetime(2018, 7, 10, 10, 8, 39)) self.assertEqual(t, t2)
sponsee_dict, "Valid") except: print("error: %s" % processed_memo) if False: #check all trx datasets print("check trx dataset") for op in data: if op["status"] == "Valid": try: share_type = op["share_type"] sponsor = op["sponsor"] sponsee = json.loads(op["sponsee"]) shares = op["shares"] share_age = 0 if isinstance(op["timestamp"], str): timestamp = formatTimeString(op["timestamp"]) else: timestamp = op["timestamp"] except: print("error at: %s" % str(op)) if False: #reset last cycle print("reset last cycle") confStorage.update({ "last_cycle": last_cycle - timedelta(seconds=60 * share_cycle_min) }) if False: # reset management shares print("Reset all mgmt trx data") trxStorage.delete_all("mgmt") shares_sum = 0 total_mgnt_shares_sum = 0
try: if settings is None: settings = accountsTrx.get(pending_vote["voter"]) if settings is not None and "upvote_comment" in settings and settings["upvote_comment"] is not None: json_metadata = {'app': 'rewarding/%s' % (rewarding_version)} reply_body = settings["upvote_comment"] reply_body = reply_body.replace("{{name}}", "@%s" % c["author"] ).replace("{{voter}}", "@%s" % pending_vote["voter"]) c.reply(reply_body, author=pending_vote["voter"], meta=json_metadata) except: print("Could not leave comment!") voteLogTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "author": c["author"], "timestamp": datetime.utcnow(), "vote_weight": vote_weight, "vote_delay_min": pending_vote["vote_delay_min"], "voted_after_min": age_min, "vp": vp, "vote_when_vp_reached": pending_vote["vote_when_vp_reached"], "trail_vote": pending_vote["trail_vote"], "main_post": pending_vote["main_post"], "voter_to_follow": pending_vote["voter_to_follow"]}) expiration = formatTimeString(reply_message["expiration"]).replace(tzinfo=None) delete_pending_votes.append({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"]}) else: expiration = datetime.utcnow() broadcastVoteTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "weight": vote_weight, "vote_delay_min": pending_vote["vote_delay_min"], "min_vp": pending_vote["min_vp"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"], "main_post": pending_vote["main_post"], "author": c["author"], "voted_after_min": 0, "created": datetime.utcnow(), "vp": settings["vp"], "down_vp": settings["down_vp"], "maximum_vote_delay_min": pending_vote["maximum_vote_delay_min"], "comment_timestamp": pending_vote["comment_timestamp"], "trail_vote": pending_vote["trail_vote"], "voter_to_follow": pending_vote["voter_to_follow"], "leave_comment": pending_vote["leave_comment"], "vote_timestamp": pending_vote["comment_timestamp"] + timedelta(seconds=pending_vote["vote_delay_min"]/60), "max_votes_per_day": pending_vote["max_votes_per_day"], "max_votes_per_week": pending_vote["max_votes_per_week"]}) delete_pending_votes.append({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"]})
try: if settings is None: settings = accountsTrx.get(voter_acc["name"]) if settings is not None and "upvote_comment" in settings and settings["upvote_comment"] is not None: json_metadata = {'app': 'rewarding/%s' % (rewarding_version)} reply_body = settings["upvote_comment"] reply_body = reply_body.replace("{{name}}", "@%s" % c["author"] ).replace("{{voter}}", "@%s" % voter_acc["name"]) c.reply(reply_body, author=voter_acc["name"], meta=json_metadata) except: print("Could not leave comment!") voteLogTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "author": c["author"], "timestamp": datetime.utcnow(), "vote_weight": vote_weight, "vote_delay_min": pending_vote["vote_delay_min"], "voted_after_min": age_min, "vp": voter_acc.vp, "vote_when_vp_reached": pending_vote["vote_when_vp_reached"], "trail_vote": pending_vote["trail_vote"], "main_post": pending_vote["main_post"], "voter_to_follow": pending_vote["voter_to_follow"]}) broadcastVoteTrx.add({"expiration": formatTimeString(reply_message["expiration"]).replace(tzinfo=None), "authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "weight": vote_weight}) delete_pending_votes.append({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "vote_when_vp_reached": pending_vote["vote_when_vp_reached"]}) continue else: broadcastVoteTrx.add({"authorperm": pending_vote["authorperm"], "voter": pending_vote["voter"], "weight": vote_weight, "valid": True}) for pending_vote in delete_pending_votes: pendingVotesTrx.delete(pending_vote["authorperm"], pending_vote["voter"], pending_vote["vote_when_vp_reached"]) delete_pending_votes = [] print("time vote %.2f s - %d votes" % (time.time() - start_prep_time, vote_count)) votes_above_vp = 0 votes_below_vp = 0 for pending_vote in pendingVotesTrx.get_command_list_vp_reached():
backed_symbol_amount = 0 index = 0 hard_fork_reached = False year_reached = False next_year_reached = False data = init() print("duplicate indices %d" % len(duplicate_indices)) for _id in sorted(list(ops_dict.keys())): ops = ops_dict[_id] if _id in duplicate_indices: continue block = ops["block"] timestamp = ops["timestamp"].replace("T", " ") date = formatTimeString(ops["timestamp"]).strftime(date_format) trx_id = ops["trx_id"] if trx_id == "0000000000000000000000000000000000000000": trx_id = "virtual_id_" + ops["_id"] if limit_to_year and not year_reached and timestamp[:4] == str( current_year): if has_fork and hard_fork_reached: year_reached = True elif has_fork: year_reached = False else: year_reached = True if year_reached: if symbol_amount > 0: amount = Amount(symbol_amount,
def update_account_hist(self): if self.hist_account is None: return votes = [] daily_curation = 0 daily_author_SP = 0 daily_author_SBD = 0 daily_author_STEEM = 0 self.append_account_hist() new_op_found = False start_block = self.account_hist_info["start_block"] if start_block == 0: first_call = True else: first_call = False trx_ids = self.account_hist_info["trx_ids"] for op in self.account_history: if op["block"] < start_block: # last_block = op["block"] continue elif op["block"] == start_block: if op["trx_id"] in trx_ids: continue else: trx_ids.append(op["trx_id"]) else: trx_ids = [op["trx_id"]] start_block = op["block"] new_op_found = True op_timedelta = formatTimedelta( addTzInfo(datetime.utcnow()) - formatTimeString(op["timestamp"])) op_local_time = formatTimeString(op["timestamp"]).astimezone( tz.tzlocal()) # print("Read %d" % op["index"]) if op["type"] == "vote": if op["voter"] == self.hist_account["name"]: continue if op["weight"] >= 0: self.lastUpvotesListWidget.insertItem( 0, "%s - %s (%.2f %%) upvote %s" % (op_timedelta, op["voter"], op["weight"] / 100, op["permlink"])) hist_item = "%s - %s - %s (%.2f %%) upvote %s" % ( op_local_time, op["type"], op["voter"], op["weight"] / 100, op["permlink"]) tray_item = "%s - %s (%.2f %%) upvote %s" % ( op["type"], op["voter"], op["weight"] / 100, op["permlink"]) else: hist_item = "%s - %s - %s (%.2f %%) downvote %s" % ( op_local_time, op["type"], op["voter"], op["weight"] / 100, op["permlink"]) tray_item = "%s - %s (%.2f %%) downvote %s" % ( op["type"], op["voter"], op["weight"] / 100, op["permlink"]) self.accountHistListWidget.insertItem(0, hist_item) elif op["type"] == "curation_reward": curation_reward = self.stm.vests_to_sp( Amount(op["reward"], steem_instance=self.stm)) self.lastCurationListWidget.insertItem( 0, "%s - %.3f HP for %s" % (op_timedelta, curation_reward, construct_authorperm(op["comment_author"], op["comment_permlink"]))) hist_item = "%s - %s - %.3f HP for %s" % ( op_local_time, op["type"], curation_reward, construct_authorperm(op["comment_author"], op["comment_permlink"])) tray_item = "%s - %.3f HP for %s" % ( op["type"], curation_reward, construct_authorperm(op["comment_author"], op["comment_permlink"])) self.accountHistListWidget.insertItem(0, hist_item) elif op["type"] == "author_reward": sbd_payout = (Amount(op["sbd_payout"], steem_instance=self.stm)) steem_payout = (Amount(op["steem_payout"], steem_instance=self.stm)) sp_payout = self.stm.vests_to_sp( Amount(op["vesting_payout"], steem_instance=self.stm)) self.lastAuthorListWidget.insertItem( 0, "%s - %s %s %.3f HP for %s" % (op_timedelta, str(sbd_payout), str(steem_payout), sp_payout, op["permlink"])) hist_item = "%s - %s - %s %s %.3f SP for %s" % ( op_local_time, op["type"], str(sbd_payout), str(steem_payout), sp_payout, op["permlink"]) tray_item = "%s - %s %s %.3f SP for %s" % ( op["type"], str(sbd_payout), str(steem_payout), sp_payout, op["permlink"]) self.accountHistListWidget.insertItem(0, hist_item) elif op["type"] == "custom_json": hist_item = "%s - %s - %s" % (op_local_time, op["type"], op["id"]) tray_item = "%s - %s" % (op["type"], op["id"]) self.accountHistListWidget.insertItem(0, hist_item) elif op["type"] == "transfer": hist_item = "%s - %s - %s from %s" % ( op_local_time, op["type"], str(Amount(op["amount"], steem_instance=self.stm)), op["from"]) tray_item = "%s - %s from %s" % ( op["type"], str(Amount(op["amount"], steem_instance=self.stm)), op["from"]) self.accountHistListWidget.insertItem(0, hist_item) elif op["type"] == "comment": comment_type = "post" if op["parent_author"] != "": hist_item = "%s - comment on %s - %s from %s" % ( op_local_time, construct_authorperm( op["parent_author"], op["parent_permlink"]), op["title"], op["author"]) tray_item = "comment from %s: %s on %s" % ( op["author"], op["body"][:100], op["title"]) else: hist_item = "%s - post - %s from %s" % ( op_local_time, op["title"], op["author"]) tray_item = "post from %s: %s" % (op["author"], op["title"]) self.accountHistListWidget.insertItem(0, hist_item) else: hist_item = "%s - %s" % (op_local_time, op["type"]) tray_item = "%s" % (op["type"]) self.accountHistListWidget.insertItem(0, hist_item) if self.accountHistNotificationCheckBox.isChecked( ) and not first_call: self.tray.showMessage(self.hist_account["name"], tray_item) if new_op_found: self.account_hist_info["start_block"] = start_block self.account_hist_info["trx_ids"] = trx_ids for op in self.account_history: if op["type"] == "vote": if op["voter"] == self.hist_account["name"]: continue votes.append(op) elif op["type"] == "curation_reward": curation_reward = self.stm.vests_to_sp( Amount(op["reward"], steem_instance=self.stm)) daily_curation += curation_reward elif op["type"] == "author_reward": sbd_payout = (Amount(op["sbd_payout"], steem_instance=self.stm)) steem_payout = (Amount(op["steem_payout"], steem_instance=self.stm)) sp_payout = self.stm.vests_to_sp( Amount(op["vesting_payout"], steem_instance=self.stm)) daily_author_SP += sp_payout daily_author_STEEM += float(steem_payout) daily_author_SBD += float(sbd_payout) reward_text = "Curation reward (last 24 h): %.3f HP\n" % daily_curation reward_text += "Author reward (last 24 h):\n" reward_text += "%.3f HP - %.3f HIVE - %.3f HBD" % ( daily_author_SP, (daily_author_STEEM), (daily_author_SBD)) self.text2.setText(reward_text)
f["trust_score_norm"] = 0 if "trust_score_n" not in f: f["trust_score_n"] = 0 if "created_at" not in f: f["created_at"] = None accounts[f["name"]] = f else: accounts = account2TrxStorage.get_accounts() print("read all accounts") if use_mongodb: latest_created_at = account2TrxStorage.get_latest_timestamp() else: latest_created_at = followsTrxStorage.get_latest_created_at() print(latest_created_at) if latest_created_at is not None and isinstance(latest_created_at, str): latest_created_at = formatTimeString(latest_created_at) d = 0.85 trust_seed = {} trust_score = {} trust_score_old = {} sybil = {} sybil_list = [] N = len(accounts) trust_sum = 0 trust_seed_sum = 0 reverse_id = {} date_now = datetime.utcnow() last_activity_days = {}
def test_history(self): account = self.account zero_element = 0 h_all_raw = [] for h in account.history_reverse(raw_output=True): h_all_raw.append(h) index = h_all_raw[0][0] for op in h_all_raw: self.assertEqual(op[0], index) index -= 1 # h_all_raw = h_all_raw[zero_element:] zero_element = h_all_raw[-1][0] h_list = [] for h in account.history(stop=10, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) # self.assertEqual(h_list[0][0], zero_element) self.assertEqual(h_list[-1][0], 10) self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-11 + zero_element][1]['block']) h_list = [] for h in account.history(start=1, stop=9, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.history(start=start, stop=stop, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) h_list = [] for h in account.history_reverse(start=10, stop=0, use_block_num=False, batch_size=10, raw_output=False): h_list.append(h) # zero_element = h_list[-1]['index'] self.assertEqual(h_list[0]['index'], 10) # self.assertEqual(h_list[-1]['index'], zero_element) self.assertEqual(h_list[0]['block'], h_all_raw[-11 + zero_element][1]['block']) self.assertEqual(h_list[-1]['block'], h_all_raw[-1][1]['block']) h_list = [] for h in account.history_reverse(start=9, stop=1, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 9) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.history_reverse(start=start, stop=stop, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) # self.assertEqual(h_list[0][0], 8) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, order=1, raw_output=True): h_list.append(h) # self.assertEqual(h_list[0][0], zero_element) self.assertEqual(h_list[-1][0], 10) self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-11 + zero_element][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, start=1, stop=9, order=1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, start=start, stop=stop, order=1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, order=-1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 10) # self.assertEqual(h_list[-1][0], zero_element) self.assertEqual(h_list[0][1]['block'], h_all_raw[-11 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-1][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, start=9, stop=1, order=-1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 9) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.get_account_history(10, 10, start=start, stop=stop, order=-1, raw_output=True): h_list.append(h) for i in range(len(h_list)): self.assertEqual(h_list[i][0], 9 - i) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block'])
upvote_multiplier = conf_setup["upvote_multiplier"] last_paid_post = conf_setup["last_paid_post"] last_paid_comment = conf_setup["last_paid_comment"] last_delegation_check = conf_setup["last_delegation_check"] minimum_vote_threshold = conf_setup["minimum_vote_threshold"] upvote_multiplier_adjusted = conf_setup["upvote_multiplier_adjusted"] accountTrx = {} for account in accounts: if account == "steembasicincome": accountTrx["sbi"] = AccountTrx(db, "sbi") else: accountTrx[account] = AccountTrx(db, account) print("sbi_update_member_db: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) print("last_paid_post: %s - last_paid_comment: %s" % (formatTimeString(last_paid_post), formatTimeString(last_paid_comment))) if last_cycle is None: last_cycle = datetime.utcnow() - timedelta(seconds=60 * 145) confStorage.update({"last_cycle": last_cycle}) elif (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min: new_cycle = (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min current_cycle = last_cycle + timedelta(seconds=60 * share_cycle_min) print("Update member database, new cycle: %s" % str(new_cycle))
def test_history(self, node_param): if node_param == "non_appbase": account = self.account zero_element = 0 else: account = self.account_appbase zero_element = 1 # Bug in steem h_all_raw = [] for h in account.history_reverse(raw_output=True): h_all_raw.append(h) # h_all_raw = h_all_raw[zero_element:] zero_element = h_all_raw[-1][0] h_list = [] for h in account.history(stop=10, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) # self.assertEqual(h_list[0][0], zero_element) self.assertEqual(h_list[-1][0], 10) self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-11 + zero_element][1]['block']) h_list = [] for h in account.history(start=1, stop=9, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.history(start=start, stop=stop, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) h_list = [] for h in account.history_reverse(start=10, stop=0, use_block_num=False, batch_size=10, raw_output=False): h_list.append(h) # zero_element = h_list[-1]['index'] self.assertEqual(h_list[0]['index'], 10) # self.assertEqual(h_list[-1]['index'], zero_element) self.assertEqual(h_list[0]['block'], h_all_raw[-11 + zero_element][1]['block']) self.assertEqual(h_list[-1]['block'], h_all_raw[-1][1]['block']) h_list = [] for h in account.history_reverse(start=9, stop=1, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 9) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.history_reverse(start=start, stop=stop, use_block_num=False, batch_size=10, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 9) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, order=1, raw_output=True): h_list.append(h) # self.assertEqual(h_list[0][0], zero_element) self.assertEqual(h_list[-1][0], 10) self.assertEqual(h_list[0][1]['block'], h_all_raw[-1][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-11 + zero_element][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, start=1, stop=9, order=1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, start=start, stop=stop, order=1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 1) self.assertEqual(h_list[-1][0], 9) self.assertEqual(h_list[0][1]['block'], h_all_raw[-2 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-10 + zero_element][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, order=-1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 10) # self.assertEqual(h_list[-1][0], zero_element) self.assertEqual(h_list[0][1]['block'], h_all_raw[-11 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-1][1]['block']) h_list = [] for h in account.get_account_history(10, 10, use_block_num=False, start=9, stop=1, order=-1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 9) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block']) start = formatTimeString(h_list[0][1]["timestamp"]) stop = formatTimeString(h_list[-1][1]["timestamp"]) h_list = [] for h in account.get_account_history(10, 10, start=start, stop=stop, order=-1, raw_output=True): h_list.append(h) self.assertEqual(h_list[0][0], 9) self.assertEqual(h_list[-1][0], 1) self.assertEqual(h_list[0][1]['block'], h_all_raw[-10 + zero_element][1]['block']) self.assertEqual(h_list[-1][1]['block'], h_all_raw[-2 + zero_element][1]['block'])
databaseConnector2 = config_data["databaseConnector2"] start_prep_time = time.time() # sqlDataBaseFile = os.path.join(path, database) # databaseConnector = "sqlite:///" + sqlDataBaseFile db = dataset.connect(databaseConnector) db2 = dataset.connect(databaseConnector2) accountStorage = AccountsDB(db2) accounts = accountStorage.get() other_accounts = accountStorage.get_transfer() confStorage = ConfigurationDB(db2) conf_setup = confStorage.get() last_cycle = conf_setup["last_cycle"] share_cycle_min = conf_setup["share_cycle_min"] print("sbi_store_ops_db: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) if last_cycle is not None and (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min: # Update current node list from @fullnodeupdate nodes = NodeList() nodes.update_nodes() # nodes.update_nodes(weights={"hist": 1}) stm = Steem(node=nodes.get_nodes()) # print(str(stm)) print("Fetch new account history ops.") blockchain = Blockchain(steem_instance=stm)
accountTrx[account] = AccountTrx(db, account) # Create keyStorage trxStorage = TrxDB(db2) memberStorage = MemberDB(db2) keyStorage = KeysDB(db2) transactionStorage = TransactionMemoDB(db2) transactionOutStorage = TransactionOutDB(db2) confStorage = ConfigurationDB(db2) conf_setup = confStorage.get() last_cycle = conf_setup["last_cycle"] share_cycle_min = conf_setup["share_cycle_min"] print("sbi_transfer: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) if last_cycle is not None and (datetime.utcnow() - last_cycle ).total_seconds() > 60 * share_cycle_min: key_list = [] print("Parse new transfers.") key = keyStorage.get("steembasicincome", "memo") if key is not None: key_list.append(key["wif"]) #print(key_list) nodes = NodeList() try: nodes.update_nodes() except:
last_paid_comment = conf_setup["last_paid_comment"] last_delegation_check = conf_setup["last_delegation_check"] minimum_vote_threshold = conf_setup["minimum_vote_threshold"] upvote_multiplier_adjusted = conf_setup["upvote_multiplier_adjusted"] comment_vote_divider = conf_setup["comment_vote_divider"] accountTrx = {} for account in accounts: if account == "steembasicincome": accountTrx["sbi"] = AccountTrx(db, "sbi") else: accountTrx[account] = AccountTrx(db, account) print("sbi_update_member_db: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) print("last_paid_post: %s - last_paid_comment: %s" % (formatTimeString(last_paid_post), formatTimeString(last_paid_comment))) if last_cycle is None: last_cycle = datetime.utcnow() - timedelta(seconds = 60 * 145) confStorage.update({"last_cycle": last_cycle}) elif (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min: new_cycle = (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min current_cycle = last_cycle + timedelta(seconds=60 * share_cycle_min) print("Update member database, new cycle: %s" % str(new_cycle)) # memberStorage.wipe(True) member_accounts = memberStorage.get_all_accounts() data = trxStorage.get_all_data()
n_records = 0 member_data = {} for op in data: if op["status"] == "Valid": share_type = op["share_type"] if share_type in [ "RemovedDelegation", "Delegation", "DelegationLeased", "Mgmt", "MgmtTransfer" ]: continue sponsor = op["sponsor"] sponsee = json.loads(op["sponsee"]) shares = op["shares"] share_age = 0 if isinstance(op["timestamp"], str): timestamp = formatTimeString(op["timestamp"]) else: timestamp = op["timestamp"] if shares == 0: continue if sponsor not in member_data: member = Member(sponsor, shares, timestamp) member.append_share_age(timestamp, shares) member_data[sponsor] = member else: member_data[sponsor]["latest_enrollment"] = timestamp member_data[sponsor]["shares"] += shares member_data[sponsor].append_share_age(timestamp, shares) if len(sponsee) == 0: continue for s in sponsee:
account_name = data["new_account_name"] if not fetch_account_ids or last_account_id < 965379: next_account_id = accounts_name_ids[account_name] else: acc = Account(account_name, full=False) api_calls += 1 next_account_id = acc["id"] if next_account_id == last_account_id: print("same account id") continue if next_account_id - last_account_id != 1: raise ValueError("Missing data acc %s id %d to %d - block id %d" % (account_name, last_account_id, next_account_id, block_num)) timestamp = data["timestamp"] if isinstance(timestamp, str): timestamp = formatTimeString(timestamp) account2_data = {"id": next_account_id, "name": account_name, "created_at": timestamp, "block_num": data["block_num"], "followers": 0, "following": 0, # "active_at": formatTimeString(data["timestamp"]), "cached_at": timestamp} accounts[account_name] = next_account_id db_account.append(account2_data) last_account_id = next_account_id if cnt % follow_db_batch_size == 0 and cnt > 0: print(data["timestamp"]) print("api_calls %d" % api_calls) start_time = time.time()
conf_setup = confStorage.get() last_cycle = conf_setup["last_cycle"] share_cycle_min = conf_setup["share_cycle_min"] sp_share_ratio = conf_setup["sp_share_ratio"] rshares_per_cycle = conf_setup["rshares_per_cycle"] upvote_multiplier = conf_setup["upvote_multiplier"] last_paid_post = conf_setup["last_paid_post"] last_paid_comment = conf_setup["last_paid_comment"] minimum_vote_threshold = conf_setup["minimum_vote_threshold"] comment_vote_divider = conf_setup["comment_vote_divider"] comment_vote_timeout_h = conf_setup["comment_vote_timeout_h"] print("last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) if True: last_cycle = datetime.utcnow() - timedelta(seconds=60 * 145) confStorage.update({"last_cycle": last_cycle}) print("update member database") # memberStorage.wipe(True) member_accounts = memberStorage.get_all_accounts() # Update current node list from @fullnodeupdate nodes = NodeList() nodes.update_nodes() stm = Steem(node=nodes.get_nodes()) # stm = Steem() member_data = {} n_records = 0
accountTrx[account] = AccountTrx(db, account) # Create keyStorage trxStorage = TrxDB(db2) memberStorage = MemberDB(db2) keyStorage = KeysDB(db2) transactionStorage = TransactionMemoDB(db2) transactionOutStorage = TransactionOutDB(db2) confStorage = ConfigurationDB(db2) conf_setup = confStorage.get() last_cycle = conf_setup["last_cycle"] share_cycle_min = conf_setup["share_cycle_min"] print("sbi_transfer: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) if last_cycle is not None and (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min: key_list = [] print("Parse new transfers.") key = keyStorage.get("steembasicincome", "memo") if key is not None: key_list.append(key["wif"]) #print(key_list) nodes = NodeList() try: nodes.update_nodes() except: print("could not update nodes") stm = Steem(keys=key_list, node=nodes.get_nodes(hive=hive_blockchain))
upvote_multiplier = conf_setup["upvote_multiplier"] last_paid_post = conf_setup["last_paid_post"] last_paid_comment = conf_setup["last_paid_comment"] last_delegation_check = conf_setup["last_delegation_check"] minimum_vote_threshold = conf_setup["minimum_vote_threshold"] upvote_multiplier_adjusted = conf_setup["upvote_multiplier_adjusted"] accountTrx = {} for account in accounts: if account == "steembasicincome": accountTrx["sbi"] = AccountTrx(db, "sbi") else: accountTrx[account] = AccountTrx(db, account) print("sbi_update_curation_rshares: last_cycle: %s - %.2f min" % (formatTimeString(last_cycle), (datetime.utcnow() - last_cycle).total_seconds() / 60)) print("last_paid_post: %s - last_paid_comment: %s" % (formatTimeString(last_paid_post), formatTimeString(last_paid_comment))) if (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min: new_cycle = (datetime.utcnow() - last_cycle).total_seconds() > 60 * share_cycle_min current_cycle = last_cycle + timedelta(seconds=60 * share_cycle_min) print("Update member database, new cycle: %s" % str(new_cycle)) # memberStorage.wipe(True) member_accounts = memberStorage.get_all_accounts()
op_in_trx = op["op_in_trx"] else: op_in_trx += 1 if virtual_op > 0: op_in_trx = 0 if trx_in_block > 255: trx_in_block = 0 d = { "block": op["block"], "op_acc_index": start_index, "op_acc_name": account["name"], "trx_in_block": trx_in_block, "op_in_trx": op_in_trx, "virtual_op": virtual_op, "timestamp": formatTimeString(op["timestamp"]), "type": op["type"], "op_dict": json.dumps(op) } #op_in_trx += 1 start_index += 1 last_block = op["block"] last_trx = trx_in_block data.append(d) if cnt % 1000 == 0: print(op["timestamp"]) accountTrx[account_name].add_batch(data) data = [] cnt += 1 if len(data) > 0: print(op["timestamp"])