def run(self): b = Blockchain(steem_instance=self.stm) yesterday = date.today() - timedelta(days=1) yesterday_0_0_0 = datetime(yesterday.year, yesterday.month, yesterday.day) yesterday_23_59_59 = datetime(yesterday.year, yesterday.month, yesterday.day, 23, 59, 59) start_block = b.get_estimated_block_num(addTzInfo(yesterday_0_0_0)) stop_block = b.get_estimated_block_num(addTzInfo(yesterday_23_59_59)) logger.info("Check token transfer from %s..." % self.config["scot_account"]) token_sent_last_24_h = self.get_token_transfer_last_24_h() if token_sent_last_24_h > 0: logger.warning("Token were already sent today...") return logger.info("No token transfer were found, continue...") token_per_100_vote = self.get_token_holder() logger.info("%d token holder were found." % len(token_per_100_vote)) token_to_authors = self.get_token_to_sent(start_block, stop_block, token_per_100_vote) token_to_authors = self.adapt_to_precision(token_to_authors) token_amount_to_sent = self.count_token(token_to_authors) logger.info("Start to send %f token to %d accounts" % (token_amount_to_sent, len(token_to_authors))) self.send_token(token_to_authors)
class Crawler: def __init__(self, name, blockchain): self.__name = name if blockchain == "hive": self._blockchain = Blockchain( blockchain_instance=blockchains.HIVE_INSTANCE) pass elif blockchain == "steem": self._blockchain = Blockchain( blockchain_instance=blockchains.STEEM_INSTANCE) else: raise NotImplementedError def _stream(self, opNames=[]): if hasattr(self, "start") and hasattr(self, "stop"): # get starting block id start_block_id = self._blockchain.get_estimated_block_num( self.start, accurate=True) stop_block_id = self._blockchain.get_estimated_block_num( self.stop, accurate=True) # looping trough generator for op_json in self._blockchain.stream(opNames=opNames, start=start_block_id, stop=stop_block_id): yield op_json else: print( "[DEBUG] Timeframe was not set in criteria, direct streaming used" ) # looping trough generator for op_json in self._blockchain.stream(opNames=opNames): yield op_json def set_timeframe(self, start, stop): if isinstance(start, datetime.datetime): self.start = start else: raise TypeError( "start argument must be an instance of datetime.datetime") if isinstance(stop, datetime.datetime): self.stop = stop else: raise TypeError( "stop argument must be an instance of datetime.datetime") if start >= stop: raise RuntimeError("stop({}) < start({})".format(stop, start))
def test_estimate_block_num(self): bts = self.bts b = Blockchain(steem_instance=bts) last_block = b.get_current_block() num = last_block.identifier old_block = Block(num - 60, steem_instance=bts) date = old_block.time() est_block_num = b.get_estimated_block_num(date, accurate=False) self.assertTrue((est_block_num - (old_block.identifier)) < 10) est_block_num = b.get_estimated_block_num(date, accurate=True) self.assertTrue((est_block_num - (old_block.identifier)) < 2) est_block_num = b.get_estimated_block_num(date, estimateForwards=True, accurate=True) self.assertTrue((est_block_num - (old_block.identifier)) < 2) est_block_num = b.get_estimated_block_num(date, estimateForwards=True, accurate=False)
class SteemStream: def __init__(self, operations=[]): self.blockchain = Blockchain(mode="head") self.operations = operations self.last_streamed_block = 0 self.max_batch_size = 50 self.threading = False def run(self, callback=default_callback, lookback=0, start=-1, days=-1): try: if lookback > 0 or start > 0 or days > 0: if self.last_streamed_block == 0: if start > 0: start_block = start elif lookback > 0: start_block = self.blockchain.get_current_block_num() - int(lookback) #200000 else: start_date = days_ago(days) start_block = self.blockchain.get_estimated_block_num(start_date) else: start_block = self.last_streamed_block + 1 stop_block = self.blockchain.get_current_block_num() logger.info("Streaming for operations {} has started from block {} to {}".format(self.operations, start_block, stop_block)) for ops in self.blockchain.stream(opNames=self.operations, start=start_block, stop=stop_block, max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8): try: callback(ops) except: logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc())) else: logger.info("Streaming for operations {} has started from the latest blocks".format(self.operations)) for ops in self.blockchain.stream(opNames=self.operations, max_batch_size=self.max_batch_size, threading=self.threading, thread_num=8): try: callback(ops) except: logger.error("Failed when procssing operation {} with error: {}".format(ops, traceback.format_exc())) except: logger.error("Failed when streaming operations {} with error: {}".format(self.operations, traceback.format_exc()))
import shelve from beem import Steem from beem.blockchain import Blockchain from datetime import datetime, timedelta year = 2019 month = int(sys.argv[1]) day = int(sys.argv[2]) start_date = datetime(year, month, day) stop_date = start_date + timedelta(days=1) DW_IDs = ['dw-heist', 'drugwars', 'dw-unit', 'dw-upgrade', 'dw-char'] stm = None #Steem(node=['https://api.steemit.com']) b = Blockchain(steem_instance=stm) start_block = b.get_estimated_block_num(start_date) stop_block = b.get_estimated_block_num(stop_date) transactions = [] for blk in b.blocks(start=start_block, stop=stop_block, threading=True): # max_batch_size=50): sys.stdout.write("%s\r" % (blk['timestamp'])) for trx in blk.transactions: keep = False for op in trx['operations']: if op['type'] == 'custom_json_operation' and \ op['value']['id'] in DW_IDs: keep = True if keep: transactions.append(trx)
for vote in c["active_votes"]: cnt3 += 1 if int(vote["rshares"]) == 0: continue if (addTzInfo(datetime.utcnow()) - (vote["time"]) ).total_seconds() / 60 / 60 / 24 <= 7: continue if vote["voter"] not in member_data: continue if authorperm in comments_transfer and stm.rshares_to_sbd( int(vote["rshares"])) >= 0.05: try: if cnt3 % 10 == 0: print("%d/%d votes" % (cnt3, len(c["active_votes"]))) block_num = b.get_estimated_block_num( vote["time"]) current_block_num = b.get_current_block_num() transaction = None block_search_list = [ 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5 ] block_cnt = 0 while transaction is None and block_cnt < len( block_search_list): if block_num + block_search_list[ block_cnt] > current_block_num: block_cnt += 1 continue block = Block(block_num + block_search_list[block_cnt], steem_instance=stm)
postTrx.delete_old_posts(1) # print("reading all authorperm") already_voted_posts = [] flagged_posts = [] start_block = b.get_current_block_num() - int(28800) stop_block = b.get_current_block_num() last_block_print = start_block latest_update = postTrx.get_latest_post() latest_block = postTrx.get_latest_block() if latest_block is not None and latest_block > start_block: latest_update_block = latest_block elif latest_block is not None and latest_block < start_block: latest_update_block = start_block elif latest_update is not None: latest_update_block = b.get_estimated_block_num(latest_update) else: latest_update_block = start_block print("latest update %s - %d to %d" % (str(latest_update), latest_update_block, stop_block)) start_block = max([latest_update_block, start_block]) + 1 if stop_block > start_block + 6000: stop_block = start_block + 6000 cnt = 0 updated_accounts = [] posts_dict = {} changed_member_data = [] for ops in b.stream(start=start_block, stop=stop_block, opNames=["comment"],
block_num = 33898189 timestamp = vop["trigger_date"] elif vop["trigger_date"] == datetime( 2019, 7, 30, 20, 24, 51): timestamp = vop["trigger_date"] block_num = 35124173 else: block_num_est_method = "beem" if last_vops_timestamp is not None and abs( (vop["trigger_date"] - last_vops_timestamp).total_seconds()) < 6: print("diff to last vops %.2s" % (vop["trigger_date"] - last_vops_timestamp).total_seconds()) block_num = b.get_estimated_block_num( vop["trigger_date"]) last_vops_block_num = block_num last_vops_timestamp = vop["trigger_date"] try: block = BlockHeader(block_num, steem_instance=stm) timestamp = block['timestamp'].replace( tzinfo=None) except: timestamp = vop["trigger_date"] print("Vops estimate %s took %.2f s - %s" % (str(vop["trigger_date"]), time.time() - start_time, block_num_est_method)) table2 = connection["transactions"]
except exceptions.ContentDoesNotExistsException: print("Could not find Comment: %s" % (authorperm)) al = list() if not vote_event["voter"] in self.looked_up: al.append(vote_event["voter"]) self.looked_up.add(vote_event["voter"]) if not vote_event["author"] in self.looked_up: al.append(vote_event["author"]) self.looked_up.add(vote_event["author"]) if len(al) > 0: lookup_accounts(al) if __name__ == "__main__": wtw = WatchingTheWatchers() tb = WatchingTheWatchersBot(wtw) blockchain = Blockchain() threading = True thread_num = 16 cur_block = blockchain.get_current_block() stop = cur_block.identifier startdate = cur_block.time() - timedelta(days=1) start = blockchain.get_estimated_block_num(startdate, accurate=True) for vote in blockchain.stream(opNames=["vote"], start=start, stop=stop, threading=threading, thread_num=thread_num): tb.vote(vote) wtw.report()
accountTrx = {} accountTrx = MemberHistDB(db) b = Blockchain(steem_instance=stm) current_block = b.get_current_block() stop_time = latest_enrollment stop_time = current_block["timestamp"] start_time = stop_time - timedelta(seconds=30 * 24 * 60 * 60) blocks_per_day = 20 * 60 * 24 start_block = accountTrx.get_latest_block_num() if start_block is None: start_block = b.get_estimated_block_num(addTzInfo(start_time)) # block_id_list = [] trx_id_list = [] else: trx_id_list = accountTrx.get_block_trx_id(start_block) # end_block = current_block["id"] end_block = current_block["id"] - (20 * 10) if end_block > start_block + 6000: end_block = start_block + 6000 print("Checking member upvotes from %d to %d" % (start_block, end_block)) date_now = datetime.utcnow() date_7_before = addTzInfo(date_now - timedelta(seconds=7 * 24 * 60 * 60)) date_28_before = addTzInfo(date_now - timedelta(seconds=28 * 24 * 60 * 60)) date_72h_before = addTzInfo(date_now - timedelta(seconds=72 * 60 * 60))
class RedFisher: def __init__(self): """Initialisation.""" self.load_settings() self.nodes = [ 'https://api.steemit.com', 'https://rpc.buildteam.io', 'https://api.steem.house', 'https://steemd.steemitdev.com', 'https://steemd.steemitstage.com', 'https://steemd.steemgigs.org' ] self.s = Steem(keys=self.posting_key) self.s.set_default_nodes(self.nodes) self.b = Blockchain(self.s) set_shared_steem_instance(self.s) self.p = Pool(4) def timer_start(self): """Set a timer to restart the program at 7pm the next day.""" # Calculate how many seconds until 7pm tomorrow now = datetime.today() print(now.day) a_time = now.replace(day=now.day + 1, hour=19, minute=0, microsecond=0) d_time = a_time - now secs = d_time.seconds + 1 # Reload settings from external file self.load_settings() # Start the timer t = threading.Timer(secs, self.redfisher) t.start() print('Timer set for ' + str(secs / 3600) + ' hours.') def load_settings(self): """Load the account requrements from an external file.""" # Load the external .json settings = json.loads(open('settings.json', 'r').read()) # Import the settings to variables self.days = settings['days'] self.posts_per_week = settings['posts_per_week'] self.min_sp = settings['min_sp'] self.max_sv = settings['max_sv'] self.posting_account = settings['posting_account'] self.posting_key = settings['posting_key'] print('settings loaded\ndays: %d\nposts per week: %d\nminimum steem ' 'power: %d' % (self.days, self.posts_per_week, self.min_sp)) def redfisher(self): """ Streams through all posts within given timeframe and sends accounts to be validated. """ t1 = time.process_time() # Calculate the start block based on how many days were specified # in the settings now = datetime.now() start_date = now - timedelta(days=self.days) start_block = self.b.get_estimated_block_num(start_date) # Get the end block end_block = self.b.get_current_block_num() # Create/reset approved user dictionary self.user_list = dict() # stream comments from start block to current block print('streaming comments now') # Create checked accounts list checked_accounts = list() # Infinite while allows for many node swaps while True: try: # Start/continue the stream from start/lastchecked block for post in self.b.stream(opNames=['comment'], start=start_block, stop=end_block): # Set start block to the one currently being checked start_block = post['block_num'] # Assure that the post is not a comment if post['parent_author'] == '': author = post['author'] # Don't check accounts that have already been checked if author not in checked_accounts: # Add account to checked accounts checked_accounts.append(author) # Add account to pool to be checked self.p.enqueue(func=self.check, user=author) self.p.run() # All checks completed, break loop break except Exception as e: # Switch nodes print(e) self.failover() time.sleep(1) t2 = time.process_time() print(t2 - t1) # Wait for task pool to empty while self.p.done() == False: time.sleep(2) # Attempt to post every 20 seconds until complete while True: try: self.post() break except: time.sleep(20) def get_account_age(self, acc): """Calculate how old an account is. Keyword arguments: acc -- the account to be checked -- type: Account """ # Declare account creation types types = [ 'create_claimed_account', 'account_create_with_delegation', 'account_create' ] # Look through history until account creation is found for i in acc.history(only_ops=types): # Get account creation timestamp creation_date_raw = i['timestamp'] break print(creation_date_raw) # Convert timestamp into datetime obj creation_date = datetime.strptime(creation_date_raw, '%Y-%m-%dT%H:%M:%S') # Calculate age in days now = datetime.now() acc_age = (now - creation_date).days return acc_age def post(self): """Make the post containing all the collected info for the round.""" # Get date in string format date = datetime.today().strftime('%Y-%m-%d') print(date) # Read the post body from external file post_body = open('post_body.txt', 'r').read() # Order the approved accounts by age sorted_ul = sorted(self.user_list.keys(), key=lambda y: (self.user_list[y]['age'])) # For each approved user, add them to the bottom of the post table for username in sorted_ul: data = self.user_list[username] sp = str(round(data['sp'], 3)) age = data['age'] svp = data['svp'] post_body += '\n@%s | %s | %s | %s' % (username, sp, age, svp) print(post_body) # Broadcast post transaction self.s.post(title='Small Accounts To Support - ' + date, body=post_body, author=self.posting_account, tags=[ 'onboarding', 'retention', 'steem', 'help', 'delegation', 'community', 'giveaway' ]) print('posted') # Reset the cycle self.refresh() def check(self, user): """ Check that the users meet the requirements. user -- the account to be checked -- type: Account/str """ # If an account wasn't passed, make str into account if type(user) != Account: acc = Account(user) # Check account steempower sp = self.sp_check(acc) print(user + " sp == " + str(sp)) if sp <= float(self.min_sp): print('min sp met') # Check account posts per week ppw_check = self.post_check(acc) print(user + " posts per week check == " + str(ppw_check)) if ppw_check: print('posts per week met') # Check self vote percent self_vote_pct = self.vote_check(acc) print(user + ' self votes == %' + str(self_vote_pct)) if self_vote_pct < self.max_sv: print('self vote check met') # Check for powerups and powerdowns powered_up, powered_down = self.vest_check(acc) print(user + " powered up == " + str(powered_up)) print(user + " powered down == " + str(powered_down)) if powered_up and not powered_down: print('vest check met') # All checks completed # Get account age age = self.get_account_age(acc) # Add account to list so their data is stored for the # daily post. self.user_list[user] = { 'sp': round(sp, 3), 'age': age, 'svp': round(self_vote_pct, 2) } print(self.user_list) def sp_check(self, acc): """ Return the users steem power (ignoring outgoing delegations). acc -- the account to be checked -- type: Account """ # Get absolute steempower sp = float(acc.get_steem_power()) # Get all outgoing delegations and add them to account sp for delegation in acc.get_vesting_delegations(): vests = float(delegation['vesting_shares']['amount']) precision = delegation['vesting_shares']['precision'] dele_sp_raw = self.s.vests_to_sp(vests) dele_sp = dele_sp_raw / 10**precision sp += dele_sp return sp def vest_check(self, acc): """Check for opwer ups/downs and return powered_up (bool), powered_down (bool). acc -- the account to be checked -- type: Account """ # get all time powerup and powerdown history powered_up = False powered_down = False vest_history = acc.history_reverse( only_ops=['withdraw_vesting', 'transfer_to_vesting']) # check for powerups and powerdowns for change in vest_history: if change['type'] == 'withdraw_vesting': powered_down = True if change['type'] == 'transfer_to_vesting': powered_up = True if powered_up and powered_down: break return powered_up, powered_down def vote_check(self, acc): """Return the self vote percentage based on weight. acc -- the account to be checked -- type: Account """ # Declare variables total = 0 self_vote = 0 # Date datetime for a week ago wa = self._get_date_past() # Go through all votes in the past week for i in acc.history_reverse(only_ops=['vote'], stop=wa): # Update total total += int(i['weight']) # Update self vote total if self vote if i['author'] == i['voter']: self_vote += int(i['weight']) # No votes made results in /0 error, return 0% self vote if total == 0: return 0 self_vote_pct = (self_vote / total) * 100 return self_vote_pct def post_check(self, acc): """Return true if posts per week requirement is met, false if not. acc -- the account to be checked -- type: Account """ # Get datetime for a week ago wa = self._get_date_past() # Get comment history for the last week blog_history = acc.history_reverse(only_ops=['comment'], stop=wa) # Count how many posts were made in the last week count = 0 for post in blog_history: # Check to make sure it isn't a reply if post['parent_author'] == '': count += 1 if self.posts_per_week >= count: return True return True def _get_date_past(self, days_ago=7): """Returns the datetime for the current time, x days ago. days_ago -- how many days ago you want the datetime for -- type: int """ now = datetime.now() return now - timedelta(days=days_ago) def refresh(self): """Called when the main module has finished executing and the post has been made. It resets all the settings and resets the cycle. """ self.user_list = dict() self.load_settings() self.timer_start() def failover(self): """Switch/cycle through nodes.""" self.nodes = self.nodes[1:] + [self.nodes[0]] print('Switching nodes to ' + self.nodes[0])
if len(sys.argv) != 4: print("ERROR: Command line argument count mismatch") print("Usage: %s [year] [month] [day]" % (sys.argv[0])) exit(-1) year = int(sys.argv[1]) month = int(sys.argv[2]) day = int(sys.argv[3]) nl = NodeList() s = Steem(node=nl.get_nodes(appbase=True, normal=False)) b = Blockchain(steem_instance=s) start_date = datetime(year, month, day) end_date = start_date + timedelta(days=1) start_block = b.get_estimated_block_num(start_date) end_block = b.get_estimated_block_num(end_date) - 1 m = MongoStorage(db_name="steem", host='172.18.0.3', port=27017, user='', passwd='') m.ensure_indexes() for block in b.blocks(start=start_block, stop=end_block, max_batch_size=BATCH_SIZE): sys.stdout.write("%s - %s - %s\r" % (start_block, block.block_num, end_block))
N = len(accounts) sorted_ua = sorted(accounts_list, key=lambda x: x["trust_score"], reverse=True) rang = 0 top_acc = sorted_ua[0] nodes = NodeList() # nodes.update_nodes(weights={"block": 1}) nodes.update_nodes() node_list = nodes.get_nodes() stm = Steem(node=node_list) b = Blockchain(steem_instance=stm) #block_num = b.get_estimated_block_num(top_acc["cached_at"]) block_num = b.get_estimated_block_num(latest_created) print("Last blocknumber included: %d \n Updated at: %s (UTC)" % (block_num, formatTimeString(top_acc["cached_at"]))) top_100 = [] for f in sorted_ua[:100]: rang += 1 acc = Account(f["name"], steem_instance=stm) print("{rank: %d, account: '%s', ua: %.3f, followers: %d, following: %d, rep: %.3f}," % (rang, f["name"], f1(f["trust_score"], N), f["followers"], f["following"], acc.rep)) top_100.append({"rank": rang, "account": f["name"], "id": f["id"], "ua": f1(f["trust_score"], N), "followers": f["followers"], "following": f["following"], "rep": round(acc.rep, 2)}) if True: all_ua = [] rang = 0 for f in sorted_ua: rang += 1
### SETUP BEEM LIB STUFF UTC_TIMEZONE = tz.UTC NODELIST_INSTANCE = NodeList() NODELIST_INSTANCE.update_nodes() HIVE_INSTANCE = Hive(node=NODELIST_INSTANCE.get_hive_nodes()) BLOCKCHAIN_INSTANCE = Blockchain(blockchain_instance=HIVE_INSTANCE) not_today = datetime.datetime.now(UTC_TIMEZONE) - datetime.timedelta(days=days, hours=1) start = datetime.datetime(not_today.year, not_today.month, not_today.day) today = datetime.datetime.now(UTC_TIMEZONE) stop = datetime.datetime(today.year, today.month, today.day) start_block_id = BLOCKCHAIN_INSTANCE.get_estimated_block_num(start, accurate=True) stop_block_id = BLOCKCHAIN_INSTANCE.get_estimated_block_num(stop, accurate=True) logger.info(f"start time: {start}") logger.info(f"start block: {start_block_id}") logger.info(f"start time: {stop}") logger.info(f"start block: {stop_block_id}") laruche_subscribers = Community(laruche_community).get_subscribers() KEEPED = [] #### LOOP watched_count = 0 logger.info("STREAM STARTED")
def scan_history( hive: beem.Hive, block_num: Optional[int] = None, hours_ago: Optional[timedelta] = None, report_freq: int = 5, reports=True, use_test_node=False, quiet=False, include_unauthorized=False, include_non_podping=False, write_csv=False, ): """Scans back in history timed time delta ago, reporting with report_freq if timed is an int, treat it as hours, if report_freq is int, treat as min""" # Very first transaction from Dave Testing: """2021-05-10 13:51:58,353 INFO root MainThread : Feed Updated - 2021-05-07 20:58:33+00:00 - f0affd194524a6e0171d65d29d5c501865f0bd72 - https://feeds.transistor.fm/retail-remix""" scan_start_time = datetime.utcnow() report_timedelta = timedelta(minutes=report_freq) blockchain = Blockchain(mode="head", blockchain_instance=hive) if block_num: start_time = Block(block_num)["timestamp"].replace(tzinfo=None) elif hours_ago: start_time = datetime.utcnow() - hours_ago block_num = blockchain.get_estimated_block_num(start_time) else: raise ValueError( "scan_history: block_num or --old=<hours> required sto scan history" ) allowed_accounts = get_allowed_accounts() count_posts = 0 pings = 0 if reports: logging.info("Started catching up") # beem type doesn't have type hints # noinspection PyTypeChecker stream = blockchain.stream( opNames=["custom_json"], start=block_num, max_batch_size=50, raw_ops=False, threading=False, ) post = None for post in stream: post_time = post["timestamp"].replace(tzinfo=None) time_dif = post_time - start_time time_to_now = datetime.utcnow() - post_time count_posts += 1 if reports: if time_dif > report_timedelta: timestamp = post["timestamp"] current_block_num = post["block_num"] output_status( timestamp, pings, count_posts, time_to_now, current_block_num=current_block_num, reports=reports, quiet=quiet, ) start_time = post["timestamp"].replace(tzinfo=None) count_posts = 0 pings = 0 if allowed_op_id(post["id"]): if set(post["required_posting_auths"]) & allowed_accounts: count = output(post, quiet, use_test_node, write_csv, "data-podping") pings += count Pings.total_pings += count else: if include_unauthorized: count = output(post, quiet, use_test_node, write_csv, "data-unauthorized") pings += count Pings.total_pings += count else: if include_non_podping: output(post, quiet, use_test_node, write_csv, "data-not-podping_firehose") if time_to_now < timedelta(seconds=2): timestamp = post["timestamp"] current_block_num = post["block_num"] output_status( timestamp, pings, count_posts, time_to_now, current_block_num=current_block_num, reports=reports, quiet=quiet, ) logging.info(f"block_num: {post['block_num']}") # Break out of the for loop we've caught up. break if post: scan_time = datetime.utcnow() - scan_start_time logging.info( f"Finished catching up at block_num: {post['block_num']} in {scan_time}" )