def fetch(): organization = "OriginProtocol" credentials = (None, constants.GITHUB_KEY) repos_url = 'https://api.github.com/orgs/%s/repos' % (organization) repos = requests.get(repos_url, auth=credentials) counts = collections.defaultdict(int) pics = collections.defaultdict(str) # we don't count contributors from detached forked projects (ie. origin-docs) special_forked = ["origin-docs"] print 'checking non-forked repos first:' for repo in repos.json(): if not repo['fork'] and repo['name'] not in special_forked: # count pulls pulls_url = 'https://api.github.com/repos/%s/%s/pulls' % (organization, repo['name']) results = requests.get(pulls_url, auth=credentials) print repo['name'] stats_url = 'https://api.github.com/repos/%s/%s/stats/contributors' % (organization, repo['name']) results = requests.get(stats_url, auth=credentials) data = results.json() for author in data: # print '\t%s\t%s\t%s' % (author['author']['login'],author['total'], author['author']['avatar_url']) counts[author['author']['login']] = counts[author['author']['login']] + author['total'] pics[author['author']['login']] = author['author']['avatar_url'] # only include contributions from contributors who have also contributed to non-forked repos print 'checking forked & special-case repos:' for repo in repos.json(): if repo['fork'] or repo['name'] in special_forked: print repo['name'] stats_url = 'https://api.github.com/repos/%s/%s/stats/contributors' % (organization, repo['name']) results = requests.get(stats_url, auth=credentials) data = results.json() for author in data: # if repo['name'] in forked: if author['author']['login'] not in counts: continue counts[author['author']['login']] = counts[author['author']['login']] + author['total'] pics[author['author']['login']] = author['author']['avatar_url'] total_commits = 0 for row, value in counts.iteritems(): user = db_common.get_or_create(db.session, db_models.Contributor, username=row) user.commits = value user.avatar = pics[row] print row + '\t' + str(value) + '\t' + pics[row] total_commits += value print '%s commits' % total_commits print '%s contributors' % len(counts)
def add_contact(address, **kwargs): address = address.strip() # must look like an ETH address if not re.match("^(0x)?[0-9a-fA-F]{40}$", address): return False contact = db_common.get_or_create(db.session, db_models.EthContact, address=address.lower()) allowed_fields = [ 'name', 'email', 'phone', 'investor', 'presale_interest', 'investor_airdrop', 'dapp_user', 'employee', 'exchange', 'company_wallet', 'desc' ] for key, value in kwargs.items(): if key in allowed_fields: if value: setattr(contact, key, value) else: raise Exception("Unknown field") db.session.add(contact) db.session.commit()
def lookup_details(address): # automatically start tracking every wallet that receives OGN contact = db_common.get_or_create(db.session, db_models.EthContact, address=address.lower()) db.session.add(contact) db.session.commit() return contact
def fetch_ogn_transactions(): etherscan_url = 'http://api.etherscan.io/api?module=account&action=tokentx&contractaddress=%s&startblock=0&endblock=999999999&sort=desc&apikey=%s' % ( ogn_contract, constants.ETHERSCAN_KEY) # print etherscan_url results = call_etherscan(etherscan_url) # loop through every transaction where Origin tokens were moved for result in results['result']: tx = db_common.get_or_create(db.session, db_models.TokenTransaction, tx_hash=result['hash']) tx.from_address = result['from'].lower() tx.to_address = result['to'].lower() # intentionally using ETH instead of WEI to be more human-friendly, despite being less precise tx.amount = float(result['value']) / math.pow(10, 18) tx.block_number = result['blockNumber'] tx.timestamp = time_.fromtimestamp(result['timeStamp']) if tx.amount > 0: print "%g OGN moved in transaction %s" % (tx.amount, result['hash']) # send an email alert every time OGN tokens are moved # only alert once & ignore marketplace transactions which show up as 0 OGN if (tx.amount > 0 and not tx.notification_sent): to_details = lookup_details(tx.to_address) from_details = lookup_details(tx.from_address) if from_details.name and to_details.name: subject = "%s moved %g OGN to %s" % ( from_details.name, tx.amount, to_details.name) elif from_details.name: subject = "%s moved %g OGN" % (from_details.name, tx.amount) elif to_details.name: subject = "%g OGN moved to %s" % (tx.amount, to_details.name) else: subject = "%g OGN moved" % (tx.amount) body = u""" {amount} OGN <a href='https://etherscan.io/tx/{tx_hash}'>moved</a> from <a href='https://etherscan.io/address/{from_address}'>{from_name}</a> to <a href='https://etherscan.io/address/{to_address}'>{to_name}</a> """.format(amount='{0:g}'.format(float(tx.amount)), tx_hash=tx.tx_hash, from_name=from_details.name if from_details.name else tx.from_address, from_address=tx.from_address, to_name=to_details.name if to_details.name else tx.to_address, to_address=tx.to_address) print subject sgw.notify_founders(body, subject) tx.notification_sent = True db.session.add(tx) db.session.commit()
def fill_missing_txs(do_it): # last_tx_db = db_models.TokenTransaction.query.order_by( # db_models.TokenTransaction.block_number.desc() # ).first() last_entry = db_models.CirculatingSupply.query.order_by( db_models.CirculatingSupply.snapshot_date.desc() ).first() if not last_entry: print("Couldn't find any entry on DB") return start_block = 10178155 # the last one in DB, same as last_tx_db.block_number new_supply = last_entry.supply_amount etherscan_url = ( "http://api.etherscan.io/api?module=account&action=tokentx&contractaddress=%s&startblock=%s&endblock=999999999&sort=asc&apikey=%s" % (ogn_contract, start_block, constants.ETHERSCAN_KEY) ) raw_json = requests.get(etherscan_url) results = raw_json.json() update = 0 for result in results["result"]: from_address = result["from"].lower() to_address = result["to"].lower() amount = float(result["value"]) / math.pow(10, 18) if from_address in reserved_addresses and to_address not in reserved_addresses: new_supply = new_supply + amount elif from_address not in reserved_addresses and to_address in reserved_addresses: new_supply = new_supply - amount else: continue update = update + 1 instance = db_common.get_or_create( db.session, db_models.CirculatingSupply, snapshot_date=time_.fromtimestamp(result["timeStamp"]) ) instance.supply_amount = new_supply db.session.add(instance) if do_it: db.session.commit() print("Have parsed {} transactions".format(update)) print("Circulating supply at the end of all txs: {}".format(new_supply))
def fetch_wallet_balance(wallet): print("Checking the balance of wallet {}".format(wallet, )) url = "https://api.ethplorer.io/getAddressInfo/%s" % (wallet) results = call_ethplorer(url) contact = db_common.get_or_create(db.session, db_models.EthContact, address=wallet) if "error" in results: print("Error while fetching balance") print(results["error"]["message"]) raise ValueError(results["error"]["message"]) contact.eth_balance = results["ETH"]["balance"] contact.transaction_count = results["countTxs"] print("ETH balance of {} is {}".format(wallet, results["ETH"]["balance"])) if "tokens" in results: contact.tokens = results["tokens"] # update the OGN & DAI balance for token in results["tokens"]: if token["tokenInfo"]["address"] == token_stats.ogn_contract: contact.ogn_balance = float(token["balance"]) / math.pow( 10, 18) print("OGN balance of {} is {}".format(wallet, contact.ogn_balance)) elif token["tokenInfo"]["address"] == token_stats.dai_contract: contact.dai_balance = float(token["balance"]) / math.pow( 10, 18) print("DAI balance of %s is %s".format(wallet, contact.dai_balance)) contact.token_count = len(results["tokens"]) else: print("OGN balance of {} is {}".format(wallet, 0)) contact.ogn_balance = 0 contact.dai_balance = 0 contact.last_updated = datetime.utcnow() db.session.add(contact) db.session.commit() return contact
def fetch_meta_tx_balance(): print "Fetching meta tx purse balance" try: url = "http://api.ethplorer.io/getAddressInfo/%s" % (meta_tx_purse) results = call_ethplorer(url) contact = db_common.get_or_create(db.session, db_models.EthContact, address=meta_tx_purse) contact.eth_balance = results['ETH']['balance'] contact.transaction_count = results['countTxs'] if 'tokens' in results: contact.tokens = results['tokens'] # update the OGN & DAI balance for token in results['tokens']: if token['tokenInfo']['address'] == ogn_contract: contact.ogn_balance = float(token['balance']) / math.pow( 10, 18) elif token['tokenInfo']['address'] == dai_contract: contact.dai_balance = float(token['balance']) / math.pow( 10, 18) contact.token_count = len(results['tokens']) contact.last_updated = datetime.utcnow() print(contact.eth_balance) if contact.eth_balance < 1: print 'Low balance. Notifying.' subject = "Meta-transactions purse is running low. %s ETH remaining" % ( contact.eth_balance) body = "Please send more ETH to %s" % (meta_tx_purse) print(body) print(subject) sgw.notify_founders(body, subject) db.session.add(contact) db.session.commit() except Exception as e: print e
def add_contact(address, **kwargs): # nothing to do here, bail if not address: return False address = address.strip() # must look like an ETH address if not re.match("^(0x)?[0-9a-fA-F]{40}$", address): return False contact = db_common.get_or_create(db.session, db_models.EthContact, address=address.lower()) allowed_fields = [ "name", "email", "phone", "investor", "presale_interest", "investor_airdrop", "dapp_user", "employee", "exchange", "company_wallet", "desc", "country_code", ] for key, value in kwargs.items(): if key in allowed_fields: if value: # Normalize email to lower case before storing in the DB. if key == "email": value = value.lower() setattr(contact, key, value) else: raise Exception("Unknown field") db.session.add(contact) db.session.commit()
def fetch_ogn_token_info(): print "Checking OGN token info" url = "http://api.ethplorer.io/getTokenInfo/%s" % (ogn_contract) results = call_ethplorer(url) if "error" in results: print("Error while fetching token info") print(results["error"]["message"]) raise ValueError(results["error"]["message"]) token_info = db_common.get_or_create(db.session, db_models.TokenInfo) token_info.total_supply = results["totalSupply"] token_info.holders = results["holdersCount"] token_info.transfers_count = results["transfersCount"] db.session.add(token_info) db.session.commit() return token_info
def update_circulating_supply(circulating_supply): snapshot_date = datetime.utcnow() supply_snapshot = db_common.get_or_create(db.session, db_models.CirculatingSupply, snapshot_date=snapshot_date) supply_snapshot.supply_amount = circulating_supply db.session.commit() supply_data = db.engine.execute(""" select timewin, max(s.supply_amount) from generate_series(now() - interval '12 month', now(), '1 day') as timewin left outer join (select * from circulating_supply where snapshot_date > now() - interval '12 month' and snapshot_date > '2020-01-01'::date order by snapshot_date desc) s on s.snapshot_date < timewin and s.snapshot_date >= timewin - (interval '1 day') where timewin > '2020-01-01'::date group by timewin order by timewin desc """) out = [] supply_data_list = list(supply_data) latest_supply = supply_data_list[0][1] for row in supply_data_list: if row[1] is not None: latest_supply = row[1] out.append( dict([("supply_amount", row[1] or latest_supply), ("snapshot_date", row[0].strftime("%Y/%m/%d %H:%M:%S"))])) print( "Updated current circulating supply to {}".format(circulating_supply)) return out
def fill_missing_txs(do_it): start_block = 10176690 new_supply = 58821352 etherscan_url = ( "http://api.etherscan.io/api?module=account&action=tokentx&contractaddress=%s&startblock=%s&endblock=999999999&sort=asc&apikey=%s" % (ogn_contract, start_block, constants.ETHERSCAN_KEY)) raw_json = requests.get(etherscan_url) results = raw_json.json() update = 0 for result in results["result"]: from_address = result["from"].lower() to_address = result["to"].lower() amount = float(result["value"]) / math.pow(10, 18) if from_address in reserved_addresses and to_address not in reserved_addresses: new_supply = new_supply + amount elif from_address not in reserved_addresses and to_address in reserved_addresses: new_supply = new_supply - amount else: continue update = update + 1 instance = db_common.get_or_create(db.session, db_models.CirculatingSupply, snapshot_date=time_.fromtimestamp( result["timeStamp"])) instance.supply_amount = new_supply db.session.add(instance) print("{} {}".format(time_.fromtimestamp(result["timeStamp"]), new_supply)) if do_it: db.session.commit() print("Have parsed {}/{} transactions".format(update, len(results["result"]))) print("Circulating supply at the end of all txs: {}".format(new_supply))