def updateRegistryNonces(username): folder = os.path.join(zen.DATA, username) registries = [n for n in os.listdir(folder) if n.endswith(".registry")] if not len(registries): return False KEYS01, KEYS02 = getKeys(username) wallet = rest.GET.api.wallets(username).get("data", {}) nonce = int(wallet["nonce"]) + 1 if KEYS01 and len(wallet): for name in registries: full_registry = loadJson(name, folder=folder) registry = loadJson(name+".milestone", folder=folder) if len(registry): logMsg("updating transaction nonces in %s..." % (name+".milestone")) for tx in list(registry.values()): old_id = tx["id"] new_tx = dposlib.core.Transaction(tx) new_tx.nonce = nonce nonce += 1 new_tx.signWithKeys(KEYS01["publicKey"], KEYS01["privateKey"]) if KEYS02 is not None: new_tx.signSignWithKey(KEYS02["privateKey"]) new_tx.identify() registry.pop(old_id) full_registry.pop(old_id) registry[new_tx["id"]] = new_tx full_registry[new_tx["id"]] = new_tx dumpJson(registry, name+".milestone", folder=folder) dumpJson(full_registry, name, folder=folder) return True
def faq(): registered_usernames = [ name.split("-")[0] for name in os.listdir(zen.JSON) if name.endswith("-webhook.json") ] data = dict((k, v) for k, v in dposlib.core.cfg.__dict__.items() if not k.startswith('_')) data["begintime"] = data["begintime"].strftime("%Y-%m-%dT%H:%M:%S.000Z") try: data["blocktime"] = dposlib.core.mixin.deltas()["real blocktime"] except Exception as e: zen.logMsg('error occured computing deltas : %s' % e) delegates = dict([ username, dict(zen.loadJson(username + ".json", zen.JSON), **dposlib.rest.GET.api.delegates(username, returnKey="data")) ] for username in registered_usernames) for username in registered_usernames: minvote = delegates[username].get("minimum_vote", 0.) * 100000000 maxvote = delegates[username].get("maximum_vote", None) if isinstance(maxvote, (int, float)): maxvote *= 100000000 _max = lambda v, maxi=maxvote: min(maxi, v) else: _max = lambda v, maxi=maxvote: v _min = lambda v, mini=minvote: v if v >= mini else 0 delegates[username]["votes"] = sum([ _min(_max(float(v["balance"]))) for v in zen.misc.loadPages( dposlib.rest.GET.api.delegates.__getattr__(username).voters) ]) / 100000000 return flask.render_template("faq.html", info=data, delegates=delegates)
def loadPages(endpoint, pages=None, quiet=True, nb_tries=10, peer=None, condition=[]): if not isinstance(endpoint, zen.rest.EndPoint): raise Exception("Invalid endpoint class") count, pageCount, data = 0, 1, [] while count < pageCount: req = endpoint.__call__(page=count + 1, peer=peer) if req.get("error", False): nb_tries -= 1 if not quiet: zen.logMsg("Api error occured... [%d tries left]" % nb_tries) if nb_tries <= 0: raise Exception("Api error occured: %r" % req) else: pageCount = req["meta"]["pageCount"] if isinstance(pages, int): pageCount = min(pages, pageCount) if not quiet: zen.logMsg("reading page %s over %s" % (count + 1, pageCount)) data.extend(req.get("data", [])) count += 1 return data
def run(self): # controled infinite loop while not TaskExecutioner.STOP.is_set(): try: # compute new block when it comes computeDelegateBlock(*TaskExecutioner.JOB.get()) except Exception as error: logMsg("%r" % error)
def _launcher(func): name = func.__name__ try: REPORT[name] = func() except Exception as error: zen.logMsg("%s exception:\n%r\n%s" % (func.__name__, error, traceback.format_exc())) REPORT[name] = "%r" % error
def checkNode(): global REPORT REPORT = {} root = zen.loadJson("root.json") config = root.get("tasks-enabled", {}).get("checkNode", {}) seed_peer = config.get("seed-peer", False) if not seed_peer: zen.logMsg("no seed peer defined") return "no seed peer defined" monitored_peer = config.get("monitored-peer", zen.API_PEER) syncing = _GET.api.node.syncing(peer=monitored_peer).get("data", {}) status = _GET.api.node.status(peer=seed_peer).get("data", {}) notification_delay = config.get("notification-delay", 5 * 60) if syncing == {}: msg = "%s not responding" % monitored_peer now = time.time() if now - REPORT.get("not responding", now) > notification_delay: REPORT["not responding"] = time.time() zen.logMsg(msg) zen.misc.notify(msg) return msg elif REPORT.pop("not responding", False): msg = "%s is active again" % monitored_peer zen.logMsg(msg) zen.misc.notify(msg) if root.get("env", False): height = int( subprocess.check_output( shlex.split( '/usr/bin/psql -qtAX -d ark_mainnet -c ' '"SELECT height FROM blocks ORDER BY height DESC LIMIT 1"') ).strip()) else: height = syncing["height"] height_diff = status.get("now", height) - height if syncing.get("syncing", False): msg = "%s not synced: height diff %s" % (monitored_peer, height_diff) now = time.time() if now - REPORT.get("not synced", now) > notification_delay: REPORT["not synced"] = time.time() zen.logMsg(msg) zen.misc.notify(msg) elif REPORT.pop("not responding", False): msg = "%s synced at height %s" % (monitored_peer, height_diff) zen.logMsg(msg) zen.misc.notify(msg) else: msg = "%s synced @ height %s" % (monitored_peer, height) return msg
def waitForPeer(peer): try: while dposlib.rest.GET(peer=peer, timeout=2).get("status", False) != 200: time.sleep(2) zen.logMsg("wating for peer %s..." % peer) return True except KeyboardInterrupt: return False
def show_hidden(*args, **kwargs): report = {} for key, value in [(k, v) for k, v in zen.biom.__dict__.items() if "#" in k]: username, num = key.replace("_", "").split("#") if value is not None: report[username] = report.get(username, []) + ["puk#" + num] msg = "Loaded private keys = %s" % json.dumps(report) zen.logMsg(msg) zen.misc.notify(msg)
def askSecret(account, cmp_key="publicKey"): if account.get(cmp_key, False): keys = dposlib.core.crypto.getKeys("01") while keys["publicKey"] != account[cmp_key]: try: secret = getpass.getpass("> enter %s secret for %s: " % (account["username"], cmp_key)) keys = dposlib.core.crypto.getKeys(secret) except KeyboardInterrupt: printNewLine() logMsg("delegate configuration skipped") sys.exit(1) return keys["privateKey"]
def faq(): data = dict((k,v) for k,v in dposlib.core.cfg.__dict__.items() if not k.startswith('_')) data["begintime"] = data["begintime"].strftime("%Y-%m-%dT%H:%M:%S.000Z") try: data["blocktime"] = dposlib.core.mixin.deltas()["real blocktime"] except Exception as e: zen.logMsg('error occured computing deltas : %s' % e) delegates = dict( [username, dict(zen.loadJson(username+".json", zen.JSON), **dposlib.rest.GET.api.delegates(username, returnKey="data"))] for \ username in [name.split("-")[0] for name in os.listdir(zen.JSON) if name.endswith("-webhook.json")] ) return flask.render_template("faq.html", info=data, delegates=delegates)
def notify(body): title = "ark-zen@%s" % zen.dposlib.core.cfg.network body = body.decode("utf-8") if isinstance(body, bytes) else body for func in [ freemobile_sendmsg, pushbullet_pushes, pushover_messages, twilio_messages ]: response = func(title, body) if isinstance(response, dict): zen.logMsg("notification response:\n%s" % response) if response.get("status", 1000) < 300: return response
def checkIfForging(): config = zen.loadJson("root.json").get("tasks-enabled", {}).get("checkIfForging", {}) active_delegates = zen.biom.dposlib.rest.cfg.activeDelegates notification_delay = config.get("notification-delay", 10 * 60) monitored_peer = config.get("monitored-peer", zen.API_PEER) usernames = [ name.split("-")[0] for name in os.listdir(zen.JSON) if name.endswith("-webhook.json") ] for user in usernames: dlgt = _GET.api.delegates(user, peer=monitored_peer).get("data", {}) last_block = dlgt.get("blocks", {}).get("last", {}) last_computed_block = zen.loadJson("%s.last.block" % user, folder=zen.DATA) if dlgt and last_computed_block == {}: zen.dumpJson(last_block, "%s.last.block" % user, folder=zen.DATA) return False elif dlgt.get("rank", 52) <= active_delegates: blkchn = _GET.api.blockchain(peer=monitored_peer).get("data", {}) height = blkchn.get("block", {}).get("height", 0) current_round = (height - 1) // active_delegates dlgt_round = (last_block["height"] - 1) // active_delegates diff = current_round - dlgt_round missed = last_computed_block.get("missed", 0) if diff > 1: now = time.time() last_computed_block["missed"] = missed + 1 delay = now - last_computed_block.get("notification", 0) msg = ("%s just missed a block" % user) if missed < 1 else ( "%s is missing blocks (total %d)" % (user, missed + 1)) if delay > notification_delay: zen.misc.notify(msg) last_computed_block["notification"] = now elif missed > 0: msg = "%s is forging again" % user zen.misc.notify(msg) last_computed_block.pop("notification", False) last_computed_block.pop("missed", False) else: msg = "%s is forging" % user last_computed_block.update(last_block) zen.dumpJson(last_computed_block, "%s.last.block" % user, folder=zen.DATA) zen.logMsg("round check [delegate:%d | blockchain:%d] - %s" % (dlgt_round, current_round, msg)) return True
def pushBackKeys(): module = sys.modules[__name__] delegates = [] for key, value in [(k, v) for k, v in module.__dict__.items() if k[-2:] in "#1#2"]: username, num = key.replace("_", "").split("#") num = "#" + num config = zen.loadJson("%s.json" % username) config[num] = value zen.dumpJson(config, "%s.json" % username) delegates.append(username) for username in set(delegates): zen.logMsg("%s secrets pushed back" % username)
def broadcast(username, chunk_size=30): # initialize options tbw = loadJson("tbw.json") chunk_size = max(5, tbw.get("chunk_size", chunk_size)) folder = os.path.join(zen.DATA, username) # proceed all registry file found in username folder for name in [n for n in os.listdir(folder) if n.endswith(".registry")]: registry = loadJson(name, folder=folder) transactions = sorted(list(registry.values()), key=lambda t:t["nonce"]) for chunk in (transactions[x:x+chunk_size] for x in range(0, len(transactions), chunk_size)): response = rest.POST.api.transactions(transactions=chunk, **({"peer":zen.API_PEER} if zen.misc.delegateIsForging(username) else {})) logMsg("broadcasting chunk of transactions...\n%s" % json.dumps(response, indent=2)) zen.misc.notify("New payroll started : %d transactions sent to delegate node..." % len(transactions))
def regenerateUnapplied(username, filename): registry = zen.loadJson("%s.registry" % filename, os.path.join(zen.DATA, username)) tbw = zen.loadJson("%s.tbw" % filename, os.path.join(zen.TBW, username, "history")) for tx in registry.values(): if not transactionApplied(tx["id"]): zen.logMsg('tx %(id)s [%(amount)s --> %(recipientId)s] unapplied' % tx) else: tbw["weight"].pop(tx["recipientId"], False) zen.dumpJson(tbw, '%s-unapplied.tbw' % filename, os.path.join(zen.TBW, username))
def adjust(username, value): if getPublicKeyFromUsername(username): folder = os.path.join(zen.DATA, username) forgery = loadJson("%s.forgery" % username, folder=folder) total = sum(forgery["contributions"].values()) dumpJson( { "fees": forgery.get("fees", 0.), "blocks": forgery.get("blocks", 0), "contributions": OrderedDict(sorted([[a, v/total*value] for a,v in forgery["contributions"].items()], key=lambda e:e[-1], reverse=True)) }, "%s.forgery" % username, folder=folder ) else: logMsg("%s username does not exist" % username)
def pullKeys(): module = sys.modules[__name__] for username in [ n.replace("-webhook.json", "") for n in next(os.walk(zen.JSON))[-1] if n.endswith("-webhook.json") ]: config = zen.loadJson("%s.json" % username) hide = False if "#1" in config: setattr(module, "_%s_#1" % username, config.pop("#1")) hide = True if "#2" in config: setattr(module, "_%s_#2" % username, config.pop("#2")) hide = True if hide: zen.logMsg("%s secrets pulled" % username) zen.dumpJson(config, "%s.json" % username)
def generateCharts(): try: delegates = dict( [username, dict(zen.loadJson(username+".json", zen.JSON), **zen.dposlib.rest.GET.api.delegates(username, returnKey="data"))] for \ username in [name.split("-")[0] for name in os.listdir(zen.JSON) if name.endswith("-webhook.json")] ) real_blocktime = mixin.deltas()["real blocktime"] [ zen.misc.chartAir(delegates[username]["share"], 50, username, real_blocktime) for username in delegates ] [zen.misc.generateChart(username) for username in delegates] zen.misc.chartAir(1., 50, "", real_blocktime) zen.logMsg("charts successfully generated") except Exception as e: zen.logMsg("chart generation error:\n%r\n%s" % (e, traceback.format_exc()))
def generateCharts(): delegates = dict([ username, dict(zen.loadJson(username + ".json", zen.JSON), **_GET.api.delegates(username, returnKey="data")) ] for username in [ name.split("-")[0] for name in os.listdir(zen.JSON) if name.endswith("-webhook.json") ]) real_blocktime = mixin.deltas()["real blocktime"] [ zen.misc.chartAir(delegates[username].get("share", 1.0), 50, username, real_blocktime) for username in delegates ] [zen.misc.generateChart(username) for username in delegates] zen.misc.chartAir(1., 50, "", real_blocktime) zen.logMsg("charts successfully generated") return True
def checkVersion(): peers = _GET.api.peers(orderBy="version:desc") peers = [ p for p in peers.get("data", []) if int(p["version"].split(".")[1]) < 10 ] node = _GET.api.node.configuration(peer=zen.API_PEER).get("data", {})\ .get("core", {}).get("version", False) if node and len(peers): versions = set([p["version"].split("-")[0] for p in peers]) last = sorted([int(e) for e in v.split(".")] for v in versions)[-1] last = ".".join(["%s" % i for i in last]) if node != last: zen.logMsg("your node have to be updated to %s" % last) zen.misc.notify("your node have to be upgraded to %s" % last) return False else: zen.logMsg("your node is up to date (%s)" % last) return True
def checkApplied(username): folder = os.path.join(zen.DATA, username) sqlite = initDb(username) cursor = sqlite.cursor() for name in [n for n in os.listdir(folder) if n.endswith(".registry")]: full_registry = loadJson(name, folder=folder) # try to lad a milestone first, if no one exists registry = loadJson(name+".milestone", folder=folder) # if void dict returned by loadJson, then load registry file if not len(registry): registry = dict(full_registry) #loadJson(name, folder=folder) logMsg("starting transaction check from %s..." % name) else: logMsg("resuming transaction check from %s..." % (name+".milestone")) start = time.time() transactions = list(registry.values()) for tx in transactions: if zen.misc.transactionApplied(tx["id"]): logMsg("transaction %(id)s <type %(type)s> applied" % registry.pop(tx["id"])) if "payments" in tx.get("asset", {}): for record in tx["asset"]["payments"]: cursor.execute( "INSERT OR REPLACE INTO transactions(filename, timestamp, amount, address, id) VALUES(?,?,?,?,?);", (os.path.splitext(name)[0], tx["timestamp"], int(record["amount"])/100000000., record["recipientId"], tx["id"]) ) # set a milestone every 5 seconds if (time.time() - start) > 5.: sqlite.commit() dumpJson(registry, name+".milestone", folder=folder) logMsg("milestone set (%d transaction left to check)" % len(registry)) start = time.time() dumpJson(registry, name+".milestone", folder=folder) if len(registry) == 0: dumpJson(full_registry, name, folder=os.path.join(folder, "backup")) try: os.remove(os.path.join(folder, name)) os.remove(os.path.join(folder, name+".milestone")) except: pass checked_tx = full_registry.values() zen.misc.notify("Payroll successfully broadcasted !\n%.8f %s sent trough %d transactions" % ( sum([sum([int(rec["amount"]) for rec in tx["asset"].get("payments", [])]) for tx in checked_tx])/100000000., dposlib.rest.cfg.symbol, len(checked_tx) )) else: zen.misc.notify("Transactions are still to be checked (%d)..." % len(registry)) sqlite.commit()
def configure(**kwargs): if "username" in kwargs: username = kwargs.pop("username") if getPublicKeyFromUsername(username): if not os.path.exists( os.path.join(zen.JSON, "%s-webhook.json" % username)): if not setDelegate(username, peer=kwargs.get("webhook_peer", None)): return zen.logMsg("%s delegate webhook set" % username) else: # load update and save in a row zen.dumpJson( dict(zen.loadJson("%s.json" % username), **kwargs), "%s.json" % username) zen.logMsg("%s delegate set" % username) else: zen.logMsg("can not find delegate %s" % username) elif not len(kwargs): root = zen.loadJson("root.json") if "env" in root: delegates = zen.loadJson("delegates.json", os.path.dirname(root["env"])) for secret in delegates["secrets"]: setDelegate(dposlib.core.crypto.getKeys(secret)["publicKey"])
def backupData(): config = zen.loadJson("root.json").get("tasks-enabled", {}).get("backupData", {}) backup_type = config.get("backup-type", ".tar.bz2") backup_number = config.get("backup-number", 5) backup_folder = os.path.abspath( config.get("backup-folder", zen.__path__[0])) os.makedirs(backup_folder, exist_ok=True) if not zen.biom.archive_data(backup_folder, backup_type): zen.logMsg("data backup successfully done") backups = [ os.path.join(backup_folder, name) for name in os.listdir(backup_folder) if ".tar" in name ] if len(backups) > backup_number: backups.sort(key=lambda p: os.stat(p).st_mtime) for path in backups[:-backup_number]: os.remove(path) return backups[-1] else: zen.logMsg("data backup failed") return False
def start(): info = zen.loadJson("root.json") tasks = info.get("tasks-enabled", {}) sleep_time = info.get( "sleep-time", zen.tbw.rest.cfg.blocktime * zen.tbw.rest.cfg.activeDelegates) daemons = [] for task, params in tasks.items(): func = getattr(zen.task, task) if callable(func): daemons.append(setInterval(params["interval"])(_launcher)(func)) zen.logMsg("%s daemon set: interval=%ss" % (task, params["interval"])) importlib.reload(zen.task) RELEASE.clear() while not RELEASE.is_set(): RELEASE.wait(timeout=float(sleep_time)) zen.logMsg("Sleep time finished :\n%s" % json.dumps(REPORT, indent=2)) for daemon in daemons: daemon.set()
def removeDelegate(username): webhook = zen.loadJson("%s-webhook.json" % username) if len(webhook): resp = deleteWebhook(webhook["id"], peer=webhook["peer"]) if resp.get("status", 500) < 300: zen.logMsg("webhook subscription removed") if input("Remove %s data ?[Y/n] " % username) in "yY": zen.dropJson("%s.json" % username) zen.dropJson("%s-webhook.json" % username) shutil.rmtree(os.path.join(zen.DATA, username), ignore_errors=True) shutil.rmtree(os.path.join(zen.TBW, username), ignore_errors=True) try: os.remove(os.path.join(zen.ROOT, "%s.db" % username)) except Exception: pass zen.logMsg("%s data removed" % username) else: zen.logMsg("%s not found" % username)
def checkVersion(): try: peers = zen.dposlib.rest.GET.api.peers(orderBy="version:desc", returnKey="data") if len(peers): versions = set([p["version"] for p in peers[1:]]) # pop the very first update last = sorted([int(e) for e in v.split(".")] for v in versions)[-1] last = ".".join(["%s" % i for i in last]) if (last.encode("utf-8") if zen.PY3 else last) not in subprocess.check_output(["ark", "version" ]).split()[0]: zen.logMsg("your node have to be updated to %s" % last) zen.misc.notify("your node have to be upgraded to %s" % last) else: zen.logMsg("your node is up to date (%s)" % last) except Exception as e: zen.logMsg("version check error:\n%r\n%s" % (e, traceback.format_exc()))
def checkRegistries(): for username in [ name for name in os.listdir(zen.DATA) if os.path.isdir(os.path.join(zen.DATA, name)) ]: block_delay = zen.loadJson("%s.json" % username).get( "block_delay", False) blocks = zen.loadJson("%s.forgery" % username, folder=os.path.join(zen.DATA, username)).get("blocks", 0) if block_delay and blocks >= block_delay: zen.logMsg("%s payroll triggered by block delay : %s [>= %s]" % (username, blocks, block_delay)) zen.tbw.extract(username) else: zen.tbw.checkApplied(username) zen.logMsg("%s registry checked : %s [< %s]" % (username, blocks, block_delay)) if zen.tbw.dumpRegistry(username): zen.tbw.broadcast(username) zen.logMsg("%s registry dumped and broadcasted" % username)
def checkNode(): global IS_SYNCING, STATUS, SEED_STATUS, CHECK_RESULT api_port = zen.rest.cfg.ports["core-api"] IS_SYNCING = zen.rest.GET.api.node.syncing(peer="http://127.0.0.1:%s" % api_port).get("data", {}) STATUS = zen.rest.GET.api.node.status(peer="http://127.0.0.1:%s" % api_port).get("data", {}) SEED_STATUS = zen.rest.GET.api.node.status( peer="https://explorer.ark.io:8443").get("data", {}) try: if STATUS == {}: CHECK_RESULT["not responding"] = True zen.misc.notify("Your node is not responding") zen.logMsg("node is not responding") elif CHECK_RESULT.get("not responding", False): CHECK_RESULT.pop("not responding") zen.misc.notify("Your node is back online") else: zen.logMsg(zen.json.dumps(STATUS)) if not STATUS.get("synced"): if IS_SYNCING["syncing"]: zen.misc.notify( "Your node is syncing... %d blocks from network height" % IS_SYNCING["blocks"]) else: CHECK_RESULT["not syncing"] = True zen.misc.notify( "Your node is not synced and seems stoped at height %d, network is at height %d" % (IS_SYNCING["height"], SEED_STATUS["now"])) elif CHECK_RESULT.get("not syncing", False): CHECK_RESULT.pop("not syncing") zen.misc.notify("Your node had recovered network height %d" % STATUS["now"]) except Exception as e: zen.logMsg("node check error:\n%r\n%s" % (e, traceback.format_exc()))
def start(): global DAEMON, IS_SYNCING, STATUS, SEED_STATUS sleep_time = zen.rest.cfg.blocktime * zen.rest.cfg.activeDelegates data = zen.loadJson("bg-marker.json") data["stop"] = False zen.dumpJson(data, "bg-marker.json") DAEMON = threading.Event() # check health status every minutes daemon_1 = setInterval(60)(checkNode)() # generate svg charts every round daemon_2 = setInterval(sleep_time)(generateCharts)() # check all registries daemon_3 = setInterval(sleep_time)(checkRegistries)() # check updates daemon_4 = setInterval(5 * sleep_time)(checkVersion)() # check forge daemon_5 = setInterval(30)(checkIfForging)() zen.logMsg("Background tasks started !") zen.misc.notify("Background tasks started !") try: while not DAEMON.is_set(): time.sleep(sleep_time) zen.logMsg("sleep time finished :\n%s" % zen.json.dumps(CHECK_RESULT)) except KeyboardInterrupt: zen.logMsg("Background tasks interrupted !") daemon_1.set() daemon_2.set() daemon_3.set() daemon_4.set() daemon_5.set() zen.misc.notify("Background tasks stoped !")
def checkRegistries(): try: for username in [ name for name in os.listdir(zen.DATA) if os.path.isdir(os.path.join(zen.DATA, name)) ]: block_delay = zen.loadJson("%s.json" % username).get( "block_delay", False) blocks = zen.loadJson("%s.forgery" % username, folder=os.path.join(zen.DATA, username)).get( "blocks", 0) if block_delay and blocks >= block_delay: zen.logMsg("%s payroll triggered by block delay : %s [>= %s]" % (username, blocks, block_delay)) zen.tbw.extract(username) zen.tbw.dumpRegistry(username) zen.tbw.broadcast(username) else: zen.tbw.checkApplied(username) zen.logMsg("%s registry checked : %s [< %s]" % (username, blocks, block_delay)) except Exception as e: zen.logMsg("transaction check error:\n%r\n%s" % (e, traceback.format_exc()))