def getSeed(pin=False): try: value = lystener.loadJson("salt")["salt"] except KeyError: if not os.path.exists(SALT): startSeed() value = lystener.loadJson("salt")["salt"] else: if time.time() - os.path.getmtime(SALT) > 60: startSeed() return str(int(value, 16))[:6] if pin else value
def destroy_listener(args={}, **options): """ unlink ark blockchain event from a python function. """ # maybee not ! # # connect to database in order to remove webhook associated data # sqlite = initDB() # cursor = sqlite.cursor() function = args.get("<function>", options.get("function", False)) if not function: listeners = [ name.replace(".json", "") for name in os.listdir(lystener.JSON) if name.endswith(".json") ] function = lystener.chooseItem("Select listener to destroy:", *listeners) if not function: return json_name = "%s.json" % function # load webhook configuration webhook = lystener.loadJson(json_name) # condition bellow checks if webhook configurations is found if webhook.get("peer", False): # delete webhook usong its id and parent peer rest.DELETE.api.webhooks("%s" % webhook["id"], peer=webhook["peer"]) # delete the webhook configuration os.remove(os.path.join(lystener.JSON, json_name)) # cursor.execute("SELECT FROM history WHERE autorization = ?", (webhook["token"][:32],)).fetchall() lystener.logMsg("%s webhook destroyed" % function) else: lystener.logMsg("%s webhook not found" % function)
def catch(mod, func, **kwargs): "Create a new job and return simple message." body = kwargs.get("data", {}) auth = kwargs.get("headers", {}).get("authorization", "") # try to save fingerprint, fails if fingerprint already exists else accept # and send data to TaskChecker try: webhook_name = task.webhookName(auth, mod, func) token = loadJson(webhook_name)["token"] content = body.get("data", {}) task.trace(CURSOR, token, content) except KeyError as error: logMsg("%r" % error) msg = {"status": 404, "msg": "no listener for %s.%s" % (mod, func)} except task.sqlite3.IntegrityError as error: logMsg("%r" % error) msg = {"status": 409, "msg": "data already parsed"} else: task.TaskChecker.JOB.put([mod, func, auth, content]) msg = { "status": 200, "msg": "task set: %s.%s(%s)" % (mod, func, content) } finally: CURSOR.commit() return msg
def checkRemoteAuth(**args): try: headers = args.get("headers", {}) task.MessageLogger.JOB.put("received secured headers: %r" % headers) # check if publick key defined in header is an authorized one # auth is a list of public keys. Because loadJson returns a void # dict if no file found, the if condition acts same as if it was a list publicKey = headers.get("public-key", "?") if publicKey not in lystener.loadJson("auth"): return {"status": 401, "msg": "not authorized"} # Note that client has to define its own salt value and get a # 1-min-valid random seed from lystener server. # See /salt endpoint sig = secp256k1.HexSig.from_der(headers["signature"]) puk = secp256k1.PublicKey.decode(publicKey) msg = secp256k1.hash_sha256(headers["salt"] + getSeed()) if not secp256k1._ecdsa.verify(msg, puk.x, puk.y, sig.r, sig.s): return {"status": 403, "msg": "bad signature"} return {"status": 200, "msg": "access granted"} except Exception as error: return { "status": 500, "msg": "checkPutDelete response: %s\n%s" % ("%r" % error, traceback.format_exc()) }
def notifyWhaleMove(data): task.MessageLogger.log('data received :\n%s' % json.dumps(data, indent=2)) params = loadJson("notifyWhaleMove.param", folder=lystener.DATA) sender = getAddress(data["senderPublicKey"], data["network"]) receiver = data["recipientId"] task.MessageLogger.log('Big move from %s to %s!' % (sender, receiver)) for exchange, wallets in params["hot wallets"].items(): task.MessageLogger.log( ' checking %s wallets : %s...' % (exchange, wallets) ) if receiver in wallets: task.FunctionCaller.call( notify.send, "Whale alert", "%s sent %.2f token to %s" % (sender, float(data["amount"])/100000000, exchange) ) return {"success": True} elif sender in wallets: task.FunctionCaller.call( notify.send, "Whale alert", "%s sent %.2f token to %s" % (exchange, float(data["amount"])/100000000, receiver) ) return {"success": True} return {"success": False, "response": "Nothing hapen !"}
def deploy_listener(args={}, **options): """ link ark blockchain event to a python function. """ function = args.get("<function>", options.get("function", False)) regexp = args.get("<regexp>", options.get("regexp", False)) event = args.get("<event>", options.get("event", False)) json_name = "%s.json" % function # build peers and target url webhook_peer = options.get( "webhook", "%(scheme)s://%(ip)s:%(port)s" % rest.WEBHOOK_PEER) listener_peer = options.get( "listener", "%(scheme)s://%(ip)s:%(port)s" % rest.LISTENER_PEER) target_url = listener_peer + "/" + function.replace(".", "/") # compute listener condition # if only a regexp is givent compute condition on vendorField if regexp: condition = { "key": "vendorField", "condition": "regexp", "value": args["<regexp>"] } # else create a condition. # Ark webhook api will manage condition errors elif len(options): condition = { "field": options["field"], "condition": options["condition"], "value": options["value"] } # load webhook configuration if already set webhook = lystener.loadJson(json_name) # lystener.loadJson returns void dict if json_name not found, # the if clause bellow will be true then if not webhook.get("token", False): # create the webhook req = rest.POST.api.webhooks(event=event, peer=webhook_peer, target=target_url, conditions=[condition]) # parse request result if no error messages if not req.get("error", False): webhook = req["data"] # save the used peer to be able to delete it later webhook["peer"] = webhook_peer webhook["hub"] = _endpoints(options.get("endpoints", "")) # save webhook configuration in JSON folder lystener.dumpJson(webhook, json_name) lystener.logMsg("%s webhook set" % function) else: lystener.logMsg("%r" % req) lystener.logMsg("%s webhook not set" % function) else: lystener.logMsg("webhook already set for %s" % function)
def run(self): # sqlite db opened within thread TaskChecker.DB = initDB() logMsg("TaskChecker is running in background...") # run until Task.killall() called while not Task.STOP.is_set(): skip = True # wait until a job is given module, name, auth, content = TaskChecker.JOB.get() # get webhook data webhook = loadJson(webhookName(auth, module, name)) # compute security hash if not isGenuineWebhook(auth, webhook): msg = "not authorized here\n%s" % json.dumps(content, indent=2) else: # build a signature signature = "%s@%s.%s[%s]" % (webhook["event"], module, name, jsonHash(content)) skip = False if not skip: # import asked module try: obj = importlib.import_module("lystener." + module) except Exception as exception: skip = True msg = "%r\ncan not import python module %s" % \ (exception, module) else: # try to get function by name TaskExecutioner.MODULES.add(obj) func = getattr(obj, name, False) if callable(func): TaskExecutioner.JOB.put( [func, content, webhook["token"], signature]) msg = "forwarded: " + signature else: skip = True msg = "python definition %s not found in %s or is " \ "not callable" % (name, module) if skip and "token" in webhook: Task.LOCK.acquire() # ATOMIC ACTION ----------------------------------------------- untrace(TaskChecker.DB, webhook["token"], content) TaskChecker.DB.commit() # END ATOMIC ACTION ------------------------------------------- Task.LOCK.release() # push msg MessageLogger.JOB.put(msg) logMsg("exiting TaskChecker...")
def start_listening(args={}, **options): # persistent options effect # modifying the pm2 app.json configuration app_folder = os.path.abspath(os.path.dirname(lystener.__path__[0])) app = lystener.loadJson("app.json", folder=app_folder) app["apps"][0]["args"] = " ".join( ["--{0:s}={1:s}".format(*item) for item in options.items()]) lystener.dumpJson(app, "app.json", folder=app_folder) # execute pm2 command lines os.system(""" if [ "$(pm2 id lystener-server) " = "[] " ]; then cd %(abspath)s pm2 start app.json else pm2 restart lystener-server fi """ % {"abspath": app_folder})
def vacuumDB(): sqlite = sqlite3.connect(os.path.join(DATA, "database.db"), isolation_level=None) sqlite.row_factory = sqlite3.Row token = [ row["token"] for row in sqlite.execute( "SELECT DISTINCT token FROM history").fetchall() ] for tok in [ loadJson(name).get("token", None) for name in os.listdir(JSON) if name.endswith(".json") ]: if tok not in token: cleanDB(sqlite, tok) sqlite.execute("VACUUM") sqlite.commit() sqlite.close()
def index(): if os.path.exists(os.path.join(lystener.ROOT, ".json")): json_list = [ loadJson(name) for name in os.listdir(os.path.join(lystener.ROOT, ".json")) if name.endswith(".json") ] else: json_list = [] if app.config.ini.has_section("Autorizations"): tiny_list = dict(app.config.ini.items("Autorizations", vars={})) else: tiny_list = {} cursor = connect() return flask.render_template( "listener.html", counts=dict( cursor.execute( "SELECT autorization, count(*) FROM history GROUP BY autorization" ).fetchall()), webhooks=json_list, tinies=tiny_list)
def index(): if os.path.exists(os.path.join(lystener.ROOT, ".json")): json_list = [ lystener.loadJson(name) for name in os.listdir(os.path.join(lystener.ROOT, ".json")) if name.endswith(".json") ] else: json_list = [] counts = dict( CURSOR.execute("SELECT authorization, count(*) " "FROM history GROUP BY authorization").fetchall()) data = [] for webhook in json_list: info = {} info["counts"] = counts.get(webhook["token"], 0) info["id"] = webhook["id"] info["event"] = webhook["event"] info["call"] = ".".join(webhook["target"].split("/")[-2:]) info["conditions"] = webhook["conditions"] data.append(info) return data
# doesn't even have to be reachable s.connect(('10.255.255.255', 1)) PUBLIC_IP = s.getsockname()[0] except Exception: PUBLIC_IP = '127.0.0.1' finally: s.close() return PUBLIC_IP PUBLIC_IP = GET.plain(peer="https://www.ipecho.net").get("raw", getPublicIp()) # default peer configuration LISTENER_PEER = {"scheme": "http", "ip": PUBLIC_IP, "port": ":5001"} WEBHOOK_PEER = {"scheme": "http", "ip": "127.0.0.1", "port": ":4004"} # generate defaults peers if needed peers = lystener.loadJson("peer.json", folder=lystener.ROOT) LISTENER_PEER.update(peers.get("listener", {})) WEBHOOK_PEER.update(peers.get("webhook", {})) # dump peer.json on first import lystener.dumpJson({ "listener": LISTENER_PEER, "webhook": WEBHOOK_PEER }, "peer.json", folder=lystener.ROOT) req.EndPoint.peer = "%(scheme)s://%(ip)s%(port)s" % WEBHOOK_PEER
""" import re import json import requests import lystener # default peer configuration LISTENER_PEER = {"scheme": "http", "ip": "127.0.0.1", "port": 5001} WEBHOOK_PEER = {"scheme": "http", "ip": "127.0.0.1", "port": 4004} # global var used by REST requests HEADERS = {"Content-Type": "application/json"} TIMEOUT = 7 # generate defaults peers peers = lystener.loadJson("peer.json", folder=lystener.__path__[0]) LISTENER_PEER.update(peers.get("listener", {})) WEBHOOK_PEER.update(peers.get("webhook", {})) # dump peer.json on first import lystener.dumpJson({ "listener": LISTENER_PEER, "webhook": WEBHOOK_PEER }, "peer.json", folder=lystener.__path__[0]) class EndPoint(object): @staticmethod def _manageResponse(req): try:
def execute(module, name): if flask.request.method == "POST": data = json.loads(flask.request.data).get("data", False) # check the data sent by webhook # TESTED --> OK if not data: logMsg("no data provided") return json.dumps({ "success": False, "message": "no data provided" }) # check autorization and exit if bad one # TESTED --> OK autorization = flask.request.headers.get("Authorization", "?") webhook = loadJson("%s.%s.json" % (module, name)) half_token = webhook.get("token", 32 * " ")[:32] if app.config.ini.has_section("Autorizations"): ini_autorizations = app.config.ini.options("Autorizations") if autorization == "?" or (half_token != autorization and autorization not in ini_autorizations): logMsg("not autorized here") return json.dumps({ "success": False, "message": "not autorized here" }) # use sqlite database to check if data already parsed once # TESTED --> OK cursor = connect() signature = data.get("signature", False) if not signature: # remove all trailing spaces, new lines, tabs etc... # and generate sha 256 hash as signature raw = re.sub(r"[\s]*", "", json.dumps(sameDataSort(data))) h = hashlib.sha256(raw.encode("utf-8")).hexdigest() signature = h.decode() if isinstance(h, bytes) else h # check if signature already in database cursor.execute("SELECT count(*) FROM history WHERE signature = ?", (signature, )) if cursor.fetchone()[0] == 0: # insert signature if no one found in database cursor.execute( "INSERT OR REPLACE INTO history(signature, autorization) VALUES(?,?);", (signature, autorization)) else: # exit if signature found in database logMsg("data already parsed") return json.dumps({ "success": False, "message": "data already parsed" }) ### NEED PRODUCER CONSMER PATTERN ### TESTED # # act as a hub endpoints list found # endpoints = webhook.get("hub", []) # # or if config file has a [Hub] section # if app.config.ini.has_section("Hub"): # endpoints.extend([item[-1] for item in app.config.ini.items("Hub", vars={})]) # if len(endpoints): # result = [] # for endpoint in endpoints: # try: # req = requests.post(endpoint, data=flask.request.data, headers=flask.request.headers, timeout=5, verify=True) # except Exception as error: # result.append({"success":False,"error":"%r"%error,"except":True}) # else: # result.append(req.text) # msg = "event broadcasted to hub :\n%s" % json.dumps(dict(zip(endpoints, result)), indent=2) # logMsg(msg) # # if node is used as a hub, should not have to execute something # # so exit here # return json.dumps({"success": True, "message": msg}) try: # import asked module obj = import_module("lystener." + module) except ImportError as error: msg = "%r\ncan not import python element %s" % (error, module) logMsg(msg) return json.dumps({"success": False, "message": msg}) # get asked function and execute with data provided by webhook func = getattr(obj, name, False) if func: ### NEED PRODUCER CONSMER PATTERN MAYBE... ### if the python code takes too long, ### connection will be broken ### push to FIFO1 (func, data) ### get from FIFO2 with 10s timeout # JOB_QUEUE.put((func, data)) # try: # msg = RESPONSE_QUEUE.get(timeout=5) # logMsg("%s response:\n%s" % (func.__name__, msg)) # except queue.Empty: # msg = "%s.%s response time reached..." % (module, name) # logMsg(msg) response = func(data) logMsg("%s response:\n%s" % (name, response)) else: msg = "python definition %s not found in %s" % (name, module) logMsg(msg) return json.dumps({"success": False, "message": msg}) # remove the module so if code is modified it will be updated without # a listener restart sys.modules.pop(obj.__name__, False) del obj return json.dumps({"success": True, "message": response})