def catch(mod, func, **kwargs): "Create a new job and return simple message." body = kwargs.get("data", {}) auth = kwargs.get("headers", {}).get("authorization", "") # try to save fingerprint, fails if fingerprint already exists else accept # and send data to TaskChecker try: webhook_name = task.webhookName(auth, mod, func) token = loadJson(webhook_name)["token"] content = body.get("data", {}) task.trace(CURSOR, token, content) except KeyError as error: logMsg("%r" % error) msg = {"status": 404, "msg": "no listener for %s.%s" % (mod, func)} except task.sqlite3.IntegrityError as error: logMsg("%r" % error) msg = {"status": 409, "msg": "data already parsed"} else: task.TaskChecker.JOB.put([mod, func, auth, content]) msg = { "status": 200, "msg": "task set: %s.%s(%s)" % (mod, func, content) } finally: CURSOR.commit() return msg
def destroy_listener(args={}, **options): """ unlink ark blockchain event from a python function. """ # maybee not ! # # connect to database in order to remove webhook associated data # sqlite = initDB() # cursor = sqlite.cursor() function = args.get("<function>", options.get("function", False)) if not function: listeners = [ name.replace(".json", "") for name in os.listdir(lystener.JSON) if name.endswith(".json") ] function = lystener.chooseItem("Select listener to destroy:", *listeners) if not function: return json_name = "%s.json" % function # load webhook configuration webhook = lystener.loadJson(json_name) # condition bellow checks if webhook configurations is found if webhook.get("peer", False): # delete webhook usong its id and parent peer rest.DELETE.api.webhooks("%s" % webhook["id"], peer=webhook["peer"]) # delete the webhook configuration os.remove(os.path.join(lystener.JSON, json_name)) # cursor.execute("SELECT FROM history WHERE autorization = ?", (webhook["token"][:32],)).fetchall() lystener.logMsg("%s webhook destroyed" % function) else: lystener.logMsg("%s webhook not found" % function)
def start(self, *args, **kwargs): while True: func, data = JOB_QUEUE.get() try: resp = {"success": True, "message": func(data)} except Exception as error: resp = {"except": True, "error": "%r" % error} logMsg("%s response:\n%s" % (func.__name__, resp)) RESPONSE_QUEUE.put(resp)
def run(self): logMsg("TaskExecutioner is running in background...") TaskExecutioner.DB = initDB() while not Task.STOP.is_set(): error = True response = {} # wait until a job is given func, data, token, sig = TaskExecutioner.JOB.get() try: response = func(data) except Exception as exception: msg = "%s response:\n%s\n%s" % \ (func, "%r" % exception, traceback.format_exc()) else: error = False msg = "%s response:\n%s" % (func, response) # push msg MessageLogger.JOB.put(msg) # daemon waits here to log results, update database and clean # memory try: Task.LOCK.acquire() # ATOMIC ACTION ----------------------------------------------- if not error and response.get("success", False): TaskExecutioner.DB.execute( "INSERT OR REPLACE INTO history(signature, token) " "VALUES(?, ?);", (sig, token)) # remove the module if all jobs done so if code is modified it # will be updated without a listener restart if TaskExecutioner.JOB.empty(): empty = False while not empty: try: obj = TaskExecutioner.MODULES.pop() except Exception: empty = True else: sys.modules.pop(obj.__name__, False) del obj except Exception as exception: MessageLogger.JOB.put( "Internal error occured:\n%s\n%s" % ("%r" % exception, traceback.format_exc())) finally: if error: untrace(TaskExecutioner.DB, token, data) TaskExecutioner.DB.commit() # END ATOMIC ACTION ------------------------------------------- Task.LOCK.release() logMsg("exiting TaskExecutioner...")
def run(self): # sqlite db opened within thread TaskChecker.DB = initDB() logMsg("TaskChecker is running in background...") # run until Task.killall() called while not Task.STOP.is_set(): skip = True # wait until a job is given module, name, auth, content = TaskChecker.JOB.get() # get webhook data webhook = loadJson(webhookName(auth, module, name)) # compute security hash if not isGenuineWebhook(auth, webhook): msg = "not authorized here\n%s" % json.dumps(content, indent=2) else: # build a signature signature = "%s@%s.%s[%s]" % (webhook["event"], module, name, jsonHash(content)) skip = False if not skip: # import asked module try: obj = importlib.import_module("lystener." + module) except Exception as exception: skip = True msg = "%r\ncan not import python module %s" % \ (exception, module) else: # try to get function by name TaskExecutioner.MODULES.add(obj) func = getattr(obj, name, False) if callable(func): TaskExecutioner.JOB.put( [func, content, webhook["token"], signature]) msg = "forwarded: " + signature else: skip = True msg = "python definition %s not found in %s or is " \ "not callable" % (name, module) if skip and "token" in webhook: Task.LOCK.acquire() # ATOMIC ACTION ----------------------------------------------- untrace(TaskChecker.DB, webhook["token"], content) TaskChecker.DB.commit() # END ATOMIC ACTION ------------------------------------------- Task.LOCK.release() # push msg MessageLogger.JOB.put(msg) logMsg("exiting TaskChecker...")
def deploy_listener(args={}, **options): """ link ark blockchain event to a python function. """ function = args.get("<function>", options.get("function", False)) regexp = args.get("<regexp>", options.get("regexp", False)) event = args.get("<event>", options.get("event", False)) json_name = "%s.json" % function # build peers and target url webhook_peer = options.get( "webhook", "%(scheme)s://%(ip)s:%(port)s" % rest.WEBHOOK_PEER) listener_peer = options.get( "listener", "%(scheme)s://%(ip)s:%(port)s" % rest.LISTENER_PEER) target_url = listener_peer + "/" + function.replace(".", "/") # compute listener condition # if only a regexp is givent compute condition on vendorField if regexp: condition = { "key": "vendorField", "condition": "regexp", "value": args["<regexp>"] } # else create a condition. # Ark webhook api will manage condition errors elif len(options): condition = { "field": options["field"], "condition": options["condition"], "value": options["value"] } # load webhook configuration if already set webhook = lystener.loadJson(json_name) # lystener.loadJson returns void dict if json_name not found, # the if clause bellow will be true then if not webhook.get("token", False): # create the webhook req = rest.POST.api.webhooks(event=event, peer=webhook_peer, target=target_url, conditions=[condition]) # parse request result if no error messages if not req.get("error", False): webhook = req["data"] # save the used peer to be able to delete it later webhook["peer"] = webhook_peer webhook["hub"] = _endpoints(options.get("endpoints", "")) # save webhook configuration in JSON folder lystener.dumpJson(webhook, json_name) lystener.logMsg("%s webhook set" % function) else: lystener.logMsg("%r" % req) lystener.logMsg("%s webhook not set" % function) else: lystener.logMsg("webhook already set for %s" % function)
def run(self): logMsg("FunctionCaller is running in background...") while not Task.STOP.is_set(): func, args, kwargs = FunctionCaller.JOB.get() try: Task.LOCK.acquire() response = func(*args, **kwargs) except Exception as exception: msg = "%s response:\n%r\n%s" % \ (func, exception, traceback.format_exc()) else: msg = "%s response:\n%r" % (func, response) finally: Task.LOCK.release() # push msg MessageLogger.JOB.put(msg) logMsg("exiting FunctionCaller...")
def run(self): logMsg("MessageLogger is running in background...") while not Task.STOP.is_set(): msg = MessageLogger.JOB.get() try: Task.LOCK.acquire() logMsg(msg) except Exception as exception: msg = "log error:\n%r\n%s" % \ (exception, traceback.format_exc()) finally: Task.LOCK.release() logMsg("exiting MessageLogger...")
def cleanDB(sqlite, token): logMsg("removing hitory of token %s..." % token) sqlite.execute("DELETE FROM fingerprint WHERE token=?", (token, )) sqlite.execute("DELETE FROM history WHERE token=?", (token, ))
def logSomething(data): logMsg('Transaction sent :\n%s' % json.dumps(data, indent=2)) return json.dumps({"success": True})
def execute(module, name): if flask.request.method == "POST": data = json.loads(flask.request.data).get("data", False) # check the data sent by webhook # TESTED --> OK if not data: logMsg("no data provided") return json.dumps({ "success": False, "message": "no data provided" }) # check autorization and exit if bad one # TESTED --> OK autorization = flask.request.headers.get("Authorization", "?") webhook = loadJson("%s.%s.json" % (module, name)) half_token = webhook.get("token", 32 * " ")[:32] if app.config.ini.has_section("Autorizations"): ini_autorizations = app.config.ini.options("Autorizations") if autorization == "?" or (half_token != autorization and autorization not in ini_autorizations): logMsg("not autorized here") return json.dumps({ "success": False, "message": "not autorized here" }) # use sqlite database to check if data already parsed once # TESTED --> OK cursor = connect() signature = data.get("signature", False) if not signature: # remove all trailing spaces, new lines, tabs etc... # and generate sha 256 hash as signature raw = re.sub(r"[\s]*", "", json.dumps(sameDataSort(data))) h = hashlib.sha256(raw.encode("utf-8")).hexdigest() signature = h.decode() if isinstance(h, bytes) else h # check if signature already in database cursor.execute("SELECT count(*) FROM history WHERE signature = ?", (signature, )) if cursor.fetchone()[0] == 0: # insert signature if no one found in database cursor.execute( "INSERT OR REPLACE INTO history(signature, autorization) VALUES(?,?);", (signature, autorization)) else: # exit if signature found in database logMsg("data already parsed") return json.dumps({ "success": False, "message": "data already parsed" }) ### NEED PRODUCER CONSMER PATTERN ### TESTED # # act as a hub endpoints list found # endpoints = webhook.get("hub", []) # # or if config file has a [Hub] section # if app.config.ini.has_section("Hub"): # endpoints.extend([item[-1] for item in app.config.ini.items("Hub", vars={})]) # if len(endpoints): # result = [] # for endpoint in endpoints: # try: # req = requests.post(endpoint, data=flask.request.data, headers=flask.request.headers, timeout=5, verify=True) # except Exception as error: # result.append({"success":False,"error":"%r"%error,"except":True}) # else: # result.append(req.text) # msg = "event broadcasted to hub :\n%s" % json.dumps(dict(zip(endpoints, result)), indent=2) # logMsg(msg) # # if node is used as a hub, should not have to execute something # # so exit here # return json.dumps({"success": True, "message": msg}) try: # import asked module obj = import_module("lystener." + module) except ImportError as error: msg = "%r\ncan not import python element %s" % (error, module) logMsg(msg) return json.dumps({"success": False, "message": msg}) # get asked function and execute with data provided by webhook func = getattr(obj, name, False) if func: ### NEED PRODUCER CONSMER PATTERN MAYBE... ### if the python code takes too long, ### connection will be broken ### push to FIFO1 (func, data) ### get from FIFO2 with 10s timeout # JOB_QUEUE.put((func, data)) # try: # msg = RESPONSE_QUEUE.get(timeout=5) # logMsg("%s response:\n%s" % (func.__name__, msg)) # except queue.Empty: # msg = "%s.%s response time reached..." % (module, name) # logMsg(msg) response = func(data) logMsg("%s response:\n%s" % (name, response)) else: msg = "python definition %s not found in %s" % (name, module) logMsg(msg) return json.dumps({"success": False, "message": msg}) # remove the module so if code is modified it will be updated without # a listener restart sys.modules.pop(obj.__name__, False) del obj return json.dumps({"success": True, "message": response})
def deploy_listener(args={}, **options): """ link blockchain event to a python function. """ function = args.get("<function>", options.get("function", "")) regexp = args.get("<regexp>", options.get("regexp", None)) event = args.get("<event>", options.get("event", "null")) target_url = \ ("%(scheme)s://%(ip)s%(port)s" % rest.LISTENER_PEER) + \ "/" + function.replace(".", "/") # compute listener condition # if only a regexp is givent compute condition on vendorField if regexp is not None: conditions = [{ "key": "vendorField", "condition": "regexp", "value": regexp }] # else create a condition. # Ark webhook api will manage condition errors else: conditions = list({ "key": k, "condition": c, "value": v } for k, c, v in zip( args.get("<field>", options.get("field", [])), args.get("<condition>", options.get("condition", [])), args.get("<value>", options.get("value", [])))) # check if remotly called remotly = options.get("remote", False) if remotly: _POST = POST target_peer = options.get("peer", "http://127.0.0.1") link() else: _POST = rest.POST target_peer = args.get("<node>", options.get("node", rest.req.EndPoint.peer)) # create the webhook req = _POST.api.webhooks(event=event, peer=target_peer, target=target_url, conditions=conditions) # parse request result if no error messages try: if req.get("status", req.get("statusCode", 500)) < 300: webhook = req["data"] # save the used peer to be able to delete it later webhook["peer"] = target_peer # build the security hash and keep only second token part webhook["hash"] = hashlib.sha256( webhook["token"].encode("utf-8")).hexdigest() # create a unique name based on first token part and module.name # so when a content is received, authorisation in POST header can # be used with module and name mentioned in the url webhook_name = task.webhookName(webhook["token"][:32], *function.split(".")) logMsg("token: %s" % webhook["token"]) webhook["token"] = webhook["token"][32:] dumpJson(webhook, webhook_name) logMsg("%s webhook set" % function) else: logMsg("%s webhook not set:\n%r" % (function, req)) except Exception as error: logMsg("%s" % req) logMsg("%r\n%s" % (error, traceback.format_exc()))