def catch(mod, func, **kwargs):
"Create a new job and return simple message."
body = kwargs.get("data", {})
auth = kwargs.get("headers", {}).get("authorization", "")
# try to save fingerprint, fails if fingerprint already exists else accept
# and send data to TaskChecker
try:
webhook_name = task.webhookName(auth, mod, func)
token = loadJson(webhook_name)["token"]
content = body.get("data", {})
task.trace(CURSOR, token, content)
except KeyError as error:
logMsg("%r" % error)
msg = {"status": 404, "msg": "no listener for %s.%s" % (mod, func)}
except task.sqlite3.IntegrityError as error:
logMsg("%r" % error)
msg = {"status": 409, "msg": "data already parsed"}
else:
task.TaskChecker.JOB.put([mod, func, auth, content])
msg = {
"status": 200,
"msg": "task set: %s.%s(%s)" % (mod, func, content)
}
finally:
CURSOR.commit()
return msg
def destroy_listener(args={}, **options):
"""
unlink ark blockchain event from a python function.
"""
# maybee not !
# # connect to database in order to remove webhook associated data
# sqlite = initDB()
# cursor = sqlite.cursor()
function = args.get("<function>", options.get("function", False))
if not function:
listeners = [
name.replace(".json", "") for name in os.listdir(lystener.JSON)
if name.endswith(".json")
]
function = lystener.chooseItem("Select listener to destroy:",
*listeners)
if not function: return
json_name = "%s.json" % function
# load webhook configuration
webhook = lystener.loadJson(json_name)
# condition bellow checks if webhook configurations is found
if webhook.get("peer", False):
# delete webhook usong its id and parent peer
rest.DELETE.api.webhooks("%s" % webhook["id"], peer=webhook["peer"])
# delete the webhook configuration
os.remove(os.path.join(lystener.JSON, json_name))
# cursor.execute("SELECT FROM history WHERE autorization = ?", (webhook["token"][:32],)).fetchall()
lystener.logMsg("%s webhook destroyed" % function)
else:
lystener.logMsg("%s webhook not found" % function)
def start(self, *args, **kwargs):
while True:
func, data = JOB_QUEUE.get()
try:
resp = {"success": True, "message": func(data)}
except Exception as error:
resp = {"except": True, "error": "%r" % error}
logMsg("%s response:\n%s" % (func.__name__, resp))
RESPONSE_QUEUE.put(resp)
def run(self):
logMsg("TaskExecutioner is running in background...")
TaskExecutioner.DB = initDB()
while not Task.STOP.is_set():
error = True
response = {}
# wait until a job is given
func, data, token, sig = TaskExecutioner.JOB.get()
try:
response = func(data)
except Exception as exception:
msg = "%s response:\n%s\n%s" % \
(func, "%r" % exception, traceback.format_exc())
else:
error = False
msg = "%s response:\n%s" % (func, response)
# push msg
MessageLogger.JOB.put(msg)
# daemon waits here to log results, update database and clean
# memory
try:
Task.LOCK.acquire()
# ATOMIC ACTION -----------------------------------------------
if not error and response.get("success", False):
TaskExecutioner.DB.execute(
"INSERT OR REPLACE INTO history(signature, token) "
"VALUES(?, ?);", (sig, token))
# remove the module if all jobs done so if code is modified it
# will be updated without a listener restart
if TaskExecutioner.JOB.empty():
empty = False
while not empty:
try:
obj = TaskExecutioner.MODULES.pop()
except Exception:
empty = True
else:
sys.modules.pop(obj.__name__, False)
del obj
except Exception as exception:
MessageLogger.JOB.put(
"Internal error occured:\n%s\n%s" %
("%r" % exception, traceback.format_exc()))
finally:
if error:
untrace(TaskExecutioner.DB, token, data)
TaskExecutioner.DB.commit()
# END ATOMIC ACTION -------------------------------------------
Task.LOCK.release()
logMsg("exiting TaskExecutioner...")
def run(self):
# sqlite db opened within thread
TaskChecker.DB = initDB()
logMsg("TaskChecker is running in background...")
# run until Task.killall() called
while not Task.STOP.is_set():
skip = True
# wait until a job is given
module, name, auth, content = TaskChecker.JOB.get()
# get webhook data
webhook = loadJson(webhookName(auth, module, name))
# compute security hash
if not isGenuineWebhook(auth, webhook):
msg = "not authorized here\n%s" % json.dumps(content, indent=2)
else:
# build a signature
signature = "%s@%s.%s[%s]" % (webhook["event"], module, name,
jsonHash(content))
skip = False
if not skip:
# import asked module
try:
obj = importlib.import_module("lystener." + module)
except Exception as exception:
skip = True
msg = "%r\ncan not import python module %s" % \
(exception, module)
else:
# try to get function by name
TaskExecutioner.MODULES.add(obj)
func = getattr(obj, name, False)
if callable(func):
TaskExecutioner.JOB.put(
[func, content, webhook["token"], signature])
msg = "forwarded: " + signature
else:
skip = True
msg = "python definition %s not found in %s or is " \
"not callable" % (name, module)
if skip and "token" in webhook:
Task.LOCK.acquire()
# ATOMIC ACTION -----------------------------------------------
untrace(TaskChecker.DB, webhook["token"], content)
TaskChecker.DB.commit()
# END ATOMIC ACTION -------------------------------------------
Task.LOCK.release()
# push msg
MessageLogger.JOB.put(msg)
logMsg("exiting TaskChecker...")
def deploy_listener(args={}, **options):
"""
link ark blockchain event to a python function.
"""
function = args.get("<function>", options.get("function", False))
regexp = args.get("<regexp>", options.get("regexp", False))
event = args.get("<event>", options.get("event", False))
json_name = "%s.json" % function
# build peers and target url
webhook_peer = options.get(
"webhook", "%(scheme)s://%(ip)s:%(port)s" % rest.WEBHOOK_PEER)
listener_peer = options.get(
"listener", "%(scheme)s://%(ip)s:%(port)s" % rest.LISTENER_PEER)
target_url = listener_peer + "/" + function.replace(".", "/")
# compute listener condition
# if only a regexp is givent compute condition on vendorField
if regexp:
condition = {
"key": "vendorField",
"condition": "regexp",
"value": args["<regexp>"]
}
# else create a condition.
# Ark webhook api will manage condition errors
elif len(options):
condition = {
"field": options["field"],
"condition": options["condition"],
"value": options["value"]
}
# load webhook configuration if already set
webhook = lystener.loadJson(json_name)
# lystener.loadJson returns void dict if json_name not found,
# the if clause bellow will be true then
if not webhook.get("token", False):
# create the webhook
req = rest.POST.api.webhooks(event=event,
peer=webhook_peer,
target=target_url,
conditions=[condition])
# parse request result if no error messages
if not req.get("error", False):
webhook = req["data"]
# save the used peer to be able to delete it later
webhook["peer"] = webhook_peer
webhook["hub"] = _endpoints(options.get("endpoints", ""))
# save webhook configuration in JSON folder
lystener.dumpJson(webhook, json_name)
lystener.logMsg("%s webhook set" % function)
else:
lystener.logMsg("%r" % req)
lystener.logMsg("%s webhook not set" % function)
else:
lystener.logMsg("webhook already set for %s" % function)
def run(self):
logMsg("FunctionCaller is running in background...")
while not Task.STOP.is_set():
func, args, kwargs = FunctionCaller.JOB.get()
try:
Task.LOCK.acquire()
response = func(*args, **kwargs)
except Exception as exception:
msg = "%s response:\n%r\n%s" % \
(func, exception, traceback.format_exc())
else:
msg = "%s response:\n%r" % (func, response)
finally:
Task.LOCK.release()
# push msg
MessageLogger.JOB.put(msg)
logMsg("exiting FunctionCaller...")
def run(self):
logMsg("MessageLogger is running in background...")
while not Task.STOP.is_set():
msg = MessageLogger.JOB.get()
try:
Task.LOCK.acquire()
logMsg(msg)
except Exception as exception:
msg = "log error:\n%r\n%s" % \
(exception, traceback.format_exc())
finally:
Task.LOCK.release()
logMsg("exiting MessageLogger...")
def cleanDB(sqlite, token):
logMsg("removing hitory of token %s..." % token)
sqlite.execute("DELETE FROM fingerprint WHERE token=?", (token, ))
sqlite.execute("DELETE FROM history WHERE token=?", (token, ))
def logSomething(data):
logMsg('Transaction sent :\n%s' % json.dumps(data, indent=2))
return json.dumps({"success": True})
def execute(module, name):
if flask.request.method == "POST":
data = json.loads(flask.request.data).get("data", False)
# check the data sent by webhook
# TESTED --> OK
if not data:
logMsg("no data provided")
return json.dumps({
"success": False,
"message": "no data provided"
})
# check autorization and exit if bad one
# TESTED --> OK
autorization = flask.request.headers.get("Authorization", "?")
webhook = loadJson("%s.%s.json" % (module, name))
half_token = webhook.get("token", 32 * " ")[:32]
if app.config.ini.has_section("Autorizations"):
ini_autorizations = app.config.ini.options("Autorizations")
if autorization == "?" or (half_token != autorization
and autorization not in ini_autorizations):
logMsg("not autorized here")
return json.dumps({
"success": False,
"message": "not autorized here"
})
# use sqlite database to check if data already parsed once
# TESTED --> OK
cursor = connect()
signature = data.get("signature", False)
if not signature:
# remove all trailing spaces, new lines, tabs etc...
# and generate sha 256 hash as signature
raw = re.sub(r"[\s]*", "", json.dumps(sameDataSort(data)))
h = hashlib.sha256(raw.encode("utf-8")).hexdigest()
signature = h.decode() if isinstance(h, bytes) else h
# check if signature already in database
cursor.execute("SELECT count(*) FROM history WHERE signature = ?",
(signature, ))
if cursor.fetchone()[0] == 0:
# insert signature if no one found in database
cursor.execute(
"INSERT OR REPLACE INTO history(signature, autorization) VALUES(?,?);",
(signature, autorization))
else:
# exit if signature found in database
logMsg("data already parsed")
return json.dumps({
"success": False,
"message": "data already parsed"
})
### NEED PRODUCER CONSMER PATTERN
### TESTED
# # act as a hub endpoints list found
# endpoints = webhook.get("hub", [])
# # or if config file has a [Hub] section
# if app.config.ini.has_section("Hub"):
# endpoints.extend([item[-1] for item in app.config.ini.items("Hub", vars={})])
# if len(endpoints):
# result = []
# for endpoint in endpoints:
# try:
# req = requests.post(endpoint, data=flask.request.data, headers=flask.request.headers, timeout=5, verify=True)
# except Exception as error:
# result.append({"success":False,"error":"%r"%error,"except":True})
# else:
# result.append(req.text)
# msg = "event broadcasted to hub :\n%s" % json.dumps(dict(zip(endpoints, result)), indent=2)
# logMsg(msg)
# # if node is used as a hub, should not have to execute something
# # so exit here
# return json.dumps({"success": True, "message": msg})
try:
# import asked module
obj = import_module("lystener." + module)
except ImportError as error:
msg = "%r\ncan not import python element %s" % (error, module)
logMsg(msg)
return json.dumps({"success": False, "message": msg})
# get asked function and execute with data provided by webhook
func = getattr(obj, name, False)
if func:
### NEED PRODUCER CONSMER PATTERN MAYBE...
### if the python code takes too long,
### connection will be broken
### push to FIFO1 (func, data)
### get from FIFO2 with 10s timeout
# JOB_QUEUE.put((func, data))
# try:
# msg = RESPONSE_QUEUE.get(timeout=5)
# logMsg("%s response:\n%s" % (func.__name__, msg))
# except queue.Empty:
# msg = "%s.%s response time reached..." % (module, name)
# logMsg(msg)
response = func(data)
logMsg("%s response:\n%s" % (name, response))
else:
msg = "python definition %s not found in %s" % (name, module)
logMsg(msg)
return json.dumps({"success": False, "message": msg})
# remove the module so if code is modified it will be updated without
# a listener restart
sys.modules.pop(obj.__name__, False)
del obj
return json.dumps({"success": True, "message": response})
def deploy_listener(args={}, **options):
"""
link blockchain event to a python function.
"""
function = args.get("<function>", options.get("function", ""))
regexp = args.get("<regexp>", options.get("regexp", None))
event = args.get("<event>", options.get("event", "null"))
target_url = \
("%(scheme)s://%(ip)s%(port)s" % rest.LISTENER_PEER) + \
"/" + function.replace(".", "/")
# compute listener condition
# if only a regexp is givent compute condition on vendorField
if regexp is not None:
conditions = [{
"key": "vendorField",
"condition": "regexp",
"value": regexp
}]
# else create a condition.
# Ark webhook api will manage condition errors
else:
conditions = list({
"key": k,
"condition": c,
"value": v
} for k, c, v in zip(
args.get("<field>", options.get("field", [])),
args.get("<condition>", options.get("condition", [])),
args.get("<value>", options.get("value", []))))
# check if remotly called
remotly = options.get("remote", False)
if remotly:
_POST = POST
target_peer = options.get("peer", "http://127.0.0.1")
link()
else:
_POST = rest.POST
target_peer = args.get("<node>",
options.get("node", rest.req.EndPoint.peer))
# create the webhook
req = _POST.api.webhooks(event=event,
peer=target_peer,
target=target_url,
conditions=conditions)
# parse request result if no error messages
try:
if req.get("status", req.get("statusCode", 500)) < 300:
webhook = req["data"]
# save the used peer to be able to delete it later
webhook["peer"] = target_peer
# build the security hash and keep only second token part
webhook["hash"] = hashlib.sha256(
webhook["token"].encode("utf-8")).hexdigest()
# create a unique name based on first token part and module.name
# so when a content is received, authorisation in POST header can
# be used with module and name mentioned in the url
webhook_name = task.webhookName(webhook["token"][:32],
*function.split("."))
logMsg("token: %s" % webhook["token"])
webhook["token"] = webhook["token"][32:]
dumpJson(webhook, webhook_name)
logMsg("%s webhook set" % function)
else:
logMsg("%s webhook not set:\n%r" % (function, req))
except Exception as error:
logMsg("%s" % req)
logMsg("%r\n%s" % (error, traceback.format_exc()))
