def getSeed(pin=False):
try:
value = lystener.loadJson("salt")["salt"]
except KeyError:
if not os.path.exists(SALT):
startSeed()
value = lystener.loadJson("salt")["salt"]
else:
if time.time() - os.path.getmtime(SALT) > 60:
startSeed()
return str(int(value, 16))[:6] if pin else value
def destroy_listener(args={}, **options):
"""
unlink ark blockchain event from a python function.
"""
# maybee not !
# # connect to database in order to remove webhook associated data
# sqlite = initDB()
# cursor = sqlite.cursor()
function = args.get("<function>", options.get("function", False))
if not function:
listeners = [
name.replace(".json", "") for name in os.listdir(lystener.JSON)
if name.endswith(".json")
]
function = lystener.chooseItem("Select listener to destroy:",
*listeners)
if not function: return
json_name = "%s.json" % function
# load webhook configuration
webhook = lystener.loadJson(json_name)
# condition bellow checks if webhook configurations is found
if webhook.get("peer", False):
# delete webhook usong its id and parent peer
rest.DELETE.api.webhooks("%s" % webhook["id"], peer=webhook["peer"])
# delete the webhook configuration
os.remove(os.path.join(lystener.JSON, json_name))
# cursor.execute("SELECT FROM history WHERE autorization = ?", (webhook["token"][:32],)).fetchall()
lystener.logMsg("%s webhook destroyed" % function)
else:
lystener.logMsg("%s webhook not found" % function)
def catch(mod, func, **kwargs):
"Create a new job and return simple message."
body = kwargs.get("data", {})
auth = kwargs.get("headers", {}).get("authorization", "")
# try to save fingerprint, fails if fingerprint already exists else accept
# and send data to TaskChecker
try:
webhook_name = task.webhookName(auth, mod, func)
token = loadJson(webhook_name)["token"]
content = body.get("data", {})
task.trace(CURSOR, token, content)
except KeyError as error:
logMsg("%r" % error)
msg = {"status": 404, "msg": "no listener for %s.%s" % (mod, func)}
except task.sqlite3.IntegrityError as error:
logMsg("%r" % error)
msg = {"status": 409, "msg": "data already parsed"}
else:
task.TaskChecker.JOB.put([mod, func, auth, content])
msg = {
"status": 200,
"msg": "task set: %s.%s(%s)" % (mod, func, content)
}
finally:
CURSOR.commit()
return msg
def checkRemoteAuth(**args):
try:
headers = args.get("headers", {})
task.MessageLogger.JOB.put("received secured headers: %r" % headers)
# check if publick key defined in header is an authorized one
# auth is a list of public keys. Because loadJson returns a void
# dict if no file found, the if condition acts same as if it was a list
publicKey = headers.get("public-key", "?")
if publicKey not in lystener.loadJson("auth"):
return {"status": 401, "msg": "not authorized"}
# Note that client has to define its own salt value and get a
# 1-min-valid random seed from lystener server.
# See /salt endpoint
sig = secp256k1.HexSig.from_der(headers["signature"])
puk = secp256k1.PublicKey.decode(publicKey)
msg = secp256k1.hash_sha256(headers["salt"] + getSeed())
if not secp256k1._ecdsa.verify(msg, puk.x, puk.y, sig.r, sig.s):
return {"status": 403, "msg": "bad signature"}
return {"status": 200, "msg": "access granted"}
except Exception as error:
return {
"status":
500,
"msg":
"checkPutDelete response: %s\n%s" %
("%r" % error, traceback.format_exc())
}
def notifyWhaleMove(data):
task.MessageLogger.log('data received :\n%s' % json.dumps(data, indent=2))
params = loadJson("notifyWhaleMove.param", folder=lystener.DATA)
sender = getAddress(data["senderPublicKey"], data["network"])
receiver = data["recipientId"]
task.MessageLogger.log('Big move from %s to %s!' % (sender, receiver))
for exchange, wallets in params["hot wallets"].items():
task.MessageLogger.log(
' checking %s wallets : %s...' % (exchange, wallets)
)
if receiver in wallets:
task.FunctionCaller.call(
notify.send,
"Whale alert",
"%s sent %.2f token to %s" %
(sender, float(data["amount"])/100000000, exchange)
)
return {"success": True}
elif sender in wallets:
task.FunctionCaller.call(
notify.send,
"Whale alert",
"%s sent %.2f token to %s" %
(exchange, float(data["amount"])/100000000, receiver)
)
return {"success": True}
return {"success": False, "response": "Nothing hapen !"}
def deploy_listener(args={}, **options):
"""
link ark blockchain event to a python function.
"""
function = args.get("<function>", options.get("function", False))
regexp = args.get("<regexp>", options.get("regexp", False))
event = args.get("<event>", options.get("event", False))
json_name = "%s.json" % function
# build peers and target url
webhook_peer = options.get(
"webhook", "%(scheme)s://%(ip)s:%(port)s" % rest.WEBHOOK_PEER)
listener_peer = options.get(
"listener", "%(scheme)s://%(ip)s:%(port)s" % rest.LISTENER_PEER)
target_url = listener_peer + "/" + function.replace(".", "/")
# compute listener condition
# if only a regexp is givent compute condition on vendorField
if regexp:
condition = {
"key": "vendorField",
"condition": "regexp",
"value": args["<regexp>"]
}
# else create a condition.
# Ark webhook api will manage condition errors
elif len(options):
condition = {
"field": options["field"],
"condition": options["condition"],
"value": options["value"]
}
# load webhook configuration if already set
webhook = lystener.loadJson(json_name)
# lystener.loadJson returns void dict if json_name not found,
# the if clause bellow will be true then
if not webhook.get("token", False):
# create the webhook
req = rest.POST.api.webhooks(event=event,
peer=webhook_peer,
target=target_url,
conditions=[condition])
# parse request result if no error messages
if not req.get("error", False):
webhook = req["data"]
# save the used peer to be able to delete it later
webhook["peer"] = webhook_peer
webhook["hub"] = _endpoints(options.get("endpoints", ""))
# save webhook configuration in JSON folder
lystener.dumpJson(webhook, json_name)
lystener.logMsg("%s webhook set" % function)
else:
lystener.logMsg("%r" % req)
lystener.logMsg("%s webhook not set" % function)
else:
lystener.logMsg("webhook already set for %s" % function)
def run(self):
# sqlite db opened within thread
TaskChecker.DB = initDB()
logMsg("TaskChecker is running in background...")
# run until Task.killall() called
while not Task.STOP.is_set():
skip = True
# wait until a job is given
module, name, auth, content = TaskChecker.JOB.get()
# get webhook data
webhook = loadJson(webhookName(auth, module, name))
# compute security hash
if not isGenuineWebhook(auth, webhook):
msg = "not authorized here\n%s" % json.dumps(content, indent=2)
else:
# build a signature
signature = "%s@%s.%s[%s]" % (webhook["event"], module, name,
jsonHash(content))
skip = False
if not skip:
# import asked module
try:
obj = importlib.import_module("lystener." + module)
except Exception as exception:
skip = True
msg = "%r\ncan not import python module %s" % \
(exception, module)
else:
# try to get function by name
TaskExecutioner.MODULES.add(obj)
func = getattr(obj, name, False)
if callable(func):
TaskExecutioner.JOB.put(
[func, content, webhook["token"], signature])
msg = "forwarded: " + signature
else:
skip = True
msg = "python definition %s not found in %s or is " \
"not callable" % (name, module)
if skip and "token" in webhook:
Task.LOCK.acquire()
# ATOMIC ACTION -----------------------------------------------
untrace(TaskChecker.DB, webhook["token"], content)
TaskChecker.DB.commit()
# END ATOMIC ACTION -------------------------------------------
Task.LOCK.release()
# push msg
MessageLogger.JOB.put(msg)
logMsg("exiting TaskChecker...")
def start_listening(args={}, **options):
# persistent options effect
# modifying the pm2 app.json configuration
app_folder = os.path.abspath(os.path.dirname(lystener.__path__[0]))
app = lystener.loadJson("app.json", folder=app_folder)
app["apps"][0]["args"] = " ".join(
["--{0:s}={1:s}".format(*item) for item in options.items()])
lystener.dumpJson(app, "app.json", folder=app_folder)
# execute pm2 command lines
os.system("""
if [ "$(pm2 id lystener-server) " = "[] " ]; then
cd %(abspath)s
pm2 start app.json
else
pm2 restart lystener-server
fi
""" % {"abspath": app_folder})
def vacuumDB():
sqlite = sqlite3.connect(os.path.join(DATA, "database.db"),
isolation_level=None)
sqlite.row_factory = sqlite3.Row
token = [
row["token"] for row in sqlite.execute(
"SELECT DISTINCT token FROM history").fetchall()
]
for tok in [
loadJson(name).get("token", None) for name in os.listdir(JSON)
if name.endswith(".json")
]:
if tok not in token:
cleanDB(sqlite, tok)
sqlite.execute("VACUUM")
sqlite.commit()
sqlite.close()
def index():
if os.path.exists(os.path.join(lystener.ROOT, ".json")):
json_list = [
loadJson(name)
for name in os.listdir(os.path.join(lystener.ROOT, ".json"))
if name.endswith(".json")
]
else:
json_list = []
if app.config.ini.has_section("Autorizations"):
tiny_list = dict(app.config.ini.items("Autorizations", vars={}))
else:
tiny_list = {}
cursor = connect()
return flask.render_template(
"listener.html",
counts=dict(
cursor.execute(
"SELECT autorization, count(*) FROM history GROUP BY autorization"
).fetchall()),
webhooks=json_list,
tinies=tiny_list)
def index():
if os.path.exists(os.path.join(lystener.ROOT, ".json")):
json_list = [
lystener.loadJson(name)
for name in os.listdir(os.path.join(lystener.ROOT, ".json"))
if name.endswith(".json")
]
else:
json_list = []
counts = dict(
CURSOR.execute("SELECT authorization, count(*) "
"FROM history GROUP BY authorization").fetchall())
data = []
for webhook in json_list:
info = {}
info["counts"] = counts.get(webhook["token"], 0)
info["id"] = webhook["id"]
info["event"] = webhook["event"]
info["call"] = ".".join(webhook["target"].split("/")[-2:])
info["conditions"] = webhook["conditions"]
data.append(info)
return data
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
PUBLIC_IP = s.getsockname()[0]
except Exception:
PUBLIC_IP = '127.0.0.1'
finally:
s.close()
return PUBLIC_IP
PUBLIC_IP = GET.plain(peer="https://www.ipecho.net").get("raw", getPublicIp())
# default peer configuration
LISTENER_PEER = {"scheme": "http", "ip": PUBLIC_IP, "port": ":5001"}
WEBHOOK_PEER = {"scheme": "http", "ip": "127.0.0.1", "port": ":4004"}
# generate defaults peers if needed
peers = lystener.loadJson("peer.json", folder=lystener.ROOT)
LISTENER_PEER.update(peers.get("listener", {}))
WEBHOOK_PEER.update(peers.get("webhook", {}))
# dump peer.json on first import
lystener.dumpJson({
"listener": LISTENER_PEER,
"webhook": WEBHOOK_PEER
},
"peer.json",
folder=lystener.ROOT)
req.EndPoint.peer = "%(scheme)s://%(ip)s%(port)s" % WEBHOOK_PEER
"""
import re
import json
import requests
import lystener
# default peer configuration
LISTENER_PEER = {"scheme": "http", "ip": "127.0.0.1", "port": 5001}
WEBHOOK_PEER = {"scheme": "http", "ip": "127.0.0.1", "port": 4004}
# global var used by REST requests
HEADERS = {"Content-Type": "application/json"}
TIMEOUT = 7
# generate defaults peers
peers = lystener.loadJson("peer.json", folder=lystener.__path__[0])
LISTENER_PEER.update(peers.get("listener", {}))
WEBHOOK_PEER.update(peers.get("webhook", {}))
# dump peer.json on first import
lystener.dumpJson({
"listener": LISTENER_PEER,
"webhook": WEBHOOK_PEER
},
"peer.json",
folder=lystener.__path__[0])
class EndPoint(object):
@staticmethod
def _manageResponse(req):
try:
def execute(module, name):
if flask.request.method == "POST":
data = json.loads(flask.request.data).get("data", False)
# check the data sent by webhook
# TESTED --> OK
if not data:
logMsg("no data provided")
return json.dumps({
"success": False,
"message": "no data provided"
})
# check autorization and exit if bad one
# TESTED --> OK
autorization = flask.request.headers.get("Authorization", "?")
webhook = loadJson("%s.%s.json" % (module, name))
half_token = webhook.get("token", 32 * " ")[:32]
if app.config.ini.has_section("Autorizations"):
ini_autorizations = app.config.ini.options("Autorizations")
if autorization == "?" or (half_token != autorization
and autorization not in ini_autorizations):
logMsg("not autorized here")
return json.dumps({
"success": False,
"message": "not autorized here"
})
# use sqlite database to check if data already parsed once
# TESTED --> OK
cursor = connect()
signature = data.get("signature", False)
if not signature:
# remove all trailing spaces, new lines, tabs etc...
# and generate sha 256 hash as signature
raw = re.sub(r"[\s]*", "", json.dumps(sameDataSort(data)))
h = hashlib.sha256(raw.encode("utf-8")).hexdigest()
signature = h.decode() if isinstance(h, bytes) else h
# check if signature already in database
cursor.execute("SELECT count(*) FROM history WHERE signature = ?",
(signature, ))
if cursor.fetchone()[0] == 0:
# insert signature if no one found in database
cursor.execute(
"INSERT OR REPLACE INTO history(signature, autorization) VALUES(?,?);",
(signature, autorization))
else:
# exit if signature found in database
logMsg("data already parsed")
return json.dumps({
"success": False,
"message": "data already parsed"
})
### NEED PRODUCER CONSMER PATTERN
### TESTED
# # act as a hub endpoints list found
# endpoints = webhook.get("hub", [])
# # or if config file has a [Hub] section
# if app.config.ini.has_section("Hub"):
# endpoints.extend([item[-1] for item in app.config.ini.items("Hub", vars={})])
# if len(endpoints):
# result = []
# for endpoint in endpoints:
# try:
# req = requests.post(endpoint, data=flask.request.data, headers=flask.request.headers, timeout=5, verify=True)
# except Exception as error:
# result.append({"success":False,"error":"%r"%error,"except":True})
# else:
# result.append(req.text)
# msg = "event broadcasted to hub :\n%s" % json.dumps(dict(zip(endpoints, result)), indent=2)
# logMsg(msg)
# # if node is used as a hub, should not have to execute something
# # so exit here
# return json.dumps({"success": True, "message": msg})
try:
# import asked module
obj = import_module("lystener." + module)
except ImportError as error:
msg = "%r\ncan not import python element %s" % (error, module)
logMsg(msg)
return json.dumps({"success": False, "message": msg})
# get asked function and execute with data provided by webhook
func = getattr(obj, name, False)
if func:
### NEED PRODUCER CONSMER PATTERN MAYBE...
### if the python code takes too long,
### connection will be broken
### push to FIFO1 (func, data)
### get from FIFO2 with 10s timeout
# JOB_QUEUE.put((func, data))
# try:
# msg = RESPONSE_QUEUE.get(timeout=5)
# logMsg("%s response:\n%s" % (func.__name__, msg))
# except queue.Empty:
# msg = "%s.%s response time reached..." % (module, name)
# logMsg(msg)
response = func(data)
logMsg("%s response:\n%s" % (name, response))
else:
msg = "python definition %s not found in %s" % (name, module)
logMsg(msg)
return json.dumps({"success": False, "message": msg})
# remove the module so if code is modified it will be updated without
# a listener restart
sys.modules.pop(obj.__name__, False)
del obj
return json.dumps({"success": True, "message": response})
