def test(apispec):
if apispec.method == POST:
fn = requests_post
else:
fn = requests_get
# try:
if apispec.header and type(apispec.header) is dict:
apispec.header.update(__header)
else:
apispec.header = __header
res = fn(
apispec.fulluri,
auth=apispec.auth,
cookies=apispec.cookie,
data=apispec.data,
headers=apispec.header, # {**apispec.header, **__header},
json=apispec.json,
params=apispec.param,
verify=apispec.verify,
timeout=__timeout,
stream=False)
res.raise_for_status()
# except Exception as e:
# out.warn(f"unsuccessful test HTTP {apisec.method} to "
# f"\"{apisec.fulluri}\": {e}")
# else:
out.debug(
f"successful test HTTP {apispec.method} to \"{apispec.fulluri}\"")
res.close()
del res
return
def tryf(file):
try:
with open(file, mode='rb') as fd:
out.debug(f"file \"{file}\" valid")
except:
out.error(f"cannot open file \"{file}\"")
return True
def validff(file):
for f in file:
try:
with open(f, mode='rb') as fd:
out.debug(f"file \"{f}\" valid")
except:
return False
return True
def openf(file, mode='rb', debug=True):
try:
fd = open(file, mode=mode)
if debug:
out.debug(f"file \"{file}\" valid")
return fd
except:
out.error(f"cannot open file \"{file}\"")
def _jsonload(json):
if type(json) is dict:
return json
elif type(json) is str:
try:
tree = loads(json)
except Exception as e:
out.error(
f"type \"{type(json)}\" invalid for parsing through JSON:"
f" {e}")
else:
return tree
else:
out.debug(f"type \"{type(json)}\" invalid for parsing through JSON")
def inarg():
global ina, pause
try:
pause = int(arg["--pause"])
if int(pause) < 0:
pause = 0
except ValueError:
out.warn(f"invalid pause interval integer value \"{pause}\", "
f"defaulting to zero")
if arg["--test"]:
ina = base.Service.get_apifn()
elif arg["--find"]:
ina = arg["<input>"]
elif arg["--quota"]:
ina = __NOINPUT
else:
# domain
if arg["--domain"]:
ina = web.parse_domainl(*arg["<input>"])
if not ina:
out.error("all input domain names are invalid")
# ipaddr
elif arg["--ipaddr"]:
ina = web.parse_ipaddrl(*arg["<input>"])
if not ina:
out.error("all input IP addresses are invalid")
# url
elif arg["--url"]:
ina = web.parse_fullurll(*arg["<input>"])
if not ina:
out.error("all input URLs are invalid")
# file, hash
else:
if arg["--submit"]:
ina = file.new(*arg["<input>"])
elif arg["--report"]:
# ina = arg["<input>"]
ina = crypto.parse_hashl(*arg["<input>"])
else: # arg["--download"]
if arg["<input>"]:
ina = crypto.parse_hashl(*arg["<input>"])
else:
ina = __NOINPUT
out.debug("ina", obj=ina)
if ina != __NOINPUT and not arg["--test"] and len(ina) != len(
arg["<input>"]):
rw.yn("one or more input arguments are invalid")
def loadserv():
from malsub.service import serv as _serv
global anserv
out.debug("_serv", obj=_serv)
if arg["--analysis"].lower() != "all":
from re import findall
anserv = list(findall(r"\w+", arg["--analysis"].lower()))
inv = [s for s in anserv if s not in _serv]
if inv:
out.error(f"input service(s) \"{','.join(inv)}\" unknown")
anserv = serv.ServiceList([s for s in _serv if s in anserv])
else:
anserv = serv.ServiceList([s for s in _serv])
out.debug("anserv", obj=anserv)
def _request(fn,
uri,
auth=None,
cookie=None,
data=None,
file=None,
header=None,
json_req=None,
param=None,
verify=True,
bin=False,
json=False):
# Quicksand.io has a broken cert
if 'quicksand' in uri:
verify = False
elif header and type(header) is dict:
header.update(__header)
else:
header = __header
# try:
res = fn(
uri,
auth=auth, # HTTPBasicAuth
cookies=cookie,
data=data,
files=file,
headers=header, # {**header, **__header},
json=json_req,
params=param,
verify=verify,
timeout=__timeout,
stream=False)
out.debug("res.headers", obj=dict(res.headers))
out.debug("res.text", obj=res.text)
res.raise_for_status()
# except ConnectionError as e:
# # out.warn(f"HTTP connection error to \"{uri}\": {e}")
# raise ConnectionError(e)
# except HTTPError as e:
# # out.warn(f"HTTP response error to \"{uri}\": {e}")
# raise HTTPError(e)
# except Timeout as e:
# # out.warn(f"HTTP request time out to \"{uri}\": {e}")
# raise Timeout(e)
# except TooManyRedirects as e:
# # out.warn(f"too many HTTP redirects to \"{uri}\": {e}")
# raise TooManyRedirects(e)
# else:
out.debug(f"HTTP {res.status_code} \"{res.reason}\" -- \"{uri}\"")
if res.status_code == 204 or res.headers.get("Content-Length") == 0:
raise Exception(f"HTTP {res.status_code} \"{res.reason}\""
f" -- \"{uri}\": no content")
filename = regex.http_filename(res.headers.get("Content-Disposition"))
if bin:
return res.content, filename
if json:
return res.json(), filename
return res.text, filename
def loadkey():
from yaml import safe_load as loadyaml
from yaml.scanner import ScannerError
global anserv
try:
apikey = {n.lower(): k for n, k in
loadyaml(open(meta.APIKEY_PATH)).items() if
n.lower() in anserv}
except IOError as e:
out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}")
except ScannerError as e:
out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}")
else:
for n, k in apikey.items():
if type(k) is not dict or \
k.get("apikey") is None or k.get("apikey") == "":
out.error(f"service \"{anserv[n].name}\" missing "
f"a valid API key \"{k}\"")
else:
anserv[n].set_apikey(k)
out.debug("apikey", obj=apikey)
def _request(fn,
uri,
header=None,
cookie=None,
data=None,
json=None,
file=None,
param=None,
verify=True,
bin=False):
if header and type(header) is dict:
header.update(__header)
else:
header = __header
# try:
res = fn(
uri,
cookies=cookie,
data=data,
json=json,
files=file,
params=param,
headers=header, # {**header, **__header},
verify=verify,
timeout=__timeout,
stream=False)
out.debug("res.headers", obj=dict(res.headers))
out.debug("res.text", obj=res.text)
res.raise_for_status()
# except ConnectionError as e:
# # out.warn(f"HTTP connection error to \"{uri}\": {e}")
# raise ConnectionError(e)
# except HTTPError as e:
# # out.warn(f"HTTP response error to \"{uri}\": {e}")
# raise HTTPError(e)
# except Timeout as e:
# # out.warn(f"HTTP request time out to \"{uri}\": {e}")
# raise Timeout(e)
# except TooManyRedirects as e:
# # out.warn(f"too many HTTP redirects to \"{uri}\": {e}")
# raise TooManyRedirects(e)
# else:
out.debug(f"successfull HTTP response from \"{uri}\": "
f"{res.status_code} \"{res.reason}\"")
if res.status_code == __http_nocontent or res.headers.get(
"Content-Length") == 0:
raise NoContentException
filename = regex.http_filename(res.headers.get("Content-Disposition"))
if bin:
return res.content, filename
return res.text, filename
def run(arg):
def loadserv():
from malsub.service import serv as _serv
global anserv
out.debug("_serv", obj=_serv)
if arg["--analysis"].lower() != "all":
from re import findall
anserv = list(findall(r"\w+", arg["--analysis"].lower()))
inv = [s for s in anserv if s not in _serv]
if inv:
out.error(f"input service(s) \"{','.join(inv)}\" unknown")
anserv = serv.ServiceList([s for s in _serv if s in anserv])
else:
anserv = serv.ServiceList([s for s in _serv])
out.debug("anserv", obj=anserv)
def loadkey():
from yaml import safe_load as loadyaml
from yaml.scanner import ScannerError
global anserv
try:
apikey = {
n.lower(): k
for n, k in loadyaml(open(meta.APIKEY_PATH)).items()
if n.lower() in anserv
}
except IOError as e:
out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}")
except ScannerError as e:
out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}")
else:
for n, k in apikey.items():
if type(k) is not dict or k.get("apikey") is None:
out.error(f"service \"{anserv[n].name}\" missing "
f"a valid API key \"{k}\"")
else:
anserv[n].set_apikey(k)
out.debug("apikey", obj=apikey)
def inarg():
global ina, pause
try:
pause = int(arg["--pause"])
if int(pause) < 0:
pause = 0
except ValueError:
out.warn(f"invalid pause interval integer value \"{pause}\", "
f"defaulting to zero")
if arg["--test"]:
ina = base.Service.get_apifn()
elif arg["--find"]:
ina = arg["<input>"]
elif arg["--quota"]:
ina = __NOINPUT
else:
# domain
if arg["--domain"]:
ina = web.parse_domainl(*arg["<input>"])
if not ina:
out.error("all input domain names are invalid")
# ipaddr
elif arg["--ipaddr"]:
ina = web.parse_ipaddrl(*arg["<input>"])
if not ina:
out.error("all input IP addresses are invalid")
# url
elif arg["--url"]:
ina = web.parse_fullurll(*arg["<input>"])
if not ina:
out.error("all input URLs are invalid")
# file, hash
else:
if arg["--submit"]:
ina = file.new(*arg["<input>"])
elif arg["--report"]:
# ina = arg["<input>"]
ina = crypto.parse_hashl(*arg["<input>"])
else: # arg["--download"]
if arg["<input>"]:
ina = crypto.parse_hashl(*arg["<input>"])
else:
ina = __NOINPUT
out.debug("ina", obj=ina)
if ina != __NOINPUT and not arg["--test"] and len(ina) != len(
arg["<input>"]):
rw.yn("one or more input arguments are invalid")
if arg["--verbose"] == 1:
out.LEVEL = out.log.verb
if arg["--verbose"] > 1:
out.LEVEL = out.log.debug
out.debug("arg", obj=arg)
loadserv()
loadkey()
ascii.banner()
inarg()
from time import sleep, time
from malsub.core.work import exec
summ = []
for i, j in enumerate(ina):
start = time()
fn, kwargs = "", {}
if arg["--test"]:
fn = base.TEST_API
kwargs = {"apifn": j}
elif arg["--find"]:
fn = base.SEARCH
kwargs = {"srch": j}
elif arg["--quota"]:
fn = base.QUOTA
else:
# app
if arg["--appl"]:
kwargs = {"hash": j}
fn = base.REPORT_APP
# domain
elif arg["--domain"]:
kwargs = {"dom": j}
if arg["--report"]:
fn = base.REPORT_DOM
# ipaddr
elif arg["--ipaddr"]:
kwargs = {"ip": j}
if arg["--report"]:
fn = base.REPORT_IP
# url
elif arg["--url"]:
kwargs = {"url": j}
if arg["--submit"]:
fn = base.SUBMIT_URL
elif arg["--report"]:
fn = base.REPORT_URL
# file, hash
else:
if arg["--submit"]:
fn = base.SUBMIT_FILE
kwargs = {"file": j}
elif arg["--report"]:
fn = base.REPORT_FILE
kwargs = {"hash": j}
elif arg["--download"]:
fn = base.DOWNLOAD_FILE
kwargs = {"hash": j}
if fn: # and kwargs:
summ += [[i + 1, util.trunc(j), *exec(anserv, fn, kwargs)]]
if i < len(ina) - 1 and time() - start < pause:
nap = pause - (time() - start)
out.verb(f"sleeping for {nap} seconds")
sleep(nap)
file.close(ina)
from malsub.common import frmt
header = ["#", "input"] + [s.obj.name for s in anserv]
out.verb(f"{meta.MALSUB_NAME} finished with results:\n"
f"{frmt.tablevert(header, summ)}")
return
from sys import modules
from malsub.common import color, out, util
from malsub.core import meta
try:
from pyfiglet import figlet_format, Figlet
except:
out.debug("\"pyfiglet\" import error, using ASCII art defaults")
_hline = f" {meta.MALSUB_NAME} {meta.MALSUB_VERSION}\n"
_default = [
f"\n{_hline}\n {util.asciibin(meta.MALSUB_NAME)}\n", f"""
_ _
| | | |
_ __ ___ __ _| |___ _ _| |__ +--+
| '_ ` _ \ / _` | / __| | | | '_ \\ +--+|
| | | | | | (_| | \__ \ |_| | |_) | | |+
|_| |_| |_|\__,_|_|___/\__,_|_.__/ +--+
""" + "\n" + _hline
]
"""
+------+
/| /|
m------a |
| u- ~ |-b
|/ |/
s------l
+------+
def inarg():
if (arg["--ipaddr"] or arg["--domain"] or arg["--appl"] or arg["--url"]) \
and not (arg["--download"] or arg["--find"] or arg["--find"]
or arg["--report"] or arg["--submit"]):
print(usage)
exit(0)
global iarg, pause
try:
pause = int(arg["--pause"])
if int(pause) < 0:
raise ValueError
except ValueError:
out.warn(f"invalid pause interval integer value \"{pause}\", "
f"defaulting to zero")
pause = 0
if arg["--test"]:
iarg = base.Service.get_apifn()
elif arg["--find"]:
iarg = arg["<input>"]
elif arg["--quota"]:
iarg = __NOINPUT
else:
# domain
if arg["--domain"]:
iarg = web.parse_domainl(*arg["<input>"])
if not iarg:
out.error("all input domain names are invalid")
# ipaddr
elif arg["--ipaddr"]:
iarg = web.parse_ipaddrl(*arg["<input>"])
if not iarg:
out.error("all input IP addresses are invalid")
# url
elif arg["--url"]:
iarg = web.parse_fullurll(*arg["<input>"])
if not iarg:
out.error("all input URLs are invalid")
# file, hash
else:
from glob import glob
from os.path import isdir, isfile, normpath
if arg["--recursive"]:
tmp = [f for f in arg["<input>"] if isfile(f)]
for f in [normpath(f) for f in arg["<input>"] if isdir(f)]:
tmp += [f for f in
glob(f + meta.SEP + "**", recursive=True) if
isfile(f)]
tmp += [h for h in arg["<input>"] if
not isfile(h) and not isdir(h)]
arg["<input>"] = tmp
if arg["--submit"]:
iarg = file.new(*arg["<input>"])
elif arg["--report"]: # file, hash
iarg = [f for f in arg["<input>"] if not isfile(f)]
tmp = [f for f in arg["<input>"] if isfile(f)]
if rw.validff(tmp): # file
iarg += [crypto.sha256(f) for f in tmp]
iarg = crypto.parse_hashl(*iarg) # hash
else: # arg["--download"]
if arg["<input>"]:
iarg = crypto.parse_hashl(*arg["<input>"])
else:
iarg = __NOINPUT
out.debug("iarg", obj=iarg)
if iarg != __NOINPUT and \
not arg["--test"] and len(iarg) != len(arg["<input>"]):
rw.yn("one or more input arguments are invalid")
def run(arg, usage):
def inarg():
if (arg["--ipaddr"] or arg["--domain"] or arg["--appl"] or arg["--url"]) \
and not (arg["--download"] or arg["--find"] or arg["--find"]
or arg["--report"] or arg["--submit"]):
print(usage)
exit(0)
global iarg, pause
try:
pause = int(arg["--pause"])
if int(pause) < 0:
raise ValueError
except ValueError:
out.warn(f"invalid pause interval integer value \"{pause}\", "
f"defaulting to zero")
pause = 0
if arg["--test"]:
iarg = base.Service.get_apifn()
elif arg["--find"]:
iarg = arg["<input>"]
elif arg["--quota"]:
iarg = __NOINPUT
else:
# domain
if arg["--domain"]:
iarg = web.parse_domainl(*arg["<input>"])
if not iarg:
out.error("all input domain names are invalid")
# ipaddr
elif arg["--ipaddr"]:
iarg = web.parse_ipaddrl(*arg["<input>"])
if not iarg:
out.error("all input IP addresses are invalid")
# url
elif arg["--url"]:
iarg = web.parse_fullurll(*arg["<input>"])
if not iarg:
out.error("all input URLs are invalid")
# file, hash
else:
from glob import glob
from os.path import isdir, isfile, normpath
if arg["--recursive"]:
tmp = [f for f in arg["<input>"] if isfile(f)]
for f in [normpath(f) for f in arg["<input>"] if isdir(f)]:
tmp += [f for f in
glob(f + meta.SEP + "**", recursive=True) if
isfile(f)]
tmp += [h for h in arg["<input>"] if
not isfile(h) and not isdir(h)]
arg["<input>"] = tmp
if arg["--submit"]:
iarg = file.new(*arg["<input>"])
elif arg["--report"]: # file, hash
iarg = [f for f in arg["<input>"] if not isfile(f)]
tmp = [f for f in arg["<input>"] if isfile(f)]
if rw.validff(tmp): # file
iarg += [crypto.sha256(f) for f in tmp]
iarg = crypto.parse_hashl(*iarg) # hash
else: # arg["--download"]
if arg["<input>"]:
iarg = crypto.parse_hashl(*arg["<input>"])
else:
iarg = __NOINPUT
out.debug("iarg", obj=iarg)
if iarg != __NOINPUT and \
not arg["--test"] and len(iarg) != len(arg["<input>"]):
rw.yn("one or more input arguments are invalid")
def loadserv():
from malsub.service import serv as _serv
global anserv
out.debug("_serv", obj=_serv)
if arg["--analysis"].lower() != "all":
from re import findall
anserv = [s for s in
list(findall(r"[\w-]+", arg["--analysis"].lower()))]
notanserv = [s[1:] for s in anserv if s.startswith("-")]
if "all" in anserv:
anserv = [s for s in _serv if s not in notanserv]
else:
anserv = [s for s in anserv if s[1:] not in notanserv]
inv = [s for s in anserv if s not in _serv]
if inv:
out.error(f"input service(s) \"{','.join(inv)}\" unknown")
anserv = serv.ServiceList([s for s in _serv if s in anserv])
else:
anserv = serv.ServiceList([s for s in _serv])
out.debug("anserv", obj=anserv)
def loadkey():
from yaml import safe_load as loadyaml
from yaml.scanner import ScannerError
global anserv
try:
apikey = {n.lower(): k for n, k in
loadyaml(open(meta.APIKEY_PATH)).items() if
n.lower() in anserv}
except IOError as e:
out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}")
except ScannerError as e:
out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}")
else:
for n, k in apikey.items():
if type(k) is not dict or \
k.get("apikey") is None or k.get("apikey") == "":
out.error(f"service \"{anserv[n].name}\" missing "
f"a valid API key \"{k}\"")
else:
anserv[n].set_apikey(k)
out.debug("apikey", obj=apikey)
if arg["--verbose"] == 1:
out.LEVEL = out.log.verb
elif arg["--verbose"] > 1:
out.LEVEL = out.log.debug
out.debug("arg", obj=arg)
inarg()
loadserv()
loadkey()
ascii.banner()
from malsub.common import frmt
if arg["--servhelp"]:
header = ["service", "description", "subscription", "url", "api"]
summ = []
for s in anserv:
summ += [[s.obj.name] + s.obj.help()]
pass
out.info("\n" + frmt.tablevert(header, summ))
else:
from time import sleep, time
from malsub.core.work import exec
summ = []
for i, j in enumerate(iarg):
start = time()
fn, kwarg = "", {}
if arg["--test"]:
fn = base.TEST_API
kwarg = {"apifn": j}
elif arg["--find"]:
fn = base.SEARCH
kwarg = {"srch": j}
elif arg["--quota"]:
fn = base.QUOTA
else:
# app
if arg["--appl"]:
kwarg = {"hash": j}
fn = base.REPORT_APP
# domain
elif arg["--domain"]:
kwarg = {"dom": j}
if arg["--report"]:
fn = base.REPORT_DOM
# ipaddr
elif arg["--ipaddr"]:
kwarg = {"ip": j}
if arg["--report"]:
fn = base.REPORT_IP
# url
elif arg["--url"]:
kwarg = {"url": j}
if arg["--submit"]:
fn = base.SUBMIT_URL
elif arg["--report"]:
fn = base.REPORT_URL
# file, hash
else:
if arg["--submit"]:
fn = base.SUBMIT_FILE
kwarg = {"file": j}
elif arg["--report"]:
fn = base.REPORT_FILE
kwarg = {"hash": j}
elif arg["--download"]:
fn = base.DOWNLOAD_FILE
kwarg = {"hash": j}
if fn: # and kwarg:
summ += [[i + 1, util.trunc(j), *exec(anserv, fn, kwarg)]]
if i < len(iarg) - 1 and time() - start < pause:
nap = pause - (time() - start)
out.verb(f"sleeping for {nap} seconds")
sleep(nap)
file.close(iarg)
header = ["#", "input"] + [s.obj.name for s in anserv]
out.verb(f"{meta.MALSUB_NAME} finished with results:\n"
f"{frmt.tablevert(header, summ)}")
return
from os.path import basename, dirname
from sys import modules
from pkgutil import iter_modules
from importlib import import_module
from inspect import getfile
from malsub.common import out
from malsub.core import serv as service
from malsub.service import base
for module_loader, name, ispkg in iter_modules([dirname(__file__)]):
import_module("." + name, __package__)
serv = service.ServiceList()
out.debug("base.Service.__subclasses__", obj=base.Service.__subclasses__())
for cls in base.Service.__subclasses__():
if cls.__module__ in modules:
if not hasattr(cls, base.UNUSED):
s = f"class \"{cls.__name__}\" {cls} in " \
f"\"{basename(getfile(cls))}\""
try:
cls()
except Exception as e:
out.error(s + f" does not implement all abstract methods: {e}")
else:
out.debug(s + " valid")
serv += service.Service(cls)
# serv[cls.__name__.lower()] = (cls.__name__.lower(), cls)
# serv[cls.sname.lower()] = (cls.__name__.lower(), cls)
if not serv:
