def exec(anserv, fn, kwarg):
anserv.setfn(fn)
fn = [s for s in anserv if not s.fnhasattr(base.UNSUPPORTED)]
# fn = {n: getattr(s(), fn) for n, s in anserv.items() if
# not hasattr(getattr(s, fn), base.UNSUPPORTED)}
# kwarg = {n: {"apikey": s.get_apikey()} for n, s in anserv.items()}
summ = {s.name: color.grayb("unsupported") for s in anserv}
if fn:
with ThreadPoolExecutor(__maxt) as tp:
# fut = {tp.submit(f, **kwarg[n], **kwargs): n for n, f in fn.items()}
fut = {tp.submit(s.fn, **kwarg): s for s in fn}
# timeout=60 (sec) # does not work like this
for f in as_completed(fut, timeout=None):
try:
data = f.result()
except base.UnsupportedAttr:
pass
except Exception as e:
summ[fut[f].name] = color.redb("unsuccessful")
out.warn(f"\"{fut[f].name}\" -- \"{fut[f].fnname}\" "
f"error: {e}\n{color.red(format_exc())}")
else:
summ[fut[f].name] = color.greenb("successful")
out.info(f"\"{fut[f].name}\" -- \"{fut[f].fnname}\" "
f"completed:\n{data}")
return [summ[s.name] for s in anserv]
def run(arg, usage):
def inarg():
if (arg["--ipaddr"] or arg["--domain"] or arg["--appl"] or arg["--url"]) \
and not (arg["--download"] or arg["--find"] or arg["--find"]
or arg["--report"] or arg["--submit"]):
print(usage)
exit(0)
global iarg, pause
try:
pause = int(arg["--pause"])
if int(pause) < 0:
raise ValueError
except ValueError:
out.warn(f"invalid pause interval integer value \"{pause}\", "
f"defaulting to zero")
pause = 0
if arg["--test"]:
iarg = base.Service.get_apifn()
elif arg["--find"]:
iarg = arg["<input>"]
elif arg["--quota"]:
iarg = __NOINPUT
else:
# domain
if arg["--domain"]:
iarg = web.parse_domainl(*arg["<input>"])
if not iarg:
out.error("all input domain names are invalid")
# ipaddr
elif arg["--ipaddr"]:
iarg = web.parse_ipaddrl(*arg["<input>"])
if not iarg:
out.error("all input IP addresses are invalid")
# url
elif arg["--url"]:
iarg = web.parse_fullurll(*arg["<input>"])
if not iarg:
out.error("all input URLs are invalid")
# file, hash
else:
from glob import glob
from os.path import isdir, isfile, normpath
if arg["--recursive"]:
tmp = [f for f in arg["<input>"] if isfile(f)]
for f in [normpath(f) for f in arg["<input>"] if isdir(f)]:
tmp += [f for f in
glob(f + meta.SEP + "**", recursive=True) if
isfile(f)]
tmp += [h for h in arg["<input>"] if
not isfile(h) and not isdir(h)]
arg["<input>"] = tmp
if arg["--submit"]:
iarg = file.new(*arg["<input>"])
elif arg["--report"]: # file, hash
iarg = [f for f in arg["<input>"] if not isfile(f)]
tmp = [f for f in arg["<input>"] if isfile(f)]
if rw.validff(tmp): # file
iarg += [crypto.sha256(f) for f in tmp]
iarg = crypto.parse_hashl(*iarg) # hash
else: # arg["--download"]
if arg["<input>"]:
iarg = crypto.parse_hashl(*arg["<input>"])
else:
iarg = __NOINPUT
out.debug("iarg", obj=iarg)
if iarg != __NOINPUT and \
not arg["--test"] and len(iarg) != len(arg["<input>"]):
rw.yn("one or more input arguments are invalid")
def loadserv():
from malsub.service import serv as _serv
global anserv
out.debug("_serv", obj=_serv)
if arg["--analysis"].lower() != "all":
from re import findall
anserv = [s for s in
list(findall(r"[\w-]+", arg["--analysis"].lower()))]
notanserv = [s[1:] for s in anserv if s.startswith("-")]
if "all" in anserv:
anserv = [s for s in _serv if s not in notanserv]
else:
anserv = [s for s in anserv if s[1:] not in notanserv]
inv = [s for s in anserv if s not in _serv]
if inv:
out.error(f"input service(s) \"{','.join(inv)}\" unknown")
anserv = serv.ServiceList([s for s in _serv if s in anserv])
else:
anserv = serv.ServiceList([s for s in _serv])
out.debug("anserv", obj=anserv)
def loadkey():
from yaml import safe_load as loadyaml
from yaml.scanner import ScannerError
global anserv
try:
apikey = {n.lower(): k for n, k in
loadyaml(open(meta.APIKEY_PATH)).items() if
n.lower() in anserv}
except IOError as e:
out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}")
except ScannerError as e:
out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}")
else:
for n, k in apikey.items():
if type(k) is not dict or \
k.get("apikey") is None or k.get("apikey") == "":
out.error(f"service \"{anserv[n].name}\" missing "
f"a valid API key \"{k}\"")
else:
anserv[n].set_apikey(k)
out.debug("apikey", obj=apikey)
if arg["--verbose"] == 1:
out.LEVEL = out.log.verb
elif arg["--verbose"] > 1:
out.LEVEL = out.log.debug
out.debug("arg", obj=arg)
inarg()
loadserv()
loadkey()
ascii.banner()
from malsub.common import frmt
if arg["--servhelp"]:
header = ["service", "description", "subscription", "url", "api"]
summ = []
for s in anserv:
summ += [[s.obj.name] + s.obj.help()]
pass
out.info("\n" + frmt.tablevert(header, summ))
else:
from time import sleep, time
from malsub.core.work import exec
summ = []
for i, j in enumerate(iarg):
start = time()
fn, kwarg = "", {}
if arg["--test"]:
fn = base.TEST_API
kwarg = {"apifn": j}
elif arg["--find"]:
fn = base.SEARCH
kwarg = {"srch": j}
elif arg["--quota"]:
fn = base.QUOTA
else:
# app
if arg["--appl"]:
kwarg = {"hash": j}
fn = base.REPORT_APP
# domain
elif arg["--domain"]:
kwarg = {"dom": j}
if arg["--report"]:
fn = base.REPORT_DOM
# ipaddr
elif arg["--ipaddr"]:
kwarg = {"ip": j}
if arg["--report"]:
fn = base.REPORT_IP
# url
elif arg["--url"]:
kwarg = {"url": j}
if arg["--submit"]:
fn = base.SUBMIT_URL
elif arg["--report"]:
fn = base.REPORT_URL
# file, hash
else:
if arg["--submit"]:
fn = base.SUBMIT_FILE
kwarg = {"file": j}
elif arg["--report"]:
fn = base.REPORT_FILE
kwarg = {"hash": j}
elif arg["--download"]:
fn = base.DOWNLOAD_FILE
kwarg = {"hash": j}
if fn: # and kwarg:
summ += [[i + 1, util.trunc(j), *exec(anserv, fn, kwarg)]]
if i < len(iarg) - 1 and time() - start < pause:
nap = pause - (time() - start)
out.verb(f"sleeping for {nap} seconds")
sleep(nap)
file.close(iarg)
header = ["#", "input"] + [s.obj.name for s in anserv]
out.verb(f"{meta.MALSUB_NAME} finished with results:\n"
f"{frmt.tablevert(header, summ)}")
return