def test(apispec): if apispec.method == POST: fn = requests_post else: fn = requests_get # try: if apispec.header and type(apispec.header) is dict: apispec.header.update(__header) else: apispec.header = __header res = fn( apispec.fulluri, auth=apispec.auth, cookies=apispec.cookie, data=apispec.data, headers=apispec.header, # {**apispec.header, **__header}, json=apispec.json, params=apispec.param, verify=apispec.verify, timeout=__timeout, stream=False) res.raise_for_status() # except Exception as e: # out.warn(f"unsuccessful test HTTP {apisec.method} to " # f"\"{apisec.fulluri}\": {e}") # else: out.debug( f"successful test HTTP {apispec.method} to \"{apispec.fulluri}\"") res.close() del res return
def tryf(file): try: with open(file, mode='rb') as fd: out.debug(f"file \"{file}\" valid") except: out.error(f"cannot open file \"{file}\"") return True
def validff(file): for f in file: try: with open(f, mode='rb') as fd: out.debug(f"file \"{f}\" valid") except: return False return True
def openf(file, mode='rb', debug=True): try: fd = open(file, mode=mode) if debug: out.debug(f"file \"{file}\" valid") return fd except: out.error(f"cannot open file \"{file}\"")
def _jsonload(json): if type(json) is dict: return json elif type(json) is str: try: tree = loads(json) except Exception as e: out.error( f"type \"{type(json)}\" invalid for parsing through JSON:" f" {e}") else: return tree else: out.debug(f"type \"{type(json)}\" invalid for parsing through JSON")
def inarg(): global ina, pause try: pause = int(arg["--pause"]) if int(pause) < 0: pause = 0 except ValueError: out.warn(f"invalid pause interval integer value \"{pause}\", " f"defaulting to zero") if arg["--test"]: ina = base.Service.get_apifn() elif arg["--find"]: ina = arg["<input>"] elif arg["--quota"]: ina = __NOINPUT else: # domain if arg["--domain"]: ina = web.parse_domainl(*arg["<input>"]) if not ina: out.error("all input domain names are invalid") # ipaddr elif arg["--ipaddr"]: ina = web.parse_ipaddrl(*arg["<input>"]) if not ina: out.error("all input IP addresses are invalid") # url elif arg["--url"]: ina = web.parse_fullurll(*arg["<input>"]) if not ina: out.error("all input URLs are invalid") # file, hash else: if arg["--submit"]: ina = file.new(*arg["<input>"]) elif arg["--report"]: # ina = arg["<input>"] ina = crypto.parse_hashl(*arg["<input>"]) else: # arg["--download"] if arg["<input>"]: ina = crypto.parse_hashl(*arg["<input>"]) else: ina = __NOINPUT out.debug("ina", obj=ina) if ina != __NOINPUT and not arg["--test"] and len(ina) != len( arg["<input>"]): rw.yn("one or more input arguments are invalid")
def loadserv(): from malsub.service import serv as _serv global anserv out.debug("_serv", obj=_serv) if arg["--analysis"].lower() != "all": from re import findall anserv = list(findall(r"\w+", arg["--analysis"].lower())) inv = [s for s in anserv if s not in _serv] if inv: out.error(f"input service(s) \"{','.join(inv)}\" unknown") anserv = serv.ServiceList([s for s in _serv if s in anserv]) else: anserv = serv.ServiceList([s for s in _serv]) out.debug("anserv", obj=anserv)
def _request(fn, uri, auth=None, cookie=None, data=None, file=None, header=None, json_req=None, param=None, verify=True, bin=False, json=False): # Quicksand.io has a broken cert if 'quicksand' in uri: verify = False elif header and type(header) is dict: header.update(__header) else: header = __header # try: res = fn( uri, auth=auth, # HTTPBasicAuth cookies=cookie, data=data, files=file, headers=header, # {**header, **__header}, json=json_req, params=param, verify=verify, timeout=__timeout, stream=False) out.debug("res.headers", obj=dict(res.headers)) out.debug("res.text", obj=res.text) res.raise_for_status() # except ConnectionError as e: # # out.warn(f"HTTP connection error to \"{uri}\": {e}") # raise ConnectionError(e) # except HTTPError as e: # # out.warn(f"HTTP response error to \"{uri}\": {e}") # raise HTTPError(e) # except Timeout as e: # # out.warn(f"HTTP request time out to \"{uri}\": {e}") # raise Timeout(e) # except TooManyRedirects as e: # # out.warn(f"too many HTTP redirects to \"{uri}\": {e}") # raise TooManyRedirects(e) # else: out.debug(f"HTTP {res.status_code} \"{res.reason}\" -- \"{uri}\"") if res.status_code == 204 or res.headers.get("Content-Length") == 0: raise Exception(f"HTTP {res.status_code} \"{res.reason}\"" f" -- \"{uri}\": no content") filename = regex.http_filename(res.headers.get("Content-Disposition")) if bin: return res.content, filename if json: return res.json(), filename return res.text, filename
def loadkey(): from yaml import safe_load as loadyaml from yaml.scanner import ScannerError global anserv try: apikey = {n.lower(): k for n, k in loadyaml(open(meta.APIKEY_PATH)).items() if n.lower() in anserv} except IOError as e: out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}") except ScannerError as e: out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}") else: for n, k in apikey.items(): if type(k) is not dict or \ k.get("apikey") is None or k.get("apikey") == "": out.error(f"service \"{anserv[n].name}\" missing " f"a valid API key \"{k}\"") else: anserv[n].set_apikey(k) out.debug("apikey", obj=apikey)
def _request(fn, uri, header=None, cookie=None, data=None, json=None, file=None, param=None, verify=True, bin=False): if header and type(header) is dict: header.update(__header) else: header = __header # try: res = fn( uri, cookies=cookie, data=data, json=json, files=file, params=param, headers=header, # {**header, **__header}, verify=verify, timeout=__timeout, stream=False) out.debug("res.headers", obj=dict(res.headers)) out.debug("res.text", obj=res.text) res.raise_for_status() # except ConnectionError as e: # # out.warn(f"HTTP connection error to \"{uri}\": {e}") # raise ConnectionError(e) # except HTTPError as e: # # out.warn(f"HTTP response error to \"{uri}\": {e}") # raise HTTPError(e) # except Timeout as e: # # out.warn(f"HTTP request time out to \"{uri}\": {e}") # raise Timeout(e) # except TooManyRedirects as e: # # out.warn(f"too many HTTP redirects to \"{uri}\": {e}") # raise TooManyRedirects(e) # else: out.debug(f"successfull HTTP response from \"{uri}\": " f"{res.status_code} \"{res.reason}\"") if res.status_code == __http_nocontent or res.headers.get( "Content-Length") == 0: raise NoContentException filename = regex.http_filename(res.headers.get("Content-Disposition")) if bin: return res.content, filename return res.text, filename
def run(arg): def loadserv(): from malsub.service import serv as _serv global anserv out.debug("_serv", obj=_serv) if arg["--analysis"].lower() != "all": from re import findall anserv = list(findall(r"\w+", arg["--analysis"].lower())) inv = [s for s in anserv if s not in _serv] if inv: out.error(f"input service(s) \"{','.join(inv)}\" unknown") anserv = serv.ServiceList([s for s in _serv if s in anserv]) else: anserv = serv.ServiceList([s for s in _serv]) out.debug("anserv", obj=anserv) def loadkey(): from yaml import safe_load as loadyaml from yaml.scanner import ScannerError global anserv try: apikey = { n.lower(): k for n, k in loadyaml(open(meta.APIKEY_PATH)).items() if n.lower() in anserv } except IOError as e: out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}") except ScannerError as e: out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}") else: for n, k in apikey.items(): if type(k) is not dict or k.get("apikey") is None: out.error(f"service \"{anserv[n].name}\" missing " f"a valid API key \"{k}\"") else: anserv[n].set_apikey(k) out.debug("apikey", obj=apikey) def inarg(): global ina, pause try: pause = int(arg["--pause"]) if int(pause) < 0: pause = 0 except ValueError: out.warn(f"invalid pause interval integer value \"{pause}\", " f"defaulting to zero") if arg["--test"]: ina = base.Service.get_apifn() elif arg["--find"]: ina = arg["<input>"] elif arg["--quota"]: ina = __NOINPUT else: # domain if arg["--domain"]: ina = web.parse_domainl(*arg["<input>"]) if not ina: out.error("all input domain names are invalid") # ipaddr elif arg["--ipaddr"]: ina = web.parse_ipaddrl(*arg["<input>"]) if not ina: out.error("all input IP addresses are invalid") # url elif arg["--url"]: ina = web.parse_fullurll(*arg["<input>"]) if not ina: out.error("all input URLs are invalid") # file, hash else: if arg["--submit"]: ina = file.new(*arg["<input>"]) elif arg["--report"]: # ina = arg["<input>"] ina = crypto.parse_hashl(*arg["<input>"]) else: # arg["--download"] if arg["<input>"]: ina = crypto.parse_hashl(*arg["<input>"]) else: ina = __NOINPUT out.debug("ina", obj=ina) if ina != __NOINPUT and not arg["--test"] and len(ina) != len( arg["<input>"]): rw.yn("one or more input arguments are invalid") if arg["--verbose"] == 1: out.LEVEL = out.log.verb if arg["--verbose"] > 1: out.LEVEL = out.log.debug out.debug("arg", obj=arg) loadserv() loadkey() ascii.banner() inarg() from time import sleep, time from malsub.core.work import exec summ = [] for i, j in enumerate(ina): start = time() fn, kwargs = "", {} if arg["--test"]: fn = base.TEST_API kwargs = {"apifn": j} elif arg["--find"]: fn = base.SEARCH kwargs = {"srch": j} elif arg["--quota"]: fn = base.QUOTA else: # app if arg["--appl"]: kwargs = {"hash": j} fn = base.REPORT_APP # domain elif arg["--domain"]: kwargs = {"dom": j} if arg["--report"]: fn = base.REPORT_DOM # ipaddr elif arg["--ipaddr"]: kwargs = {"ip": j} if arg["--report"]: fn = base.REPORT_IP # url elif arg["--url"]: kwargs = {"url": j} if arg["--submit"]: fn = base.SUBMIT_URL elif arg["--report"]: fn = base.REPORT_URL # file, hash else: if arg["--submit"]: fn = base.SUBMIT_FILE kwargs = {"file": j} elif arg["--report"]: fn = base.REPORT_FILE kwargs = {"hash": j} elif arg["--download"]: fn = base.DOWNLOAD_FILE kwargs = {"hash": j} if fn: # and kwargs: summ += [[i + 1, util.trunc(j), *exec(anserv, fn, kwargs)]] if i < len(ina) - 1 and time() - start < pause: nap = pause - (time() - start) out.verb(f"sleeping for {nap} seconds") sleep(nap) file.close(ina) from malsub.common import frmt header = ["#", "input"] + [s.obj.name for s in anserv] out.verb(f"{meta.MALSUB_NAME} finished with results:\n" f"{frmt.tablevert(header, summ)}") return
from sys import modules from malsub.common import color, out, util from malsub.core import meta try: from pyfiglet import figlet_format, Figlet except: out.debug("\"pyfiglet\" import error, using ASCII art defaults") _hline = f" {meta.MALSUB_NAME} {meta.MALSUB_VERSION}\n" _default = [ f"\n{_hline}\n {util.asciibin(meta.MALSUB_NAME)}\n", f""" _ _ | | | | _ __ ___ __ _| |___ _ _| |__ +--+ | '_ ` _ \ / _` | / __| | | | '_ \\ +--+| | | | | | | (_| | \__ \ |_| | |_) | | |+ |_| |_| |_|\__,_|_|___/\__,_|_.__/ +--+ """ + "\n" + _hline ] """ +------+ /| /| m------a | | u- ~ |-b |/ |/ s------l +------+
def inarg(): if (arg["--ipaddr"] or arg["--domain"] or arg["--appl"] or arg["--url"]) \ and not (arg["--download"] or arg["--find"] or arg["--find"] or arg["--report"] or arg["--submit"]): print(usage) exit(0) global iarg, pause try: pause = int(arg["--pause"]) if int(pause) < 0: raise ValueError except ValueError: out.warn(f"invalid pause interval integer value \"{pause}\", " f"defaulting to zero") pause = 0 if arg["--test"]: iarg = base.Service.get_apifn() elif arg["--find"]: iarg = arg["<input>"] elif arg["--quota"]: iarg = __NOINPUT else: # domain if arg["--domain"]: iarg = web.parse_domainl(*arg["<input>"]) if not iarg: out.error("all input domain names are invalid") # ipaddr elif arg["--ipaddr"]: iarg = web.parse_ipaddrl(*arg["<input>"]) if not iarg: out.error("all input IP addresses are invalid") # url elif arg["--url"]: iarg = web.parse_fullurll(*arg["<input>"]) if not iarg: out.error("all input URLs are invalid") # file, hash else: from glob import glob from os.path import isdir, isfile, normpath if arg["--recursive"]: tmp = [f for f in arg["<input>"] if isfile(f)] for f in [normpath(f) for f in arg["<input>"] if isdir(f)]: tmp += [f for f in glob(f + meta.SEP + "**", recursive=True) if isfile(f)] tmp += [h for h in arg["<input>"] if not isfile(h) and not isdir(h)] arg["<input>"] = tmp if arg["--submit"]: iarg = file.new(*arg["<input>"]) elif arg["--report"]: # file, hash iarg = [f for f in arg["<input>"] if not isfile(f)] tmp = [f for f in arg["<input>"] if isfile(f)] if rw.validff(tmp): # file iarg += [crypto.sha256(f) for f in tmp] iarg = crypto.parse_hashl(*iarg) # hash else: # arg["--download"] if arg["<input>"]: iarg = crypto.parse_hashl(*arg["<input>"]) else: iarg = __NOINPUT out.debug("iarg", obj=iarg) if iarg != __NOINPUT and \ not arg["--test"] and len(iarg) != len(arg["<input>"]): rw.yn("one or more input arguments are invalid")
def run(arg, usage): def inarg(): if (arg["--ipaddr"] or arg["--domain"] or arg["--appl"] or arg["--url"]) \ and not (arg["--download"] or arg["--find"] or arg["--find"] or arg["--report"] or arg["--submit"]): print(usage) exit(0) global iarg, pause try: pause = int(arg["--pause"]) if int(pause) < 0: raise ValueError except ValueError: out.warn(f"invalid pause interval integer value \"{pause}\", " f"defaulting to zero") pause = 0 if arg["--test"]: iarg = base.Service.get_apifn() elif arg["--find"]: iarg = arg["<input>"] elif arg["--quota"]: iarg = __NOINPUT else: # domain if arg["--domain"]: iarg = web.parse_domainl(*arg["<input>"]) if not iarg: out.error("all input domain names are invalid") # ipaddr elif arg["--ipaddr"]: iarg = web.parse_ipaddrl(*arg["<input>"]) if not iarg: out.error("all input IP addresses are invalid") # url elif arg["--url"]: iarg = web.parse_fullurll(*arg["<input>"]) if not iarg: out.error("all input URLs are invalid") # file, hash else: from glob import glob from os.path import isdir, isfile, normpath if arg["--recursive"]: tmp = [f for f in arg["<input>"] if isfile(f)] for f in [normpath(f) for f in arg["<input>"] if isdir(f)]: tmp += [f for f in glob(f + meta.SEP + "**", recursive=True) if isfile(f)] tmp += [h for h in arg["<input>"] if not isfile(h) and not isdir(h)] arg["<input>"] = tmp if arg["--submit"]: iarg = file.new(*arg["<input>"]) elif arg["--report"]: # file, hash iarg = [f for f in arg["<input>"] if not isfile(f)] tmp = [f for f in arg["<input>"] if isfile(f)] if rw.validff(tmp): # file iarg += [crypto.sha256(f) for f in tmp] iarg = crypto.parse_hashl(*iarg) # hash else: # arg["--download"] if arg["<input>"]: iarg = crypto.parse_hashl(*arg["<input>"]) else: iarg = __NOINPUT out.debug("iarg", obj=iarg) if iarg != __NOINPUT and \ not arg["--test"] and len(iarg) != len(arg["<input>"]): rw.yn("one or more input arguments are invalid") def loadserv(): from malsub.service import serv as _serv global anserv out.debug("_serv", obj=_serv) if arg["--analysis"].lower() != "all": from re import findall anserv = [s for s in list(findall(r"[\w-]+", arg["--analysis"].lower()))] notanserv = [s[1:] for s in anserv if s.startswith("-")] if "all" in anserv: anserv = [s for s in _serv if s not in notanserv] else: anserv = [s for s in anserv if s[1:] not in notanserv] inv = [s for s in anserv if s not in _serv] if inv: out.error(f"input service(s) \"{','.join(inv)}\" unknown") anserv = serv.ServiceList([s for s in _serv if s in anserv]) else: anserv = serv.ServiceList([s for s in _serv]) out.debug("anserv", obj=anserv) def loadkey(): from yaml import safe_load as loadyaml from yaml.scanner import ScannerError global anserv try: apikey = {n.lower(): k for n, k in loadyaml(open(meta.APIKEY_PATH)).items() if n.lower() in anserv} except IOError as e: out.error(f"cannot open API keys file \"{meta.APIKEY_PATH}\": {e}") except ScannerError as e: out.error(f"cannot load API keys file \"{meta.APIKEY_PATH}\": {e}") else: for n, k in apikey.items(): if type(k) is not dict or \ k.get("apikey") is None or k.get("apikey") == "": out.error(f"service \"{anserv[n].name}\" missing " f"a valid API key \"{k}\"") else: anserv[n].set_apikey(k) out.debug("apikey", obj=apikey) if arg["--verbose"] == 1: out.LEVEL = out.log.verb elif arg["--verbose"] > 1: out.LEVEL = out.log.debug out.debug("arg", obj=arg) inarg() loadserv() loadkey() ascii.banner() from malsub.common import frmt if arg["--servhelp"]: header = ["service", "description", "subscription", "url", "api"] summ = [] for s in anserv: summ += [[s.obj.name] + s.obj.help()] pass out.info("\n" + frmt.tablevert(header, summ)) else: from time import sleep, time from malsub.core.work import exec summ = [] for i, j in enumerate(iarg): start = time() fn, kwarg = "", {} if arg["--test"]: fn = base.TEST_API kwarg = {"apifn": j} elif arg["--find"]: fn = base.SEARCH kwarg = {"srch": j} elif arg["--quota"]: fn = base.QUOTA else: # app if arg["--appl"]: kwarg = {"hash": j} fn = base.REPORT_APP # domain elif arg["--domain"]: kwarg = {"dom": j} if arg["--report"]: fn = base.REPORT_DOM # ipaddr elif arg["--ipaddr"]: kwarg = {"ip": j} if arg["--report"]: fn = base.REPORT_IP # url elif arg["--url"]: kwarg = {"url": j} if arg["--submit"]: fn = base.SUBMIT_URL elif arg["--report"]: fn = base.REPORT_URL # file, hash else: if arg["--submit"]: fn = base.SUBMIT_FILE kwarg = {"file": j} elif arg["--report"]: fn = base.REPORT_FILE kwarg = {"hash": j} elif arg["--download"]: fn = base.DOWNLOAD_FILE kwarg = {"hash": j} if fn: # and kwarg: summ += [[i + 1, util.trunc(j), *exec(anserv, fn, kwarg)]] if i < len(iarg) - 1 and time() - start < pause: nap = pause - (time() - start) out.verb(f"sleeping for {nap} seconds") sleep(nap) file.close(iarg) header = ["#", "input"] + [s.obj.name for s in anserv] out.verb(f"{meta.MALSUB_NAME} finished with results:\n" f"{frmt.tablevert(header, summ)}") return
from os.path import basename, dirname from sys import modules from pkgutil import iter_modules from importlib import import_module from inspect import getfile from malsub.common import out from malsub.core import serv as service from malsub.service import base for module_loader, name, ispkg in iter_modules([dirname(__file__)]): import_module("." + name, __package__) serv = service.ServiceList() out.debug("base.Service.__subclasses__", obj=base.Service.__subclasses__()) for cls in base.Service.__subclasses__(): if cls.__module__ in modules: if not hasattr(cls, base.UNUSED): s = f"class \"{cls.__name__}\" {cls} in " \ f"\"{basename(getfile(cls))}\"" try: cls() except Exception as e: out.error(s + f" does not implement all abstract methods: {e}") else: out.debug(s + " valid") serv += service.Service(cls) # serv[cls.__name__.lower()] = (cls.__name__.lower(), cls) # serv[cls.sname.lower()] = (cls.__name__.lower(), cls) if not serv: