def do_info(self, args):
"""List the details of a code item (tool or component).
code info NAME_OR_ID
"""
parser = self._argparser()
parser.add_argument("name_or_id")
parser.add_argument("--tools",
"-t",
default=False,
action="store_true")
parser.add_argument("--components",
"-c",
default=False,
action="store_true")
args = parser.parse_args(shlex.split(args))
if args.tools and args.components:
raise errors.TalusApiError("must specify only -t or -c")
search = {}
if args.tools:
search["type"] = "tool"
if args.components:
search["type"] = "component"
code = self._talus_client.code_find(args.name_or_id, **search)
if code is None:
raise errors.TalusApiError(
"could not find talus code named {!r}".format(args.name_or_id))
print("""
ID: {id}
Name: {name}
Bases: {bases}
Type: {type}
""".format(id=code.id,
name=code.name,
bases=" ".join(code.bases),
type=code.type.capitalize())[0:-3])
params_nice = []
for param in code.params:
if param["type"]["type"] == "native":
param_type = param["type"]["name"]
elif param["type"]["type"] == "component":
param_type = "Component({})".format(param["type"]["name"])
else:
param_type = param["type"]["name"]
params_nice.append([param_type, param["name"], param["desc"]])
params = tabulate(params_nice, headers=["type", "name", "desc"])
print("Params:\n\n" + "\n".join(" " + p
for p in params.split("\n")) + "\n")
def do_get(self, args):
"""Get the file(s) from the corpus with the provided id(s), saving to the
current directory if no destination path is provided (works like cp and mv)
Example:
To fetch a single file and save it into /tmp:
corpus get 55cdcbaedd18da0008caa793 /tmp
To fetch multiple files and save to /tmp:
corpus get 55cdcbaedd18da0008caa793 55cdcbaedd18da0008caa794 55cdcbaedd18da0008caa795 /tmp
"""
parts = shlex.split(args)
if len(parts) == 0:
raise errors.TalusApiError("At least one id must be provided")
if len(parts) > 1:
dest = parts[-1]
file_ids = parts[:-1]
else:
dest = None
file_ids = [parts[0]]
if dest is None:
dest = "./"
full_dest = os.path.abspath(os.path.expanduser(dest))
# it needs to be a directory for this to work
if len(file_ids) > 1 and (not os.path.exists(full_dest)
or not os.path.isdir(full_dest)):
raise errors.TalusApiError(
"Destination for multiple files must exist _and_ be a directory!"
)
for file_id in file_ids:
fname, data = self._talus_client.corpus_get(file_id)
if len(file_ids) > 1 or os.path.isdir(full_dest):
unexpanded_dest = os.path.join(dest, fname)
write_dest = os.path.join(full_dest, fname)
else:
unexpanded_dest = dest
write_dest = full_dest
with open(write_dest, "wb") as f:
f.write(data)
print("{} saved to {} ({} bytes)".format(file_id, unexpanded_dest,
len(data)))
def do_edit(self, args):
"""Edit an existing image. Interactive mode only
"""
if args.strip() == "":
raise errors.TalusApiError("you must provide a name/id of an image to edit it")
parts = shlex.split(args)
leftover = []
image_id_or_name = None
search = self._search_terms(parts, out_leftover=leftover)
if len(leftover) > 0:
image_id_or_name = leftover[0]
image = self._resolve_one_model(image_id_or_name, Image, search)
if image is None:
raise errors.TalusApiError("could not find talus image with id {!r}".format(image_id_or_name))
while True:
model_cmd = self._make_model_cmd(image)
cancelled = model_cmd.cmdloop()
if cancelled:
break
error = False
if image.os is None:
self.err("You must specify the os")
error = True
if image.name is None or image.name == "":
self.err("You must specify a name for the image")
error = True
if image.base_image is None:
self.err("You must specify the base_image for your new image")
error = True
if error:
continue
try:
image.timestamps = {"modified": time.time()}
image.save()
self.ok("edited image {}".format(image.id))
self.ok("note that this DOES NOT start the image for configuring!")
except errors.TalusApiError as e:
self.err(e.message)
return
def delete(self):
"""Delete this model
"""
res = utils.json_request(requests.delete, self._id_url())
if res.status_code // 100 != 2:
raise errors.TalusApiError("Could not delete model", error=res.text)
self._fields = {}
def _resolve_one_model(self,
id_or_name,
model,
search,
sort="-timestamps.created",
default_id_search=None):
if default_id_search is None:
default_id_search = ["id", "name"]
if id_or_name is not None and not id_or_name.startswith("+"):
for default_compare in default_id_search:
res = model.find_one(**{default_compare: id_or_name})
if res is not None:
return res
return None
if id_or_name is None:
skip = 0
else:
if not re.match(r'^\+\d+$', id_or_name):
raise errors.TalusApiError(
"Git-like referencing must be a plus sign followed by digits"
)
skip = int(id_or_name.replace("+", "")) - 1
search["skip"] = skip
search["num"] = 1
search["sort"] = sort
return model.find_one(**search)
def do_delete(self, args):
"""Delete the file specified by the provided id
"""
parts = shlex.split(args)
if len(parts) == 0:
raise errors.TalusApiError(
"You must provide the id of the file to delete")
file_id = parts[0]
res = self._talus_client.corpus_delete(file_id)
if "error" in res:
raise errors.TalusApiError(
"Could not delete file id {!r}: {}".format(
file_id,
res["error"],
))
print("deleted")
def refresh(self):
"""Refresh the current model
"""
if "id" not in self._fields:
return
matches = self.objects_raw(api_base=self.api_base, id=self.id)
if len(matches) == 0:
raise errors.TalusApiError("Error! current model no longer exists!")
update = matches[0]
self._populate(update)
def do_create(self, args):
"""Create new code in the repository. This will create the code in the talus
repository, as well as in the database.
code create NAME -t or -c
-t,--tool Create a tool
-c,--component Create a component
"""
args = shlex.split(args)
parser = self._argparser()
parser.add_argument("name")
parser.add_argument("--tool", "-t", default=False, action="store_true")
parser.add_argument("--component",
"-c",
default=False,
action="store_true")
parser.add_argument("--tag", dest="tags", action="append")
args = parser.parse_args(args)
if not args.tool and not args.component:
raise errors.TalusApiError(
"must specify either a tool or a component, non specified")
if args.tool and args.component:
raise errors.TalusApiError(
"must specify either a tool or a component, non both")
if args.tool:
code_type = "tool"
else:
code_type = "component"
res = self._talus_client.code_create(code_name=args.name,
code_type=code_type,
tags=args.tags)
if res["status"] == "error":
self.err(res["message"])
else:
self.ok(res["message"])
def do_info(self, args):
"""List details about a task
"""
if args.strip() == "":
raise errors.TalusApiError(
"you must provide a name/id of a task to show info about it")
parts = shlex.split(args)
leftover = []
task_id_or_name = None
search = self._search_terms(parts, out_leftover=leftover)
if len(leftover) > 0:
task_id_or_name = leftover[0]
task = self._resolve_one_model(task_id_or_name, Task, search)
if task is None:
raise errors.TalusApiError(
"could not find talus task with id {!r}".format(
task_id_or_name))
def json_request(method, *args, **params):
content_type = "application/json"
if "data" in params and hasattr(params["data"], "content_type"):
content_type = params["data"].content_type
params.setdefault("headers", {}).setdefault("content-type", content_type)
try:
res = method(*args, **params)
except requests.ConnectionError as e:
raise errors.TalusApiError("Could not connect to {}".format(args[0]))
except Exception as e:
return None
return res
def do_edit(self, args):
"""Edit an existing task in Talus. Interactive mode only
"""
if args.strip() == "":
raise errors.TalusApiError(
"you must provide a name/id of a task to edit it")
parts = shlex.split(args)
leftover = []
task_id_or_name = None
search = self._search_terms(parts, out_leftover=leftover)
if len(leftover) > 0:
task_id_or_name = leftover[0]
task = self._resolve_one_model(task_id_or_name, Task, search)
self._interactive_loop(task)
def save(self):
"""Save this model's fields
"""
files = None
data = json.dumps(self._filtered_fields())
if "id" in self._fields:
res = utils.json_request(
requests.put,
self._id_url(),
data=data
)
else:
res = utils.json_request(
requests.post,
self.api_url(self.api_base),
data=data
)
# yes, that's intentional (the //) - look it up
if res.status_code // 100 != 2:
raise errors.TalusApiError("Could not save model", error=res.text)
self._populate(res.json())
def do_create(self, args):
"""Create a new image in talus using an existing base image. Anything not explicitly
specified will be inherited from the base image, except for the name, which is required.
create -n NAME -b BASEID_NAME [-d DESC] [-t TAG1,TAG2,..] [-u USER] [-p PASS] [-o OSID] [-i]
-o,--os ID or name of the operating system model
-b,--base ID or name of the base image
-n,--name The name of the resulting image (default: basename(FILE))
-d,--desc A description of the image (default: "")
-t,--tags Tags associated with the image (default: [])
--shell Forcefully drop into an interactive shell
-v,--vagrantfile A vagrant file that will be used to congfigure the image
-i,--interactive To interact with the imported image for setup (default: False)
Examples:
To create a new image based on the image with id 222222222222222222222222 and adding
a new description and allowing for manual user setup:
image create -b 222222222222222222222222 -d "some new description" -i
"""
args = shlex.split(args)
if self._go_interactive(args):
image = Image()
self._prep_model(image)
image.username = "******"
image.password = "******"
image.md5 = " "
image.desc = "some description"
image.status = {
"name": "create",
"vagrantfile": None,
"user_interaction": True
}
while True:
model_cmd = self._make_model_cmd(image)
model_cmd.add_field(
"interactive",
Field(True),
lambda x,v: x.status.update({"user_interaction": v}),
lambda x: x.status["user_interaction"],
desc="If the image requires user interaction for configuration",
)
model_cmd.add_field(
"vagrantfile",
Field(str),
lambda x,v: x.status.update({"vagrantfile": open(v).read()}),
lambda x: x.status["vagrantfile"],
desc="The path to the vagrantfile that will configure the image"
)
cancelled = model_cmd.cmdloop()
if cancelled:
break
error = False
if image.os is None:
self.err("You must specify the os")
error = True
if image.name is None or image.name == "":
self.err("You must specify a name for the image")
error = True
if image.base_image is None:
self.err("You must specify the base_image for your new image")
error = True
if error:
continue
try:
image.timestamps = {"created": time.time()}
image.save()
self.ok("created new image {}".format(image.id))
except errors.TalusApiError as e:
self.err(e.message)
else:
self._wait_for_image(image, image.status["user_interaction"])
return
parser = self._argparser()
parser.add_argument("--os", "-o", default=None)
parser.add_argument("--base", "-b", default=None)
parser.add_argument("--name", "-n", default=None)
parser.add_argument("--desc", "-d", default="")
parser.add_argument("--tags", "-t", default="")
parser.add_argument("--vagrantfile", "-v", default=None, type=argparse.FileType("rb"))
parser.add_argument("--interactive", "-i", action="store_true", default=False)
args = parser.parse_args(args)
if args.name is None:
raise errors.TalusApiError("You must specify an image name")
vagrantfile_contents = None
if args.vagrantfile is not None:
vagrantfile_contents = args.vagrantfile.read()
if args.tags is not None:
args.tags = args.tags.split(",")
error = False
validation = {
"os" : "You must set the os",
"base" : "You must set the base",
"name" : "You must set the name",
}
error = False
for k,v in validation.iteritems():
if getattr(args, k) is None:
self.err(v)
error = True
if error:
parser.print_help()
return
image = self._talus_client.image_create(
image_name = args.name,
base_image_id_or_name = args.base,
os_id = args.os,
desc = args.desc,
tags = args.tags,
vagrantfile = vagrantfile_contents,
user_interaction = args.interactive
)
self._wait_for_image(image, args.interactive)
def do_export(self, args):
"""Export crash information to the target directory. Crashes are identified using
git-like syntax, ids, and/or search queries, as with the info commands:
crash export --tags IE +2
The above command will export the 2nd most recent crash (+2) that belongs to you and
contains the tag "IE".
By default crashes will be saved into the current working directory. Use the --dest
argument to specify a different output directory:
crash export +1 --all --tags adobe --dest adobe_crashes
The more complicated example below will search among all crashes (--all, vs only
those tagged with your username) for ones that have an exploitability category of
EXPLOITABLE and crashing module of libxml. The second crash (+2) will be chosen
after sorting by data.registers.eax
crash export --all --exploitability EXPLOITABLE --crashing_module libxml --sort data.registers.eax +2
"""
if args.strip() == "":
raise errors.TalusApiError(
"you must provide a name/id/git-thing of a crash to export it")
parts = shlex.split(args)
leftover = []
crash_id_or_name = None
search = self._search_terms(
parts,
out_leftover=leftover,
no_hex_keys=["hash_major", "hash_minor", "hash"])
root_level_items = [
"created", "tags", "job", "tool", "$where", "sort", "num", "dest"
]
new_search = {}
new_search["type"] = "crash"
for k, v in search.iteritems():
if k in root_level_items:
new_search[k] = v
else:
new_search["data." + k] = v
search = new_search
dest_dir = search.setdefault("dest", os.getcwd())
dest_dir = os.path.expanduser(dest_dir)
del search["dest"]
if len(leftover) > 0:
crash_id_or_name = leftover[0]
crash = self._resolve_one_model(crash_id_or_name,
Result,
search,
sort="-created")
if crash is None:
raise errors.TalusApiError(
"could not find a crash with that id/search")
self.ok("exporting crash {} from job {}".format(
crash.id, self._nice_name(crash, "job"), crash.tags))
first_num = int(crash.data["hash_major"], 16)
second_num = int(crash.data["hash_minor"], 16)
adj = utils.ADJECTIVES[first_num % len(utils.ADJECTIVES)]
noun = utils.NOUNS[second_num % len(utils.NOUNS)]
dest_name = "{}_{}_{}".format(adj, noun, crash.id)
dest_path = os.path.join(dest_dir, dest_name)
self.ok("saving to {}".format(dest_path))
if os.path.exists(dest_path):
self.warn(
"export path ({}) already exists! not gonna overwrite it, bailing"
.format(dest_path))
return
os.makedirs(dest_path)
file_path = os.path.join(dest_path, "crash.json")
self.out(file_path)
with open(file_path, "wb") as f:
f.write(
json.dumps(crash._filtered_fields(),
indent=4,
separators=(",", ": ")).encode("utf-8"))
file_path = os.path.join(dest_path, "crash.txt")
self.out(file_path)
with open(file_path, "wb") as f:
txt_info = self.do_info("",
return_string=True,
crash=crash,
show_details=True)
txt_info = utils.strip_color(txt_info)
f.write(txt_info.encode("utf-8"))
for file_id in crash.data.setdefault("repro", []):
fname, data = self._talus_client.corpus_get(file_id)
if fname is None:
fname = file_id
file_path = os.path.join(dest_path, "repro", fname)
if not os.path.exists(os.path.dirname(file_path)):
os.makedirs(os.path.dirname(file_path))
self.out(file_path)
with open(file_path, "wb") as f:
f.write(data.encode("utf-8"))
self.ok("done exporting crash")
def do_info(self,
args,
return_string=False,
crash=None,
show_details=False):
"""List detailed information about the crash.
Git-like syntax can also be used here to refer to the most recently created
crash result. E.g. the command below will show info about the 2nd most recent crash:
crash info +2
Search information can also be used. If git-like syntax is omitted, only
the first entry returned from the database will be displayed.
crash info --all --registers.eip 0x41414141 --sort registers.eax +1
The example above will show information about the crash with the lowest eax value
(+2 would show the 2nd lowest) that has an eip 0f 0x41414141. Omitting --all will
cause the search to be performed only among _your_ crashes.
To view _all_ of the details about a crash, add the --details flag.
"""
if crash is None:
if args.strip() == "":
raise errors.TalusApiError(
"you must provide a name/id/git-thing of a crash to show info about it"
)
parts = shlex.split(args)
if "--details" in parts:
parts.remove("--details")
show_details = True
leftover = []
crash_id_or_name = None
search = self._search_terms(
parts,
out_leftover=leftover,
no_hex_keys=["hash_major", "hash_minor", "hash"])
root_level_items = [
"created", "tags", "job", "tool", "$where", "sort", "num"
]
new_search = {}
new_search["type"] = "crash"
for k, v in search.iteritems():
if k in root_level_items:
new_search[k] = v
else:
new_search["data." + k] = v
search = new_search
if len(leftover) > 0:
crash_id_or_name = leftover[0]
crash = self._resolve_one_model(crash_id_or_name,
Result,
search,
sort="-created")
if crash is None:
raise errors.TalusApiError(
"could not find a crash with that id/search")
crashing_instr = None
asm = crash.data.setdefault("disassembly", ["?"])
for x in asm:
if "->" in x:
match = re.match(r'^-+>(.*$)', x)
crashing_instr = match.group(1).strip()
# if we don't find the arrow, we'll say it's the last instruction in the
# list
if crashing_instr is None:
crashing_instr = x
crashing_instr = re.sub(r'^([a-f0-9]+\s)*(.*)$', '\\2',
crashing_instr).strip()
crashing_instr = re.sub(r'\s+', " ", crashing_instr)
colors = deque([
colorama.Fore.MAGENTA,
colorama.Fore.CYAN,
colorama.Fore.YELLOW,
colorama.Fore.GREEN,
colorama.Fore.BLUE,
colorama.Fore.RED,
])
reg_colors = {}
reg_rows = []
reg_rows_no_color = []
registers = crash.data.setdefault("registers", {})
for reg, val in registers.iteritems():
reg = reg.lower()
color = colors.popleft()
reg_colors[reg] = color
reg_rows.append(
[reg, color + "{:8x}".format(val) + colorama.Style.RESET_ALL])
reg_rows_no_color.append([reg, "{:8x}".format(val)])
colors.append(color)
split = int(math.ceil(len(reg_rows) / 2.0))
table1 = tabulate(reg_rows[:split]).split("\n")
table1_no_color = tabulate(reg_rows_no_color[:split]).split("\n")
table2 = tabulate(reg_rows[split:]).split("\n")
longest_t1 = max(len(x) for x in table1_no_color)
if len(table2) == 0:
reg_lines = table1
else:
reg_lines = []
for x in xrange(len(table1)):
if x >= len(table2):
reg_lines.append(table1[x])
else:
fmt_string = "{:" + str(longest_t1) + "} | {}"
reg_lines.append(fmt_string.format(table1[x], table2[x]))
for reg, color in reg_colors.iteritems():
crashing_instr = re.sub(r"\b" + reg + r"\b",
color + reg + colorama.Style.RESET_ALL,
crashing_instr)
indent = " "
arrow = ""
asm_text = crash.data.setdefault("disassembly", [])
for line in asm_text:
if "->" in line:
arrow = line.split()[0]
asm_rows = []
asm_rows_no_color = []
arrow_indent = " " * len(arrow)
for line in crash.data.setdefault("disassembly", []):
line = line.strip()
line = re.sub(r'\s+', " ", line)
if not line.startswith(arrow):
line = " " * len(arrow) + line
line_no_color = line
for reg, color in reg_colors.iteritems():
line = re.sub(r"\b" + reg + r"\b",
color + reg + colorama.Style.RESET_ALL, line)
has_arrow = (arrow in line)
line = line.replace(arrow, "")
line_no_color = line_no_color.replace(arrow, "")
match = re.match(r'^\s+([a-f0-9]+)\s+([a-f0-9]+)\s+(.*)$', line)
no_color_match = re.match(r'^\s+([a-f0-9]+)\s+([a-f0-9]+)\s+(.*)$',
line_no_color)
if match is None:
match2 = re.match(r'^\s+([a-f0-9]+)\s+(.*)$', line)
no_color_match2 = re.match(r'^\s+([a-f0-9]+)\s+(.*)$',
line_no_color)
if match2 is None:
asm_rows.append(["", "", "", line])
asm_rows_no_color.append(["", "", "", line_no_color])
else:
asm_rows.append([
"-->" if has_arrow else "", "",
match2.group(1),
match2.group(2)
])
asm_rows_no_color.append([
"-->" if has_arrow else "", "",
no_color_match2.group(1),
no_color_match2.group(2)
])
else:
asm_rows.append([
"-->" if has_arrow else "",
match.group(1),
match.group(2),
match.group(3)
])
asm_rows_no_color.append([
"-->" if has_arrow else "",
no_color_match.group(1),
no_color_match.group(2),
no_color_match.group(3)
])
table1 = asm_lines = tabulate(asm_rows).split("\n")
table1_no_color = tabulate(asm_rows_no_color).split("\n")
table2 = reg_lines
longest_t1 = max(len(x) for x in table1_no_color)
asm_and_regs = []
if len(table2) == 0:
asm_and_regs = table1
else:
asm_and_regs = []
for x in xrange(len(table1)):
if x >= len(table2):
asm_and_regs.append(table1[x])
else:
no_color_diff = len(table1[x]) - len(table1_no_color[x])
fmt_string = "{:" + str(longest_t1 +
no_color_diff) + "} | {}"
asm_and_regs.append(fmt_string.format(
table1[x], table2[x]))
details = ""
if show_details:
if isinstance(crash.data.setdefault("backtrace", ""), list):
crash.data["backtrace"] = "\n".join(crash.data["backtrace"])
detail_indent = " " * 4
details += """
Stack: \n{stack}
Loaded Modules: \n{loaded_mods}
Backtrace: \n{backtrace}
Exploitability Details: \n{exploit_details}
""".format(
stack="\n".join(
detail_indent + x
for x in crash.data.setdefault("stack", "").split("\n")),
loaded_mods="\n".join(detail_indent + x
for x in crash.data.setdefault(
"loaded_modules", "").split("\n")),
backtrace="\n".join(detail_indent + x
for x in crash.data.setdefault(
"backtrace", "").split("\n")),
exploit_details="\n".join(
detail_indent + x for x in crash.data.setdefault(
"exploitability_details", "").split("\n")),
)
res = """
ID: {id}
Tags: {tags}
Job: {job}
Exploitability: {expl}
Hash Major/Minor: {hash_major} {hash_minor}
Crash Instr: {crash_instr}
Crash Module: {crash_module}
Exception Code: {exception_code:8x}
{asm_and_regs}{details}
""".format(
id=crash.id,
tags=",".join(crash.tags),
job=self._nice_name(crash, "job"),
expl=crash.data.setdefault("exploitability", "None"),
hash_major=crash.data.setdefault("hash_major", "None"),
hash_minor=crash.data.setdefault("hash_minor", "None"),
crash_instr=crashing_instr,
crash_module=crash.data.setdefault("crash_module", ""),
exception_code=crash.data.setdefault("exception_code", 0),
reg_tables="\n".join(indent + x for x in reg_lines),
asm_and_regs="\n".join(" " + x for x in asm_and_regs),
details=details,
)
if not return_string:
print(res)
else:
return res
def _search_terms(self,
parts,
key_remap=None,
user_default_filter=True,
out_leftover=None,
no_hex_keys=None):
"""Return a dictionary of search terms"""
if no_hex_keys is None:
no_hex_keys = []
search = {}
key = None
if key_remap is None:
key_remap = {}
key_map = {"status": "status.name"}
key_map.update(key_remap)
found_all = False
for item in parts:
if key is None:
if not item.startswith("--"):
if out_leftover is not None:
out_leftover.append(item)
continue
else:
raise errors.TalusApiError(
"args must be alternating search item/value pairs!"
)
item = item[2:].replace("-", "_")
key = item
if key == "all":
found_all = True
key = None
continue
if key in key_map:
key = key_map[key]
if key.endswith("__type") or key.endswith(".type"):
key += "_"
elif key is not None:
# hex conversion
if re.match(r'^0x[0-9a-f]+$', item,
re.IGNORECASE) is not None and key.split(
"__")[0] not in no_hex_keys:
item = int(item, 16)
if key in search and not isinstance(search[key], list):
search[key] = [search[key]]
if key in search and isinstance(search[key], list):
search[key].append(item)
else:
search[key] = item
self.out("searching for {} = {}".format(key, item))
# reset this
key = None
if user_default_filter and not found_all and self._talus_user is not None:
# default filter by username tag
self.out("default filtering by username (searching for tags = {})".
format(self._talus_user))
self.out("use --all to view all models")
if "tags" in search and not isinstance(search["tags"], list):
search["tags"] = [search["tags"]]
if "tags" in search and isinstance(search["tags"], list):
search["tags"].append(self._talus_user)
else:
search["tags"] = self._talus_user
if out_leftover is not None and key is not None:
out_leftover.append(key)
return search
