class Report(ReportTemplate):
"""
This report displays all of the found email addresses.
"""
name = "EmailReport"
markdown = ["####", "-"]
def __init__(self, db):
self.User = UserRepository(db)
def set_options(self):
super(Report, self).set_options()
self.options.add_argument("-t", "--tool", help="Source tool")
def run(self, args):
# Cidrs = self.CIDR.
results = {}
res = []
users = self.User.all()
for u in users:
if u.email is not None and u.email is not "None":
if args.tool:
# pdb.set_trace()
if not u.meta.get(args.tool, False):
continue
if u.domain:
domain = u.domain.domain
else:
domain = u.email.split("@")[1]
if not results.get(domain, False):
results[domain] = []
results[domain].append(u.email.lower())
for d in results.keys():
res.append(d)
for e in sorted(results[d]):
res.append("\t" + e)
self.process_output(res, args)
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
class Module(ModuleTemplate):
"""
Unfortunately, this is a Windows only deal. So we output a list of commands
to be run, and have a way of ingesting the output. Since metadata is more
junk then good data, items have to be manually verified
"""
name = "PowerMeta"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-c",
"--command",
help="Print out a list of commands to run",
action="store_true",
)
self.options.add_argument(
"-ps", "--powershell", help="Filename for a .ps1 command output"
)
self.options.add_argument(
"-i", "--import_file", help="import CSV file generated by PowerMeta"
)
self.options.add_argument(
"-f", "--import_folder", help="import CSV files generated by PowerMeta"
)
def run(self, args):
# pdb.set_trace()
if args.command:
res = []
res.append("Import-Module PowerMeta.ps1")
res += [
"Invoke-PowerMeta -TargetDomain %s -Download -Extract -ExtractAllToCsv %s.csv"
% (b.domain, b.domain)
for b in self.BaseDomain.all()
]
if args.powershell:
open(args.powershell, "w").write("\r\n".join(res))
print("Command saved as %s" % args.powershell)
else:
print("\n".join(res))
if args.import_file:
self.process_domain(args.import_file, args)
if args.import_folder:
if args.import_folder[-1] == "/":
args.import_folder = args.import_folder[:-1]
files = glob.glob(args.import_folder + "/*.csv")
for f in files:
print("Processing %s" % f)
self.process_domain(f, args)
def process_domain(self, csvfile, args):
domain = csvfile.split("/")[-1].split(".csv")[0]
domain_obj = self.BaseDomain.find(domain=domain)
if not domain_obj:
print("Error: Domain not found: %s" % domain)
return
csvdata = open(csvfile).read().decode("utf-16").encode("utf-8")
csvreader = csv.reader(BytesIO(csvdata), delimiter=",", quotechar='"')
headers = csvreader.next()
searchable_headers = ["Author", "Creator"]
indexes = [headers.index(s) for s in searchable_headers if s in headers]
data = []
for row in csvreader:
for i in indexes:
data.append(row[i])
data = list(set(data)) # Dedup
for d in data:
if d.strip():
res = six.input("Is %s a valid name? [y/N] " % d)
if res and res[0].lower() == "y":
if " " in d:
if ", " in d:
first_name = d.split(", ")[1]
last_name = d.split(", ")[0]
else:
first_name = d.split(" ")[0]
last_name = " ".join(d.split(" ")[1:])
created, user = self.User.find_or_create(
first_name=first_name, last_name=last_name
)
if created:
print("New user created")
user.domain = domain_obj
# else:
# created, user = self.User.find_or_create(username=d)
# if created:
# print("New username created")
# user.domain = domain_obj
user.update()
self.User.commit()
class Module(ToolTemplate):
'''
PyMeta is a tool used for searching domains on various search engines, finding all of the relevant
documents, and raiding the exif data to find users.
'''
name = "PyMeta"
binary_name = "pymeta"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d", "--domain", help="Domain to brute force")
self.options.add_argument("-f", "--file", help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
targets.append(args.domain)
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
targets.append(d)
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name, scope_type="passive")
for d in domains:
targets.append(d.domain)
res = []
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:]
)
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path
)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
output = os.path.join(output_path)
res.append({"target": t, "output": output})
return res
def build_cmd(self, args):
cmd = self.binary + " -o {output} -d {target} -csv -f full "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
output_path = cmd["output"]
created, domain_obj = self.BaseDomain.find_or_create(domain=cmd["target"])
try:
csvreader = csv.reader(open(os.path.join(cmd["output"], "pymeta_{}.csv".format(cmd["target"]))))
if sys.version[0] == '2':
headers = csvreader.next()
else:
headers = csvreader.__next__()
searchable_headers = ["Author", "Creator", "Producer"]
indexes = [headers.index(s) for s in searchable_headers if s in headers]
data = []
for row in csvreader:
for i in indexes:
data.append(row[i])
data = list(set(data)) # Dedup
for d in data:
# pdb.set_trace()
if d.strip() and len(d.split(' '))==2:
res = raw_input("Is %s a valid name? [y/N] " % d)
if res and res[0].lower() == "y":
if " " in d:
if ", " in d:
first_name = d.split(", ")[1]
last_name = d.split(", ")[0]
else:
first_name = d.split(" ")[0]
last_name = " ".join(d.split(" ")[1:])
created, user = self.User.find_or_create(
first_name=first_name, last_name=last_name
)
if created:
print("New user created")
user.domain = domain_obj
elif '@' in d:
res = raw_input("Is %s a valid email address? [y/N] " % d)
if res and res[0].lower() == 'y':
created, user = self.User.find_or_create(
email=d.strip())
if created:
print("New user created")
user.domain = domain_obj
except IOError:
pass
except Exception as e:
display_error("Error processing pymeta_{}.csv: {}".format(cmd["target"], e))
self.User.commit()
def __init__(self, db):
self.User = UserRepository(db)
class Module(ModuleTemplate):
name = "LinkedInt"
binary_name = "linkedint.py"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument(
"-b",
"--binary",
help="Path to binary for LinkedInt",
default=self.binary_name,
)
self.options.add_argument("-d",
"--domain",
help="Domain to add onto email")
self.options.add_argument("-c",
"--company_id",
help="Company ID to search")
self.options.add_argument("-C",
"--restrict",
help="Restrict to company filter",
action="store_true")
self.options.add_argument(
"-e",
"--email_format",
help=
"Format for emails: auto,full,firstlast,firstmlast,flast,first.last,fmlast,lastfirst, default is auto",
default="auto",
)
self.options.add_argument("-k",
"--keywords",
help="Keywords to search for")
self.options.add_argument(
"-o",
"--output_path",
help=
"Path which will contain program output (relative to base_path in config",
default=self.name,
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
self.options.add_argument(
"--smart_shuffle",
help=
"Provide a list of keywords. The tool will run once with all of the keywords, then run again excluding all of the keywords. This is useful for bypassing the 1k limit. Keywords must be comma separated.",
)
self.options.add_argument(
"--top",
help="Use the top X keywords from the job titles for smart shuffle"
)
self.options.add_argument(
"--auto_keyword",
help=
"Generate a list of keywords from titles already discovered, and search repeatedly using the top x number of results (specified with --top).",
action="store_true")
def run(self, args):
# pdb.set_trace()
if not args.binary:
self.binary = which.run("LinkedInt.py")
else:
self.binary = which.run(args.binary)
if not self.binary:
display_error(
"LinkedInt binary not found. Please explicitly provide path with --binary"
)
if args.domain:
created, domain = self.BaseDomain.find_or_create(
domain=args.domain)
if args.top:
titles = [
user.job_title.split(" at ")[0] for user in domain.users
if user.job_title
]
words = []
for t in titles:
words += [w.lower() for w in get_words(t)]
word_count = Counter(words).most_common()
display("Using the top %s words:" % args.top)
res = []
for w in word_count[:int(args.top)]:
display("\t{}\t{}".format(w[0], w[1]))
res.append(w[0])
# pdb.set_trace()
args.smart_shuffle = ",".join(res)
if args.auto_keyword:
if not args.top:
display_error(
"You must specify the top number of keywords using --top"
)
else:
if os.path.isfile('/tmp/armory_linkedinsearchqueries'):
blacklist = open('/tmp/armory_linkedinsearchqueries'
).read().split('\n')
else:
blacklist = []
bfile = open('/tmp/armory_linkedinsearchqueries', 'a')
for w in args.smart_shuffle.split(','):
if w not in blacklist:
args.keywords = w
self.process_domain(domain, args)
self.BaseDomain.commit()
bfile.write('{}\n'.format(w))
else:
display(
"Skipped {} due to it already being searched.".
format(w))
bfile.close()
elif args.smart_shuffle:
args.keywords = " OR ".join(
['"{}"'.format(i) for i in args.smart_shuffle.split(",")])
self.process_domain(domain, args)
self.BaseDomain.commit()
args.keywords = " AND ".join(
['-"{}"'.format(i) for i in args.smart_shuffle.split(",")])
self.process_domain(domain, args)
self.BaseDomain.commit()
else:
self.process_domain(domain, args)
self.BaseDomain.commit()
self.BaseDomain.commit()
def process_domain(self, domain_obj, args):
domain = domain_obj.domain
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
output_path = os.path.join(output_path,
"%s-linkedint" % domain.replace(".", "_"))
command_args = " -o %s" % output_path
command_args += " -e %s" % domain
if args.keywords:
command_args += " -u '%s'" % args.keywords
if args.company_id:
command_args += " -i %s " % args.company_id
if args.restrict:
command_args += " -c "
# if args.threads:
# command_args += " -t " + args.threads
if args.email_format:
command_args += " -f " + args.email_format
current_dir = os.getcwd()
new_dir = "/".join(self.binary.split("/")[:-1])
os.chdir(new_dir)
cmd = shlex.split("python2 " + self.binary + command_args)
print("Executing: %s" % " ".join(cmd))
subprocess.Popen(cmd).wait()
os.chdir(current_dir)
count = 0
with open(output_path + ".csv") as csvfile:
csvreader = csv.reader(csvfile, delimiter=",", quotechar='"')
for row in csvreader:
count += 1
created, user = self.User.find_or_create(
email=remove_binary(row[3]))
user.first_name = remove_binary(row[0])
user.last_name = remove_binary(row[1]).split(',')[0]
user.job_title = remove_binary(row[4])
user.location = remove_binary(row[5])
if created:
user.domain = domain_obj
print("New user: %s %s" %
(remove_binary(row[0]), remove_binary(row[1])))
user.update()
print("%s found and imported" % count)
self.User.commit()
def __init__(self, db):
self.BaseDomain = BaseDomainRepository(db)
self.User = UserRepository(db)
self.Cred = CredRepository(db)
class Module(ToolTemplate):
name = "TheHarvester"
binary_name = "theharvester"
def __init__(self, db):
self.db = db
self.BaseDomain = BaseDomainRepository(db, self.name)
self.Domain = DomainRepository(db, self.name)
self.User = UserRepository(db, self.name)
def set_options(self):
super(Module, self).set_options()
self.options.add_argument("-d", "--domain", help="Domain to harvest")
self.options.add_argument("-f",
"--file",
help="Import domains from file")
self.options.add_argument(
"-i",
"--import_database",
help="Import domains from database",
action="store_true",
)
self.options.add_argument(
"-s",
"--rescan",
help="Rescan domains that have already been scanned",
action="store_true",
)
def get_targets(self, args):
targets = []
if args.domain:
targets.append({"target": args.domain})
elif args.file:
domains = open(args.file).read().split("\n")
for d in domains:
if d:
created, domain = self.BaseDomain.find_or_create(domain=d)
targets.append({"target": domain.domain})
elif args.import_database:
if args.rescan:
domains = self.BaseDomain.all(scope_type="passive")
else:
domains = self.BaseDomain.all(tool=self.name,
scope_type="passive")
for d in domains:
targets.append({"target": d.domain})
if args.output_path[0] == "/":
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path[1:])
else:
output_path = os.path.join(
self.base_config["PROJECT"]["base_path"], args.output_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
for t in targets:
t["output"] = os.path.join(
output_path, "%s-theharvester" % t["target"].replace(".", "_"))
return targets
def build_cmd(self, args):
cmd = self.binary + " -f {output} -b default -d {target} "
if args.tool_args:
cmd += args.tool_args
return cmd
def process_output(self, cmds):
for cmd in cmds:
try:
data = xmltodict.parse(open(cmd["output"] + ".xml").read())
except Exception as e:
# display_error("Error with {}: {}".format(cmd["output"], e))
data = None
if data:
if data["theHarvester"].get("email", False):
if type(data["theHarvester"]["email"]) == list:
emails = data["theHarvester"]["email"]
else:
emails = [data["theHarvester"]["email"]]
for e in emails:
display("Processing E-mail: {}".format(e))
created, user = self.User.find_or_create(email=e)
_, domain = self.BaseDomain.find_or_create(
domain=e.split("@")[1])
user.domain = domain
user.update()
if created:
display_new("New email: %s" % e)
if data["theHarvester"].get("host", False):
if type(data["theHarvester"]["host"]) == list:
hosts = data["theHarvester"]["host"]
else:
hosts = [data["theHarvester"]["host"]]
for d in hosts:
created, domain = self.Domain.find_or_create(
domain=d["hostname"])
if data["theHarvester"].get("vhost", False):
if type(data["theHarvester"]["vhost"]) == list:
hosts = data["theHarvester"]["vhost"]
else:
hosts = [data["theHarvester"]["vhost"]]
for d in hosts:
created, domain = self.Domain.find_or_create(
domain=d["hostname"])
self.BaseDomain.commit()