def batch(batch_name, task_type):
"""
Prints out some numbers which describe the state of tasks in certain batch
"""
headers = ['Batch',
'Total',
'Completed (flag)',
'Percent (flag)',
'Answers',
'Percent (answers)',
'Breakdown (answers: tasks)']
pt = VeryPrettyTable(headers)
pt.align = 'l'
pt.left_padding_width = 1
pt.hrules = ALL
for k, v in _stats.batch_completeness(batch_name, task_type).items():
values = [k,
v['total'],
v['flag'],
'{:5.1f} %'.format(v['flag_percent']),
v['answers'],
'{:5.1f} %'.format(v['answers_percent']),
v['breakdown']]
pt.add_row(values)
print(pt)
def options(self):
import inspect
from veryprettytable import VeryPrettyTable
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
for attr in [attr for attr in dir(self) if inspect.ismethod(getattr(self, attr))]:
if attr not in ["options", "__init__"]:
# print("%s\t\t\t%s" % (attr, getattr(self, attr).__doc__))
commands.add_row([attr, getattr(self, attr)()])
return commands
def usage(self):
"""Usage Coomment String"""
import inspect
from Core.banners import Banners
from veryprettytable import VeryPrettyTable
banner = Banners()
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
banner.randomize()
for attr in [attr for attr in dir(self) if inspect.ismethod(getattr(self, attr))]:
if attr not in ["usage", "__init__", "__del__", "__str__", "save_state", "methods", "sprint"]:
commands.add_row([attr, getattr(self, attr).__doc__])
return commands
def options(self):
import inspect
from veryprettytable import VeryPrettyTable
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
for attr in [
attr for attr in dir(self)
if inspect.ismethod(getattr(self, attr))
]:
if attr not in ["options", "__init__"]:
# print("%s\t\t\t%s" % (attr, getattr(self, attr).__doc__))
commands.add_row([attr, getattr(self, attr)()])
return commands
def usage(self):
"""Usage Coomment String"""
import inspect
from Core.banners import Banners
from veryprettytable import VeryPrettyTable
banner = Banners()
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
banner.randomize()
for attr in [attr for attr in dir(self) if inspect.ismethod(getattr(self, attr))]:
if attr not in ["usage", "__init__", "__del__", "__str__", "methods", "sprint"]:
# print("%s\t\t\t%s" % (attr, getattr(self, attr).__doc__))
commands.add_row([attr, getattr(self, attr).__doc__])
return commands
def list(self):
"""[ LIST ] Me My Targets (last 20)? """
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url", "notes"])
table.align = "l"
table.padding_width = 2
try:
self.sprint("\r\n")
db = mongo["targets"]["targets"].find()
for target in db:
table.add_row([target["target_pinned"], target["_id"], target["target_name"], target["target_ip"], target["target_url"], target["target_note"]])
except Exception as e:
print(str(e))
finally:
db.close()
print("\n{}\n".format(table))
def list(self):
"""[ LIST ] Me My Targets (last 20)? """
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url", "notes"])
table.align = "l"
table.padding_width = 2
try:
self.sprint("\r\n")
db = mongo["targets"]["targets"].find()
for target in db:
table.add_row([
target["target_pinned"], target["_id"],
target["target_name"], target["target_ip"],
target["target_url"], target["target_note"]
])
except Exception as e:
print(str(e))
finally:
db.close()
print("\n{}\n".format(table))
def list(self):
"""[ LIST_FIND ] List targets in Crawler|Mode and use"""
import re
from veryprettytable import VeryPrettyTable
table = VeryPrettyTable(["crawled", "id", "name", "ip", "url", "notes"])
table.align = "l"
table.padding_width = 2
# pre-defined key_commands
key_command = ["save", "crawl", "queue"]
# pre-defined find filter with key and values
find_filter = {}
# get the complete command minus the command itself
# print(self.command.split())
# check is something is filled
if len(self.command.split()[1:]) >= 1:
# there are arguments given
# check how many arguments ar given
# print(len(self.command.split()[1:]))
# loop over the arguments
for i in self.command.split()[1:]:
''' *** START: key commands *** '''
# check if argument is a key_command argument
if i in key_command:
# argument [ i ] is a key_command
# print("This is a Key Command: {}".format(i))
if i == "save":
""" Save to my _saved_list """
print("Save this to the Target-List")
print("[ Not Working Yet ]")
elif i == "discover":
""" Send to Spider to .discover() """
print("Trying to Discover if there is a known CMS or Framework installed")
elif i == "crawl":
""" Send to Spider to .crawl() """
self.sprint(" [!] Sending Spiders to the Target", mode=self.modus)
# check in which index number the given command i is indexed
num = self.command.split()[1:].index(i)
# print index number as integer in array of command .split()
# print(num)
# pop the command out of the array, So there will only be a string IF given key:value is 1 argument
doc = self.command.split().pop(num)
# get Key and Value of the String
k, v = doc.split(":")[0], doc.split(":", 1)[1]
# cursor to get the url value fore the given filter
urls = db.find({"target_{}".format(k):v})
# loop over the given documents to get the url value of each document
for url in urls:
# firstly print the gotten url
self.sprint(" [!] Found URL and Queue: {}".format(url["target_url"]), mode=self.modus)
# send directly to the crawler to crawl the URL
self._queue_crawler.queue.append(url["target_url"])
self.sprint(" [!] ... appended URL to the Queue", mode=self.modus)
self.sprint(" [!] ... Send to Spiders")
# start the crawl with the spiders
self.crawl()
elif i == "queue":
""" Update Tasksand update _queue_crawler"""
print("Put into the Queue to process when possible")
# check in which index number the given command i is indexed
num = self.command.split()[1:].index(i)
# print index number as integer in array of command .split()
print(num)
# pop the command out of the array, So there will only be a string IF given key:value is 1 argument
doc = self.command.split().pop(num)
# get Key and Value of the String
k, v = doc.split(":")[0], doc.split(":", 1)[1]
# cursor to get the url value fore the given filter
urls = db.find({"target_{}".format(k):v})
# loop over the given documents to get the url value of each document
for url in urls:
# firstly print the gotten url
print(" [!] Found URL and Queue: {}".format(url["target_url"]))
# send directly to the crawler to crawl the URL
# self.crawl(url=url["target_url"])
self._queue_crawler.queue.append(url["target_url"])
print(" ... ... appended URL to the Queue")
continue
# else
# check if argument is a Key:Value
elif re.search(":", i):
# argument is a Key:Value
# print(i)
# create a Key and Value of the given argument
k, v = i.split(":")[0], i.split(":", 1)[1]
# print(" Printing the Key: {}".format(k))
# print(" Printing the Value: {}".format(v))
found = db.find({"target_{}".format(k):v})
for i in found:
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"], i["target_note"]])
print("\n{}\n".format(table))
continue
print("Unknown command")
else:
# there are None arguments given
self.sprint("No arguments are given!", mode=self.modus)
try:
self.sprint("\r\n")
docs = db.find()
for target in docs:
table.add_row([target["target_pinned"], target["_id"], target["target_name"], target["target_ip"], target["target_url"], target["target_note"]])
except Exception as e:
print(str(e))
print("\n{}\n".format(table))
def remove(self):
"""[ REM ] Remove from Hitlist"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# remember all objects by _id
object_id = []
# pre-defined find filter
# this is meant to find the requested records easier from the database
rem_filter = {}
# get command and check if there are multiple arguments minus self.command[0]
if self.command.split()[1:]:
# loop over the arguments and check if there is a Key:Value
for a in self.command.split()[1:]:
print(a)
print("-"*50)
# check k:v
if re.search(":", a):
k, v = a.split(":")[0], a.split(":", 1)[1]
print(k)
print(v)
print("-"*50)
# update the filter dictionary
rem_filter.update({"target_{}".format(k):v})
# get the Findings from the Database
findings = db.find(rem_filter)
print(findings)
print("-"*50)
# loop over every document you have found and display it in a Table on CLI
for i in findings:
print(i)
# append the object_id array by _id as key
object_id.append(i["_id"])
# add every document as row to the table
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"]])
# print the table
print("\n{}\n".format(table))
# loop for acknowledge
while True:
# ask if these documents are to be removed ?
acknowledge = input("Are these document(s) the ones you want to remove ?: Y/n ").lower()
if acknowledge == "y".lower() or acknowledge == "yes".lower():
# if yes loop over the array with all object _id(s)
for v in object_id:
doc = db.find_one({"_id":v})
archive.insert_one(doc)
db.remove(doc)
print("all done")
break
elif acknowledge == "n".lower() or acknowledge == "no".lower():
print("Shheeeuuuhh")
break
else:
print("Try again!")
continue
else:
self.sprint("You forgot to add options", mode=self.modus)
def edit(self):
"""[ EDIT ] Edit my a Target"""
""" Psuedo Code
Check If Something is filled
edit .... ?
check if there is a something filled
edit ip:123.123.321.22 ... ?
|_ IF yes get Records for PART1
|_ IF no return Err Message
check if is > sign
edit ip:123.123.321.22 > ...?
|_ IF yes AND no PART2 Err Message: You didnt give me anything to update it with;
|_ IF yes ... break it into PART1 (filter) and PART2 (update// $set)
# TARGET --shell
edit [ ...:... ...:... ] > [ ...:... ]
remove [ ...:... ]
|- return record(s)
|_ Are you sure?
add ip:... name:... url:...
find [ ...:... ...:... ]
# ADD
vpn [ ..:.. ]
server [ ..:.. ]
user [ ..:.. ]
# OPTIONS
modules <- return a list of available modules
modules:crawler <- return a list of available options crawler module
modules:crawler xpath:True base_only:True tag_forms:True
plugin
user
proxy
vpn
target
template
## CliDl
command category:type:name:additional limit filter filter -f function_name()
"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# pre-defined object variable to connect the gotten record and update it by _id
# there can be multiple _id(s) to be updated so therefore it will be a array
object_id = []
if self.command.split()[1:]:
# there is more then index[0]
# check if there is a > $gt
# if self.command.find(">"):
if re.search(">", self.command):
# there is a $gt sign and can be split up
splitted = self.command.split(">", 1)
# splitted PART1 contains the original command
# so only split and get the given filters
# Also how many filters are given ?
part1 = splitted[0].split()
# print how many filters are given MINUS the original command ?
# print(len(part1[1:]))
# print everything after the original command
# print(part1[1:])
# create a dictionary to fill with the complete query
filter_part1 = {}
# Loop over the filters if there are more the 1
if len(part1[1:]) >= 1:
# loop over every filter
for f in part1[1:]:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part1.update({"target_{}".format(k):v})
# save foundings into records variable
records = db.find(filter_part1)
# loop over findings
for i in records:
# update the object_id array
# which holds the given objects to be updated
object_id.append(i["_id"])
# and display the finding into the pre-defined tables
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"]])
# print the table for the user
# print("\n{}\n".format(table))
# print the given object id(s)
# print(object_id)
# we got the first part and record(s)
# now for the second part to update the requested record(s)
part2 = splitted[1].split()
# print the length of how many field we have to update
# print(len(part2))
# create a dictionary to fill with the complete update_query
filter_part2 = {}
# check if length update arguments are more then 1
if len(part2) >= 1:
# for every field update get key, value
for f in part2:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part2.update({"target_{}".format(k):v})
# get the objects to be updated by object_id array
# get the fields from part2 which have to be updated
# do a findAndUpdate of the documents with $set
# do not update full documents, just update fields or add non-existing fields to the document
# update the records in the database as Many documents
db.update_many(filter_part1, {"$set": filter_part2})
else:
if re.search(":", self.command[1:]):
# get records for PART1 but there is no PART2!
print("getting records from database")
# pre-defined find filter for part1 without part2
# this is meant to find the requested records easier then directly updating the database
find_filter = {}
# get all filters minus indexx[0] the command
find = self.command.split()[1:]
# print the length of the filters
print(len(find))
# do a loop to get a complete key, value query to send to the database
for f in find:
# for each loop get the k, v and update it to the dictionary
k, v = f.split(":")[0], f.split(":", 1)[1]
# update the dictionary
find_filter.update({"target_{}".format(k):v})
# get records from the database
findings = db.find(find_filter)
# loop over the records to create a table at cli
for i in findings:
# and display the finding into the pre-defined tables
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"]])
# print the table for the user
print("\n{}\n".format(table))
else:
print("Your arguments to filter from the database are not correct. I need a Key:Value")
else:
self.sprint("You forgot to add options", mode=self.modus)
def sg_lookup(profile, region, debug):
"""
Summary.
Returns securitygroup user selection in given region
Args:
:profile (str): profile_name from local awscli configuration
:region (str): AWS region code
Returns:
securitygroup ID chosen by user
"""
padding = 2
field_max_width = 50
max_gn, max_desc = 10, 10 # starting value to find max length of a table field (chars)
x = VeryPrettyTable(border=True, header=True, padding_width=padding)
sgs = profile_securitygroups(profile, region)
for index, row in enumerate(sgs):
for k, v in row.items():
if len(v['GroupName']) > max_gn:
max_gn = len(v['GroupName'])
if len(v['Description']) > max_desc:
max_desc = len(v['GroupName'])
if debug:
print('max_gn = {}'.format(max_gn))
print('max_desc = {}'.format(max_desc))
# GroupName header
tabspaces_gn = int(max_gn / 4) - int(len('GroupName') / 2) + padding
tab_gn = '\t'.expandtabs(tabspaces_gn)
# Description header
tabspaces_desc = int(max_desc / 4) - int(len('Description') / 2) + padding
tab_desc = '\t'.expandtabs(tabspaces_desc)
x.field_names = [
bd + ' # ' + frame, bd + 'GroupId' + frame,
tab_gn + bd + 'GroupName' + frame, bd + 'VpcId' + frame,
tab_desc + bd + 'Description' + frame
]
# cell alignment
x.align = 'c'
x.align[tab_gn + bd + 'GroupName' + frame] = 'l'
x.align[tab_desc + bd + 'Description' + frame] = 'l'
# populate table
lookup = {}
for index, row in enumerate(sgs):
for k, v in row.items():
lookup[index] = k
x.add_row([
rst + userchoice_mapping(index) + '.' + frame, rst + k + frame,
rst + v['GroupName'][:field_max_width] + frame,
rst + v['VpcId'] + frame,
rst + v['Description'][:field_max_width] + frame
])
# Table showing selections
print(
f'\n\tSecurity Groups in region {bd + region + rst}\n'.expandtabs(30))
display_table(x)
return choose_resource(lookup)
def consoleLabel(deduper, additional_columns=[]): # pragma: no cover
'''
Command line interface for presenting and labeling training pairs
by the user
Argument :
A deduper object
'''
finished = False
use_previous = False
fields = unique(field.field for field in deduper.data_model.primary_fields)
buffer_len = 1 # Max number of previous operations
examples_buffer = []
uncertain_pairs = []
while not finished:
if use_previous:
record_pair, _ = examples_buffer.pop(0)
use_previous = False
else:
if not uncertain_pairs:
uncertain_pairs = deduper.uncertainPairs()
record_pair = uncertain_pairs.pop()
n_match = (len(deduper.training_pairs['match']) +
sum(label == 'match' for _, label in examples_buffer))
n_distinct = (len(deduper.training_pairs['distinct']) +
sum(label == 'distinct' for _, label in examples_buffer))
x = VeryPrettyTable()
columns = fields + additional_columns
x.field_names = columns
x.align = "l"
for pair in record_pair:
x.add_row(list(pair[field] for field in columns))
print(x)
print("{0}/10 positive, {1}/10 negative".format(n_match, n_distinct),
file=sys.stderr)
print('Do these records refer to the same thing?', file=sys.stderr)
valid_response = False
user_input = ''
while not valid_response:
if examples_buffer:
prompt = '(y)es / (n)o / (u)nsure / (f)inished / (p)revious'
valid_responses = {'y', 'n', 'u', 'f', 'p'}
else:
prompt = '(y)es / (n)o / (u)nsure / (f)inished'
valid_responses = {'y', 'n', 'u', 'f'}
print(prompt, file=sys.stderr)
user_input = input()
if user_input in valid_responses:
valid_response = True
if user_input == 'y':
examples_buffer.insert(0, (record_pair, 'match'))
elif user_input == 'n':
examples_buffer.insert(0, (record_pair, 'distinct'))
elif user_input == 'u':
examples_buffer.insert(0, (record_pair, 'uncertain'))
elif user_input == 'f':
print('Finished labeling', file=sys.stderr)
finished = True
elif user_input == 'p':
use_previous = True
uncertain_pairs.append(record_pair)
if len(examples_buffer) > buffer_len:
record_pair, label = examples_buffer.pop()
if label in ['distinct', 'match']:
examples = {'distinct': [], 'match': []}
examples[label].append(record_pair)
deduper.markPairs(examples)
for record_pair, label in examples_buffer:
if label in ['distinct', 'match']:
examples = {'distinct': [], 'match': []}
examples[label].append(record_pair)
deduper.markPairs(examples)
def remove(self):
"""[ REM ] Remove from Hitlist"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# remember all objects by _id
object_id = []
# pre-defined find filter
# this is meant to find the requested records easier from the database
rem_filter = {}
# get command and check if there are multiple arguments minus self.command[0]
if self.command.split()[1:]:
# loop over the arguments and check if there is a Key:Value
for a in self.command.split()[1:]:
print(a)
print("-" * 50)
# check k:v
if re.search(":", a):
k, v = a.split(":")[0], a.split(":", 1)[1]
print(k)
print(v)
print("-" * 50)
# update the filter dictionary
rem_filter.update({"target_{}".format(k): v})
# get the Findings from the Database
findings = db.find(rem_filter)
print(findings)
print("-" * 50)
# loop over every document you have found and display it in a Table on CLI
for i in findings:
print(i)
# append the object_id array by _id as key
object_id.append(i["_id"])
# add every document as row to the table
table.add_row([
i["target_pinned"], i["_id"], i["target_name"],
i["target_ip"], i["target_url"]
])
# print the table
print("\n{}\n".format(table))
# loop for acknowledge
while True:
# ask if these documents are to be removed ?
acknowledge = input(
"Are these document(s) the ones you want to remove ?: Y/n "
).lower()
if acknowledge == "y".lower() or acknowledge == "yes".lower():
# if yes loop over the array with all object _id(s)
for v in object_id:
doc = db.find_one({"_id": v})
archive.insert_one(doc)
db.remove(doc)
print("all done")
break
elif acknowledge == "n".lower() or acknowledge == "no".lower():
print("Shheeeuuuhh")
break
else:
print("Try again!")
continue
else:
self.sprint("You forgot to add options", mode=self.modus)
def edit(self):
"""[ EDIT ] Edit my a Target"""
""" Psuedo Code
Check If Something is filled
edit .... ?
check if there is a something filled
edit ip:123.123.321.22 ... ?
|_ IF yes get Records for PART1
|_ IF no return Err Message
check if is > sign
edit ip:123.123.321.22 > ...?
|_ IF yes AND no PART2 Err Message: You didnt give me anything to update it with;
|_ IF yes ... break it into PART1 (filter) and PART2 (update// $set)
# TARGET --shell
edit [ ...:... ...:... ] > [ ...:... ]
remove [ ...:... ]
|- return record(s)
|_ Are you sure?
add ip:... name:... url:...
find [ ...:... ...:... ]
# ADD
vpn [ ..:.. ]
server [ ..:.. ]
user [ ..:.. ]
# OPTIONS
modules <- return a list of available modules
modules:crawler <- return a list of available options crawler module
modules:crawler xpath:True base_only:True tag_forms:True
plugin
user
proxy
vpn
target
template
## CliDl
command category:type:name:additional limit filter filter -f function_name()
"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# pre-defined object variable to connect the gotten record and update it by _id
# there can be multiple _id(s) to be updated so therefore it will be a array
object_id = []
if self.command.split()[1:]:
# there is more then index[0]
# check if there is a > $gt
# if self.command.find(">"):
if re.search(">", self.command):
# there is a $gt sign and can be split up
splitted = self.command.split(">", 1)
# splitted PART1 contains the original command
# so only split and get the given filters
# Also how many filters are given ?
part1 = splitted[0].split()
# print how many filters are given MINUS the original command ?
# print(len(part1[1:]))
# print everything after the original command
# print(part1[1:])
# create a dictionary to fill with the complete query
filter_part1 = {}
# Loop over the filters if there are more the 1
if len(part1[1:]) >= 1:
# loop over every filter
for f in part1[1:]:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part1.update({"target_{}".format(k): v})
# save foundings into records variable
records = db.find(filter_part1)
# loop over findings
for i in records:
# update the object_id array
# which holds the given objects to be updated
object_id.append(i["_id"])
# and display the finding into the pre-defined tables
table.add_row([
i["target_pinned"], i["_id"], i["target_name"],
i["target_ip"], i["target_url"]
])
# print the table for the user
# print("\n{}\n".format(table))
# print the given object id(s)
# print(object_id)
# we got the first part and record(s)
# now for the second part to update the requested record(s)
part2 = splitted[1].split()
# print the length of how many field we have to update
# print(len(part2))
# create a dictionary to fill with the complete update_query
filter_part2 = {}
# check if length update arguments are more then 1
if len(part2) >= 1:
# for every field update get key, value
for f in part2:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part2.update({"target_{}".format(k): v})
# get the objects to be updated by object_id array
# get the fields from part2 which have to be updated
# do a findAndUpdate of the documents with $set
# do not update full documents, just update fields or add non-existing fields to the document
# update the records in the database as Many documents
db.update_many(filter_part1, {"$set": filter_part2})
else:
if re.search(":", self.command[1:]):
# get records for PART1 but there is no PART2!
print("getting records from database")
# pre-defined find filter for part1 without part2
# this is meant to find the requested records easier then directly updating the database
find_filter = {}
# get all filters minus indexx[0] the command
find = self.command.split()[1:]
# print the length of the filters
print(len(find))
# do a loop to get a complete key, value query to send to the database
for f in find:
# for each loop get the k, v and update it to the dictionary
k, v = f.split(":")[0], f.split(":", 1)[1]
# update the dictionary
find_filter.update({"target_{}".format(k): v})
# get records from the database
findings = db.find(find_filter)
# loop over the records to create a table at cli
for i in findings:
# and display the finding into the pre-defined tables
table.add_row([
i["target_pinned"], i["_id"], i["target_name"],
i["target_ip"], i["target_url"]
])
# print the table for the user
print("\n{}\n".format(table))
else:
print(
"Your arguments to filter from the database are not correct. I need a Key:Value"
)
else:
self.sprint("You forgot to add options", mode=self.modus)