def batch(batch_name, task_type):
"""
Prints out some numbers which describe the state of tasks in certain batch
"""
headers = ['Batch',
'Total',
'Completed (flag)',
'Percent (flag)',
'Answers',
'Percent (answers)',
'Breakdown (answers: tasks)']
pt = VeryPrettyTable(headers)
pt.align = 'l'
pt.left_padding_width = 1
pt.hrules = ALL
for k, v in _stats.batch_completeness(batch_name, task_type).items():
values = [k,
v['total'],
v['flag'],
'{:5.1f} %'.format(v['flag_percent']),
v['answers'],
'{:5.1f} %'.format(v['answers_percent']),
v['breakdown']]
pt.add_row(values)
print(pt)
def print_table(table, col_names, title):
result = VeryPrettyTable()
result.title = title
result.field_names = col_names
for row in table:
result.add_row([format(i, ".2f") for i in row])
print(result)
def userdata_lookup(debug):
"""
Summary.
Instance Profile role user selection
Returns:
iam instance profile role ARN (str) or None
"""
# setup table
x = VeryPrettyTable(border=True, header=True, padding_width=2)
field_max_width = 70
x.field_names = [
bd + '#' + frame,
bd + 'Filename' + frame,
bd + 'Path' + frame,
bd + 'CreateDate' + frame,
bd + 'LastModified' + frame
]
# cell alignment
x.align[bd + '#' + frame] = 'c'
x.align[bd + 'Filename' + frame] = 'c'
x.align[bd + 'Path' + frame] = 'l'
x.align[bd + 'CreateDate' + frame] = 'c'
x.align[bd + 'LastModified' + frame] = 'c'
filenames = source_local_userdata()
paths = source_local_userdata(paths=True)
ctimes = [time.ctime(os.path.getctime(x)) for x in paths]
mtimes = [time.ctime(os.path.getmtime(x)) for x in paths]
# populate table
lookup = {}
for index, path in enumerate(paths):
lookup[index] = paths[index]
x.add_row(
[
rst + userchoice_mapping(index) + '.' + frame,
rst + filenames[index] + frame,
rst + path + frame,
rst + ctimes[index] + frame,
rst + mtimes[index] + frame
]
)
# Table showing selections
print(f'\n\tUserdata Scripts (local filesystem: ~/.config/ec2tools/userdata)\n'.expandtabs(26))
display_table(x, tabspaces=4)
return choose_resource(lookup)
def setup_table(pv, pypi, inc):
"""
Renders Table containing data elements via cli stdout
"""
# setup table
x = VeryPrettyTable(border=tablespec['border'],
header=tablespec['header'],
padding_width=tablespec['padding'])
title_cell1 = 'Current Project'
title_cell2 = 'pypi.python.org'
title_cell3 = 'Next Increment'
x.field_names = [
titlec + title_cell1 + frame,
titlec + title_cell2 + frame,
titlec + title_cell3 + frame,
]
# cell max width
x.max_width[titlec + title_cell1 + frame] = column_widths['project']
x.max_width[titlec + title_cell2 + frame] = column_widths['pypi']
x.max_width[titlec + title_cell3 + frame] = column_widths['incremental']
# cell min = max width
x.min_width[titlec + title_cell1 + frame] = column_widths['project']
x.min_width[titlec + title_cell2 + frame] = column_widths['pypi']
x.min_width[titlec + title_cell3 + frame] = column_widths['incremental']
# cell alignment
x.align[titlec + title_cell1 + frame] = 'c'
x.align[titlec + title_cell2 + frame] = 'c'
x.align[titlec + title_cell3 + frame] = 'c'
# populate table
# key credentials are either expired (age > KEYAGE_MAX) or valid
project_version = c.BOLD + c.BRIGHT_BLUE + pv + rst
pypi_version = c.BOLD + c.BRIGHT_BLUE + pypi + rst
inc_version = c.BOLD + c.BRIGHT_BLUE + inc + rst
x.add_row([
rst + project_version + frame,
rst + pypi_version + frame,
rst + inc_version + frame,
])
# Table
vtab_int = 20
vtab = '\t'.expandtabs(vtab_int)
msg = '{}PROJECT VERSION SOURCES{}{}|{}'.format(btext, rst + frame,
' ' + vtab, rst)
print_header(title=msg, indent=10, spacing=vtab_int)
display_table(x, tabspaces=4)
return _postprocessing()
def _list(self, args):
"""List the supported CVEs.
"""
table = VeryPrettyTable()
table.field_names = ["CVE", "Description", "Versions affected"]
for exploit in self.exploits:
table.add_row([exploit[0], exploit[1], exploit[2]])
self.logger.handle(table.get_string(), None)
def table(data):
x = VeryPrettyTable()
x.field_names = HEADERS
temp_kelvin = data['main']['temp']
temp_celcius = celcius(temp_kelvin)
temp_farenheit = farenheit(temp_kelvin)
x.add_row([
data['name'], temp_celcius, temp_farenheit, data['weather'][0]['main'],
data['weather'][0]['description']
])
return x
def options(self):
import inspect
from veryprettytable import VeryPrettyTable
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
for attr in [attr for attr in dir(self) if inspect.ismethod(getattr(self, attr))]:
if attr not in ["options", "__init__"]:
# print("%s\t\t\t%s" % (attr, getattr(self, attr).__doc__))
commands.add_row([attr, getattr(self, attr)()])
return commands
def ip_lookup(profile, region, debug):
"""
Summary.
Instance Profile role user selection
Returns:
iam instance profile role ARN (str) or None
"""
now = datetime.datetime.utcnow()
# setup table
x = VeryPrettyTable(border=True, header=True, padding_width=2)
field_max_width = 70
x.field_names = [
bd + '#' + frame, bd + 'InstanceProfileName' + frame,
bd + 'RoleArn' + frame, bd + 'CreateDate' + frame
]
# cell alignment
x.align[bd + '#' + frame] = 'c'
x.align[bd + 'InstanceProfileName' + frame] = 'l'
x.align[bd + 'RoleArn' + frame] = 'l'
x.align[bd + 'CreateDate' + frame] = 'c'
roles = source_instanceprofiles(parse_profiles(profile))
# populate table
lookup = {}
for index, iprofile in enumerate(roles):
lookup[index] = iprofile['Arn']
x.add_row([
rst + str(index) + '.' + frame,
rst + iprofile['InstanceProfileName'] + frame,
rst + iprofile['Arn'][:field_max_width] + frame,
rst + iprofile['CreateDate'] + frame
])
# add default choice (None)
lookup[index + 1] = None
x.add_row([
rst + str(index + 1) + '.' + frame, rst + 'Default' + frame, None,
rst + now.strftime('%Y-%m-%dT%H:%M:%S') + frame
])
# Table showing selections
print(f'\n\tInstance Profile Roles (global directory)\n'.expandtabs(26))
display_table(x, tabspaces=4)
return choose_resource(lookup, selector='numbers', default=(index + 1))
def usage(self):
"""Usage Coomment String"""
import inspect
from Core.banners import Banners
from veryprettytable import VeryPrettyTable
banner = Banners()
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
banner.randomize()
for attr in [attr for attr in dir(self) if inspect.ismethod(getattr(self, attr))]:
if attr not in ["usage", "__init__", "__del__", "__str__", "save_state", "methods", "sprint"]:
commands.add_row([attr, getattr(self, attr).__doc__])
return commands
def options(self):
import inspect
from veryprettytable import VeryPrettyTable
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
for attr in [
attr for attr in dir(self)
if inspect.ismethod(getattr(self, attr))
]:
if attr not in ["options", "__init__"]:
# print("%s\t\t\t%s" % (attr, getattr(self, attr).__doc__))
commands.add_row([attr, getattr(self, attr)()])
return commands
def usage(self):
"""Usage Coomment String"""
import inspect
from Core.banners import Banners
from veryprettytable import VeryPrettyTable
banner = Banners()
commands = VeryPrettyTable(["Command", "Description"])
commands.align = "l"
commands.padding_width = 2
banner.randomize()
for attr in [attr for attr in dir(self) if inspect.ismethod(getattr(self, attr))]:
if attr not in ["usage", "__init__", "__del__", "__str__", "methods", "sprint"]:
# print("%s\t\t\t%s" % (attr, getattr(self, attr).__doc__))
commands.add_row([attr, getattr(self, attr).__doc__])
return commands
def get_subnet(profile, region, debug):
"""
Summary.
Returns subnet user selection in given region
Args:
:profile (str): profile_name from local awscli configuration
:region (str): AWS region code
Returns:
subnet id chosen by user
"""
# setup table
x = VeryPrettyTable(border=True, header=True, padding_width=2)
field_max_width = 30
x.field_names = [
bd + '#' + frame, bd + 'SubnetId' + frame, bd + 'AZ' + frame,
bd + 'CIDR' + frame, bd + 'Ip Assign' + frame, bd + 'State' + frame,
bd + 'VpcId' + frame
]
#subnets = get_contents(account_file)[region]['Subnets']
subnets = profile_subnets(profile, region)
# populate table
lookup = {}
for index, row in enumerate(subnets):
for k, v in row.items():
lookup[index] = k
x.add_row([
rst + userchoice_mapping(index) + '.' + frame, rst + k + frame,
rst + v['AvailabilityZone'] + frame,
rst + v['CidrBlock'] + frame, rst + v['IpAddresses'] + frame,
rst + v['State'] + frame, rst + v['VpcId'] + frame
])
# Table showing selections
print(f'\n\tSubnets in region {bd + region + rst}\n'.expandtabs(30))
display_table(x)
return choose_resource(lookup)
class demo(object):
"""flightaware demo"""
def __init__(self, flight):
super(demo, self).__init__()
self.data, self.data2, self.flight_name = flightaware().get(flight)
self.table_header = VeryPrettyTable()
self.table_header.field_names = ['Flight info','Data']
for v, k in self.flight_name.iteritems():
self.table_header.add_row([v,k])
self.table = self.create(self.data, self.data2)
print self.table_header
print self.table
def create(self, departure, arrival):
table = VeryPrettyTable()
table.field_names = ['Departure',' ','Arrival',' ']
for de,ar, in zip(departure,arrival):
table.add_row([de,departure[de],ar,arrival[ar]])
return table
def list(self):
"""[ LIST ] Me My Targets (last 20)? """
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url", "notes"])
table.align = "l"
table.padding_width = 2
try:
self.sprint("\r\n")
db = mongo["targets"]["targets"].find()
for target in db:
table.add_row([target["target_pinned"], target["_id"], target["target_name"], target["target_ip"], target["target_url"], target["target_note"]])
except Exception as e:
print(str(e))
finally:
db.close()
print("\n{}\n".format(table))
def keypair_lookup(profile, region, debug):
"""
Summary.
Returns name of keypair user selection in given region
Args:
:profile (str): profile_name from local awscli configuration
:region (str): AWS region code
Returns:
keypair name chosen by user
"""
# setup table
x = VeryPrettyTable(border=True, header=True, padding_width=2)
field_max_width = 30
x.field_names = [bd + '#' + frame, bd + 'Keypair' + frame]
# cell alignment
x.align[bd + '#' + frame] = 'c'
x.align[bd + 'Keypair' + frame] = 'l'
keypairs = profile_keypairs(parse_profiles(profile), region)[region]
# populate table
lookup = {}
for index, keypair in enumerate(keypairs):
lookup[index] = keypair
x.add_row([
rst + userchoice_mapping(index) + '.' + frame,
rst + keypair + frame
])
# Table showing selections
print(f'\n\tKeypairs in region {bd + region + rst}\n'.expandtabs(26))
display_table(x, tabspaces=16)
return choose_resource(lookup)
def format_dict_to_table(data: dict, delete=None, show_type=False):
delete = delete or []
table = VeryPrettyTable()
field_names = ["key", "value"]
if show_type:
field_names += ["type"]
table.field_names = field_names
table.align['key'] = 'r'
table.align['value'] = 'l'
if show_type:
table.align['type'] = 'l'
for _key, _value in data.items():
if _key in delete:
continue
row = [_key, _value] if not show_type else [_key, _value, f"{type(_value)}"]
table.add_row(row)
return f"{table.get_string()}"
def changelog(self):
"""
You need to create in your rule table with next structure:
Version Reason Author Date Comment
separator for fields specified by cl_separator
:return: table with changes
:rtype VeryPrettyTable()
"""
prettytable = VeryPrettyTable()
changelog = []
for i in range(0, len(self.allrule)):
if self.allrule[i].lower().startswith('\'change log'):
self.log_start = i
if self.allrule[i].startswith('\'_') and i > self.log_start:
self.log_end = i
break
self.log_start += self.cl_intendant # intending from change log string to table
for i in range(self.log_start, self.log_end):
self.allrule[i] = self.allrule[i].replace(' ', '\t')
comment_line = [
item for item in self.allrule[i][1:].strip().split(
self.cl_separator) if item
]
version = comment_line.pop(0)
reason = comment_line.pop(0)
author = comment_line.pop(0)
data = comment_line.pop(0)
comment = " ".join(comment_line)
changelog.append([version, reason, author, data, comment])
prettytable.field_names = [
"Version", 'Reason', 'Author', 'Date', 'Comment'
]
prettytable.align['Comment'] = 'l'
for change in changelog:
prettytable.add_row(change)
return prettytable
def list(self):
"""[ LIST ] Me My Targets (last 20)? """
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url", "notes"])
table.align = "l"
table.padding_width = 2
try:
self.sprint("\r\n")
db = mongo["targets"]["targets"].find()
for target in db:
table.add_row([
target["target_pinned"], target["_id"],
target["target_name"], target["target_ip"],
target["target_url"], target["target_note"]
])
except Exception as e:
print(str(e))
finally:
db.close()
print("\n{}\n".format(table))
def test_file(csv_file, debug):
import csv
from veryprettytable import VeryPrettyTable
pt = VeryPrettyTable([" ", "Positive", "Negative"])
with open(csv_file, "r") as fp:
r = csv.DictReader(fp)
res = {True: {True: 0, False: 0}, False: {True: 0, False: 0}}
for l in r:
expected = l["ground truth"].lower() in ["true", "1", "on"]
predicted = full_compare(l["name1"], l["name2"])
if predicted != expected and debug:
print(predicted, expected, l["name1"], l["name2"])
res[predicted][expected] += 1
for predicted in [True, False]:
pt.add_row(
[
"Predicted positive" if predicted else "Predicted negative",
res[predicted][True],
res[predicted][False],
]
)
precision = res[True][True] / (res[True][True] + res[True][False])
recall = res[True][True] / (res[True][True] + res[False][True])
f1 = 2 * precision * recall / (precision + recall)
print(pt)
print("Precision: {:5.2f}".format(precision))
print("Recall: {:5.2f}".format(recall))
print("F1 score: {:5.2f}".format(f1))
def wordpress(self):
from veryprettytable import VeryPrettyTable
x = VeryPrettyTable()
x.field_names = ["City name", "Area", "Population", "Annual Rainfall"]
x.add_row(["Adelaide",1295, 1158259, 600.5])
x.add_row(["Brisbane",5905, 1857594, 1146.4])
x.add_row(["Darwin", 112, 120900, 1714.7])
x.add_row(["Hobart", 1357, 205556, 619.5])
x.add_row(["Sydney", 2058, 4336374, 1214.8])
x.add_row(["Melbourne", 1566, 3806092, 646.9])
x.add_row(["Perth", 5386, 1554769, 869.4])
return print(x)
for field in courses.columns.values:
print(field, end="\t\t")
print()
for index, row in courses.iterrows():
for col in row:
print(col, end="\t\t")
print()
if __name__ == '__main__':
while True:
dept_no = input("請輸入系所代號 (如要離開，請輸入-1) : ")
if (dept_no == "-1"):
break
try:
crawler = NckuCourseCrawler(dept_no=dept_no)
html = crawler.get_raw_HTML()
parser = NckuCourseParser(html)
parser.include_fields = ["系號", "序號", "餘額", "課程名稱(連結課程地圖)", "學分", "教師姓名*:主負責老師", "時間"]
courses = parser.parse(sort=True)
table = VeryPrettyTable()
table.field_names = courses.columns.values
for i in courses.iterrows():
table.add_row(i[1])
print(table)
except NoCourseAvailableError as e:
print(e)
def consoleLabel(deduper, additional_columns=[]): # pragma: no cover
'''
Command line interface for presenting and labeling training pairs
by the user
Argument :
A deduper object
'''
finished = False
use_previous = False
fields = unique(field.field for field in deduper.data_model.primary_fields)
buffer_len = 1 # Max number of previous operations
examples_buffer = []
uncertain_pairs = []
while not finished:
if use_previous:
record_pair, _ = examples_buffer.pop(0)
use_previous = False
else:
if not uncertain_pairs:
uncertain_pairs = deduper.uncertainPairs()
record_pair = uncertain_pairs.pop()
n_match = (len(deduper.training_pairs['match']) +
sum(label == 'match' for _, label in examples_buffer))
n_distinct = (len(deduper.training_pairs['distinct']) +
sum(label == 'distinct' for _, label in examples_buffer))
x = VeryPrettyTable()
columns = fields + additional_columns
x.field_names = columns
x.align = "l"
for pair in record_pair:
x.add_row(list(pair[field] for field in columns))
print(x)
print("{0}/10 positive, {1}/10 negative".format(n_match, n_distinct),
file=sys.stderr)
print('Do these records refer to the same thing?', file=sys.stderr)
valid_response = False
user_input = ''
while not valid_response:
if examples_buffer:
prompt = '(y)es / (n)o / (u)nsure / (f)inished / (p)revious'
valid_responses = {'y', 'n', 'u', 'f', 'p'}
else:
prompt = '(y)es / (n)o / (u)nsure / (f)inished'
valid_responses = {'y', 'n', 'u', 'f'}
print(prompt, file=sys.stderr)
user_input = input()
if user_input in valid_responses:
valid_response = True
if user_input == 'y':
examples_buffer.insert(0, (record_pair, 'match'))
elif user_input == 'n':
examples_buffer.insert(0, (record_pair, 'distinct'))
elif user_input == 'u':
examples_buffer.insert(0, (record_pair, 'uncertain'))
elif user_input == 'f':
print('Finished labeling', file=sys.stderr)
finished = True
elif user_input == 'p':
use_previous = True
uncertain_pairs.append(record_pair)
if len(examples_buffer) > buffer_len:
record_pair, label = examples_buffer.pop()
if label in ['distinct', 'match']:
examples = {'distinct': [], 'match': []}
examples[label].append(record_pair)
deduper.markPairs(examples)
for record_pair, label in examples_buffer:
if label in ['distinct', 'match']:
examples = {'distinct': [], 'match': []}
examples[label].append(record_pair)
deduper.markPairs(examples)
def edit(self):
"""[ EDIT ] Edit my a Target"""
""" Psuedo Code
Check If Something is filled
edit .... ?
check if there is a something filled
edit ip:123.123.321.22 ... ?
|_ IF yes get Records for PART1
|_ IF no return Err Message
check if is > sign
edit ip:123.123.321.22 > ...?
|_ IF yes AND no PART2 Err Message: You didnt give me anything to update it with;
|_ IF yes ... break it into PART1 (filter) and PART2 (update// $set)
# TARGET --shell
edit [ ...:... ...:... ] > [ ...:... ]
remove [ ...:... ]
|- return record(s)
|_ Are you sure?
add ip:... name:... url:...
find [ ...:... ...:... ]
# ADD
vpn [ ..:.. ]
server [ ..:.. ]
user [ ..:.. ]
# OPTIONS
modules <- return a list of available modules
modules:crawler <- return a list of available options crawler module
modules:crawler xpath:True base_only:True tag_forms:True
plugin
user
proxy
vpn
target
template
## CliDl
command category:type:name:additional limit filter filter -f function_name()
"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# pre-defined object variable to connect the gotten record and update it by _id
# there can be multiple _id(s) to be updated so therefore it will be a array
object_id = []
if self.command.split()[1:]:
# there is more then index[0]
# check if there is a > $gt
# if self.command.find(">"):
if re.search(">", self.command):
# there is a $gt sign and can be split up
splitted = self.command.split(">", 1)
# splitted PART1 contains the original command
# so only split and get the given filters
# Also how many filters are given ?
part1 = splitted[0].split()
# print how many filters are given MINUS the original command ?
# print(len(part1[1:]))
# print everything after the original command
# print(part1[1:])
# create a dictionary to fill with the complete query
filter_part1 = {}
# Loop over the filters if there are more the 1
if len(part1[1:]) >= 1:
# loop over every filter
for f in part1[1:]:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part1.update({"target_{}".format(k): v})
# save foundings into records variable
records = db.find(filter_part1)
# loop over findings
for i in records:
# update the object_id array
# which holds the given objects to be updated
object_id.append(i["_id"])
# and display the finding into the pre-defined tables
table.add_row([
i["target_pinned"], i["_id"], i["target_name"],
i["target_ip"], i["target_url"]
])
# print the table for the user
# print("\n{}\n".format(table))
# print the given object id(s)
# print(object_id)
# we got the first part and record(s)
# now for the second part to update the requested record(s)
part2 = splitted[1].split()
# print the length of how many field we have to update
# print(len(part2))
# create a dictionary to fill with the complete update_query
filter_part2 = {}
# check if length update arguments are more then 1
if len(part2) >= 1:
# for every field update get key, value
for f in part2:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part2.update({"target_{}".format(k): v})
# get the objects to be updated by object_id array
# get the fields from part2 which have to be updated
# do a findAndUpdate of the documents with $set
# do not update full documents, just update fields or add non-existing fields to the document
# update the records in the database as Many documents
db.update_many(filter_part1, {"$set": filter_part2})
else:
if re.search(":", self.command[1:]):
# get records for PART1 but there is no PART2!
print("getting records from database")
# pre-defined find filter for part1 without part2
# this is meant to find the requested records easier then directly updating the database
find_filter = {}
# get all filters minus indexx[0] the command
find = self.command.split()[1:]
# print the length of the filters
print(len(find))
# do a loop to get a complete key, value query to send to the database
for f in find:
# for each loop get the k, v and update it to the dictionary
k, v = f.split(":")[0], f.split(":", 1)[1]
# update the dictionary
find_filter.update({"target_{}".format(k): v})
# get records from the database
findings = db.find(find_filter)
# loop over the records to create a table at cli
for i in findings:
# and display the finding into the pre-defined tables
table.add_row([
i["target_pinned"], i["_id"], i["target_name"],
i["target_ip"], i["target_url"]
])
# print the table for the user
print("\n{}\n".format(table))
else:
print(
"Your arguments to filter from the database are not correct. I need a Key:Value"
)
else:
self.sprint("You forgot to add options", mode=self.modus)
def list(self):
"""[ LIST_FIND ] List targets in Crawler|Mode and use"""
import re
from veryprettytable import VeryPrettyTable
table = VeryPrettyTable(["crawled", "id", "name", "ip", "url", "notes"])
table.align = "l"
table.padding_width = 2
# pre-defined key_commands
key_command = ["save", "crawl", "queue"]
# pre-defined find filter with key and values
find_filter = {}
# get the complete command minus the command itself
# print(self.command.split())
# check is something is filled
if len(self.command.split()[1:]) >= 1:
# there are arguments given
# check how many arguments ar given
# print(len(self.command.split()[1:]))
# loop over the arguments
for i in self.command.split()[1:]:
''' *** START: key commands *** '''
# check if argument is a key_command argument
if i in key_command:
# argument [ i ] is a key_command
# print("This is a Key Command: {}".format(i))
if i == "save":
""" Save to my _saved_list """
print("Save this to the Target-List")
print("[ Not Working Yet ]")
elif i == "discover":
""" Send to Spider to .discover() """
print("Trying to Discover if there is a known CMS or Framework installed")
elif i == "crawl":
""" Send to Spider to .crawl() """
self.sprint(" [!] Sending Spiders to the Target", mode=self.modus)
# check in which index number the given command i is indexed
num = self.command.split()[1:].index(i)
# print index number as integer in array of command .split()
# print(num)
# pop the command out of the array, So there will only be a string IF given key:value is 1 argument
doc = self.command.split().pop(num)
# get Key and Value of the String
k, v = doc.split(":")[0], doc.split(":", 1)[1]
# cursor to get the url value fore the given filter
urls = db.find({"target_{}".format(k):v})
# loop over the given documents to get the url value of each document
for url in urls:
# firstly print the gotten url
self.sprint(" [!] Found URL and Queue: {}".format(url["target_url"]), mode=self.modus)
# send directly to the crawler to crawl the URL
self._queue_crawler.queue.append(url["target_url"])
self.sprint(" [!] ... appended URL to the Queue", mode=self.modus)
self.sprint(" [!] ... Send to Spiders")
# start the crawl with the spiders
self.crawl()
elif i == "queue":
""" Update Tasksand update _queue_crawler"""
print("Put into the Queue to process when possible")
# check in which index number the given command i is indexed
num = self.command.split()[1:].index(i)
# print index number as integer in array of command .split()
print(num)
# pop the command out of the array, So there will only be a string IF given key:value is 1 argument
doc = self.command.split().pop(num)
# get Key and Value of the String
k, v = doc.split(":")[0], doc.split(":", 1)[1]
# cursor to get the url value fore the given filter
urls = db.find({"target_{}".format(k):v})
# loop over the given documents to get the url value of each document
for url in urls:
# firstly print the gotten url
print(" [!] Found URL and Queue: {}".format(url["target_url"]))
# send directly to the crawler to crawl the URL
# self.crawl(url=url["target_url"])
self._queue_crawler.queue.append(url["target_url"])
print(" ... ... appended URL to the Queue")
continue
# else
# check if argument is a Key:Value
elif re.search(":", i):
# argument is a Key:Value
# print(i)
# create a Key and Value of the given argument
k, v = i.split(":")[0], i.split(":", 1)[1]
# print(" Printing the Key: {}".format(k))
# print(" Printing the Value: {}".format(v))
found = db.find({"target_{}".format(k):v})
for i in found:
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"], i["target_note"]])
print("\n{}\n".format(table))
continue
print("Unknown command")
else:
# there are None arguments given
self.sprint("No arguments are given!", mode=self.modus)
try:
self.sprint("\r\n")
docs = db.find()
for target in docs:
table.add_row([target["target_pinned"], target["_id"], target["target_name"], target["target_ip"], target["target_url"], target["target_note"]])
except Exception as e:
print(str(e))
print("\n{}\n".format(table))
net_total = total_net(netsalarylist)
roundednettotal = round(net_total, 2)
sal_total = total_sal(salaries)
roundedsaltotal = round(sal_total, 2)
ret_total = total_ret(retdollarlist)
roundedrettotal = round(ret_total, 2)
#rounding the lists themselves to two decimal places
roundedretdollarlist = [round(x, 2) for x in retdollarlist]
roundednetsalarylist = [round(x, 2) for x in netsalarylist]
roundedsalarieslist = [round(x, 2) for x in salaries]
#defining space as a place holder to make room in the pretty table
space = ' '
#adding the information to the pretty table column by column
x.add_column("Employee Code", employee_code)
x.add_column("Salary($)", roundedsalarieslist)
x.add_column("Retirement($)", roundedretdollarlist)
x.add_column("Net Salary($)", roundednetsalarylist)
#adding a blank row to created space in the pretty table between the values and the totals
x.add_row([" ", space, space, space])
#adding the last total row to the pretty table
x.add_row(["Total($):", roundednettotal, roundedrettotal, roundedsaltotal])
#printing the pretty table with the title "Title Boxing Franchise"
print(x.get_string(title="Title Boxing Franchise"))
#printing the employee with the largest salary
print("Note! Employee: %s has the largest salary!" % maxemployee)
from veryprettytable import VeryPrettyTable table = VeryPrettyTable(["animal", "ferocity"]) table.add_row(["wolverine", 100]) table.add_row(["grizzly", 87]) table.add_row(["cat", -1]) table.add_row(["dolphin", 63]) print(table)
def remove(self):
"""[ REM ] Remove from Hitlist"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# remember all objects by _id
object_id = []
# pre-defined find filter
# this is meant to find the requested records easier from the database
rem_filter = {}
# get command and check if there are multiple arguments minus self.command[0]
if self.command.split()[1:]:
# loop over the arguments and check if there is a Key:Value
for a in self.command.split()[1:]:
print(a)
print("-"*50)
# check k:v
if re.search(":", a):
k, v = a.split(":")[0], a.split(":", 1)[1]
print(k)
print(v)
print("-"*50)
# update the filter dictionary
rem_filter.update({"target_{}".format(k):v})
# get the Findings from the Database
findings = db.find(rem_filter)
print(findings)
print("-"*50)
# loop over every document you have found and display it in a Table on CLI
for i in findings:
print(i)
# append the object_id array by _id as key
object_id.append(i["_id"])
# add every document as row to the table
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"]])
# print the table
print("\n{}\n".format(table))
# loop for acknowledge
while True:
# ask if these documents are to be removed ?
acknowledge = input("Are these document(s) the ones you want to remove ?: Y/n ").lower()
if acknowledge == "y".lower() or acknowledge == "yes".lower():
# if yes loop over the array with all object _id(s)
for v in object_id:
doc = db.find_one({"_id":v})
archive.insert_one(doc)
db.remove(doc)
print("all done")
break
elif acknowledge == "n".lower() or acknowledge == "no".lower():
print("Shheeeuuuhh")
break
else:
print("Try again!")
continue
else:
self.sprint("You forgot to add options", mode=self.modus)
def edit(self):
"""[ EDIT ] Edit my a Target"""
""" Psuedo Code
Check If Something is filled
edit .... ?
check if there is a something filled
edit ip:123.123.321.22 ... ?
|_ IF yes get Records for PART1
|_ IF no return Err Message
check if is > sign
edit ip:123.123.321.22 > ...?
|_ IF yes AND no PART2 Err Message: You didnt give me anything to update it with;
|_ IF yes ... break it into PART1 (filter) and PART2 (update// $set)
# TARGET --shell
edit [ ...:... ...:... ] > [ ...:... ]
remove [ ...:... ]
|- return record(s)
|_ Are you sure?
add ip:... name:... url:...
find [ ...:... ...:... ]
# ADD
vpn [ ..:.. ]
server [ ..:.. ]
user [ ..:.. ]
# OPTIONS
modules <- return a list of available modules
modules:crawler <- return a list of available options crawler module
modules:crawler xpath:True base_only:True tag_forms:True
plugin
user
proxy
vpn
target
template
## CliDl
command category:type:name:additional limit filter filter -f function_name()
"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# pre-defined object variable to connect the gotten record and update it by _id
# there can be multiple _id(s) to be updated so therefore it will be a array
object_id = []
if self.command.split()[1:]:
# there is more then index[0]
# check if there is a > $gt
# if self.command.find(">"):
if re.search(">", self.command):
# there is a $gt sign and can be split up
splitted = self.command.split(">", 1)
# splitted PART1 contains the original command
# so only split and get the given filters
# Also how many filters are given ?
part1 = splitted[0].split()
# print how many filters are given MINUS the original command ?
# print(len(part1[1:]))
# print everything after the original command
# print(part1[1:])
# create a dictionary to fill with the complete query
filter_part1 = {}
# Loop over the filters if there are more the 1
if len(part1[1:]) >= 1:
# loop over every filter
for f in part1[1:]:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part1.update({"target_{}".format(k):v})
# save foundings into records variable
records = db.find(filter_part1)
# loop over findings
for i in records:
# update the object_id array
# which holds the given objects to be updated
object_id.append(i["_id"])
# and display the finding into the pre-defined tables
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"]])
# print the table for the user
# print("\n{}\n".format(table))
# print the given object id(s)
# print(object_id)
# we got the first part and record(s)
# now for the second part to update the requested record(s)
part2 = splitted[1].split()
# print the length of how many field we have to update
# print(len(part2))
# create a dictionary to fill with the complete update_query
filter_part2 = {}
# check if length update arguments are more then 1
if len(part2) >= 1:
# for every field update get key, value
for f in part2:
# get the key, value for each filter
k, v = f.split(":")[0], f.split(":", 1)[1]
# and update the dictionary to get the records IF exists
# WATCH the target_{} string
filter_part2.update({"target_{}".format(k):v})
# get the objects to be updated by object_id array
# get the fields from part2 which have to be updated
# do a findAndUpdate of the documents with $set
# do not update full documents, just update fields or add non-existing fields to the document
# update the records in the database as Many documents
db.update_many(filter_part1, {"$set": filter_part2})
else:
if re.search(":", self.command[1:]):
# get records for PART1 but there is no PART2!
print("getting records from database")
# pre-defined find filter for part1 without part2
# this is meant to find the requested records easier then directly updating the database
find_filter = {}
# get all filters minus indexx[0] the command
find = self.command.split()[1:]
# print the length of the filters
print(len(find))
# do a loop to get a complete key, value query to send to the database
for f in find:
# for each loop get the k, v and update it to the dictionary
k, v = f.split(":")[0], f.split(":", 1)[1]
# update the dictionary
find_filter.update({"target_{}".format(k):v})
# get records from the database
findings = db.find(find_filter)
# loop over the records to create a table at cli
for i in findings:
# and display the finding into the pre-defined tables
table.add_row([i["target_pinned"], i["_id"], i["target_name"], i["target_ip"], i["target_url"]])
# print the table for the user
print("\n{}\n".format(table))
else:
print("Your arguments to filter from the database are not correct. I need a Key:Value")
else:
self.sprint("You forgot to add options", mode=self.modus)
def remove(self):
"""[ REM ] Remove from Hitlist"""
import re
# pre-defined tables with headers IF we find any data
table = VeryPrettyTable(["pinned", "id", "name", "ip", "url"])
table.align = "l"
table.padding_width = 2
# remember all objects by _id
object_id = []
# pre-defined find filter
# this is meant to find the requested records easier from the database
rem_filter = {}
# get command and check if there are multiple arguments minus self.command[0]
if self.command.split()[1:]:
# loop over the arguments and check if there is a Key:Value
for a in self.command.split()[1:]:
print(a)
print("-" * 50)
# check k:v
if re.search(":", a):
k, v = a.split(":")[0], a.split(":", 1)[1]
print(k)
print(v)
print("-" * 50)
# update the filter dictionary
rem_filter.update({"target_{}".format(k): v})
# get the Findings from the Database
findings = db.find(rem_filter)
print(findings)
print("-" * 50)
# loop over every document you have found and display it in a Table on CLI
for i in findings:
print(i)
# append the object_id array by _id as key
object_id.append(i["_id"])
# add every document as row to the table
table.add_row([
i["target_pinned"], i["_id"], i["target_name"],
i["target_ip"], i["target_url"]
])
# print the table
print("\n{}\n".format(table))
# loop for acknowledge
while True:
# ask if these documents are to be removed ?
acknowledge = input(
"Are these document(s) the ones you want to remove ?: Y/n "
).lower()
if acknowledge == "y".lower() or acknowledge == "yes".lower():
# if yes loop over the array with all object _id(s)
for v in object_id:
doc = db.find_one({"_id": v})
archive.insert_one(doc)
db.remove(doc)
print("all done")
break
elif acknowledge == "n".lower() or acknowledge == "no".lower():
print("Shheeeuuuhh")
break
else:
print("Try again!")
continue
else:
self.sprint("You forgot to add options", mode=self.modus)
def sg_lookup(profile, region, debug):
"""
Summary.
Returns securitygroup user selection in given region
Args:
:profile (str): profile_name from local awscli configuration
:region (str): AWS region code
Returns:
securitygroup ID chosen by user
"""
padding = 2
field_max_width = 50
max_gn, max_desc = 10, 10 # starting value to find max length of a table field (chars)
x = VeryPrettyTable(border=True, header=True, padding_width=padding)
sgs = profile_securitygroups(profile, region)
for index, row in enumerate(sgs):
for k, v in row.items():
if len(v['GroupName']) > max_gn:
max_gn = len(v['GroupName'])
if len(v['Description']) > max_desc:
max_desc = len(v['GroupName'])
if debug:
print('max_gn = {}'.format(max_gn))
print('max_desc = {}'.format(max_desc))
# GroupName header
tabspaces_gn = int(max_gn / 4) - int(len('GroupName') / 2) + padding
tab_gn = '\t'.expandtabs(tabspaces_gn)
# Description header
tabspaces_desc = int(max_desc / 4) - int(len('Description') / 2) + padding
tab_desc = '\t'.expandtabs(tabspaces_desc)
x.field_names = [
bd + ' # ' + frame, bd + 'GroupId' + frame,
tab_gn + bd + 'GroupName' + frame, bd + 'VpcId' + frame,
tab_desc + bd + 'Description' + frame
]
# cell alignment
x.align = 'c'
x.align[tab_gn + bd + 'GroupName' + frame] = 'l'
x.align[tab_desc + bd + 'Description' + frame] = 'l'
# populate table
lookup = {}
for index, row in enumerate(sgs):
for k, v in row.items():
lookup[index] = k
x.add_row([
rst + userchoice_mapping(index) + '.' + frame, rst + k + frame,
rst + v['GroupName'][:field_max_width] + frame,
rst + v['VpcId'] + frame,
rst + v['Description'][:field_max_width] + frame
])
# Table showing selections
print(
f'\n\tSecurity Groups in region {bd + region + rst}\n'.expandtabs(30))
display_table(x)
return choose_resource(lookup)
def create(self, departure, arrival):
table = VeryPrettyTable()
table.field_names = ['Departure',' ','Arrival',' ']
for de,ar, in zip(departure,arrival):
table.add_row([de,departure[de],ar,arrival[ar]])
return table
r = csv.DictReader(fp)
res = {True: {True: 0, False: 0}, False: {True: 0, False: 0}}
for l in r:
expected = l["ground truth"].lower() in ["true", "1", "on"]
predicted = full_compare(l["name1"], l["name2"])
if predicted != expected:
print(predicted, expected, l["name1"], l["name2"])
res[predicted][expected] += 1
for predicted in [True, False]:
pt.add_row([
"Predicted positive" if predicted else "Predicted negative",
res[predicted][True],
res[predicted][False],
])
precision = res[True][True] / (res[True][True] + res[True][False])
recall = res[True][True] / (res[True][True] + res[False][True])
f1 = 2 * precision * recall / (precision + recall)
print(pt)
print("Precision: {:5.2f}".format(precision))
print("Recall: {:5.2f}".format(recall))
print("F1 score: {:5.2f}".format(f1))
else:
print(