def _status(self, final=False):
if not final:
new = (green(len(self._completed)),
white(len(self._running)),
yellow(len(self._queued)),
green('finished'),
white('running'),
yellow('queued'))
if hasattr(self, 'last_status') and new == self.last_status:
return
self.last_status = (green(len(self._completed)),
white(len(self._running)),
yellow(len(self._queued)),
green('finished'),
white('running'),
yellow('queued'))
print WHIPE, "[%s/%s/%s] %s, %s, %s" % new
else:
print "\n[ %s OK / %s ERROR ] in %s seconds" % (
green(self._num_of_jobs - self._errors, True),
red(self._errors),
time.time() - self._time_start)
if self._errors:
print red("Failures:", True)
for job in self._completed:
if job.exitcode != 0:
print red(job.name)
sys.stdout.flush()
def do_list(self, *args):
"""List all available sessions."""
# List with all active hosts.
hosts = []
# Length of the bigggest text string[card].
biggest = None
for index, connection in enumerate(self.__sessions):
# Append host card to list.
hosts.append('{} - {} - {}'.format(index, *connection[1:]))
if index == 0:
# The first bigger card.
biggest = len(hosts[0])
else:
if len(hosts[index]) > biggest:
# Set the new biggest card.
biggest = len(hosts[index])
else:
if self.server_started:
# Any socket has connected the server.
hosts.append('Any available connections!')
else:
# The server has not been started.
hosts.append('Server still not started. Use "listen" to start a remote server.')
biggest = len(hosts[0])
# Print the top stick.
print(utils.yellow('{}'.format('-' * (biggest + 4))))
# Print each host.
for host in hosts:
print(utils.blue('| {} |'.format(host + ' ' * (biggest - len(host)))))
# Print the bottom stick.
print(utils.yellow('{}'.format('-' * (biggest + 4))))
def open(self):
os.system("clear")
print blue("Welcome to %s" % green(self.name))
print blue("========================")
while True:
print red("How may I help you?")
print yellow("1. Buy a ticket\n"
" 2. Book a ticket by phone\n"
" 3. Book a ticket\n"
" 4. Cancel booking\n"
" 5. See how many money do you have\n"
" 6. Goodbye!\n")
option = raw_input(green("I want to: "))
if option == "1":
self.buy_ticket()
if option == "2":
self.book_ticket_by_phone()
if option == "3":
self.book_ticket()
if option == "4":
self.cancel_booking()
if option == "5":
self.money()
if option == "6":
break
def start(self):
os.system("clear")
while True:
print green("How can I help you, %s?" % self.user)
print yellow("1. Transfer some money\n"
" 2. Pay\n"
" 3. Retrieve\n"
" 4. Show balance\n"
" 5. Logout\n")
option = int(raw_input(blue("I want to: ")))
if option == 1 or option == 2:
to = raw_input(red("to: "))
money = raw_input(red("sum: "))
try:
print self.bank.transfer_from(self.user, to, money)
except ValueError as ve:
print red(ve)
elif option == 3:
money = raw_input(red("sum: "))
try:
if self.bank.transfer_from(self.user, self.name, money):
print green("Operation finished with success!\n")
else:
print red("Something went wrong...try again later\n")
except ValueError as ve:
print red(ve)
elif option == 4:
print self.bank.show_balance(self.user)
elif option == 5:
self.user = None
return
def gather_facts(host, inventory=None, user=None):
if inventory is None:
inventory = get_inventory()
# Gather facts
try:
# ... temporary playbook file
playbook_file = tempfile.NamedTemporaryFile()
playbook_file.write(SETUP_PLAYBOOK.format(host=host))
playbook_file.seek(0)
# ... run setup module
stats = ansible.callbacks.AggregateStats()
playbook = PlayBook(
playbook=playbook_file.name,
inventory=inventory,
callbacks=Callbacks(),
runner_callbacks=Callbacks(),
remote_user=user or C.DEFAULT_REMOTE_USER,
stats=stats,
)
results = playbook.run()
# ... notify the user of failures
for host, result in results.iteritems():
if result.get('unreachable') or result.get('failures'):
yellow('Unable to gather facts for host "{}"'.format(host))
finally:
playbook_file.close()
return playbook.SETUP_CACHE
def wrapper(*args, **kwargs):
custom_dir = join(dirname(dirname(__file__)), 'fabsetup_custom')
presetting_dir = join(dirname(dirname(__file__)), 'fabfile_data',
'presetting_fabsetup_custom')
if not isdir(custom_dir):
print(yellow('\n** ** Init ') + yellow('fabsetup_custom', bold=True) + yellow(' ** **\n'))
print(yellow('** Create files in dir fabsetup_custom **'))
local(flo('mkdir -p {custom_dir}'))
local(flo('cp -r --no-clobber {presetting_dir}/. {custom_dir}'))
else:
with quiet():
local(flo('cp -r --no-clobber {presetting_dir}/. {custom_dir}'))
if not isdir(join(custom_dir, '.git')):
print(yellow('\n** Git repo fabsetup_custom: init and first commit **'))
local(flo('cd {custom_dir} && git init'))
local(flo('cd {custom_dir} && git add .'))
local(flo('cd {custom_dir} && git commit -am "Initial commit"'))
print(yellow("** Done. Don't forget to create a backup of your fabsetup_custom repo **\n"))
print(yellow("** But do not make it public, it's custom **\n", bold=True))
else:
with quiet():
cmd = flo('cd {custom_dir} && git status --porcelain')
res = local(cmd, capture=True)
if res:
print(yellow('\n** git repo fabsetup_custom has uncommitted changes: **'))
print(cmd)
print(yellow(res, bold=True))
print(yellow("** Don't forget to commit them and make a backup of your repo **\n"))
return func(*args, **kwargs)
def print_diff(dbdict, data, removes=True):
"Print a (hopefully) human readable list of changes."
# TODO: needs work, especially on multiline properties,
# empty properties (should probably never be allowed but still)
# and probably more corner cases. Also the output format could
# use some tweaking.
try:
from collections import defaultdict
import jsonpatch
from jsonpointer import resolve_pointer, JsonPointerException
ops = defaultdict(int)
diff = jsonpatch.make_patch(dbdict, data)
for d in diff:
try:
ptr = " > ".join(decode_pointer(d["path"]))
if d["op"] == "replace":
print yellow("REPLACE:")
print yellow(ptr)
db_value = resolve_pointer(dbdict, d["path"])
print red(dump_value(db_value))
print green(dump_value(d["value"]))
ops["replace"] += 1
if d["op"] == "add":
print green("ADD:")
print green(ptr)
if d["value"]:
print green(dump_value(d["value"]))
ops["add"] += 1
if removes and d["op"] == "remove":
print red("REMOVE:")
print red(ptr)
value = resolve_pointer(dbdict, d["path"])
if value:
print red(dump_value(value))
ops["remove"] += 1
except JsonPointerException as e:
print " - Error parsing diff - report this!: %s" % e
# # The following output is a bit misleading, removing for now
# print "Total: %d operations (%d replace, %d add, %d remove)" % (
# sum(ops.values()), ops["replace"], ops["add"], ops["remove"])
return diff
except ImportError:
print >> sys.stderr, ("'jsonpatch' module not available - "
"no diff printouts for you! (Try -d instead.)")
def commit_containers(self, images, stop=True):
if stop:
self.containers_stop()
for k, v in self.containers.iteritems():
print(utils.yellow("commit {} to {}".format(v, images[k])))
docker_commit(v, images[k])
return self
def _spawn_action(self, rule, bindings):
"""Spawn a new action associated with the given rule, which just fired."""
# First, resolve the rule string
this_module = sys.modules[__name__]
for role_name, entity in bindings.items():
entity_name_with_escape_characters = entity.name.replace(
"'", "\\'").replace('"', '\\"')
setattr(this_module, "_" + role_name,
entity_name_with_escape_characters)
format_string = f"f'{rule.action_string}'".replace("{", "{_")
resolved_action_string = eval(format_string)[1:-1]
# Next, form an action entity that can be bound to the special "This" role
action_entity = Entity(name=resolved_action_string,
entity_type='Action')
if 'Action' not in self.domain:
self.domain['Action'] = []
self.domain['Action'].append(action_entity)
# Add the action into the bindings, using the special "This" role
bindings['This'] = action_entity
# Now, construct an Action object and save it to the list of actions
action_object = Action(action_name=rule.action_name,
action_string=resolved_action_string)
self.actions.append(action_object)
# Print out the action
if config.VERBOSITY >= 1:
print(utils.yellow(resolved_action_string))
return bindings
def transform_positive(self):
self.log.info("Start transforming %s from base %s to base %s" %
(self.number, self._from, self.to))
self.number = self.number.split('.')
decimal = self.number[0]
floating = self.number[1] if len(self.number) > 1 else ''
if self._from != 10:
self.log.warn(yellow("Initial base is not decimal!"))
now = time.time()
self.number = str(self.to_decimal(decimal, self._from))
if floating:
self.number += ".%s" % self.to_decimal(floating, self._from)
self.log.debug(blue("Transforming the number from base %s into decimal"
" took %f seconds" % (self._from,
round(time.time() - now, 10))))
self.log.info("Decimal representation of the number is %s" % self.number)
if self.to != 10:
now = time.time()
self.number = str(self.to_base(decimal, self.to))
if floating:
self.number += ".%s" % self._get_floating(floating, self.to)
self.log.debug(blue("Transforming the number from decimal into base %s"
" took %f seconds" % (self.to,
round(time.time() - now, 10))))
return self.number
def authenticate(config_parser):
if not config_parser.has_section('twitter'):
sys.stderr.write(utils.yellow(
"twitter is not configured; skipping...\n"
))
exit(0)
# The consumer keys can be found on your application's Details
# page located at https://dev.twitter.com/apps (under "OAuth settings")
consumer_key = config_parser.get('twitter', 'consumer_key')
consumer_secret = config_parser.get('twitter', 'consumer_secret')
# The access tokens can be found on your applications's Details
# page located at https://dev.twitter.com/apps (located
# under "Your access token")
access_token = config_parser.get('twitter', 'access_token')
access_token_secret = config_parser.get('twitter', 'access_token_secret')
# authenticate and use the api object
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
return api
def do_listen(self, *args):
"""Start a server on localhost to listen connections on given port."""
host = self._input(utils.yellow('[+] Enter the host IP > '))
port = self._input(utils.yellow('[+] Enter the port > '))
# Create a socket object from factory to accepts connections.
self.__socket = SocketFactory.server(listen=host, port=port)
# Inform user that server has started.
print(utils.green('[*] Started a remote server on {}:{}'.format(host, port)))
# Start to accept the incoming connections.
self.connection_acceptor.start()
# Set the server has started.
self.server_started = True
def on_skip(self, input_file, **kwargs):
source = kwargs.get('source', '')
self._skipped += 1
filename = self.format_filename(input_file.replace(source, ''))
print("%s %18s %30s" % (filename, yellow("SKIP"), "-"))
def open(self):
os.system("clear")
print blue("Welcome to ") + yellow(self.bank.name)
print blue("========================\n\n")
while True:
print green("Please choose one of the action:")
print red("1. Register\n"
" 2. Login\n\n")
option = int(raw_input(yellow("I want to: ")))
if option == 1:
self.register()
elif option == 2:
self.login()
else:
print red("I dont't understand you! Please repeat")
def debug_grid(grid, flashes):
print(f"┌{'─'*len(grid[0])}┐")
for i, row in enumerate(grid):
print("│" + "".join(
u.yellow(val % 10, True) if
(i, j) in flashes else u.purple(val % 10, True)
for j, val in enumerate(row)) + "│")
print(f"└{'─'*len(grid[0])}┘")
def build_images(self, reset=None):
self.reset(reset)
existing = self.get_real_images()
for image in self.images_names:
if image not in existing:
print(utils.yellow("Build image {}".format(image)))
docker_build(os.path.join(self.images_rootdir, 'images'), image, image)
return self
def docker_run(image, container, host=None, parameters=None):
cmd = 'docker run -d '
cmd += '--name {} '.format(container)
cmd += '-h {} '.format(host or container)
if parameters:
cmd += parameters + ' '
cmd += image
print(utils.yellow(cmd))
return not utils.command(cmd)
def install_packages(packages,
what_for='for a complete setup to work properly'):
'''Try to install .deb packages given by list.
Return True, if packages could be installed or are installed already, or if
they cannot be installed but the user gives feedback to continue.
Else return False.
'''
res = True
non_installed_packages = _non_installed(packages)
packages_str = ' '.join(non_installed_packages)
if non_installed_packages:
with quiet():
dpkg = _has_dpkg()
hint = ' (You may have to install them manually)'
do_install = False
go_on = True
if dpkg:
if _is_sudoer('Want to install dpkg packages'):
do_install = True
else:
do_install is False # cannot install anything
info = yellow(' '.join([
'This deb packages are missing to be installed',
flo("{what_for}: "),
', '.join(non_installed_packages),
]))
question = ' Continue anyway?'
go_on = query_yes_no(info + hint + question, default='no')
else:
# dpkg == False, unable to determine if packages are installed
do_install = False # cannot install anything
info = yellow(' '.join([
flo('Required {what_for}: '),
', '.join(non_installed_packages),
]))
go_on = query_yes_no(info + hint + ' Continue?', default='yes')
if not go_on:
sys.exit('Abort')
if do_install:
command = flo('sudo apt-get install {packages_str}')
res = run(command).return_code == 0
return res
def install_packages(packages,
what_for='for a complete setup to work properly'):
'''Try to install .deb packages given by list.
Return True, if packages could be installed or are installed already, or if
they cannot be installed but the user gives feedback to continue.
Else return False.
'''
res = True
non_installed_packages = _non_installed(packages)
packages_str = ' '.join(non_installed_packages)
if non_installed_packages:
with quiet():
dpkg = _has_dpkg()
hint = ' (You may have to install them manually)'
do_install = False
go_on = True
if dpkg:
if _is_sudoer('Want to install dpkg packages'):
do_install = True
else:
do_install is False # cannot install anything
info = yellow(' '.join([
'This deb packages are missing to be installed',
flo("{what_for}: "), ', '.join(non_installed_packages),
]))
question = ' Continue anyway?'
go_on = query_yes_no(info + hint + question, default='no')
else:
# dpkg == False, unable to determine if packages are installed
do_install = False # cannot install anything
info = yellow(' '.join([
flo('Required {what_for}: '),
', '.join(non_installed_packages),
]))
go_on = query_yes_no(info + hint + ' Continue?', default='yes')
if not go_on:
sys.exit('Abort')
if do_install:
command = flo('sudo apt-get install {packages_str}')
res = run(command).return_code == 0
return res
def __call__(self):
if _col_name is None:
all_df = self.df.empty()
self.selected = self.selected.union(self.kept)
for i in self.selected:
all_df.append(self.df.rows[i])
if verbose:
if len(all_df) == 0:
print(yellow("no row selected"))
else:
print("{} out of {} row(s) selected".format(len(all_df), len(self.df)))
return all_df
else:
col_ind = self.df.head.index(_col_name)
for row_ind in self.indices():
self.df.rows[row_ind][col_ind] = _value
if verbose:
if len(self.indices()) == 0:
print(yellow("no row updated"))
else:
print("{} out of {} row(s) updated".format(len(self.indices()), len(self.df)))
return self.df
def _training_info(self, total_rewards, average_speed, loop_speed,
update_stats):
last_ten = np.mean(total_rewards[-20:]) if len(total_rewards) else 0.
logger_msg = "Ran {0} steps, at {1:.3f} fps (avg {2:.3f} fps), last 20 episodes avg {3:.5f}"
lines = [
'',
]
lines.append(
logger_msg.format(self.global_step, loop_speed, average_speed,
last_ten))
lines.append(str(update_stats))
logging.info(yellow('\n'.join(lines)))
def _training_info(self, total_rewards, average_speed, loop_speed,
moving_averages, grad_norms):
last_ten = np.mean(total_rewards[-10:]) if len(total_rewards) else 0.
logger_msg = "Ran {} steps, at {} steps/s ({} steps/s avg), last 10 rewards avg {}"
lines = [
'',
]
lines.append(
logger_msg.format(self.global_step, loop_speed, average_speed,
last_ten))
lines.append(str(moving_averages))
lines.append('grad_norm: {}'.format(grad_norms))
logging.info(yellow('\n'.join(lines)))
def render_loop(self):
"""
Provides a base behavior for blender render loops
"""
self.initalize_imgnr_iter()
self.assert_before_loop()
if self.pre_loop_messages:
utils.print_boxed("Output information:", *self.pre_loop_messages)
if self.wait:
input("Press enter to start rendering\n")
utils.print_boxed("Rendering initialized")
len_iter = len(self.imgnr_iter)
interval_flag: bool = False # To make Pylance happy
for iternum, imgnr in enumerate(self.imgnr_iter):
self.setup_scene(imgnr, **self.setup_scene_kwargs)
imgfilepath = self.imgpath + str(imgnr)
print(f"Starting to render imgnr {imgnr}")
utils.render_and_save(imgfilepath)
print(f"Returned from rendering imgnr {imgnr}")
try:
assert_image_saved(imgfilepath, self.view_mode)
except FileNotFoundError as e:
print(e)
print("Breaking render loop")
interval_flag == False # Will enable callback after the loop
break
self.iter_callback(imgnr)
# Only commit in intervals
interval_flag = not imgnr % self.interval
if interval_flag:
self.interval_callback(imgnr)
print("Progress: ", utils.yellow(f"{iternum+1} / {len_iter}"))
# If loop exited without commiting remaining stuff
# This if test is kinda redundant, but idk man
if interval_flag == False:
self.interval_callback(imgnr)
self.end_callback()
def _is_sudoer(what_for=''):
'''Return True if current user is a sudoer, else False.
Should be called non-eager if sudo is wanted only.
'''
if env.get('nosudo', None) is None:
if what_for:
print(yellow(what_for))
with quiet():
# possible outputs:
# en: "Sorry, user winhost-tester may not run sudo on <hostname>"
# en: "sudo: a password is required" (=> is sudoer)
# de: "sudo: Ein Passwort ist notwendig" (=> is sudoer)
output = run('sudo -nv', capture=True)
env.nosudo = not (output.startswith('sudo: ') or output == '')
if env.nosudo:
print('Cannot execute sudo-commands')
return not env.nosudo
def _is_sudoer(what_for=''):
'''Return True if current user is a sudoer, else False.
Should be called non-eager if sudo is wanted only.
'''
if env.get('nosudo', None) is None:
if what_for:
print(yellow(what_for))
with quiet():
# possible outputs:
# en: "Sorry, user winhost-tester may not run sudo on <hostname>"
# en: "sudo: a password is required" (=> is sudoer)
# de: "sudo: Ein Passwort ist notwendig" (=> is sudoer)
output = run('sudo -nv', capture=True)
env.nosudo = not (output.startswith('sudo: ') or output == '')
if env.nosudo:
print('Cannot execute sudo-commands')
return not env.nosudo
def print(self, n=-1, highlight_rows=None):
if highlight_rows is None:
highlight_rows = []
elif type(highlight_rows) == int:
highlight_rows = [highlight_rows]
if len(self.rows) == 0:
print(green(join(self.head, " ")))
print("(empty)")
print()
else:
def get_col_width(lst):
return max(list(map(lambda x: len(str(x)), lst)))
col_width_list = list(map(lambda col: get_col_width(self[col]), self.head))
delta_list = []
for i in range(len(self.head)):
delta = col_width_list[i] - len(self.head[i])
delta_list.append(delta)
head = ""
for index, col_name in enumerate(self.head):
head += col_name
head += " " * max(delta_list[index] + 2, 2)
print(green(head))
i = 0
for r in self.rows:
if i == n:
break
else:
i += 1
_row = ""
for j in range(len(self.head)):
head_len = len(self.head[j] + " " * max(delta_list[j] + 2, 2))
space_num = head_len - len(str(r[j]))
_row += str(r[j]) + " " * space_num
if i - 1 in highlight_rows:
print(yellow(_row))
else:
print(_row)
print()
return self
def dn_cn_of_certificate_with_san(domain):
'''Return the Common Name (cn) from the Distinguished Name (dn) of the
certificate which contains the `domain` in its Subject Alternativ Name (san)
list.
Needs repo ~/.fabsetup-custom.
Return None if no certificate is configured with `domain` in SAN.
'''
cn_dn = None
from config import domain_groups
cns = [domains[0] for domains in domain_groups if domain in domains]
if cns:
if len(cns) > 1:
print_msg(
yellow(
flo('Several certificates are configured to '
'contain {domain} '
'(You should clean-up your config.py)\n')))
cn_dn = cns[0]
return cn_dn
def profile_leaks(shell, queries, count=1, rounds=1, supp_file=None):
report = {}
for name, query in queries.iteritems():
print("Analyzing leaks in query: %s" % query)
# Apply count (optionally run the query several times).
summary = check_leaks(shell, query, count, supp_file)
display = []
for key in summary:
output = summary[key]
if output is not None and output[0] != "0":
# Add some fun colored output if leaking.
if key == "definitely":
output = utils.red(output)
report[name] = "LEAKING"
if key == "indirectly":
output = utils.yellow(output)
report[name] = "WARNING"
else:
report[name] = "SAFE"
display.append("%s: %s" % (key, output))
print(" %s" % "; ".join(display))
return report
def dn_cn_of_certificate_with_san(domain):
'''Return the Common Name (cn) from the Distinguished Name (dn) of the
certificate which contains the `domain` in its Subject Alternativ Name (san)
list.
Needs repo ~/.fabsetup-custom.
Return None if no certificate is configured with `domain` in SAN.
'''
cn_dn = None
from config import domain_groups
cns = [domains[0]
for domains
in domain_groups
if domain in domains]
if cns:
if len(cns) > 1:
print_msg(yellow(flo('Several certificates are configured to '
'contain {domain} '
'(You should clean-up your config.py)\n')))
cn_dn = cns[0]
return cn_dn
def __init__(self, prompt, intro=None, max_connections=5):
"""
Constructor of ShellBox class.
:param prompt: The shell prompt.
:param max_connections: The maximum of connections the server accepts.
:return: The ShellBox object.
"""
cmd.Cmd.__init__(self)
self.prompt = utils.yellow(prompt.strip() + '> ')
self.server_started = False
self.intro = intro
self.__socket = None
self.__sessions = []
# Create the thread to accept connections every 2 seconds.
self.connection_acceptor = Pooler(2, self._accept_connection)
# Create the thread to resolve connections every 5 seconds.
self.connection_resolver = Pooler(5, self._resolve_connections)
# Starts the manager.
self.connection_resolver.start()
def main(
data_labels_dir: str,
wait: bool,
view_mode: str,
*,
imgnrs: Optional[Iterable[int]] = None,
predfile: Optional[str] = None,
imgrange: Optional[Tuple[int, int]] = None,
):
n_assert_arg = (imgnrs, predfile, imgrange).count(None)
if n_assert_arg == 0:
raise AssertionError(
"Expected one of ('imgnrs', 'predfile', 'imgrange') to be specified, but got none"
)
elif n_assert_arg != 2:
raise AssertionError(
"Expected ONLY one of ('imgnrs', 'predfile', 'imgrange') to be specified,"
f" but got {3-n_assert_arg}")
check_generated_datadir(data_labels_dir, cng.BBOX_DB_FILE)
check_or_create_datadir(cng.LABELCHECK_DATA_DIR, cng.LABELCHECK_DB_FILE)
loader = recon.Sceneloader(data_labels_dir)
imgpath = str(dirpath / cng.LABELCHECK_DATA_DIR /
cng.LABELCHECK_IMAGE_DIR / cng.LABELCHECK_IMAGE_NAME)
con = db.connect(
str(dirpath / cng.LABELCHECK_DATA_DIR / cng.LABELCHECK_DB_FILE))
cursor = con.cursor()
output_info = []
labeliter: Union[Iterable[int], range]
if imgnrs:
# renderloop_imgnrs(imgnrs, imgpath, wait, view_mode, con, cursor)
labeliter = imgnrs
output_info.append(f"Rendering given imgnrs")
elif predfile:
renderloop_predfile(predfile)
elif imgrange:
assert len(imgrange) == 2
assert isinstance(imgrange[0], int)
assert isinstance(imgrange[1], int)
# renderloop_imgrange(imgrange, imgpath, wait, view_mode, con, cursor)
labeliter = range(imgrange[0], imgrange[1])
output_info.append(f"Rendering given imgrange: {labeliter}")
utils.print_boxed(
"Output information:",
f"Imgs to render: {len(labeliter)}",
*output_info,
f"Saves images at: {os.path.join(cng.LABELCHECK_DATA_DIR, cng.LABELCHECK_IMAGE_DIR)}",
f"Sqlite3 DB at: {os.path.join(cng.LABELCHECK_DATA_DIR, cng.LABELCHECK_DB_FILE)}",
)
if wait:
input("Press enter to start rendering\n")
utils.print_boxed("Rendering initalized")
commit_flag: bool = False # To make Pylance happy
for i, nr in enumerate(labeliter):
print(f"Reacreating imgnr: {nr}")
loader.clear()
loader.reconstruct_scene_from_db(nr)
imgfilepath = imgpath + str(nr)
print(f"Starting to render imgnr {nr}")
utils.render_and_save(imgfilepath)
print(f"Returned from rendering imgnr {nr}")
try:
mainfile.assert_image_saved(imgfilepath, view_mode)
except FileNotFoundError as e:
print(e)
print("Breaking render loop")
commit_flag == False # Will enable commit after the loop
break
# Only commit in intervals
commit_flag = not i % cng.COMMIT_INTERVAL
if commit_flag:
con.commit()
cursor.execute(
f"INSERT OR REPLACE INTO {cng.LABELCHECK_DB_TABLE} VALUES ({nr})")
print("Progress: ", utils.yellow(f"{i+1} / {len(labeliter)}"))
# If loop exited without commiting remaining stuff
if commit_flag == False:
con.commit()
import os
import sys
import datetime
import csv
# third party
import stackexchange
# local
import utils
# authenticate to the API
config_parser = utils.get_config_parser()
if not config_parser.has_section('stackoverflow'):
sys.stderr.write(utils.yellow(
"stackoverflow is not configured; skipping...\n"
))
exit(0)
# authenticate. if this gives problems exceeding request limits,
# you'll need to obtain an API key
# https://github.com/lucjon/Py-StackExchange/tree/updating-2.0#api-keys
so = stackexchange.Site(stackexchange.StackOverflow)
so.impose_throttling = True
user = so.user(config_parser.get('stackoverflow', 'user_id'))
timeline = user.timeline.fetch()
# timeline = user.timeline.fetch( # i think this is the format
# fromdate=datetime.datetime.now(),
# todate=datetime.datetime.now()-datetime.timedelta(days=365),
# )
def containers_stop(self):
for container in self.get_real_containers():
print(utils.yellow("Stop container {}".format(container)))
container_stop(container)
return self
def containers_delete(self):
for container in self.get_real_containers(True):
print(utils.yellow("Delete container {}".format(container)))
container_delete(container)
return self
def docker_build(context, image, tag=None):
cmd = 'docker build -f {}/Dockerfile{} .'.format(image, ' -t {}'.format(tag) if tag else '')
print(utils.yellow(cmd))
with utils.cd(context):
return utils.Command(cmd, show='Build: ').returncode
def debug_explored(dict_explored):
for dice, status in dict_explored.items():
u.yellow("-".join(str(d) for d in dice))
positions, scores = status
print(f"positions :", positions)
print(f"scores :", scores)
def wrapper(*args, **kwargs):
if not os.path.exists(FABSETUP_CUSTOM_DIR):
msg = '''\
Git repository ~/.fabsetup-custom with configurations does not exist.
This configs are required to use fabsetup.
Clone it if you already have your own fabsetup-custom repository:
git clone <user>@<hostname>:/path/to/fabsetup-custom.git ~/.fabetup-custom
Else, initialize a new repository.
Init a new repository `~/.fabsetup-custom`?'''
if not query_yes_no(msg, default='yes'):
sys.exit('abort')
custom_dir = FABSETUP_CUSTOM_DIR
presetting_dir = join(FABFILE_DATA_DIR,
'presetting-fabsetup-custom')
if not isdir(custom_dir):
print(
yellow('\n** ** Init ') +
yellow('~/.fabsetup-custom', bold=True) +
yellow(' ** **\n'))
print(yellow(flo('** Create files in dir {custom_dir} **')))
local(flo('mkdir -p {custom_dir}'))
local(
flo('cp -r --no-clobber {presetting_dir}/. {custom_dir}'))
import_fabsetup_custom(globals())
else:
with quiet():
local(
flo('cp -r --no-clobber {presetting_dir}/. {custom_dir}'
))
if not isdir(join(custom_dir, '.git')):
print(
yellow('\n** Git repo ~/.fabsetup-custom: '
'init and first commit **'))
local(flo('cd {custom_dir} && git init'))
local(flo('cd {custom_dir} && git add .'))
local(
flo('cd {custom_dir} && git commit -am "Initial commit"'))
print(
yellow("** Done. Don't forget to create a backup of your "
'~/.fabsetup-custom repo **\n'))
print(
yellow("** But do not make it public, it's custom **\n",
bold=True))
else:
with quiet():
cmd = flo('cd {custom_dir} && git status --porcelain')
res = local(cmd, capture=True)
if res:
print(
yellow('\n** git repo ') +
magenta('~/.fabsetup-custom ') +
yellow('has uncommitted changes: **'))
print(cmd)
print(yellow(res, bold=True))
print(
yellow("** Don't forget to commit them and make a "
"backup of your repo **\n"))
return func(*args, **kwargs)
def main(
n: int,
bbox_modes: Sequence[str],
wait: bool,
stdbboxcam: bpy.types.Object,
view_mode: str,
nspawnrange: Tuple[int, int],
) -> None:
"""Main function for generating data with Blender
Parameters
----------
n : int
Number of images to render
bbox_modes : Sequence[str]
Sequence of modes to save bounding boxes, given as strings. Available: xyz cps full std
wait : bool
Wait for user input before starting rendering process
stdbboxcam : bpy.types.Object
Camera object that is to be used for extracting 2D bounding boxes
view_mode : str
Essentially which cameras to render from,
"""
check_or_create_datadir(cng.GENERATED_DATA_DIR, cng.BBOX_DB_FILE)
scene = gen.Scenemaker()
gen.create_metadata(scene)
con = db.connect(str(dirpath / cng.GENERATED_DATA_DIR / cng.BBOX_DB_FILE))
cursor = con.cursor()
datavisitor = gen.DatadumpVisitor(stdbboxcam=stdbboxcam, bbox_modes=bbox_modes, cursor=cursor)
maxids = [
gen.get_max_imgid(cursor, table) for table in (cng.BBOX_DB_TABLE_CPS, cng.BBOX_DB_TABLE_XYZ)
]
maxid = max(maxids)
if maxid < 0:
maxid = 0
else:
maxid += 1
imgpath = str(dirpath / cng.GENERATED_DATA_DIR / cng.IMAGE_DIR / cng.IMAGE_NAME)
utils.print_boxed(
"Output information:",
f"Imgs to render: {n}",
f"Starting at index: {maxid}",
f"Ends at index: {maxid+n-1}",
f"Saves images at: {os.path.join(cng.GENERATED_DATA_DIR, cng.IMAGE_DIR)}",
f"Sqlite3 DB at: {os.path.join(cng.GENERATED_DATA_DIR, cng.BBOX_DB_FILE)}",
f"Metadata at: {os.path.join(cng.GENERATED_DATA_DIR, cng.METADATA_FILE)}",
f"bbox_modes: {bbox_modes}",
)
if wait:
input("Press enter to start rendering\n")
def commit():
utils.print_boxed(f"Commited to {cng.BBOX_DB_FILE}")
con.commit()
utils.print_boxed("Rendering initialized")
commit_flag: bool = False # To make Pylance happy
for iternum, i in enumerate(range(maxid, maxid + n)):
scene.clear()
scene.generate_scene(np.random.randint(*nspawnrange))
imgfilepath = imgpath + str(i)
print(f"Starting to render imgnr {i}")
utils.render_and_save(imgfilepath)
print(f"Returned from rendering imgnr {i}")
try:
assert_image_saved(imgfilepath, view_mode)
except FileNotFoundError as e:
print(e)
print("Breaking render loop")
commit_flag == False # Will enable commit after the loop
break
datavisitor.set_n(i)
datavisitor.visit(scene)
# Only commit in intervals
commit_flag = not i % cng.COMMIT_INTERVAL
if commit_flag:
commit()
print("Progress: ", utils.yellow(f"{iternum+1} / {n}"))
# If loop exited without commiting remaining stuff
if commit_flag == False:
commit()
con.close()
def counts(self):
if self.jsonify:
utils.to_json(self.stats["counts"])
else:
header, rows = ["#", "Total", "TryHackMe", "HackTheBox", "VulnHub", "OSCPlike"], []
rows.append("___".join([x for x in [
"%s" % (utils.green("Total")),
"%s/%s (%s)" % (utils.green(self.stats["counts"]["ownedtotal"]), utils.green(self.stats["counts"]["totaltotal"]), utils.green("%.2f%%" % (self.stats["counts"]["pertotal"]))),
"%s/%s (%s)" % (utils.green(self.stats["counts"]["ownedthm"]), utils.green(self.stats["counts"]["totalthm"]), utils.green("%.2f%%" % (self.stats["counts"]["perthm"]))),
"%s/%s (%s)" % (utils.green(self.stats["counts"]["ownedhtb"]), utils.green(self.stats["counts"]["totalhtb"]), utils.green("%.2f%%" % (self.stats["counts"]["perhtb"]))),
"%s/%s (%s)" % (utils.green(self.stats["counts"]["ownedvh"]), utils.green(self.stats["counts"]["totalvh"]), utils.green("%.2f%%" % (self.stats["counts"]["pervh"]))),
"%s/%s (%s)" % (utils.red(self.stats["counts"]["ownedoscplike"]), utils.red(self.stats["counts"]["totaloscplike"]), utils.red("%.2f%%" % (self.stats["counts"]["peroscplike"]))),
]]))
rows.append("___".join([str(x) for x in [
utils.yellow("Windows"),
"%s/%s (%s)" % (utils.yellow(self.stats["counts"]["ownedwindows"]), utils.yellow(self.stats["counts"]["totalwindows"]), utils.yellow("%.2f%%" % (self.stats["counts"]["perwindows"]))),
"%s/%s (%s)" % (utils.yellow(self.stats["counts"]["ownedthmwindows"]), utils.yellow(self.stats["counts"]["thmwindows"]), utils.yellow("%.2f%%" % (self.stats["counts"]["perthmwindows"]))),
"%s/%s (%s)" % (utils.yellow(self.stats["counts"]["ownedhtbwindows"]), utils.yellow(self.stats["counts"]["htbwindows"]), utils.yellow("%.2f%%" % (self.stats["counts"]["perhtbwindows"]))),
"%s/%s (%s)" % (utils.yellow(self.stats["counts"]["ownedvhwindows"]), utils.yellow(self.stats["counts"]["vhwindows"]), utils.yellow("%.2f%%" % (self.stats["counts"]["pervhwindows"]))),
"%s/%s (%s)" % (utils.yellow(self.stats["counts"]["ownedoscplikewindows"]), utils.yellow(self.stats["counts"]["oscplikewindows"]), utils.yellow("%.2f%%" % (self.stats["counts"]["peroscplikewindows"]))),
]]))
rows.append("___".join([str(x) for x in [
utils.magenta("*nix"),
"%s/%s (%s)" % (utils.magenta(self.stats["counts"]["ownednix"]), utils.magenta(self.stats["counts"]["totalnix"]), utils.magenta("%.2f%%" % (self.stats["counts"]["pernix"]))),
"%s/%s (%s)" % (utils.magenta(self.stats["counts"]["ownedthmnix"]), utils.magenta(self.stats["counts"]["thmnix"]), utils.magenta("%.2f%%" % (self.stats["counts"]["perthmnix"]))),
"%s/%s (%s)" % (utils.magenta(self.stats["counts"]["ownedhtbnix"]), utils.magenta(self.stats["counts"]["htbnix"]), utils.magenta("%.2f%%" % (self.stats["counts"]["perhtbnix"]))),
"%s/%s (%s)" % (utils.magenta(self.stats["counts"]["ownedvhnix"]), utils.magenta(self.stats["counts"]["vhnix"]), utils.magenta("%.2f%%" % (self.stats["counts"]["pervhnix"]))),
"%s/%s (%s)" % (utils.magenta(self.stats["counts"]["ownedoscplikenix"]), utils.magenta(self.stats["counts"]["oscplikenix"]), utils.magenta("%.2f%%" % (self.stats["counts"]["peroscplikenix"]))),
]]))
rows.append("___".join([str(x) for x in [
utils.red("OSCPlike"),
"%s/%s (%s)" % (utils.red(self.stats["counts"]["ownedoscplike"]), utils.red(self.stats["counts"]["totaloscplike"]), utils.red("%.2f%%" % (self.stats["counts"]["peroscplike"]))),
"%s/%s (%s)" % (utils.red(self.stats["counts"]["ownedthmoscplike"]), utils.red(self.stats["counts"]["thmoscplike"]), utils.red("%.2f%%" % (self.stats["counts"]["perthmoscplike"]))),
"%s/%s (%s)" % (utils.red(self.stats["counts"]["ownedhtboscplike"]), utils.red(self.stats["counts"]["htboscplike"]), utils.red("%.2f%%" % (self.stats["counts"]["perhtboscplike"]))),
"%s/%s (%s)" % (utils.red(self.stats["counts"]["ownedvhoscplike"]), utils.red(self.stats["counts"]["vhoscplike"]), utils.red("%.2f%%" % (self.stats["counts"]["pervhoscplike"]))),
utils.red(""),
]]))
aligndict = {
"#": "r",
"Total": "l",
"TryHackMe": "l",
"HackTheBox": "l",
"VulnHub": "l",
"OSCPlike": "l",
}
utils.to_table(header, rows, delim="___", aligndict=aligndict)
def docker_build(image, tag=None, context=None):
cmd = 'docker build -f {}/Dockerfile -t {} .'.format(image, tag or image)
print(utils.yellow(cmd))
with utils.cd(context or os.path.join(ROOTDIR, 'images')):
return not utils.Command(cmd, show='Build: ').returncode
# standard library
import os
import sys
import csv
# third party
from github import Github
# local
import utils
# authenticate to the API
config_parser = utils.get_config_parser()
if not config_parser.has_section('github'):
sys.stderr.write(utils.yellow(
"github is not configured; skipping...\n"
))
exit(0)
api = Github(
config_parser.get("github", "username"),
config_parser.get("github", "password"),
)
# get all of the events for the authenticated user
writer = csv.writer(sys.stdout)
writer.writerow(['datetime', 'event'])
user = api.get_user(config_parser.get("github", "username"))
for event in user.get_events():
writer.writerow([event.created_at, event.type])
sys.stdout.flush()
import ConfigParser
import sys
import csv
import datetime
# third party
from fabric.api import *
# local
import utils
# get the necessary information from the config parser
config_parser = utils.get_config_parser()
if not config_parser.has_section('mercurial'):
sys.stderr.write(utils.yellow(
"mercurial is not configured; skipping...\n"
))
exit(0)
email = config_parser.get('mercurial', 'email')
host_directories = config_parser.get('mercurial', 'host_directories')
# configure fabric to use ssh configuration
# http://docs.fabfile.org/en/latest/usage/execution.html#leveraging-native-ssh-config-files
env.use_ssh_config = True
env.ssh_config_path = config_parser.get('mercurial', 'ssh_config_path')
# instantiate the writer object
writer = csv.writer(sys.stdout)
writer.writerow(['datetime', 'host', 'repo'])
# iterate over all comma-separated 'host:directory' pairs
def place_neighbor_of_locked_tile(locked: Tile, neighbor: Tile):
common_border = tuple_intersection(locked.borders, neighbor.borders)
if common_border:
direction_for_locked = locked.borders.index(common_border)
direction_for_neighbor = neighbor.borders.index(common_border)
diff = (direction_for_neighbor - direction_for_locked) % 4
if diff == 0:
neighbor.flip((direction_for_locked + 1) % 4)
elif diff == 1:
if direction_for_locked in (RIGHT, LEFT):
neighbor.rotate_clockwise(1)
elif direction_for_locked in (BOTTOM, TOP):
neighbor.rotate_clockwise(1) # direct it to top
neighbor.flip(TOP)
else:
u.yellow("todo 1")
elif diff == 2:
pass # nothing to do, they already are well oriented
elif diff == 3:
if direction_for_locked in (TOP, BOTTOM):
neighbor.rotate_clockwise(3)
elif direction_for_locked in (RIGHT, LEFT):
neighbor.rotate_clockwise(1)
neighbor.flip(TOP)
else:
u.yellow("todo 3")
if (
neighbor.borders[(direction_for_locked + 2) % 4]
!= locked.borders[direction_for_locked]
):
u.red("WRONG")
else:
locked.addNeighbor(direction_for_locked, neighbor)
neighbor.addNeighbor((direction_for_locked + 2) % 4, locked)
neighbor.locked = True
else:
common_border = tuple_intersection(
locked.borders, tuple(border[::-1] for border in neighbor.borders)
)
direction_for_locked = locked.borders.index(common_border)
direction_for_neighbor = neighbor.borders.index(common_border[::-1])
diff = (direction_for_neighbor - direction_for_locked) % 4
if neighbor.locked:
print(
f"{u.RED}SOMETHING SMELLS BAD FOR {neighbor.id} vs {locked.id}{u.NORMAL}"
)
if diff == 0:
neighbor.rotate_clockwise(2)
elif diff == 1:
if direction_for_locked in (BOTTOM, TOP):
neighbor.rotate_clockwise(1)
elif direction_for_locked in (RIGHT, LEFT):
neighbor.rotate_clockwise(1)
neighbor.flip(RIGHT)
else:
u.yellow("todo *1*")
elif diff == 2:
neighbor.flip(direction_for_locked)
elif diff == 3:
if direction_for_locked in (TOP, BOTTOM):
neighbor.rotate_clockwise(1)
neighbor.flip(LEFT)
elif direction_for_locked in (LEFT, RIGHT):
neighbor.rotate_clockwise(3)
else:
u.yellow("todo *3*")
if (
neighbor.borders[(direction_for_locked + 2) % 4]
!= locked.borders[direction_for_locked]
):
u.red(f"WRONG for {locked.id}/{neighbor.id}")
else:
locked.addNeighbor(direction_for_locked, neighbor)
neighbor.addNeighbor((direction_for_locked + 2) % 4, locked)
neighbor.locked = True
def wrapper(*args, **kwargs):
if not os.path.exists(FABSETUP_CUSTOM_DIR):
msg = '''\
Git repository ~/.fabsetup-custom with configurations does not exist.
This configs are required to use fabsetup.
Clone it if you already have your own fabsetup-custom repository:
git clone <user>@<hostname>:/path/to/fabsetup-custom.git ~/.fabetup-custom
Else, initialize a new repository.
Init a new repository `~/.fabsetup-custom`?'''
if not query_yes_no(msg, default='yes'):
sys.exit('abort')
custom_dir = FABSETUP_CUSTOM_DIR
presetting_dir = join(FABFILE_DATA_DIR,
'presetting-fabsetup-custom')
if not isdir(custom_dir):
print(yellow('\n** ** Init ') +
yellow('~/.fabsetup-custom', bold=True) +
yellow(' ** **\n'))
print(yellow(flo('** Create files in dir {custom_dir} **')))
local(flo('mkdir -p {custom_dir}'))
local(flo('cp -r --no-clobber {presetting_dir}/. {custom_dir}'))
import_fabsetup_custom(globals())
else:
with quiet():
local(flo(
'cp -r --no-clobber {presetting_dir}/. {custom_dir}'))
if not isdir(join(custom_dir, '.git')):
print(yellow(
'\n** Git repo ~/.fabsetup-custom: '
'init and first commit **'))
local(flo('cd {custom_dir} && git init'))
local(flo('cd {custom_dir} && git add .'))
local(flo('cd {custom_dir} && git commit -am "Initial commit"'))
print(yellow("** Done. Don't forget to create a backup of your "
'~/.fabsetup-custom repo **\n'))
print(yellow("** But do not make it public, it's custom **\n",
bold=True))
else:
with quiet():
cmd = flo('cd {custom_dir} && git status --porcelain')
res = local(cmd, capture=True)
if res:
print(yellow('\n** git repo ') +
magenta('~/.fabsetup-custom ') +
yellow('has uncommitted changes: **'))
print(cmd)
print(yellow(res, bold=True))
print(yellow(
"** Don't forget to commit them and make a "
"backup of your repo **\n"))
return func(*args, **kwargs)
