def write2(self, fd: TextIO):
if not fd.writable():
raise Exception(f"{fd.name} is not writable")
if self.__gc is None:
raise Exception(f"run mlps2gc before invoke this method")
for pwd, mlp, appearance, rank, cracked, cracked_ratio in tqdm(
self.__gc, desc="Saving: "):
fd.write(
f"{pwd}\t{mlp:.8f}\t{appearance}\t{rank}\t{cracked}\t{cracked_ratio:5.2f}\n"
)
self.__gc = None
pass
def cleaning(dataset: str, encoding: str, output: TextIO, valid_chr_re):
fd_dataset = open(dataset, encoding=encoding)
if not (fd_dataset.readable() and output.writable()):
print(
f"{dataset} show be readable and {output.name} should be writable",
file=sys.stderr)
sys.exit(-1)
line = ""
try:
for _line in fd_dataset:
line = _line.strip("\r\n")
if valid_chr_re.search(line):
output.write(f"{line}\n")
except UnicodeDecodeError as e:
print(f"{line}| {e.reason}", file=sys.stderr)
sys.exit(-1)
def save(uniq_lines: Set[str], save2: TextIO, order: str):
if not save2.writable() or save2.closed:
print(f"{save2.name} can not be used to write")
sys.exit(-1)
if order == 'order':
uniq_lines = sorted(uniq_lines, reverse=False)
elif order == 'reverse':
uniq_lines = sorted(uniq_lines, reverse=True)
elif order == 'random':
uniq_lines = list(uniq_lines)
shuffle(uniq_lines)
else:
sys.stderr.write(f"Unknown method: {order}")
sys.exit(-1)
for _l in uniq_lines:
save2.write(f"{_l}\n")
save2.flush()
def wrapper(dataset: TextIO, save: TextIO):
if not dataset.seekable():
raise Exception("Not seekable, do not use stdin please")
if not save.writable():
raise Exception(f"{save.name} Not writable")
total_size, len_dict = len_dist(dataset, False)
dataset.seek(0)
total_chr, chr_dict, cls_dict = chr_dist(dataset, True)
json.dump(
{
"#len": total_size,
"len": len_dict,
"#chr": total_chr,
"chr": chr_dict,
"cls": cls_dict
},
save,
indent=2)
save.close()
def v41seg(training: TextIO, test_set: TextIO, save2: TextIO) -> None:
if not save2.writable():
raise Exception(f"{save2.name} is not writable")
multiword_detector = MultiWordDetector()
for password in training:
password = password.strip("\r\n")
multiword_detector.train(password)
training.close()
pwd_counter = defaultdict(int)
for password in test_set:
password = password.strip("\r\n")
pwd_counter[password] += 1
test_set.close()
for password, num in pwd_counter.items():
section_list, found_walks = detect_keyboard_walk(password)
_ = year_detection(section_list)
"""
Note that there is a bug in context_sensitive_detection
I have fixed that and add a test case in unit_tests folder
"""
_ = context_sensitive_detection(section_list)
_, _ = alpha_detection(section_list, multiword_detector)
_ = digit_detection(section_list)
_ = other_detection(section_list)
info = [password, f"{num}"]
npass = ""
for sec, tag in section_list:
npass += sec
info.append(sec)
info.append(tag)
if password.lower() != npass.lower():
print(password)
print(section_list)
raise Exception("neq")
print("\t".join(info), end="\n", file=save2)
pass
def l33tseg(training: TextIO, test_set: TextIO, save2: TextIO) -> None:
if not save2.writable():
raise Exception(f"{save2.name} is not writable")
multiword_detector = MyMultiWordDetector()
for password in training:
password = password.strip("\r\n")
multiword_detector.train(password)
training.close()
l33t_detector = AsciiL33tDetector(multiword_detector)
l33t_detector.init_l33t(training.name, "ascii")
pwd_counter = defaultdict(int)
for password in test_set:
password = password.strip("\r\n")
pwd_counter[password] += 1
test_set.close()
for password, num in pwd_counter.items():
section_list = [(password, None)]
_ = year_detection(section_list)
section_list, _ = detect_context_sections(section_list)
section_list, _, _ = l33t_detector.parse_sections(section_list)
section_list, _, _, _, _ = multiword_detector.parse_sections(
section_list)
info = [password, f"{num}"]
npass = ""
for sec, tag in section_list:
npass += sec
info.append(sec)
info.append(tag)
if password.lower() != npass.lower():
# Note that we'll not lower X
# therefore, the best way is to compare password.lower
# with npass.lower
print(password)
print(section_list)
raise Exception("neq")
print("\t".join(info), end="\n", file=save2)
pass
def wrapper(targets: TextIO, scored_files: List[TextIO], splitter: str,
save2: TextIO):
if not save2.writable():
print(f"{save2.name} is not writable", file=sys.stderr)
sys.exit(-1)
pwd_rank = init_targets(targets)
total = sum([n for n, _ in pwd_rank.values()])
for scored in scored_files:
rs = read_scored(scored, splitter)
parse_rank(pwd_rank, model_rank=rs)
scored.close()
prev_rank = 0
cracked = 0
for pwd, (num, rank) in sorted(pwd_rank.items(),
key=lambda x: x[1][1],
reverse=False):
cracked += num
rank = round(max(rank, prev_rank + 1))
save2.write(
f"{pwd}\t{0.0}\t{num}\t{rank}\t{cracked}\t{cracked / total * 100:5.2f}\n"
)
pass
save2.flush()
save2.close()
def show_table(table: List[List[str]], fd: TextIO):
if not fd.writable():
raise Exception(f"Can not write into {fd.name}")
for row in table:
output = f" {' & '.join(row)} \\\\ {os.linesep}"
fd.write(output)
def reduce_textio(obj: TextIO):
if obj.readable() == obj.writable():
raise ValueError(
"TextIO object must be either readable or writable, but not both.")
fd = Fd(obj.fileno())
return rebuild_textio, (fd, obj.readable(), obj.writable(), obj.encoding)
def run(options: Dict[str, Union[int, float, List[str]]], file_list: List[str],
save_file: TextIO, pre_gcode: str, mid_gcode: str, post_gcode: str):
# check all the entries are present and valid
if not save_file.writable():
raise Exception('save_file must have write permissions')
if not pre_gcode.endswith('\n'):
pre_gcode += '\n'
if not mid_gcode.endswith('\n'):
mid_gcode += '\n'
assert 'gantry_z_clearance' in options
assert isinstance(options['gantry_z_clearance'],
(int, float)) and options['gantry_z_clearance'] >= 0
assert 'head_x_min' in options
assert isinstance(options['head_x_min'],
(int, float)) and options['head_x_min'] >= 0
assert 'head_x_max' in options
assert isinstance(options['head_x_max'],
(int, float)) and options['head_x_max'] >= 0
assert 'head_y_min' in options
assert isinstance(options['head_y_min'],
(int, float)) and options['head_y_min'] >= 0
assert 'head_y_max' in options
assert isinstance(options['head_y_max'],
(int, float)) and options['head_y_max'] >= 0
assert 'gantry_head_y_offset' in options
assert isinstance(options['gantry_head_y_offset'],
(int, float)) and options['gantry_head_y_offset'] >= 0
assert 'nozzle_outer_diameter' in options
assert isinstance(options['nozzle_outer_diameter'],
(int, float)) and options['nozzle_outer_diameter'] >= 0
assert 'head_z_clearance' in options
assert isinstance(options['head_z_clearance'],
(int, float)) and options['head_z_clearance'] >= 0
assert 'xy_air_gap' in options
assert isinstance(options['xy_air_gap'],
(int, float)) and options['xy_air_gap'] >= 0
assert 'bed_x_min' in options
assert isinstance(options['bed_x_min'],
(int, float)) and options['bed_x_min'] >= 0
assert 'bed_x_max' in options
assert isinstance(options['bed_x_max'],
(int, float)) and options['bed_x_max'] >= 0
assert 'bed_y_min' in options
assert isinstance(options['bed_y_min'],
(int, float)) and options['bed_y_min'] >= 0
assert 'bed_y_max' in options
assert isinstance(options['bed_y_max'],
(int, float)) and options['bed_y_max'] >= 0
assert 'bed_z_max' in options
assert isinstance(options['bed_z_max'],
(int, float)) and options['bed_z_max'] >= 0
assert 'iteration_step' in options
assert isinstance(options['iteration_step'],
(int, float)) and options['iteration_step'] >= 0
assert 'remove_commands_starting' in options
assert isinstance(options['remove_commands_starting'], list)
# the algorithm works by checking if the model axis aligned bounding box fits on the base and around other models
# it tries putting the bounding box on every point on a grid of points until it finds one that doesn't intersect other models
# step is the grid unit size and starting_coord is the origin of the grid
step = {'y': options['iteration_step']}
starting_coord = {'y': options['bed_y_min']}
# sign keeps track of which corner is started in based on the smallest dimension of the extruder head
sign = {}
if options['head_x_max'] < options['head_x_min']:
step['x'] = -options['iteration_step']
sign['x'] = -1
starting_coord['x'] = options['bed_x_max']
else:
step['x'] = options['iteration_step']
sign['x'] = 1
starting_coord['x'] = options['bed_x_min']
# a list of the axis aligned bounding boxes of the models plus an offset based on the printer settings
blocked_bounding_boxes = [
] # contains a list in the format [x_min, x_max, y_min, y_max]
# much the same as blocked_bounding_boxes but just the bounding boxes without the offset
# used to look back and find space for models that could be printed before other models
model_bouding_boxes = []
# the class instance for each model that has been placed. Used so that the gcode can be generated at the end
placed_models = []
failed_models = []
file_counter = 0
# iterate through all files in file_list
while file_counter < len(file_list):
if file_list[file_counter] == '':
# skip blank entries
file_counter += 1
continue
elif file_list[file_counter] in failed_models:
print(f'Could not find space for model {file_list[file_counter]}')
file_counter += 1
continue
# load and parse gcode file
current_model = GCode(file_list[file_counter], options)
is_placed = False
# set up the iteration grid
coord = copy.deepcopy(starting_coord)
# the dimensions of the current model
model_x_dim = current_model.max_x - current_model.min_x
model_y_dim = current_model.max_y - current_model.min_y
model_z_dim = current_model.max_z
# iterate across every point in the grid. (while is used here to make breaking out easier)
while options['bed_y_min'] <= coord['y'] <= options[
'bed_y_max'] and options[
'bed_y_min'] <= coord['y'] + model_y_dim <= options[
'bed_y_max'] and not is_placed:
while options['bed_x_min'] <= coord['x'] <= options[
'bed_x_max'] and options['bed_x_min'] <= coord[
'x'] + model_x_dim * sign['x'] <= options[
'bed_x_max'] and not is_placed:
# ^^^ check that the model fits on the bed at the current location and that it hasn't already been placed
# check that the current model location does not intersect with any other placed model
if not any([
box_collision(
coord['x'], coord['x'] + model_x_dim * sign['x'],
coord['y'], coord['y'] + model_y_dim, *box)
for box in blocked_bounding_boxes
]):
# the current position does not interfere with any other blocked bounding boxes
is_placed = True # set this so that it breaks out of the while loop on the next loop
placed_models.append(current_model)
# move the coordinates in the gcode file to the new location
if options['head_z_clearance'] < model_z_dim:
# if the height of the model is greater than the clearance between the build plate
# and the bottom of the print head when the nozzle is touching the build plate add
# a bounding box equal to the model bounding box plus the dimensions of the print head
blocked_bounding_boxes.append([
coord['x'] -
max(options['head_x_min'], options['head_x_max']) *
sign['x'],
coord['x'] + (model_x_dim + min(
options['head_x_min'], options['head_x_max']) +
options['xy_air_gap']) * sign['x'],
coord['y'] - options['head_y_max'],
coord['y'] + model_y_dim + options['head_y_min'] +
options['xy_air_gap']
])
if options['gantry_z_clearance'] < model_z_dim:
# if the model is taller than the gantry clearance then it also has the gantry to worry about
# block off the whole width of the print bed such that the gantry does not collide
blocked_bounding_boxes.append([
options['bed_x_min'], options['bed_x_max'],
coord['y'] - options['head_y_max'],
coord['y'] + model_y_dim +
options['xy_air_gap'] -
options['gantry_head_y_offset']
])
else:
# if the model can fit below the print head then add the dimension of the nozzle
blocked_bounding_boxes.append([
coord['x'] -
(options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']) * sign['x'], coord['x'] +
(model_x_dim + options['nozzle_outer_diameter'] / 2
+ options['xy_air_gap']) * sign['x'], coord['y'] -
(options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']), coord['y'] + model_y_dim +
options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']
])
elif model_z_dim < options['head_z_clearance'] and not any([
box_collision(
coord['x'] -
(options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']) * sign['x'], coord['x'] +
(model_x_dim + options['nozzle_outer_diameter'] / 2
+ options['xy_air_gap']) * sign['x'], coord['y'] -
(options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']), coord['y'] + model_y_dim +
options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap'], *box)
for box in model_bouding_boxes
]):
is_placed = True
blocked_bounding_boxes.append([
coord['x'] - (options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']) * sign['x'],
coord['x'] +
(model_x_dim + options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']) * sign['x'],
coord['y'] - (options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']), coord['y'] +
model_y_dim + options['nozzle_outer_diameter'] / 2 +
options['xy_air_gap']
])
placed_models.insert(0, current_model)
elif options['head_z_clearance'] < model_z_dim < options[
'gantry_z_clearance'] and not any([
box_collision(
coord['x'] - max(options['head_x_min'],
options['head_x_max']) *
sign['x'], coord['x'] +
(model_x_dim + min(options['head_x_min'],
options['head_x_max']) +
options['xy_air_gap']) * sign['x'],
coord['y'] - options['head_y_max'], coord['y']
+ model_y_dim + options['head_y_min'] +
options['xy_air_gap'], *box)
for box in model_bouding_boxes
]):
is_placed = True
blocked_bounding_boxes.append([
coord['x'] -
max(options['head_x_min'], options['head_x_max']) *
sign['x'], coord['x'] +
(model_x_dim +
min(options['head_x_min'], options['head_x_max']) +
options['xy_air_gap']) * sign['x'],
coord['y'] - options['head_y_max'],
coord['y'] + model_y_dim + options['head_y_min'] +
options['xy_air_gap']
])
placed_models.insert(0, current_model)
if is_placed:
model_bouding_boxes.append([
coord['x'], coord['x'] + model_x_dim * sign['x'],
coord['y'], coord['y'] + model_y_dim
])
current_model.move(
min(coord['x'], coord['x'] + model_x_dim * sign['x']) -
current_model.min_x,
min(coord['y'], coord['y'] + model_y_dim) -
current_model.min_y)
# continue to next iteration step
coord['x'] += step['x']
coord['x'] = starting_coord['x']
coord['y'] += step['y']
if not is_placed:
print(f'Could not find space for model {file_list[file_counter]}')
failed_models.append(file_list[file_counter])
file_counter += 1
max_model_height = 0
write_mid_gcode = False
save_file.write(pre_gcode)
for current_model in placed_models:
if write_mid_gcode:
# retract a little and hop up then prime the nozzle again
save_file.write(mid_gcode)
# lift the print head above the maximum model height to ensure it does not collide with any other prints
save_file.write(f'G0 Z{max_model_height + 10}\n')
# travel to the location of the new print staying at a height greater than the maximum model height
if options['head_x_max'] < options['head_x_min']:
save_file.write(
f'G0 X{current_model.max_x - options["head_x_max"]} Y{current_model.min_y + options["head_y_min"]}\n'
)
else:
save_file.write(
f'G0 X{current_model.min_x - options["head_x_min"]} Y{current_model.min_y + options["head_y_min"]}\n'
)
# add the actual model gcode that should first travel to the starting location and then start printing
save_file.write(f'{current_model.output}')
# update the max model height if the new model is taller
if max_model_height < current_model.max_z:
max_model_height = current_model.max_z
write_mid_gcode = True
save_file.write(post_gcode)