def write2(self, fd: TextIO): if not fd.writable(): raise Exception(f"{fd.name} is not writable") if self.__gc is None: raise Exception(f"run mlps2gc before invoke this method") for pwd, mlp, appearance, rank, cracked, cracked_ratio in tqdm( self.__gc, desc="Saving: "): fd.write( f"{pwd}\t{mlp:.8f}\t{appearance}\t{rank}\t{cracked}\t{cracked_ratio:5.2f}\n" ) self.__gc = None pass
def cleaning(dataset: str, encoding: str, output: TextIO, valid_chr_re): fd_dataset = open(dataset, encoding=encoding) if not (fd_dataset.readable() and output.writable()): print( f"{dataset} show be readable and {output.name} should be writable", file=sys.stderr) sys.exit(-1) line = "" try: for _line in fd_dataset: line = _line.strip("\r\n") if valid_chr_re.search(line): output.write(f"{line}\n") except UnicodeDecodeError as e: print(f"{line}| {e.reason}", file=sys.stderr) sys.exit(-1)
def save(uniq_lines: Set[str], save2: TextIO, order: str): if not save2.writable() or save2.closed: print(f"{save2.name} can not be used to write") sys.exit(-1) if order == 'order': uniq_lines = sorted(uniq_lines, reverse=False) elif order == 'reverse': uniq_lines = sorted(uniq_lines, reverse=True) elif order == 'random': uniq_lines = list(uniq_lines) shuffle(uniq_lines) else: sys.stderr.write(f"Unknown method: {order}") sys.exit(-1) for _l in uniq_lines: save2.write(f"{_l}\n") save2.flush()
def wrapper(dataset: TextIO, save: TextIO): if not dataset.seekable(): raise Exception("Not seekable, do not use stdin please") if not save.writable(): raise Exception(f"{save.name} Not writable") total_size, len_dict = len_dist(dataset, False) dataset.seek(0) total_chr, chr_dict, cls_dict = chr_dist(dataset, True) json.dump( { "#len": total_size, "len": len_dict, "#chr": total_chr, "chr": chr_dict, "cls": cls_dict }, save, indent=2) save.close()
def v41seg(training: TextIO, test_set: TextIO, save2: TextIO) -> None: if not save2.writable(): raise Exception(f"{save2.name} is not writable") multiword_detector = MultiWordDetector() for password in training: password = password.strip("\r\n") multiword_detector.train(password) training.close() pwd_counter = defaultdict(int) for password in test_set: password = password.strip("\r\n") pwd_counter[password] += 1 test_set.close() for password, num in pwd_counter.items(): section_list, found_walks = detect_keyboard_walk(password) _ = year_detection(section_list) """ Note that there is a bug in context_sensitive_detection I have fixed that and add a test case in unit_tests folder """ _ = context_sensitive_detection(section_list) _, _ = alpha_detection(section_list, multiword_detector) _ = digit_detection(section_list) _ = other_detection(section_list) info = [password, f"{num}"] npass = "" for sec, tag in section_list: npass += sec info.append(sec) info.append(tag) if password.lower() != npass.lower(): print(password) print(section_list) raise Exception("neq") print("\t".join(info), end="\n", file=save2) pass
def l33tseg(training: TextIO, test_set: TextIO, save2: TextIO) -> None: if not save2.writable(): raise Exception(f"{save2.name} is not writable") multiword_detector = MyMultiWordDetector() for password in training: password = password.strip("\r\n") multiword_detector.train(password) training.close() l33t_detector = AsciiL33tDetector(multiword_detector) l33t_detector.init_l33t(training.name, "ascii") pwd_counter = defaultdict(int) for password in test_set: password = password.strip("\r\n") pwd_counter[password] += 1 test_set.close() for password, num in pwd_counter.items(): section_list = [(password, None)] _ = year_detection(section_list) section_list, _ = detect_context_sections(section_list) section_list, _, _ = l33t_detector.parse_sections(section_list) section_list, _, _, _, _ = multiword_detector.parse_sections( section_list) info = [password, f"{num}"] npass = "" for sec, tag in section_list: npass += sec info.append(sec) info.append(tag) if password.lower() != npass.lower(): # Note that we'll not lower X # therefore, the best way is to compare password.lower # with npass.lower print(password) print(section_list) raise Exception("neq") print("\t".join(info), end="\n", file=save2) pass
def wrapper(targets: TextIO, scored_files: List[TextIO], splitter: str, save2: TextIO): if not save2.writable(): print(f"{save2.name} is not writable", file=sys.stderr) sys.exit(-1) pwd_rank = init_targets(targets) total = sum([n for n, _ in pwd_rank.values()]) for scored in scored_files: rs = read_scored(scored, splitter) parse_rank(pwd_rank, model_rank=rs) scored.close() prev_rank = 0 cracked = 0 for pwd, (num, rank) in sorted(pwd_rank.items(), key=lambda x: x[1][1], reverse=False): cracked += num rank = round(max(rank, prev_rank + 1)) save2.write( f"{pwd}\t{0.0}\t{num}\t{rank}\t{cracked}\t{cracked / total * 100:5.2f}\n" ) pass save2.flush() save2.close()
def show_table(table: List[List[str]], fd: TextIO): if not fd.writable(): raise Exception(f"Can not write into {fd.name}") for row in table: output = f" {' & '.join(row)} \\\\ {os.linesep}" fd.write(output)
def reduce_textio(obj: TextIO): if obj.readable() == obj.writable(): raise ValueError( "TextIO object must be either readable or writable, but not both.") fd = Fd(obj.fileno()) return rebuild_textio, (fd, obj.readable(), obj.writable(), obj.encoding)
def run(options: Dict[str, Union[int, float, List[str]]], file_list: List[str], save_file: TextIO, pre_gcode: str, mid_gcode: str, post_gcode: str): # check all the entries are present and valid if not save_file.writable(): raise Exception('save_file must have write permissions') if not pre_gcode.endswith('\n'): pre_gcode += '\n' if not mid_gcode.endswith('\n'): mid_gcode += '\n' assert 'gantry_z_clearance' in options assert isinstance(options['gantry_z_clearance'], (int, float)) and options['gantry_z_clearance'] >= 0 assert 'head_x_min' in options assert isinstance(options['head_x_min'], (int, float)) and options['head_x_min'] >= 0 assert 'head_x_max' in options assert isinstance(options['head_x_max'], (int, float)) and options['head_x_max'] >= 0 assert 'head_y_min' in options assert isinstance(options['head_y_min'], (int, float)) and options['head_y_min'] >= 0 assert 'head_y_max' in options assert isinstance(options['head_y_max'], (int, float)) and options['head_y_max'] >= 0 assert 'gantry_head_y_offset' in options assert isinstance(options['gantry_head_y_offset'], (int, float)) and options['gantry_head_y_offset'] >= 0 assert 'nozzle_outer_diameter' in options assert isinstance(options['nozzle_outer_diameter'], (int, float)) and options['nozzle_outer_diameter'] >= 0 assert 'head_z_clearance' in options assert isinstance(options['head_z_clearance'], (int, float)) and options['head_z_clearance'] >= 0 assert 'xy_air_gap' in options assert isinstance(options['xy_air_gap'], (int, float)) and options['xy_air_gap'] >= 0 assert 'bed_x_min' in options assert isinstance(options['bed_x_min'], (int, float)) and options['bed_x_min'] >= 0 assert 'bed_x_max' in options assert isinstance(options['bed_x_max'], (int, float)) and options['bed_x_max'] >= 0 assert 'bed_y_min' in options assert isinstance(options['bed_y_min'], (int, float)) and options['bed_y_min'] >= 0 assert 'bed_y_max' in options assert isinstance(options['bed_y_max'], (int, float)) and options['bed_y_max'] >= 0 assert 'bed_z_max' in options assert isinstance(options['bed_z_max'], (int, float)) and options['bed_z_max'] >= 0 assert 'iteration_step' in options assert isinstance(options['iteration_step'], (int, float)) and options['iteration_step'] >= 0 assert 'remove_commands_starting' in options assert isinstance(options['remove_commands_starting'], list) # the algorithm works by checking if the model axis aligned bounding box fits on the base and around other models # it tries putting the bounding box on every point on a grid of points until it finds one that doesn't intersect other models # step is the grid unit size and starting_coord is the origin of the grid step = {'y': options['iteration_step']} starting_coord = {'y': options['bed_y_min']} # sign keeps track of which corner is started in based on the smallest dimension of the extruder head sign = {} if options['head_x_max'] < options['head_x_min']: step['x'] = -options['iteration_step'] sign['x'] = -1 starting_coord['x'] = options['bed_x_max'] else: step['x'] = options['iteration_step'] sign['x'] = 1 starting_coord['x'] = options['bed_x_min'] # a list of the axis aligned bounding boxes of the models plus an offset based on the printer settings blocked_bounding_boxes = [ ] # contains a list in the format [x_min, x_max, y_min, y_max] # much the same as blocked_bounding_boxes but just the bounding boxes without the offset # used to look back and find space for models that could be printed before other models model_bouding_boxes = [] # the class instance for each model that has been placed. Used so that the gcode can be generated at the end placed_models = [] failed_models = [] file_counter = 0 # iterate through all files in file_list while file_counter < len(file_list): if file_list[file_counter] == '': # skip blank entries file_counter += 1 continue elif file_list[file_counter] in failed_models: print(f'Could not find space for model {file_list[file_counter]}') file_counter += 1 continue # load and parse gcode file current_model = GCode(file_list[file_counter], options) is_placed = False # set up the iteration grid coord = copy.deepcopy(starting_coord) # the dimensions of the current model model_x_dim = current_model.max_x - current_model.min_x model_y_dim = current_model.max_y - current_model.min_y model_z_dim = current_model.max_z # iterate across every point in the grid. (while is used here to make breaking out easier) while options['bed_y_min'] <= coord['y'] <= options[ 'bed_y_max'] and options[ 'bed_y_min'] <= coord['y'] + model_y_dim <= options[ 'bed_y_max'] and not is_placed: while options['bed_x_min'] <= coord['x'] <= options[ 'bed_x_max'] and options['bed_x_min'] <= coord[ 'x'] + model_x_dim * sign['x'] <= options[ 'bed_x_max'] and not is_placed: # ^^^ check that the model fits on the bed at the current location and that it hasn't already been placed # check that the current model location does not intersect with any other placed model if not any([ box_collision( coord['x'], coord['x'] + model_x_dim * sign['x'], coord['y'], coord['y'] + model_y_dim, *box) for box in blocked_bounding_boxes ]): # the current position does not interfere with any other blocked bounding boxes is_placed = True # set this so that it breaks out of the while loop on the next loop placed_models.append(current_model) # move the coordinates in the gcode file to the new location if options['head_z_clearance'] < model_z_dim: # if the height of the model is greater than the clearance between the build plate # and the bottom of the print head when the nozzle is touching the build plate add # a bounding box equal to the model bounding box plus the dimensions of the print head blocked_bounding_boxes.append([ coord['x'] - max(options['head_x_min'], options['head_x_max']) * sign['x'], coord['x'] + (model_x_dim + min( options['head_x_min'], options['head_x_max']) + options['xy_air_gap']) * sign['x'], coord['y'] - options['head_y_max'], coord['y'] + model_y_dim + options['head_y_min'] + options['xy_air_gap'] ]) if options['gantry_z_clearance'] < model_z_dim: # if the model is taller than the gantry clearance then it also has the gantry to worry about # block off the whole width of the print bed such that the gantry does not collide blocked_bounding_boxes.append([ options['bed_x_min'], options['bed_x_max'], coord['y'] - options['head_y_max'], coord['y'] + model_y_dim + options['xy_air_gap'] - options['gantry_head_y_offset'] ]) else: # if the model can fit below the print head then add the dimension of the nozzle blocked_bounding_boxes.append([ coord['x'] - (options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']) * sign['x'], coord['x'] + (model_x_dim + options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']) * sign['x'], coord['y'] - (options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']), coord['y'] + model_y_dim + options['nozzle_outer_diameter'] / 2 + options['xy_air_gap'] ]) elif model_z_dim < options['head_z_clearance'] and not any([ box_collision( coord['x'] - (options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']) * sign['x'], coord['x'] + (model_x_dim + options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']) * sign['x'], coord['y'] - (options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']), coord['y'] + model_y_dim + options['nozzle_outer_diameter'] / 2 + options['xy_air_gap'], *box) for box in model_bouding_boxes ]): is_placed = True blocked_bounding_boxes.append([ coord['x'] - (options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']) * sign['x'], coord['x'] + (model_x_dim + options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']) * sign['x'], coord['y'] - (options['nozzle_outer_diameter'] / 2 + options['xy_air_gap']), coord['y'] + model_y_dim + options['nozzle_outer_diameter'] / 2 + options['xy_air_gap'] ]) placed_models.insert(0, current_model) elif options['head_z_clearance'] < model_z_dim < options[ 'gantry_z_clearance'] and not any([ box_collision( coord['x'] - max(options['head_x_min'], options['head_x_max']) * sign['x'], coord['x'] + (model_x_dim + min(options['head_x_min'], options['head_x_max']) + options['xy_air_gap']) * sign['x'], coord['y'] - options['head_y_max'], coord['y'] + model_y_dim + options['head_y_min'] + options['xy_air_gap'], *box) for box in model_bouding_boxes ]): is_placed = True blocked_bounding_boxes.append([ coord['x'] - max(options['head_x_min'], options['head_x_max']) * sign['x'], coord['x'] + (model_x_dim + min(options['head_x_min'], options['head_x_max']) + options['xy_air_gap']) * sign['x'], coord['y'] - options['head_y_max'], coord['y'] + model_y_dim + options['head_y_min'] + options['xy_air_gap'] ]) placed_models.insert(0, current_model) if is_placed: model_bouding_boxes.append([ coord['x'], coord['x'] + model_x_dim * sign['x'], coord['y'], coord['y'] + model_y_dim ]) current_model.move( min(coord['x'], coord['x'] + model_x_dim * sign['x']) - current_model.min_x, min(coord['y'], coord['y'] + model_y_dim) - current_model.min_y) # continue to next iteration step coord['x'] += step['x'] coord['x'] = starting_coord['x'] coord['y'] += step['y'] if not is_placed: print(f'Could not find space for model {file_list[file_counter]}') failed_models.append(file_list[file_counter]) file_counter += 1 max_model_height = 0 write_mid_gcode = False save_file.write(pre_gcode) for current_model in placed_models: if write_mid_gcode: # retract a little and hop up then prime the nozzle again save_file.write(mid_gcode) # lift the print head above the maximum model height to ensure it does not collide with any other prints save_file.write(f'G0 Z{max_model_height + 10}\n') # travel to the location of the new print staying at a height greater than the maximum model height if options['head_x_max'] < options['head_x_min']: save_file.write( f'G0 X{current_model.max_x - options["head_x_max"]} Y{current_model.min_y + options["head_y_min"]}\n' ) else: save_file.write( f'G0 X{current_model.min_x - options["head_x_min"]} Y{current_model.min_y + options["head_y_min"]}\n' ) # add the actual model gcode that should first travel to the starting location and then start printing save_file.write(f'{current_model.output}') # update the max model height if the new model is taller if max_model_height < current_model.max_z: max_model_height = current_model.max_z write_mid_gcode = True save_file.write(post_gcode)