def save_metrics(self): with open(self.output_dir / "test_metrics.tsv", mode="w") as f: f.write("\t".join(self.metrics.names) + "\n") f.write("\t".join(map("{:.4f}".format, self.metrics.get().values())) + "\n") print_info("Metrics saved")
def convert(self, annot): name = annot['filename'] if self.verbose: print_info('Converting VIA annotations for {}'.format(name)) if not (self.input_dir / name).exists: print_error('Original image {} not found'.format(name)) return None size = Image.open(self.input_dir / name).size img = Image.new(self.mode, size, color=self.background_color) draw = ImageDraw.Draw(img) for region in annot['regions']: shape = region['shape_attributes'] if shape['name'] == 'circle': cx, cy, r = shape['cx'], shape['cy'], shape['r'] bbox = [(cx - r, cy - r), (cx + r, cy + r)] draw.ellipse(bbox, fill=self.color) elif shape['name'] == 'ellipse': cx, cy, rx, ry = shape['cx'], shape['cy'], shape['rx'], shape['ry'] bbox = [(cx - rx, cy - ry), (cx + rx, cy + ry)] draw.ellipse(bbox, fill=self.color) elif shape['name'] == 'polygon': polygon = list(zip(shape['all_points_x'], shape['all_points_y'])) draw.polygon(polygon, fill=self.color) else: raise NotImplementedError('shape "{}" not implemented'.format(shape['name'])) return img
def handle_print(command, source): sock = Session.send_command(command, source, close_socket=False) # sock.shutdown(0) download_full_path = os.path.join(os.getcwd(), parameters.download_path) if not os.path.exists(download_full_path): os.makedirs(download_full_path) file_name = source.split(parameters.sep)[-1] download_file_path = os.path.join(download_full_path, file_name) res = b"" data = sock.recv(1024) while data: res += data data = sock.recv(1024) with open(download_file_path, 'wb+') as host_file: host_file.write(res) sock.close() try: res = res.decode('utf-8') except UnicodeDecodeError: res = "Received data could not be decoded as utf-8. Probably it is not text." finally: logger.print_info(res)
def run(self): for filename in self.files: if self.verbose: print_info( 'Converting and saving as segmentation map {}'.format( filename)) img = self.convert(Image.open(filename), self.color_label_mapping) np.save(self.output_dir / filename.stem, img)
def run(self): for filename in self.files: if self.verbose: print_info( 'Converting and saving as segmentation map {}'.format( filename)) img = self.convert(np.load(filename), self.label_color_mapping) img.save(self.output_dir / '{}.{}'.format(filename.stem, self.extension))
def help(session, args): if args: command = getattr(Commands, args[0], None) if not command: logger.handle_error(Messages.wrong_command_message(args[0])) else: logger.print_info("{}: {}\n{}".format(command.name, command.usage, command.description)) else: logger.print_info(Messages.help_message())
def check_for_updates(self): check_url = "https://raw.githubusercontent.com/mission712/TextSiri/master/VERSION" ver = requests.get(check_url) if ver.status_code != 200: logger.print_warn("Couldn't check version.") return local_version = botstrings.VERSION.split(".") remote_version = ver.text.split(".") for i in range(1, len(local_version)): if int(remote_version[i]) > int(local_version[i]): logger.print_info("{color_green}New bot version found! Pass --upgrade to upgrade TextSiri at startup.")
def run(self): for image, label in self.dataset: self.single_run(image, label) print_info("Probabilities and segmentation maps computed") if self.metrics is not None: self.save_metrics() if self.save_annotations: self.save_prob_and_seg_maps() print_info("Run is over")
def update_bot(): # we won't call this function from inside the bot # Assuming you are inside the TextSiri git repository (you should be.) try: subprocess.check_output(["git", "fetch"]) subprocess.check_output(["git", "pull"]) except FileNotFoundError: logger.print_error("You must have git installed!") exit(1) logger.print_info("Updated TextSiri, restarting.") python = sys.executable sys.argv.remove("--upgrade") os.execl(python, python, *sys.argv)
def update_bot(): # we won't call this function from inside the bot # Assuming you are inside the TextSiri git repository (you should be.) try: subprocess.check_output(["git", "fetch"]) subprocess.check_output(["git", "pull"]) except FileNotFoundError: logger.print_error("You must have git installed!") exit(1) logger.print_info("Updated TextSiri, restarting.") python = sys.executable sys.argv.remove("--upgrade") os.execl(python, python, * sys.argv)
def ls(session, args): if len(args) == 0: args.append(session.get_curr_dir()) args[0] = session.resolve_full_path(args[0]) if not args[0]: errors.path_invalid(args[0]) return res = session.handle_ls(Commands.ls, args) if res is str: logger.print_info(res) else: logger.print_info('\n'.join(res))
def check_for_updates(self): check_url = "https://raw.githubusercontent.com/mission712/TextSiri/master/VERSION" ver = requests.get(check_url) if ver.status_code != 200: logger.print_warn("Couldn't check version.") return local_version = botstrings.VERSION.split(".") remote_version = ver.text.split(".") for i in range(1, len(local_version)): if int(remote_version[i]) > int(local_version[i]): logger.print_info( "{color_green}New bot version found! Pass --upgrade to upgrade TextSiri at startup." )
def handle_info(command, args): sock = Session.send_command(command, *args, close_socket=False) filename = sock.recv(1024).decode('utf-8') # ack sock.send('k'.encode('utf-8')) size = sock.recv(1024).decode('utf-8') if size != '-1': logger.print_info("{}\t{} bytes".format(filename, size)) else: logger.print_info("{}\tdir".format(filename)) # sock.shutdown(0) sock.close()
def run(self): for filename in self.files: if self.verbose: print_info('Processing {}'.format(filename.name)) pages = self.convert(filename) max_page_id = len(str(len(pages))) path = self.output_dir if self.create_sub_dir: path = path / str(filename.stem) path.mkdir() for k, page in enumerate(pages): suffix = self.suffix_fmt.format(str(k + 1).zfill(max_page_id)) page.save( path / '{}{}.{}'.format(filename.stem, suffix, self.out_ext))
def run(self, nb_train, nb_val=0.1, nb_test=0.1): if 0 < nb_val < 1: nb_val = int(nb_train * nb_val) if 0 < nb_test < 1: nb_test = int(nb_train * nb_test) shift = 0 max_len_id = len(str(nb_train + nb_val + nb_test - 1)) for name, nb in zip(['train', 'val', 'test'], [nb_train, nb_val, nb_test]): if self.verbose: print_info('Creating {} set...'.format(name)) for k in range(shift, shift + nb): if self.verbose: print_info(' Generating random lines with seed {}...'.format(k)) d = SyntheticLine(seed=k) d.save('{}'.format(k).zfill(max_len_id), self.output_dir / name) shift += nb
def save_metrics(self): metric_names = next(iter(self.metrics.values())).names all_values = [[] for _ in range(len(metric_names))] with open(self.output_dir / 'metrics.tsv', mode='w') as f: f.write('dir_name\t{}\n'.format('\t'.join(metric_names))) for name, metrics in self.metrics.items(): values = list(metrics.get().values()) f.write('{}\t{}\n'.format( name, '\t'.join(map('{:.4f}'.format, values)))) [all_values[k].append(v) for k, v in enumerate(values)] if len(self.metrics) > 1: mean_values = list(map(np.mean, all_values)) f.write('{}\t{}\n'.format( 'average', '\t'.join(map('{:.4f}'.format, mean_values)))) print_info('Metrics saved')
def __init__(self, input_dir, output_dir, suffix_fmt='-{}', out_ext='jpg', create_sub_dir=False, verbose=True): self.input_dir = coerce_to_path_and_check_exist(input_dir) self.files = get_files_from_dir(self.input_dir, valid_extensions='pdf') self.output_dir = coerce_to_path_and_create_dir(output_dir) self.suffix_fmt = suffix_fmt self.out_ext = out_ext self.create_sub_dir = create_sub_dir self.verbose = verbose if self.verbose: print_info("Pdf2Image initialised: found {} files".format( len(self.files)))
def save_prob_and_seg_maps(self): for k in range(len(self.dataset)): name = self.dataset.input_files[k].stem # saving probability maps takes a lot of space, remove comment if needed # np.save(self.prob_dir / "{}.npy".format(name), self.prob_maps[k]) pred = self.seg_maps[k] pred_img = LabeledArray2Image.convert( pred, label_color_mapping=self.dataset.label_idx_color_mapping) pred_img.save(self.seg_dir / "{}.png".format(name)) img = resize(Image.open(self.dataset.input_files[k]), pred_img.size, keep_aspect_ratio=False) blend_img = Image.blend(img, pred_img, alpha=0.4) blend_img.convert("RGB").save(self.blend_dir / "{}.jpg".format(name)) print_info("Probabilities and segmentation maps saved")
def run(self): for url in self.manifest_urls: manifest = self.get_json(url) if manifest is not None: manifest_id = Path(urlparse(manifest['@id']).path).parent.name print_info('Processing {}...'.format(manifest_id)) output_path = coerce_to_path_and_create_dir(self.output_dir / manifest_id) resources = self.get_resources(manifest) for resource_url in resources: resource_url = '/'.join(resource_url.split('/')[:-3] + [self.size] + resource_url.split('/')[-2:]) with requests.get(resource_url, stream=True) as response: response.raw.decode_content = True resrc_path = Path(urlparse(resource_url).path) name = '{}{}'.format(resrc_path.parts[-5], resrc_path.suffix) output_file = output_path / name print_info('Saving {}...'.format(output_file.relative_to(self.output_dir))) with open(output_file, mode='wb') as f: shutil.copyfileobj(response.raw, f)
def load(instance, logger, dirname): modulelist = load_modules(dirname) logger.print_info("Loading {} modules".format(len(modulelist))) for module in modulelist: try: modulelist[modulelist.index(module)] = module.Module(instance) except BaseException as e: continue for module in modulelist: if hasattr(module, "post_init"): module.post_init(modulelist) for method in dir(module): if method[:3] == "on_": try: instance.connection.add_global_handler( method[3:], getattr(module, method)) except Exception as e: logger.print_error( "Error {} occured while hooking module {}.".format( e, module.info["name"])) logger.print_info("Checking dependencies...") for module in modulelist: mods = [x.info["name"] for x in modulelist] for depend in module.info["depends"]: if depend not in mods: logger.print_error( "{0} depends on {1} but {1} does not exist! Download {1} or remove {0}." .format(module.info["name"], depend)) logger.print_info("All dependencies are included.")
def rmdir(session, args): # validate and build paths args = list(map(session.resolve_full_path, args)) for arg in args: if not arg: errors.path_invalid(arg) return is_dir = list(map(session.is_dir, args)) for i, answer in enumerate(is_dir): if not answer: errors.wrong_type(args[i], "directory") return to_remove = [] for dir in args: children = session.handle_ls(Commands.ls, [dir]) if len(children) > 2: logger.print_info("Directory {} is not empty. Do you want to remove it with all its contents?\n" "Print \"yes\" to proceed, or anything else to abort operation.".format(dir)) response = input() if response.strip(" ") != "yes": logger.print_info(f"Directory {dir} will not be removed") else: logger.print_info(f"Directory {dir} will be removed") to_remove.append(dir) else: to_remove.append(dir) CommandConfig.Actions.__n_args_handler(session.send_command, Commands.rmdir, to_remove)
def run(self, nb_train, nb_val=0.1, nb_test=0.1): if 0 < nb_val < 1: nb_val = int(nb_train * nb_val) if 0 < nb_test < 1: nb_test = int(nb_train * nb_test) shift = 0 max_len_id = len(str(nb_train + nb_val + nb_test - 1)) kwargs = {'baseline_as_label': self.baseline_as_label, 'merged_labels': self.merged_labels} for name, nb in zip(['train', 'val', 'test'], [nb_train, nb_val, nb_test]): if self.verbose: print_info('Creating {} set...'.format(name)) for k in range(shift, shift + nb): if self.verbose: print_info(' Generating random document with seed {}...'.format(k)) with use_seed(k): random_height = choice([True, False], p=[RANDOM_DOC_HEIGHT_FREQ, 1 - RANDOM_DOC_HEIGHT_FREQ]) if random_height: kwargs['height'] = None kwargs['seed'] = k d = SyntheticDocument(**kwargs) d.save('{}'.format(k).zfill(max_len_id), self.output_dir / name) shift += nb
def load(instance, logger, dirname): modulelist = load_modules(dirname) logger.print_info("Loading {} modules".format(len(modulelist))) for module in modulelist: try: modulelist[modulelist.index(module)] = module.Module(instance) except BaseException as e: continue for module in modulelist: if hasattr(module, "post_init"): module.post_init(modulelist) for method in dir(module): if method[:3] == "on_": try: instance.connection.add_global_handler(method[3:], getattr(module, method)) except Exception as e: logger.print_error("Error {} occured while hooking module {}.".format(e, module.info["name"])) logger.print_info("Checking dependencies...") for module in modulelist: mods = [x.info["name"] for x in modulelist] for depend in module.info["depends"]: if depend not in mods: logger.print_error("{0} depends on {1} but {1} does not exist! Download {1} or remove {0}.".format(module.info["name"], depend)) logger.print_info("All dependencies are included.")
def print_and_log_info(self, string): print_info(string) self.logger.info(string)
def print_curr_dir(self, *args): logger.print_info(self.get_curr_dir())
def __init__(self, n_classes, **kwargs): super().__init__() self.n_classes = n_classes self.norm_layer_kwargs = kwargs.pop('norm_layer', dict()) self.norm_layer = get_norm_layer(**self.norm_layer_kwargs) self.no_maxpool = kwargs.get('no_maxpool', False) self.conv_as_maxpool = kwargs.get('conv_as_maxpool', True) self.use_upcatconv = kwargs.get('use_upcatconv', False) self.use_deconv = kwargs.get('use_deconv', False) assert not (self.use_deconv and self.use_upcatconv) self.same_up_channels = kwargs.get('same_up_channels', False) self.use_conv1x1 = kwargs.get('use_conv1x1', False) assert not (self.conv_as_maxpool and self.no_maxpool) self.enc_name = kwargs.get('encoder_name', 'resnet18') self.reduced_layers = kwargs.get( 'reduced_layers', False) and self.enc_name not in ['resnet18, resnet34'] pretrained = kwargs.get('pretrained_encoder', True) replace_with_dilation = kwargs.get('replace_with_dilation') strides = kwargs.get('strides', 2) resnet = get_resnet_model(self.enc_name)( pretrained, progress=False, norm_layer=self.norm_layer_kwargs, strides=strides, replace_with_dilation=replace_with_dilation) self.layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu) # XXX: maxpool creates high amplitude high freq activations, removing it leads to better results if self.conv_as_maxpool: layer0_out_channels = self.get_nb_out_channels(self.layer0) self.layer1 = nn.Sequential(*[ conv3x3(layer0_out_channels, layer0_out_channels, stride=2), self.norm_layer(layer0_out_channels), nn.ReLU() ] + list(resnet.layer1.children())) elif self.no_maxpool: self.layer1 = nn.Sequential(*list(resnet.layer1.children())) else: self.layer1 = nn.Sequential(*[resnet.maxpool] + list(resnet.layer1.children())) self.layer2, self.layer3, self.layer4 = resnet.layer2, resnet.layer3, resnet.layer4 layer0_out_channels = self.get_nb_out_channels(self.layer0) layer1_out_channels = self.get_nb_out_channels(self.layer1) layer2_out_channels = self.get_nb_out_channels(self.layer2) layer3_out_channels = self.get_nb_out_channels(self.layer3) layer4_out_channels = self.get_nb_out_channels(self.layer4) if self.reduced_layers: self.layer1_red = self._reducing_layer(layer1_out_channels, LAYER1_REDUCED_CHANNELS) self.layer2_red = self._reducing_layer(layer2_out_channels, LAYER2_REDUCED_CHANNELS) self.layer3_red = self._reducing_layer(layer3_out_channels, LAYER3_REDUCED_CHANNELS) self.layer4_red = self._reducing_layer(layer4_out_channels, LAYER4_REDUCED_CHANNELS) layer1_out_channels, layer2_out_channels = LAYER1_REDUCED_CHANNELS, LAYER2_REDUCED_CHANNELS layer3_out_channels, layer4_out_channels = LAYER3_REDUCED_CHANNELS, LAYER4_REDUCED_CHANNELS self.layer4_up = self._upsampling_layer(layer4_out_channels, layer3_out_channels, layer3_out_channels) self.layer3_up = self._upsampling_layer(layer3_out_channels, layer2_out_channels, layer2_out_channels) self.layer2_up = self._upsampling_layer(layer2_out_channels, layer1_out_channels, layer1_out_channels) self.layer1_up = self._upsampling_layer(layer1_out_channels, layer0_out_channels, layer0_out_channels) self.layer0_up = self._upsampling_layer(layer0_out_channels, FINAL_LAYER_CHANNELS, INPUT_CHANNELS) self.final_layer = self._final_layer(FINAL_LAYER_CHANNELS) if not pretrained: self._init_conv_weights() print_info( "Model {} initialisated with norm_layer={}({}) and kwargs {}". format(self.name, self.norm_layer.func.__name__, self.norm_layer.keywords, kwargs))
def __init__(self): logger.print_info("Welcome to TextSiri NEO.") logger.print_info("Starting engine, version {}".format(botstrings.VERSION)) self.check_for_updates() logger.print_info("Starting config parser") logger.print_info("Config file is {}".format(botstrings.CONFIG_FILE)) self.config = json_parser.JSONParser(botstrings.CONFIG_FILE) if self.config.is_empty: logger.print_warn("Config file is empty. Generating new config.") self.config["general"] = { # General bot settings. "nickname": "TextSiri", "realname": "Your digital assistant on IRC.", "authmethod": "nickserv", # TODO add sasl method "password": "", "triggertype": "1", # Modes are: 1- Regexp, 2-Single char, 3-Text at the beginning of message. Any other will give a FATAL error. "showmotd": True, "trigger": "^{botnick}[^a-zA-Z0-9]\s(.+)" # Available replacements: {botnick} = Bot's nick. } self.config["server"] = { # Server settings. "address": "irc.freenode.net", "port": "6697", "ssl": True } self.config["channels"] = [ { "name": "#botters", "key": "", # Leave empty for no key. "joincommands": [] # MUST BE RFC2812 COMPLIANT! } ] self.config["modules"] = { # Space for module settings. Each module that needs a config should have its own dictionary with its own name here. } self.config.write() logger.print_warn("Config file is written.") logger.print_info("It looks like this is your first time running TextSiri NEO. Please edit {} in the current directory.".format(botstrings.CONFIG_FILE)) sys.exit(1) else: logger.print_info("Config file successfully read.") server_info = self.config["server"] general_info = self.config["general"] logger.print_info("Attempting to connect to {} at port {} (SSL: {})".format( server_info["address"], server_info["port"], "YES" if server_info["ssl"] else "NO")) if server_info["ssl"]: irc.bot.SingleServerIRCBot.__init__(self, [(server_info["address"], int(server_info["port"]))], general_info["nickname"], general_info["realname"], connect_factory=irc.connection.Factory(wrapper=ssl.wrap_socket)) else: irc.bot.SingleServerIRCBot.__init__(self, [(server_info["address"], int(server_info["port"]))], general_info["nickname"], general_info["realname"]) self.connection.buffer_class = irc.buffer.LenientDecodingLineBuffer logger.print_info("Attempting to load modules.") self.modules = module_loader.load(self, logger, botstrings.MODULES_DIR) logger.print_info("Finished loading modules.") self.message_buffer = [] self.message_counter = 0 self.message_slowmode_lock = 0 logger.print_info("Starting message sender.") self.msgThread = threading.Thread(target=self.message_loop, args=(self.connection,)) self.msgThread.setDaemon(True) self.msgThread.start() logger.print_info("Started message sender.") logger.print_info("Reached Target Initialization.")
def __init__(self): logger.print_info("Welcome to TextSiri NEO.") logger.print_info("Starting engine, version {}".format( botstrings.VERSION)) self.check_for_updates() logger.print_info("Starting config parser") logger.print_info("Config file is {}".format(botstrings.CONFIG_FILE)) self.config = json_parser.JSONParser(botstrings.CONFIG_FILE) if self.config.is_empty: logger.print_warn("Config file is empty. Generating new config.") self.config["general"] = { # General bot settings. "nickname": "TextSiri", "realname": "Your digital assistant on IRC.", "authmethod": "nickserv", # TODO add sasl method "password": "", "triggertype": "1", # Modes are: 1- Regexp, 2-Single char, 3-Text at the beginning of message. Any other will give a FATAL error. "showmotd": True, "trigger": "^{botnick}[^a-zA-Z0-9]\s(.+)" # Available replacements: {botnick} = Bot's nick. } self.config["server"] = { # Server settings. "address": "irc.freenode.net", "port": "6697", "ssl": True } self.config["channels"] = [{ "name": "#botters", "key": "", # Leave empty for no key. "joincommands": [] # MUST BE RFC2812 COMPLIANT! }] self.config["modules"] = { # Space for module settings. Each module that needs a config should have its own dictionary with its own name here. } self.config.write() logger.print_warn("Config file is written.") logger.print_info( "It looks like this is your first time running TextSiri NEO. Please edit {} in the current directory." .format(botstrings.CONFIG_FILE)) sys.exit(1) else: logger.print_info("Config file successfully read.") server_info = self.config["server"] general_info = self.config["general"] logger.print_info( "Attempting to connect to {} at port {} (SSL: {})".format( server_info["address"], server_info["port"], "YES" if server_info["ssl"] else "NO")) if server_info["ssl"]: irc.bot.SingleServerIRCBot.__init__( self, [(server_info["address"], int(server_info["port"]))], general_info["nickname"], general_info["realname"], connect_factory=irc.connection.Factory( wrapper=ssl.wrap_socket)) else: irc.bot.SingleServerIRCBot.__init__( self, [(server_info["address"], int(server_info["port"]))], general_info["nickname"], general_info["realname"]) self.connection.buffer_class = irc.buffer.LenientDecodingLineBuffer logger.print_info("Attempting to load modules.") self.modules = module_loader.load(self, logger, botstrings.MODULES_DIR) logger.print_info("Finished loading modules.") self.message_buffer = [] self.message_counter = 0 self.message_slowmode_lock = 0 logger.print_info("Starting message sender.") self.msgThread = threading.Thread(target=self.message_loop, args=(self.connection, )) self.msgThread.setDaemon(True) self.msgThread.start() logger.print_info("Started message sender.") logger.print_info("Reached Target Initialization.")
def print_and_log_info(self, string): self.logger.info(string) if self.verbose: print_info(string)
def pwd(session, args): logger.print_info(session.get_curr_dir())
def __init__(self, output_dir, model_path, dataset_name, dataset_kwargs=None, save_annotations=True): print_info("Tester initialized for model {} and dataset {}".format( model_path, dataset_name)) # Output directory self.output_dir = coerce_to_path_and_create_dir(output_dir) self.save_annotations = save_annotations print_info("Output dir is {}".format(self.output_dir)) # Dataset self.dataset_kwargs = dataset_kwargs or {} self.dataset = get_dataset(dataset_name)(split="test", **self.dataset_kwargs) print_info("Dataset {} loaded with kwargs {}: {} samples".format( dataset_name, self.dataset_kwargs, len(self.dataset))) # Model torch.backends.cudnn.benchmark = False # XXX: at inference, input images are usually not fixed self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.model = load_model_from_path(model_path, device=self.device) self.model.eval() print_info("Model {} created and checkpoint state loaded".format( self.model.name)) # Metrics if self.dataset.label_files is not None: self.metrics = RunningMetrics(self.dataset.restricted_labels, self.dataset.metric_labels) print_info("Labels found, metrics instantiated") else: self.metrics = None print_info( "No labels found, performance metrics won't be computed") # Outputs # saving probability maps takes a lot of space, remove comment if needed # self.prob_dir = coerce_to_path_and_create_dir(self.output_dir / "prob_map") self.prob_maps, self.seg_maps = [], [] if self.save_annotations: self.seg_dir = coerce_to_path_and_create_dir(self.output_dir / "seg_map") self.blend_dir = coerce_to_path_and_create_dir(self.output_dir / "blend")