def __health_check(self): to_delete = list() if self.__pre_processor is None: health_state = WordsDataSet.__health_state.format("") else: health_state = WordsDataSet.__health_state.format( f"_{self.__pre_processor.name}") health_path = os.path.join(self.__root_dir, health_state) if is_file(health_path): with open(health_path, 'r') as fp: to_delete = json_read(fp) else: for idx, word_meta in enumerate(self.__words): try: img, word = self[idx] stripped = right_strip(list(map(int, word)), 1) self.__check_word_length(stripped) except (cv2.error, ValueError) as e: logger.error(f"Corrupted file at index: {idx}") to_delete.append(idx) logger.debug(f"Write corrupted indices to '{health_path}'") with open(health_path, 'w') as fp: json_write(to_delete, fp) logger.info( f"WordsDataSet - Health Check: {len(to_delete)} indices={to_delete} not readable." ) self.__save_delete_indices(to_delete)
def _auth(self): """ Authenticate Amigo on GCP, fetching the credentials and saving it to a local file that can be used for the service discovery. """ # Set the credentials to be used by amigo local_cred_file = util.get_value(self.config, "local_cred_file") try: self.auth = Storage(local_cred_file) if util.is_file(util.get_value(self.config, "key_file")): creds = GoogleCredentials.get_application_default() self.auth.put(creds) return True except IOError: util.print_to_stderr( "Cannot open {0} to write, ensure you are running as root. ". format(local_cred_file)) except ApplicationDefaultCredentialsError: util.print_to_stderr("Cannot authenticate to GCP.") return False
def image_size(path_to_image: str) -> tuple: if not util.is_file(path_to_image): pfcs(f'image file e[{path_to_image}] does not exist!') return None with Image.open(path_to_image) as image: return image.size return None
def move_file(source_file, destination, create_dirs=False, new_filename=None, debug_print=True): "Custom file move method using mv command in the background" source_file = path_to_str(source_file) destination = path_to_str(destination) if not util.is_file(source_file): if debug_print: print( f'source {CSTR(source_file, "orange")} does not exist!') return False if not util.is_dir(destination) and create_dirs: os.makedirs(destination) if debug_print: print(f'move_file: created dir {CSTR(destination, "lblue")}') elif not util.is_dir(destination) and not create_dirs: if debug_print: print(f'destination {CSTR(destination, "red")} does not exists!') return False if debug_print: print(f'moving {CSTR(source_file, "lblue")}') if new_filename: command = f'mv \"{source_file}\" \"{destination}/{new_filename}\"' if debug_print: print( f'destination {CSTR(f"{destination}/{new_filename}", "lblue")}') else: command = f'mv \"{source_file}\" \"{destination}\"' if debug_print: print(f'destination {CSTR(destination, "lblue")}') if local_command(command, hide_output=True, print_info=False): if debug_print: print(CSTR('done!', 'lgreen')) return True if debug_print: print(CSTR('move failed!', 'red')) return False
def __availability_check(self): to_delete = [] for idx, word_meta in enumerate(self.__words): path = word_meta.path(self.__root_dir) if not is_file(path): logger.warning("File not found:", path) to_delete.append(idx) self.__save_delete_indices(to_delete)
def imdb_from_nfo(show_name: str): 'return the imdb-id from a tvshow.nfo, or None if unavailalble' if not util.is_dir(show_name): show_name = os.path.join(SHOW_DIR, show_name) if not util.is_dir(show_name): return None nfo_file = os.path.join(show_name, 'tvshow.nfo') if not util.is_file(nfo_file): return None return util.parse_imdbid_from_file(nfo_file)
def rename_file(source_file, destination): "Custom file move/rename method using mv command in the background" source_file = path_to_str(source_file) destination = path_to_str(destination) if not util.is_file(source_file): print( f'source {CSTR(source_file, "orange")} does not exist!') return False command = f'mv \"{source_file}\" \"{destination}\"' return local_command(command, hide_output=True, print_info=False)
def deserialize(bits): assert_type(bits, BitArray, 'deserialize') s = bits.tobytes() assert_type(s, str, 'deserialize') # print 'serialized payload from twitter: %s bytes -> %s bytes' % \ # (len(bits) / 8.0, len(s)) print 'serialized payload from twitter: %s bytes' % len(s) x = bson.loads(s) if not is_file(x) and not is_dir(x): raise ArgumentError('FATAL: bad type "%s"' % x['type']) return x
def _match_subdir(orig_path, sub_str: str): if not util.is_dir(orig_path): return "" for item in os.listdir(orig_path): if util.is_file(os.path.join(orig_path, item)): continue if is_ds_special_dir(item): continue if sub_str.lower() in item.lower(): return item # TODO: handle several matches return ""
def image_resize(path_to_image: str, width: int, heigh: int, new_file_name: None) -> bool: if not util.is_file(path_to_image): pfcs(f'image file e[{path_to_image}] does not exist!') return False old_img_path = Path(path_to_image) if not new_file_name: new_file_name = old_img_path.name.replace( old_img_path.suffix, "") + "_resized" + old_img_path.suffix elif isinstance(new_file_name, str): resized_img_path = Path(old_img_path.parent) / new_file_name elif isinstance(new_file_name, Path): resized_img_path = new_file_name else: pfcs(f'cannot determine output filename of' f'resized version of e[{path_to_image}]!') return False if util.is_file(resized_img_path): pfcs(f'output filename e[{resized_img_path}] already exist!') return False with Image.open(old_img_path) as image: resized_image = image.copy() resized_image.thumbnail((width, heigh)) resized_image.save(resized_img_path) return True
def ls(self): parser = argparse.ArgumentParser( description="List files in the specified directory") parser.add_argument("dir", nargs="?", default=self.curdir.get_dir()) args = parser.parse_args(self.args[1:]) args.dir = self.__normalize_dir(args.dir) try: for f in self.dropbox.files_list_folder(args.dir).entries: if util.is_file(f): print(coloured(f.name, 'white')) else: print(coloured(f.name, 'yellow')) except dropbox.exceptions.ApiError as e: print_error("sink ls: no such directory")
def ls(self): parser = argparse.ArgumentParser( description="List files in the specified directory") parser.add_argument("dir", nargs="?", default=self.curdir.get_dir()) args = parser.parse_args(self.args[1:]) args.dir = self.__normalize_dir(args.dir) try: for f in self.dropbox.files_list_folder(args.dir).entries: if util.is_file(f): print(coloured(f.name, 'white')) else: print(coloured(f.name, 'yellow')) except dropbox.exceptions.ApiError as e: print_error("sink ls: no such directory")
def __call__(self, meta, img_directory): root = os.path.dirname(img_directory) basename = os.path.basename(img_directory) deslant_path = meta.path( os.path.join(root, self.name.lower() + "_" + basename)) if not is_file(deslant_path): make_directories_for_file(deslant_path) original_path = meta.path(img_directory) image = cv2.imread(original_path) image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) image = self.__transform(image) cv2.imwrite(deslant_path, image) result = cv2.imread(deslant_path) return result
def unpack(payload, tweet_id, downloader, concealer, name_override=None, recur=False): assert_type(payload, dict, 'unpack') if is_file(payload): data, name, perms = payload['data'], payload['name'], int(payload['perms']) if name_override: name = name_override fatal_if_exists(name, 'file') print 'unpacked "%s" from tweet_id %s' % (name, tweet_id) write_file(name, data) print 'permissions: %s' % perms chmod(name, perms) print '' elif is_dir(payload): ids, name, perms = payload['ids'], payload['name'], int(payload['perms']) if name_override: name = name_override fatal_if_exists(name, 'directory') print 'unpacked "%s" with child tweet_ids %s' % (name, ids) write_dir(name) print 'permissions: %s' % perms chmod(name, perms) print '' if recur: chdir(name) for tweet_id in ids: payload = downloader(tweet_id) payload = concealer.reveal(payload) payload = deserialize(payload) unpack(payload, tweet_id, downloader, concealer, name_override=None, recur=recur) chdir('..')
def extract(compressed_file: 'full path', destination, create_dirs=True, overwrite=True): "Extract files with fancy color output" compressed_file = path_to_str(compressed_file) destination = path_to_str(destination) if not util.is_file(compressed_file): print( f'compressed_file {CSTR(compressed_file, "orange")} does not exist!') return False if not util.is_dir(destination): if create_dirs: os.makedirs(destination) print(f'extract: created dir {CSTR(destination, "lblue")}') else: print( f'extract: destination {CSTR(destination, "orange")} does not exist!') return False # just support rar for now file_name = util.filename_of_path(compressed_file) print(f'extracting {CSTR(file_name, "lblue")}') print(f'destination {CSTR(destination, "lblue")}') overwrite_arg = '-o+' if overwrite else '' command = shlex.split( f'unrar e {overwrite_arg} "{compressed_file}" "{destination}"') process = subprocess.Popen(command, stdout=subprocess.PIPE) while process.poll() is None: byte_line = process.stdout.readline() line = byte_line.decode() if '%' in line: percentage_done = util.parse_percent(line) print(f'\r{file_name} {CSTR(percentage_done, "lgreen")}', end='') print() if process.returncode == 0: print(CSTR('done!', 'lgreen')) return True print(CSTR('extract failed!', 'red')) return False
def test_is_file(): s = '\n' s += 'a.txt = ' + str(util.is_file('a.txt')) + '\n' s += 'b.txt = ' + str(util.is_file('b.txt')) + '\n' return s
parser.add_argument('checkpoint_file', nargs='?') parser.add_argument('--topk', action='store', default=3, type=int, help='top KK most likely classes') parser.add_argument('--category_names', action='store', help='Mapping of categories to real names') parser.add_argument('--gpu', action='store_true', default=False, help='Training via gpu') args = parser.parse_args() image_path = args.image_path checkpoint_file = args.checkpoint_file topk = args.topk category_names = args.category_names gpu = args.gpu if image_path == None: raise parser.error("Image file path was not specified") if checkpoint_file == None: raise parser.error("Model checkpoint file was not specified") if not util.is_file(checkpoint_file) or not checkpoint_file.endswith(".pth"): raise parser.error("Make sure your checkpoint file exist and has a .pth extension") model = util.load_model(checkpoint_file) probs, labels = util.predict(image_path, model, topk, gpu) if category_names != None: cat_to_names = util.get_category_names(category_names) util.show_flower_labels(probs, labels, cat_to_names) else: util.show_flower_labels(probs, labels)