def __call__(self, *args, **kwargs): PythonBatchCommandBase.__call__(self, *args, **kwargs) with open(self.file_path, "w") as wfd: utils.chown_chmod_on_fd(wfd) wfd.write(''.join( random.choice(string.ascii_lowercase + string.ascii_uppercase) for i in range(self.file_size)))
def __call__(self, *args, **kwargs) -> None: PythonBatchCommandBase.__call__(self, *args, **kwargs) self.compile_exclude_regexi() with open(self.out_file, "w") as wfd: utils.chown_chmod_on_fd(wfd) if os.path.isfile(self.folder_to_scan): file_size = os.path.getsize(self.folder_to_scan) wfd.write(f"{self.folder_to_scan}, {file_size}\n") else: folder_to_scan_name_len = len( self.folder_to_scan) + 1 # +1 for the last '\' if not self.compiled_forbidden_folder_regex.search( self.folder_to_scan): for root, dirs, files in utils.excluded_walk( self.folder_to_scan, file_exclude_regex=self. compiled_forbidden_file_regex, dir_exclude_regex=self. compiled_forbidden_folder_regex, followlinks=False): for a_file in files: full_path = os.path.join(root, a_file) file_size = os.path.getsize(full_path) partial_path = full_path[folder_to_scan_name_len:] wfd.write(f"{partial_path}, {file_size}\n")
def test_write(self): as_yaml = self.it.repr_for_yaml() as_yaml_doc = aYaml.YamlDumpDocWrap(as_yaml, '!index') as_yaml_doc.ReduceOneItemLists() with open(self.out_file_path, "w") as wfd: utils.chown_chmod_on_fd(wfd) aYaml.writeAsYaml(as_yaml_doc, wfd)
def __call__(self, *args, **kwargs): original_size = self.file_to_split.stat().st_size splits = self.calc_splits(original_size) print( f"original: {original_size}, max_size: {self.max_size}, self.num_parts: {len(splits)}, part_size: {splits[0][0]} naive total {len(splits)*splits[0][0]}" ) print("\n ".join(str(s[1]) for s in splits)) with open(self.file_to_split, "rb") as fts: for part_size, part_path in splits: with open(part_path, "wb") as pfd: utils.chown_chmod_on_fd(pfd) pfd.write(fts.read(part_size)) if self.remove_original: self.file_to_split.unlink()
def __call__(self, *args, **kwargs): if not self.file_to_join.name.endswith(".aa"): raise ValueError( f"name of file to join must end with .aa not: {self.file_to_join.name}" ) files_to_join = utils.find_split_files(self.file_to_join) joined_file_path = self.file_to_join.parent.joinpath( self.file_to_join.stem) with open(joined_file_path, "wb") as wfd: utils.chown_chmod_on_fd(wfd) for part_file in files_to_join: with open(part_file, "rb") as rfd: wfd.write(rfd.read()) if self.remove_parts: for part_file in files_to_join: os.unlink(part_file)
def read_file_or_url(in_file_or_url, config_vars, path_searcher=None, encoding='utf-8', save_to_path=None, checksum=None, connection_obj=None): need_to_download = not utils.check_file_checksum(save_to_path, checksum) if not need_to_download: # if save_to_path contains the correct data just read it by recursively # calling read_file_or_url return read_file_or_url(save_to_path, config_vars, encoding=encoding) match = protocol_header_re.match(os.fspath(in_file_or_url)) actual_file_path = in_file_or_url if not match: # it's a local file if path_searcher is not None: actual_file_path = path_searcher.find_file(actual_file_path) if actual_file_path: if 'Win' in utils.get_current_os_names(): actual_file_path = os.path.abspath(actual_file_path) else: actual_file_path = os.path.realpath(actual_file_path) else: raise FileNotFoundError( f"Could not locate local file {in_file_or_url}") if encoding is None: read_mod = "rb" else: read_mod = "r" with open(actual_file_path, "r", encoding=encoding) as rdf: buffer = rdf.read() else: assert connection_obj, "no connection_obj given" session = connection_obj.get_session(in_file_or_url) response = session.get(in_file_or_url, timeout=(33.05, 180.05)) response.raise_for_status() buffer = response.text buffer = utils.unicodify(buffer) # make sure text is unicode if save_to_path and in_file_or_url != save_to_path: with open(save_to_path, "w") as wfd: utils.chown_chmod_on_fd(wfd) wfd.write(buffer) return buffer, actual_file_path
def download_from_file_or_url(in_url, config_vars, in_target_path=None, translate_url_callback=None, cache_folder=None, expected_checksum=None): """ download a file from url and place it on a target path. Possibly also decompressed .wzip files. """ final_file_path = None cached_file_path = download_and_cache_file_or_url( in_url=in_url, config_vars=config_vars, translate_url_callback=translate_url_callback, cache_folder=cache_folder, expected_checksum=expected_checksum) if not in_target_path: in_target_path = cache_folder if in_target_path: in_target_path = utils.ExpandAndResolvePath(in_target_path) url_file_name = last_url_item(in_url) url_base_file_name, url_extension = os.path.splitext(url_file_name) need_decompress = url_extension == ".wzip" if in_target_path.is_dir(): target_file_name = url_base_file_name if need_decompress else url_file_name final_file_path = in_target_path.joinpath(target_file_name) else: final_file_path = in_target_path _, target_extension = os.path.splitext(final_file_path) if need_decompress and final_file_path.suffix == ".wzip": need_decompress = False # no need to decompress if target is expected to be compressed if need_decompress: decompressed = zlib.decompress(open(cached_file_path, "rb").read()) with open(final_file_path, "wb") as wfd: utils.chown_chmod_on_fd(wfd) wfd.write(decompressed) else: smart_copy_file(cached_file_path, final_file_path) else: final_file_path = cached_file_path return final_file_path