def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
with open(self.file_path, "w") as wfd:
utils.chown_chmod_on_fd(wfd)
wfd.write(''.join(
random.choice(string.ascii_lowercase + string.ascii_uppercase)
for i in range(self.file_size)))
def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
self.compile_exclude_regexi()
with open(self.out_file, "w") as wfd:
utils.chown_chmod_on_fd(wfd)
if os.path.isfile(self.folder_to_scan):
file_size = os.path.getsize(self.folder_to_scan)
wfd.write(f"{self.folder_to_scan}, {file_size}\n")
else:
folder_to_scan_name_len = len(
self.folder_to_scan) + 1 # +1 for the last '\'
if not self.compiled_forbidden_folder_regex.search(
self.folder_to_scan):
for root, dirs, files in utils.excluded_walk(
self.folder_to_scan,
file_exclude_regex=self.
compiled_forbidden_file_regex,
dir_exclude_regex=self.
compiled_forbidden_folder_regex,
followlinks=False):
for a_file in files:
full_path = os.path.join(root, a_file)
file_size = os.path.getsize(full_path)
partial_path = full_path[folder_to_scan_name_len:]
wfd.write(f"{partial_path}, {file_size}\n")
def test_write(self):
as_yaml = self.it.repr_for_yaml()
as_yaml_doc = aYaml.YamlDumpDocWrap(as_yaml, '!index')
as_yaml_doc.ReduceOneItemLists()
with open(self.out_file_path, "w") as wfd:
utils.chown_chmod_on_fd(wfd)
aYaml.writeAsYaml(as_yaml_doc, wfd)
def __call__(self, *args, **kwargs):
original_size = self.file_to_split.stat().st_size
splits = self.calc_splits(original_size)
print(
f"original: {original_size}, max_size: {self.max_size}, self.num_parts: {len(splits)}, part_size: {splits[0][0]} naive total {len(splits)*splits[0][0]}"
)
print("\n ".join(str(s[1]) for s in splits))
with open(self.file_to_split, "rb") as fts:
for part_size, part_path in splits:
with open(part_path, "wb") as pfd:
utils.chown_chmod_on_fd(pfd)
pfd.write(fts.read(part_size))
if self.remove_original:
self.file_to_split.unlink()
def __call__(self, *args, **kwargs):
if not self.file_to_join.name.endswith(".aa"):
raise ValueError(
f"name of file to join must end with .aa not: {self.file_to_join.name}"
)
files_to_join = utils.find_split_files(self.file_to_join)
joined_file_path = self.file_to_join.parent.joinpath(
self.file_to_join.stem)
with open(joined_file_path, "wb") as wfd:
utils.chown_chmod_on_fd(wfd)
for part_file in files_to_join:
with open(part_file, "rb") as rfd:
wfd.write(rfd.read())
if self.remove_parts:
for part_file in files_to_join:
os.unlink(part_file)
def read_file_or_url(in_file_or_url,
config_vars,
path_searcher=None,
encoding='utf-8',
save_to_path=None,
checksum=None,
connection_obj=None):
need_to_download = not utils.check_file_checksum(save_to_path, checksum)
if not need_to_download:
# if save_to_path contains the correct data just read it by recursively
# calling read_file_or_url
return read_file_or_url(save_to_path, config_vars, encoding=encoding)
match = protocol_header_re.match(os.fspath(in_file_or_url))
actual_file_path = in_file_or_url
if not match: # it's a local file
if path_searcher is not None:
actual_file_path = path_searcher.find_file(actual_file_path)
if actual_file_path:
if 'Win' in utils.get_current_os_names():
actual_file_path = os.path.abspath(actual_file_path)
else:
actual_file_path = os.path.realpath(actual_file_path)
else:
raise FileNotFoundError(
f"Could not locate local file {in_file_or_url}")
if encoding is None:
read_mod = "rb"
else:
read_mod = "r"
with open(actual_file_path, "r", encoding=encoding) as rdf:
buffer = rdf.read()
else:
assert connection_obj, "no connection_obj given"
session = connection_obj.get_session(in_file_or_url)
response = session.get(in_file_or_url, timeout=(33.05, 180.05))
response.raise_for_status()
buffer = response.text
buffer = utils.unicodify(buffer) # make sure text is unicode
if save_to_path and in_file_or_url != save_to_path:
with open(save_to_path, "w") as wfd:
utils.chown_chmod_on_fd(wfd)
wfd.write(buffer)
return buffer, actual_file_path
def download_from_file_or_url(in_url,
config_vars,
in_target_path=None,
translate_url_callback=None,
cache_folder=None,
expected_checksum=None):
"""
download a file from url and place it on a target path. Possibly also decompressed .wzip files.
"""
final_file_path = None
cached_file_path = download_and_cache_file_or_url(
in_url=in_url,
config_vars=config_vars,
translate_url_callback=translate_url_callback,
cache_folder=cache_folder,
expected_checksum=expected_checksum)
if not in_target_path:
in_target_path = cache_folder
if in_target_path:
in_target_path = utils.ExpandAndResolvePath(in_target_path)
url_file_name = last_url_item(in_url)
url_base_file_name, url_extension = os.path.splitext(url_file_name)
need_decompress = url_extension == ".wzip"
if in_target_path.is_dir():
target_file_name = url_base_file_name if need_decompress else url_file_name
final_file_path = in_target_path.joinpath(target_file_name)
else:
final_file_path = in_target_path
_, target_extension = os.path.splitext(final_file_path)
if need_decompress and final_file_path.suffix == ".wzip":
need_decompress = False # no need to decompress if target is expected to be compressed
if need_decompress:
decompressed = zlib.decompress(open(cached_file_path, "rb").read())
with open(final_file_path, "wb") as wfd:
utils.chown_chmod_on_fd(wfd)
wfd.write(decompressed)
else:
smart_copy_file(cached_file_path, final_file_path)
else:
final_file_path = cached_file_path
return final_file_path