def save_file(content, directory, filename):
if os.path.splitext(filename)[1] == "":
filename = filename + ".txt"
def save_file_task():
with open(os.path.join(str(directory), filename), "w") as outfile:
outfile.write(repr(content))
parallel_io(save_file_task)
def _save_Nmax(self, directory, filename):
def save_Nmax_task():
with open(os.path.join(str(directory), filename + ".length"),
"w") as length:
length.write(str(len(self)))
parallel_io(save_Nmax_task)
def _basic_tensor_save(tensor, directory, filename):
mpi_comm = tensor.mpi_comm()
if not has_pybind11():
mpi_comm = mpi_comm.tompi4py()
form = tensor.generator._form
# Write out generator
assert hasattr(tensor, "generator")
full_filename_generator = os.path.join(str(directory),
filename + ".generator")
form_name = wrapping.form_name(form)
def save_generator():
with open(full_filename_generator, "w") as generator_file:
generator_file.write(form_name)
parallel_io(save_generator, mpi_comm)
# Write out generator mpi size
full_filename_generator_mpi_size = os.path.join(
str(directory), filename + ".generator_mpi_size")
def save_generator_mpi_size():
with open(full_filename_generator_mpi_size,
"w") as generator_mpi_size_file:
generator_mpi_size_file.write(str(mpi_comm.size))
parallel_io(save_generator_mpi_size, mpi_comm)
# Write out generator mapping from processor dependent indices to processor independent (global_cell_index, cell_dof) tuple
_permutation_save(tensor, directory, form,
form_name + "_" + str(mpi_comm.size), mpi_comm)
# Write out content
_tensor_save(tensor, directory, filename, mpi_comm)
def test_parallel_io_with_error_1():
exception_message = "This test will fail"
def task():
raise RuntimeError(exception_message)
with pytest.raises(RuntimeError) as excinfo:
parallel_io(task)
assert str(excinfo.value) == exception_message
def _basic_tensor_load(tensor, directory, filename):
mpi_comm = tensor.mpi_comm()
form = tensor.generator._form
# Read in generator
full_filename_generator = os.path.join(str(directory), filename + ".generator")
def load_generator():
if os.path.exists(full_filename_generator):
with open(full_filename_generator, "r") as generator_file:
return generator_file.readline()
else:
raise OSError
generator_string = parallel_io(load_generator, mpi_comm)
# Read in generator mpi size
full_filename_generator_mpi_size = os.path.join(str(directory), filename + ".generator_mpi_size")
def load_generator_mpi_size():
if os.path.exists(full_filename_generator_mpi_size):
with open(full_filename_generator_mpi_size, "r") as generator_mpi_size_file:
return generator_mpi_size_file.readline()
else:
raise OSError
generator_mpi_size_string = parallel_io(load_generator_mpi_size, mpi_comm)
# Read in generator mapping from processor dependent indices (at the time of saving)
# to processor independent (global_cell_index, cell_dof) tuple
permutation = _permutation_load(tensor, directory, filename, form,
generator_string + "_" + generator_mpi_size_string, mpi_comm)
_tensor_load(tensor, directory, filename, permutation, mpi_comm)
def remove_files(directory, filename):
full_filename = os.path.join(str(directory), filename)
def remove_files_task():
if os.path.exists(full_filename + "_index.sfx"):
os.remove(full_filename + "_index.sfx")
parallel_io(remove_files_task)
def save_file(content, directory, filename):
if not filename.endswith(".npy"):
filename = filename + ".npy"
def save_file_task():
numpy.save(os.path.join(str(directory), filename), content)
parallel_io(save_file_task)
def test_parallel_io_with_error_2():
exception_message_1 = "This test"
exception_message_2 = "will fail"
def task():
raise CustomError(exception_message_1, exception_message_2)
with pytest.raises(CustomError) as excinfo:
parallel_io(task)
assert str(excinfo.value) == str((exception_message_1, exception_message_2))
def save_file(content, directory, filename):
if not filename.endswith(".sym"):
filename = filename + ".sym"
def save_file_task():
with open(os.path.join(str(directory), filename), "w") as outfile:
outfile.write(python(content))
parallel_io(save_file_task)
def save_file(content, directory, filename):
if not filename.endswith(".pkl"):
filename = filename + ".pkl"
def save_file_task():
with open(os.path.join(str(directory), filename), "wb") as outfile:
pickle.dump(content, outfile, protocol=pickle.HIGHEST_PROTOCOL)
parallel_io(save_file_task)
def save_file(content, directory, filename):
if not filename.endswith(".csv"):
filename = filename + ".csv"
def save_file_task():
with open(os.path.join(str(directory), filename), "w") as outfile:
writer = csv.writer(outfile, delimiter=";")
writer.writerows(content)
parallel_io(save_file_task)
def remove_files(directory, filename):
SolutionFile_Base.remove_files(directory, filename)
#
full_filename = os.path.join(str(directory), filename)
def remove_files_task():
if os.path.exists(full_filename + ".xdmf"):
os.remove(full_filename + ".xdmf")
os.remove(full_filename + ".h5")
os.remove(full_filename + "_checkpoint.xdmf")
os.remove(full_filename + "_checkpoint.h5")
parallel_io(remove_files_task)
def patched_export_solution_internal(self_,
folder=None,
filename=None,
*args,
**kwargs):
if str(folder) == str(snapshots_folder):
assert (hasattr(truth_problem, "_cache_file_from_kwargs")
or hasattr(truth_problem, "_cache_file"))
if hasattr(truth_problem,
"_cache_file_from_kwargs"): # differential problem
cache_filename = truth_problem._cache_file_from_kwargs(
**truth_problem._latest_solve_kwargs)
elif hasattr(truth_problem, "_cache_file"): # EIM
cache_filename = truth_problem._cache_file()
else:
raise AttributeError("Invalid cache file attribute.")
def create_links():
for cache_path in glob.iglob(
os.path.join(str(cache_folder),
cache_filename + "*")):
cache_path_filename = os.path.basename(cache_path)
cache_relpath = os.path.join(
os.path.relpath(str(cache_folder), str(folder)),
cache_path_filename)
snapshot_path = os.path.join(
str(folder),
cache_path_filename.replace(
cache_filename, filename))
if not os.path.exists(snapshot_path):
# Paraview output formats may require a light xml file that stores the path of a
# (possibly heavy) binary file. If the file is an xml file, we need to copy it and
# change the stored path. Otherwise, create a symbolic link.
try:
with open(cache_path, "r") as cache_file:
header = cache_file.read(5)
except Exception:
should_link = True
else:
should_link = (header != "<?xml")
if should_link:
os.symlink(cache_relpath, snapshot_path)
else:
with open(cache_path, "r") as cache_file, open(
snapshot_path, "w") as snapshot_file:
for l in cache_file.readlines(
): # noqa: E741
snapshot_file.write(
l.replace(cache_filename,
filename))
parallel_io(create_links)
else:
original_export_solution(folder, filename, *args, **kwargs)
def write(self, file_or_file_object):
assert isinstance(file_or_file_object, str) or file_or_file_object is sys.stdout, (
"Please provide a file name and not a file object (except for sys.stdout)")
if isinstance(file_or_file_object, str):
def write_config_parser():
with open(file_or_file_object, "w") as file_:
self._config_as_parser.write(file_)
else:
assert file_or_file_object is sys.stdout
def write_config_parser():
self._config_as_parser.write(file_or_file_object)
parallel_io(write_config_parser)
def _load_Nmax(self, directory, filename):
def load_Nmax_task():
with open(os.path.join(str(directory), filename + ".length"),
"r") as length:
return int(length.readline())
return parallel_io(load_Nmax_task)
def exists_file(directory, filename):
if not filename.endswith(".sym"):
filename = filename + ".sym"
def exists_file_task():
return os.path.exists(os.path.join(str(directory), filename))
return parallel_io(exists_file_task)
def exists_file(directory, filename):
if os.path.splitext(filename)[1] == "":
filename = filename + ".txt"
def exists_file_task():
return os.path.exists(os.path.join(str(directory), filename))
return parallel_io(exists_file_task)
def create(self):
def create_task():
if os.path.exists(self.name) and len(os.listdir(self.name)) == 0: # already created, but empty
return True
if not os.path.exists(self.name): # to be created
os.makedirs(self.name)
return True
return False
return parallel_io(create_task)
def write(self, file_):
def write_config_parser():
self._config_as_parser.write(file_)
parallel_io(write_config_parser)
def test_parallel_io_with_return_value():
def task():
return COMM_WORLD.rank
return_value = parallel_io(task)
assert return_value == 0
def test_parallel_io_without_return_value():
def task():
pass
return_value = parallel_io(task)
assert return_value is None
def run_and_compare_to_gold_function(self):
"""
Handles the comparison of test/tutorial with gold files
"""
rootdir = str(self.config.rootdir)
# Get action
action = self.config.option.action
assert action in ("compare", "regold", None)
# Get data directory
if action is not None:
data_dir = self.config.option.data_dir
assert data_dir is not None
else:
data_dir = None
# Get current and reference directory
current_dir = str(self.fspath.dirname)
if action is not None:
reference_dir = os.path.join(
current_dir.replace(rootdir, data_dir),
self.fspath.basename)
current_dir = os.path.join(current_dir, subdirectory)
reference_dir = os.path.join(reference_dir, subdirectory)
else:
reference_dir = None
# Copy training and testing sets
if action is not None:
def copy_training_and_testing_sets():
for set_ in ("testing_set", "training_set"):
set_directories = glob.glob(os.path.join(
reference_dir, "**", set_),
recursive=True)
if action == "compare":
assert len(set_directories) > 0
for set_directory in set_directories:
set_directory = os.path.relpath(
set_directory, reference_dir)
if os.path.exists(
os.path.join(reference_dir,
set_directory)):
if os.path.exists(
os.path.join(current_dir,
set_directory)):
shutil.rmtree(
os.path.join(current_dir,
set_directory))
shutil.copytree(
os.path.join(reference_dir, set_directory),
os.path.join(current_dir, set_directory))
parallel_io(copy_training_and_testing_sets)
# Run test/tutorial
runtest(self)
# Process results
def process_results():
if action == "compare":
failures = list()
filenames = glob.glob(os.path.join(reference_dir, "**",
"*.*"),
recursive=True)
assert len(filenames) > 0
for filename in filenames:
filename = os.path.relpath(filename, reference_dir)
diffs = diff(os.path.join(reference_dir, filename),
os.path.join(current_dir, filename))
if len(diffs) > 0:
failures.append(filename)
os.makedirs(os.path.dirname(
os.path.join(current_dir, filename + "_diff")),
exist_ok=True)
with open(
os.path.join(current_dir,
filename + "_diff"),
"w") as failure_file:
failure_file.writelines(diffs)
if len(failures) > 0:
raise RuntimeError(
self.name +
", comparison has failed for the following files: "
+ str(failures) + ".")
elif action == "regold":
data_dir_repo = git.Repo(data_dir)
assert not data_dir_repo.is_dirty()
# Move current files to reference directory
if os.path.exists(reference_dir):
shutil.rmtree(reference_dir)
shutil.copytree(current_dir, reference_dir)
if os.path.exists(os.path.join(reference_dir,
".gitignore")):
os.remove(os.path.join(reference_dir, ".gitignore"))
data_dir_repo.git.add([reference_dir])
# Commit changes
commit = str(git.Repo(rootdir).head.reference.commit)
relpath = os.path.relpath(str(self.fspath), rootdir)
if self.name != relpath:
message = ("Automatic regold of " + self.name +
" in " + relpath + " at upstream commit " +
commit)
else:
message = "Automatic regold of " + relpath + " at upstream commit " + commit
data_dir_repo.git.commit(message=message)
# Clean repository
data_dir_repo.git.clean("-Xdf")
parallel_io(process_results)
def _file_exists(directory, filename, mpi_comm):
def file_exists_task():
return os.path.exists(os.path.join(str(directory), filename))
return parallel_io(file_exists_task, mpi_comm)
def touch_file(self, filename):
def touch_file_task():
with open(os.path.join(self.name, filename), "a"):
os.utime(os.path.join(self.name, filename), None)
parallel_io(touch_file_task)
