def setUpClass(cls):
cls.current_dir = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
hdf5 = FileHDFio(file_name=cls.current_dir + "/test_data.h5")
with hdf5.open("content") as hdf:
hdf['key'] = "value"
copyfile(cls.current_dir + "/test_data.h5", cls.current_dir + "/test_data")
with open(cls.current_dir + '/test_data.txt', 'w') as f:
f.write("some text")
with open(cls.current_dir + '/test_data2', 'w') as f:
f.write("some text")
with open(cls.current_dir + '/test_data.csv', 'w') as f:
f.write("id,status,chemicalformula")
f.write("13,aborted,Al108")
with open(cls.current_dir + '/test_data.json', 'w') as f:
json.dump({'x': [0, 1]}, f)
def test_to_hdf(self):
filename = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../../static/dft/test_es_hdf.h5",
)
abs_filename = os.path.abspath(filename)
hdf_obj = FileHDFio(abs_filename)
es_obj_old = self.es_list[1]
es_obj_old.to_hdf(hdf_obj, group_name="written_es")
es_obj_new = ElectronicStructure()
es_obj_new.from_hdf(hdf=hdf_obj, group_name="written_es")
self.assertTrue(
np.array_equal(
hdf_obj["written_es/dos/grand_dos_matrix"],
es_obj_old.grand_dos_matrix,
))
self.assertEqual(es_obj_old.efermi, es_obj_new.efermi)
self.assertEqual(es_obj_old.is_metal, es_obj_new.is_metal)
self.assertEqual(es_obj_old.vbm, es_obj_new.vbm)
self.assertEqual(es_obj_old.cbm, es_obj_new.cbm)
self.assertTrue(
np.array_equal(es_obj_new.grand_dos_matrix,
es_obj_old.grand_dos_matrix))
self.assertTrue(
np.array_equal(es_obj_new.resolved_densities,
es_obj_old.resolved_densities))
def test_from_hdf(self):
filename = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../../static/dft/es_hdf.h5")
abs_filename = os.path.abspath(filename)
hdf_obj = FileHDFio(abs_filename)
es_obj_old = ElectronicStructure()
es_obj_old.from_hdf_old(hdf_obj, "es_old")
es_obj_new = ElectronicStructure()
es_obj_new.from_hdf(hdf=hdf_obj, group_name="es_new")
self.assertEqual(es_obj_old.efermi, es_obj_new.efermi)
self.assertEqual(es_obj_old.is_metal, es_obj_new.is_metal)
self.assertEqual(es_obj_old.vbm, es_obj_new.vbm)
self.assertEqual(es_obj_old.cbm, es_obj_new.cbm)
self.assertTrue(
np.array_equal(es_obj_new.grand_dos_matrix,
es_obj_old.grand_dos_matrix))
def load_file(fp, filetype=None, project=None):
"""
Load the file and return an appropriate object containing the data.
Args:
fp (str / file): path to the file or file object to be displayed.
filetype (str/None): File extension, if given this overwrites the assumption based on the filename.
project (pyiron-Project/None): Project calling this function, provided to all objects referring to such.
Supported file types are:
'.h5', '.hdf'
'.json'
'.txt'
'.csv'
'.ipynb'
Image extensions supported by PIL
Returns:
:class:`FileHDFio`: pointing to the file of filetype = '.h5'
dict: containing data from file of filetype = '.json'
list: of all lines from file for filetype = '.txt'
:class:`pandas.DataFrame`: containing data from file of filetype = '.csv'
"""
def _load_txt(file):
if isinstance(file, str):
with open(file, encoding="utf8") as f:
return f.readlines()
else:
return file.readlines()
def _load_ipynb(file):
return OwnNotebookNode(nbformat.read(file, as_version=4))
def _load_json(file):
if isinstance(file, str):
with open(file) as f:
return json.load(f)
else:
return json.load(file)
def _load_csv(file):
return pandas.read_csv(file)
def _load_img(file):
return Image.open(file)
def _load_default(file):
try:
return _load_txt(file)
except Exception as e:
raise IOError("File could not be loaded.") from e
def _resolve_filetype(file, _filetype):
if _filetype is None and isinstance(file, str):
_, _filetype = os.path.splitext(file)
elif _filetype is None and hasattr(file, "name"):
_, _filetype = os.path.splitext(file.name)
elif _filetype is None:
return None
elif _filetype[0] != ".":
_filetype = "." + _filetype
return _filetype.lower()
filetype = _resolve_filetype(fp, filetype)
if filetype in [".h5", ".hdf"] and isinstance(fp, str):
if project is None:
return FileHDFio(file_name=fp)
else:
return ProjectHDFio(file_name=fp, project=project)
elif filetype in [".json"]:
return _load_json(fp)
elif filetype in [".txt"]:
return _load_txt(fp)
elif filetype in [".csv"]:
return _load_csv(fp)
elif _has_imported["nbformat"] and filetype in [".ipynb"]:
return _load_ipynb(fp)
elif _has_imported["PIL"] and filetype in Image.registered_extensions():
return _load_img(fp)
else:
return _load_default(fp)
def create_table(self,
enforce_update=False,
level=3,
file=None,
job_status_list=None):
skip_table_update = False
filter_funct = self.filter_function
if job_status_list is None:
job_status_list = ["finished"]
if self._is_file():
if file is None:
file = FileHDFio(file_name=self._project.path + self.name +
".h5",
h5_path="/")
temp_user_function_dict, temp_system_function_dict = self._get_data_from_hdf5(
hdf=file)
job_update_lst = self._collect_job_update_lst(
job_status_list=job_status_list,
filter_funct=filter_funct,
job_stored_ids=self._get_job_ids())
keys_update_user_lst = [
key for key in self.add._user_function_dict.keys()
if key not in temp_user_function_dict.keys()
]
keys_update_system_lst = [
k for k, v in self.add._system_function_dict.items()
if v and not temp_system_function_dict[k]
]
if (len(job_update_lst) == 0 and len(keys_update_user_lst) == 0
and keys_update_system_lst == 0 and not enforce_update):
skip_table_update = True
else:
job_update_lst = self._collect_job_update_lst(
job_status_list=job_status_list,
filter_funct=filter_funct,
job_stored_ids=None)
keys_update_user_lst, keys_update_system_lst = [], []
if not skip_table_update and len(job_update_lst) != 0:
df_new_ids = self._iterate_over_job_lst(
job_lst=job_update_lst,
function_lst=self.add._function_lst,
level=level)
else:
df_new_ids = pandas.DataFrame({})
if not skip_table_update and (len(keys_update_user_lst) != 0
or len(keys_update_system_lst) != 0):
job_update_lst = self._collect_job_update_lst(
job_status_list=job_status_list,
filter_funct=filter_funct,
job_stored_ids=None)
function_lst = [
v for k, v in self.add._user_function_dict.items()
if k in keys_update_system_lst
] + [
funct for funct in self.add._system_function_lst
if funct.__name__ in keys_update_system_lst
]
df_new_keys = self._iterate_over_job_lst(job_lst=job_update_lst,
function_lst=function_lst,
level=level)
else:
df_new_keys = pandas.DataFrame({})
if len(self._df) > 0 and len(df_new_keys) > 0:
self._df = pandas.concat([self._df, df_new_keys],
axis=1,
sort=False).reset_index(drop=True)
if len(self._df) > 0 and len(df_new_ids) > 0:
self._df = pandas.concat([self._df, df_new_ids],
sort=False).reset_index(drop=True)
elif len(df_new_ids) > 0:
self._df = df_new_ids
def from_hdf(self):
file = FileHDFio(file_name=self._project.path + self.name + ".h5",
h5_path="/")
self.add._from_hdf(file)
def load_file(filename, filetype=None, project=None):
"""
Load the file and return an appropriate object containing the data.
Args:
filename (str): path to the file to be displayed.
filetype (str/None): File extension, if given this overwrites the assumption based on the filename.
project (pyiron-Project/None): Project calling this function, provided to all objects referring to such.
Supported file types are:
'.h5', '.hdf'
'.json'
'.txt'
'.csv'
'.ipynb'
Image extensions supported by PIL
Returns:
:class:`FileHDFio`: pointing to the file of filetype = '.h5'
dict: containing data from file of filetype = '.json'
list: of all lines from file for filetype = '.txt'
:class:`pandas.DataFrame`: containing data from file of filetype = '.csv'
"""
def _load_txt(file):
with open(file, encoding='utf8') as f:
return f.readlines()
def _load_ipynb(file):
return OwnNotebookNode(nbformat.read(file, as_version=4))
def _load_json(file):
with open(file) as f:
return json.load(f)
def _load_csv(file):
return pandas.read_csv(file)
def _load_img(file):
return Image.open(file)
def _load_default(file):
try:
return _load_txt(file)
except Exception as e:
raise IOError("File could not be loaded.") from e
if filetype is None:
_, filetype = os.path.splitext(filename)
elif filetype[0] != '.':
filetype = '.' + filetype
filetype = filetype.lower()
if filetype in ['.h5', '.hdf']:
if project is None:
return FileHDFio(file_name=filename)
else:
return ProjectHDFio(file_name=filename, project=project)
elif filetype in ['.json']:
return _load_json(filename)
elif filetype in ['.txt']:
return _load_txt(filename)
elif filetype in ['.csv']:
return _load_csv(filename)
elif _has_imported['nbformat'] and filetype in ['.ipynb']:
return _load_ipynb(filename)
elif _has_imported['PIL'] and filetype in Image.registered_extensions():
return _load_img(filename)
else:
return _load_default(filename)