def cmd_install(version, functional, protocol, traceback):
"""Install a configuration of the SSSP."""
# pylint: disable=too-many-locals
import requests
import tempfile
from aiida.common import exceptions
from aiida.common.files import md5_file
from aiida.orm import QueryBuilder
from aiida_sssp import __version__
from aiida_sssp.groups import SsspFamily
label = '{}/{}/{}/{}'.format('SSSP', version, functional, protocol)
description = 'SSSP v{} {} {} installed with aiida-sssp v{}'.format(version, functional, protocol, __version__)
try:
QueryBuilder().append(SsspFamily, filters={'label': label}).limit(1).one()
except exceptions.NotExistent:
pass
else:
echo.echo_critical('SSSP {} {} {} is already installed: {}'.format(version, functional, protocol, label))
try:
url_base = os.path.join(URL_BASE, URL_MAPPING[(version, functional, protocol)])
except KeyError:
echo.echo_critical('No SSSP available for {} {} {}'.format(version, functional, protocol))
with tempfile.TemporaryDirectory() as dirpath:
url_archive = url_base + '.tar.gz'
url_metadata = url_base + '.json'
filepath_archive = os.path.join(dirpath, 'archive.tar.gz')
filepath_metadata = os.path.join(dirpath, 'metadata.json')
with attempt('downloading selected pseudo potentials archive... ', include_traceback=traceback):
response = requests.get(url_archive)
response.raise_for_status()
with open(filepath_archive, 'wb') as handle:
handle.write(response.content)
handle.flush()
description += '\nArchive pseudos md5: {}'.format(md5_file(filepath_archive))
with attempt('downloading selected pseudo potentials metadata... ', include_traceback=traceback):
response = requests.get(url_metadata)
response.raise_for_status()
with open(filepath_metadata, 'wb') as handle:
handle.write(response.content)
handle.flush()
description += '\nPseudo metadata md5: {}'.format(md5_file(filepath_metadata))
with attempt('unpacking archive and parsing pseudos... ', include_traceback=traceback):
family = create_family_from_archive(label, filepath_archive, filepath_metadata)
family.description = description
echo.echo_success('installed `{}` containing {} pseudo potentials'.format(label, family.count()))
def get_or_create(cls, filepath, use_first=False, store_upf=True):
"""Get the `UpfData` with the same md5 of the given file, or create it if it does not yet exist.
:param filepath: an absolute filepath on disk
:param use_first: if False (default), raise an exception if more than one potential is found.
If it is True, instead, use the first available pseudopotential.
:param store_upf: boolean, if false, the `UpfData` if created will not be stored.
:return: tuple of `UpfData` and boolean indicating whether it was created.
"""
import os
from aiida.common.files import md5_file
if not os.path.isabs(filepath):
raise ValueError('filepath must be an absolute path')
pseudos = cls.from_md5(md5_file(filepath))
if not pseudos:
instance = cls(file=filepath)
if store_upf:
instance.store()
return (instance, True)
if len(pseudos) > 1:
if use_first:
return (pseudos[0], False)
raise ValueError(
'More than one copy of a pseudopotential with the same MD5 has been found in the DB. pks={}'
.format(','.join([str(i.pk) for i in pseudos])))
return (pseudos[0], False)
def set_file(self, file, filename=None):
"""Store the file in the repository and parse it to set the `element` and `md5` attributes.
:param file: filepath or filelike object of the UPF potential file to store.
Hint: Pass io.BytesIO(b"my string") to construct the file directly from a string.
:param filename: specify filename to use (defaults to name of provided file).
"""
# pylint: disable=redefined-builtin
from aiida.common.exceptions import ParsingError
from aiida.common.files import md5_file, md5_from_filelike
parsed_data = parse_upf(file)
try:
md5sum = md5_file(file)
except TypeError:
md5sum = md5_from_filelike(file)
try:
element = parsed_data['element']
except KeyError:
raise ParsingError(
"No 'element' parsed in the UPF file {}; unable to store".
format(self.filename))
super(UpfData, self).set_file(file, filename=filename)
self.set_attribute('element', str(element))
self.set_attribute('md5', md5sum)
def set_file(self, file):
"""Store the file in the repository and parse it to set the `element` and `md5` attributes.
:param file: filepath or filelike object of the UPF potential file to store.
"""
# pylint: disable=redefined-builtin
from aiida.common.exceptions import ParsingError
from aiida.common.files import md5_file, md5_from_filelike
parsed_data = parse_upf(file)
try:
md5sum = md5_file(file)
except TypeError:
md5sum = md5_from_filelike(file)
try:
element = parsed_data['element']
except KeyError:
raise ParsingError(
"No 'element' parsed in the UPF file {}; unable to store".
format(self.filename))
super(UpfData, self).set_file(file)
self.set_attribute('element', str(element))
self.set_attribute('md5', md5sum)
def _validate(self):
from aiida.common import ValidationError
super(UspData, self)._validate()
# Check again, in case things changes
usp_abspath = str(self._abs_path)
if not usp_abspath:
raise ValidationError("No valid usp file was passed")
parsed_element = get_usp_element(usp_abspath)
md5 = md5_file(usp_abspath)
attr_element = self.element
if attr_element is None:
raise ValidationError("No element is set")
attr_md5 = self.get_attribute('md5', None)
if self.md5sum is None:
raise ValidationError("attribute 'md5' not set.")
if md5 != attr_md5:
raise ValidationError(
"Mismatch between store md5 and actual md5 value")
# Warn if the parsed elemnt (if any) is not matching the attribute
if attr_element != parsed_element and parsed_element is not None:
raise ValidationError("Attribute 'element' says '{}' but '{}' was "
"parsed from file name instead.".format(
attr_element, parsed_element))
def set_file(self, file, filename=None):
"""
Extract element and compute the md5hash
"""
filename = Path(file).name
try:
element = get_usp_element(file)
except KeyError:
element = None
else:
# Only set the element if it is not there
if self.element is None:
if element is not None:
self.set_element(element)
else:
warnings.warn(
"Cannot extract element form the usp/recpot file {}."
"Please set it manually.".format(file))
else:
# The element is already set, no need to process further
pass
md5sum = md5_file(file)
self.set_attribute('md5', md5sum)
super(UspData, self).set_file(file, filename)
def store(self):
"""Write the current config to file.
.. note:: if the configuration file already exists on disk and its contents differ from those in memory, a
backup of the original file on disk will be created before overwriting it.
:return: self
"""
import tempfile
from aiida.common.files import md5_from_filelike, md5_file
# If the filepath of this configuration does not yet exist, simply write it.
if not os.path.isfile(self.filepath):
with io.open(self.filepath, 'wb') as handle:
self._write(handle)
return self
# Otherwise, we write the content to a temporary file and compare its md5 checksum with the current config on
# disk. When the checksums differ, we first create a backup and only then overwrite the existing file.
with tempfile.NamedTemporaryFile() as handle:
self._write(handle)
handle.seek(0)
if md5_from_filelike(handle) != md5_file(self.filepath):
self._backup(self.filepath)
shutil.copy(handle.name, self.filepath)
return self
def store(self):
"""Write the current config to file.
.. note:: if the configuration file already exists on disk and its contents differ from those in memory, a
backup of the original file on disk will be created before overwriting it.
:return: self
"""
from aiida.common.files import md5_from_filelike, md5_file
from .settings import DEFAULT_CONFIG_INDENT_SIZE
# If the filepath of this configuration does not yet exist, simply write it.
if not os.path.isfile(self.filepath):
self._atomic_write()
return self
# Otherwise, we write the content to a temporary file and compare its md5 checksum with the current config on
# disk. When the checksums differ, we first create a backup and only then overwrite the existing file.
with tempfile.NamedTemporaryFile() as handle:
json.dump(self.dictionary,
handle,
indent=DEFAULT_CONFIG_INDENT_SIZE)
handle.seek(0)
if md5_from_filelike(handle) != md5_file(self.filepath):
self._backup(self.filepath)
self._atomic_write()
return self
def get_or_create(cls, filename, use_first=False, store_psf=True):
"""
Pass the same parameter of the init; if a file with the same md5
is found, that PsfData is returned.
:param filename: an absolute filename on disk
:param use_first: if False (default), raise an exception if more than \
one potential is found.\
If it is True, instead, use the first available pseudopotential.
:param bool store_psf: If false, the PsfData objects are not stored in
the database. default=True.
:return (psf, created): where psf is the PsfData object, and create is either\
True if the object was created, or False if the object was retrieved\
from the DB.
"""
import os
message = ( #pylint: disable=invalid-name
'This method has been deprecated and will be removed in `v2.0.0`. Support on psf pseudos and ' +
'corresponding families is moved to the aiida_pseudo package. Use the `get_or_create` '+
'method of `aiida_pseudo.data.pseudo.psf.PsfData`.'
)
warnings.warn(message, AiidaSiestaDeprecationWarning)
if not os.path.abspath(filename):
raise ValueError("filename must be an absolute path")
md5 = md5_file(filename)
pseudos = cls.from_md5(md5)
if not pseudos:
instance = cls(file=filename)
if store_psf:
instance.store()
return (instance, True)
if len(pseudos) > 1:
if use_first:
return (pseudos[0], False)
raise ValueError(
"More than one copy of a pseudopotential "
"with the same MD5 has been found in the "
"DB. pks={}".format(",".join([str(i.pk) for i in pseudos]))
)
return (pseudos[0], False)
def set_file(self, file, filename=None):
"""
I pre-parse the file to store the attributes.
"""
from aiida.common.exceptions import ParsingError
parsed_data = parse_psml(file)
md5sum = md5_file(file)
try:
element = parsed_data['element']
except KeyError:
raise ParsingError(
"No 'element' parsed in the PSML file: unable to store")
super(PsmlData, self).set_file(file)
self.set_attribute('element', str(element))
self.set_attribute('md5', md5sum)
def get_or_create(cls,
filename,
element=None,
use_first=False,
store_usp=True):
"""
Same ase init. Check md5 in the db, it is found return a UspData.
Otherwise will store the data into the db
:return (usp, created)
"""
import aiida.common.utils
import os
# Convert the filename to an absolute path
filename = str(filename)
if filename != os.path.abspath(filename):
raise ValueError("filename must be an absolute path")
md5 = md5_file(filename)
# Check if we have got the file already
pseudos = cls.from_md5(md5)
if len(pseudos) == 0:
# No existing pseudopotential file is in the database
instance = cls(file=filename)
# If we there is an element given then I set it
if element is not None:
instance.set_element(element)
# Store the usp if requested
if store_usp is True:
instance.store()
return (instance, True)
else:
if len(pseudos) > 1:
if use_first:
return (pseudos[0], False)
else:
pks = ", ".join([str(i.pk) for i in pseudos])
raise ValueError("More than one copy of a pseudopotential"
" found. pks={}".format(pks))
else:
return (pseudos[0], False)
def get_or_create(cls, filename, use_first=False, store_psml=True):
"""
Pass the same parameter of the init; if a file with the same md5
is found, that PsmlData is returned.
:param filename: an absolute filename on disk
:param use_first: if False (default), raise an exception if more than \
one potential is found.\
If it is True, instead, use the first available pseudopotential.
:param bool store_psml: If false, the PsmlData objects are not stored in
the database. default=True.
:return (psml, created): where psml is the PsmlData object, and create is either\
True if the object was created, or False if the object was retrieved\
from the DB.
"""
import aiida.common.utils
import os
if not os.path.abspath(filename):
raise ValueError("filename must be an absolute path")
md5 = md5_file(filename)
pseudos = cls.from_md5(md5)
if len(pseudos) == 0:
if store_psml:
instance = cls(file=filename).store()
return (instance, True)
else:
instance = cls(file=filename)
return (instance, True)
else:
if len(pseudos) > 1:
if use_first:
return (pseudos[0], False)
else:
raise ValueError("More than one copy of a pseudopotential "
"with the same MD5 has been found in the "
"DB. pks={}".format(",".join(
[str(i.pk) for i in pseudos])))
else:
return (pseudos[0], False)
def get_or_create(cls, filename, use_first=False, store_cif=True):
"""
Pass the same parameter of the init; if a file with the same md5
is found, that CifData is returned.
:param filename: an absolute filename on disk
:param use_first: if False (default), raise an exception if more than \
one CIF file is found.\
If it is True, instead, use the first available CIF file.
:param bool store_cif: If false, the CifData objects are not stored in
the database. default=True.
:return (cif, created): where cif is the CifData object, and create is either\
True if the object was created, or False if the object was retrieved\
from the DB.
"""
import os
from aiida.common.files import md5_file
if not os.path.abspath(filename):
raise ValueError('filename must be an absolute path')
md5 = md5_file(filename)
cifs = cls.from_md5(md5)
if not cifs:
if store_cif:
instance = cls(file=filename).store()
return (instance, True)
instance = cls(file=filename)
return (instance, True)
if len(cifs) > 1:
if use_first:
return (cifs[0], False)
raise ValueError(
'More than one copy of a CIF file '
'with the same MD5 has been found in '
'the DB. pks={}'.format(','.join([str(i.pk) for i in cifs]))
)
return cifs[0], False
def set_file(self, filename):
"""
I pre-parse the file to store the attributes.
"""
from aiida.common.exceptions import ParsingError
import aiida.common.utils
# print("Called set_file","type of filename:",type(filename))
parsed_data = parse_psml(filename)
md5sum = md5_file(filename)
try:
element = parsed_data['element']
except KeyError:
raise ParsingError("No 'element' parsed in the PSML file {};"
" unable to store".format(self.filename))
super(PsmlData, self).set_file(filename)
self.set_attribute('element', str(element))
self.set_attribute('md5', md5sum)
def upload_upf_family(folder,
group_label,
group_description,
stop_if_existing=True):
"""Upload a set of UPF files in a given group.
:param folder: a path containing all UPF files to be added.
Only files ending in .UPF (case-insensitive) are considered.
:param group_label: the name of the group to create. If it exists and is non-empty, a UniquenessError is raised.
:param group_description: string to be set as the group description. Overwrites previous descriptions.
:param stop_if_existing: if True, check for the md5 of the files and, if the file already exists in the DB, raises a
MultipleObjectsError. If False, simply adds the existing UPFData node to the group.
"""
# pylint: disable=too-many-locals,too-many-branches
import os
from aiida import orm
from aiida.common import AIIDA_LOGGER
from aiida.common.exceptions import UniquenessError
from aiida.common.files import md5_file
if not os.path.isdir(folder):
raise ValueError('folder must be a directory')
# only files, and only those ending with .upf or .UPF;
# go to the real file if it is a symlink
filenames = [
os.path.realpath(os.path.join(folder, i)) for i in os.listdir(folder)
if os.path.isfile(os.path.join(folder, i))
and i.lower().endswith('.upf')
]
nfiles = len(filenames)
automatic_user = orm.User.objects.get_default()
group, group_created = orm.Group.objects.get_or_create(
label=group_label, type_string=UPFGROUP_TYPE, user=automatic_user)
if group.user.email != automatic_user.email:
raise UniquenessError(
'There is already a UpfFamily group with label {}'
', but it belongs to user {}, therefore you '
'cannot modify it'.format(group_label, group.user.email))
# Always update description, even if the group already existed
group.description = group_description
# NOTE: GROUP SAVED ONLY AFTER CHECKS OF UNICITY
pseudo_and_created = []
for filename in filenames:
md5sum = md5_file(filename)
builder = orm.QueryBuilder()
builder.append(UpfData, filters={'attributes.md5': {'==': md5sum}})
existing_upf = builder.first()
if existing_upf is None:
# return the upfdata instances, not stored
pseudo, created = UpfData.get_or_create(filename,
use_first=True,
store_upf=False)
# to check whether only one upf per element exists
# NOTE: actually, created has the meaning of "to_be_created"
pseudo_and_created.append((pseudo, created))
else:
if stop_if_existing:
raise ValueError('A UPF with identical MD5 to '
' {} cannot be added with stop_if_existing'
''.format(filename))
existing_upf = existing_upf[0]
pseudo_and_created.append((existing_upf, False))
# check whether pseudo are unique per element
elements = [(i[0].element, i[0].md5sum) for i in pseudo_and_created]
# If group already exists, check also that I am not inserting more than
# once the same element
if not group_created:
for aiida_n in group.nodes:
# Skip non-pseudos
if not isinstance(aiida_n, UpfData):
continue
elements.append((aiida_n.element, aiida_n.md5sum))
elements = set(elements) # Discard elements with the same MD5, that would
# not be stored twice
elements_names = [e[0] for e in elements]
if not len(elements_names) == len(set(elements_names)):
duplicates = {x for x in elements_names if elements_names.count(x) > 1}
duplicates_string = ', '.join(i for i in duplicates)
raise UniquenessError('More than one UPF found for the elements: ' +
duplicates_string + '.')
# At this point, save the group, if still unstored
if group_created:
group.store()
# save the upf in the database, and add them to group
for pseudo, created in pseudo_and_created:
if created:
pseudo.store()
AIIDA_LOGGER.debug('New node {} created for file {}'.format(
pseudo.uuid, pseudo.filename))
else:
AIIDA_LOGGER.debug('Reusing node {} for file {}'.format(
pseudo.uuid, pseudo.filename))
# Add elements to the group all togetehr
group.add_nodes([pseudo for pseudo, created in pseudo_and_created])
nuploaded = len([_ for _, created in pseudo_and_created if created])
return nfiles, nuploaded
def upload_usp_family(folder,
group_label,
group_description,
stop_if_existing=True):
"""
Upload a set of usp/recpot files in a give group
:param folder: a path containing all UPF files to be added.
Only files ending in .usp/.recpot are considered.
:param group_label: the name of the group to create. If it exists and is
non-empty, a UniquenessError is raised.
:param group_description: a string to be set as the group description.
Overwrites previous descriptions, if the group was existing.
:param stop_if_existing: if True, check for the md5 of the files and,
if the file already exists in the DB, raises a MultipleObjectsError.
If False, simply adds the existing UPFData node to the group.
"""
import os
import aiida.common
#from aiida.common import aiidalogger
from aiida.common import UniquenessError, NotExistent
from aiida.orm.querybuilder import QueryBuilder
from .otfg import OTFGGroup
files = [
os.path.realpath(os.path.join(folder, i)) for i in os.listdir(folder)
if os.path.isfile(os.path.join(folder, i)) and (
i.lower().endswith('.usp') or i.lower().endswith('recpot')
or i.lower().endswith('.uspcc'))
]
nfiles = len(files)
try:
group = OTFGGroup.get(label=group_label)
group_created = False
except NotExistent:
group = OTFGGroup(label=group_label, )
group_created = True
# Update the descript even if the group already existed
group.description = group_description
pseudo_and_created = [] # A list of records (UspData, created)
for f in files:
md5sum = md5_file(f)
qb = QueryBuilder()
qb.append(UspData, filters={'attributes.md5': {'==': md5sum}})
existing_usp = qb.first()
# Add the file if it is in the database
if existing_usp is None:
pseudo, created = UspData.get_or_create(f,
use_first=True,
store_usp=False)
pseudo_and_created.append((pseudo, created))
# The same file is there already
else:
if stop_if_existing:
raise ValueError("A usp/recpot with identical MD5 to"
" {} cannot be added with stop_if_existing"
"".format(f))
existing_usp = existing_usp[0]
pseudo_and_created.append((existing_usp, False))
# Check for unique per element
elements = [(i[0].element, i[0].md5sum) for i in pseudo_and_created]
# Check if we will duplicate after insertion
if not group_created:
for aiida_n in group.nodes:
if not isinstance(aiida_n, UspData):
continue
elements.append((aiida_n.element, aiida_n.md5sum))
# Discard duplicated pairs
elements = set(elements)
elements_names = [e[0] for e in elements]
# Check the uniqueness of the complete group
if not len(elements_names) == len(set(elements_names)):
duplicates = set(
[x for x in elements_names if elements_names.count(x) > 1])
dup_string = ", ".join(duplicates)
raise UniquenessError(
"More than one usp/recpot found for the elements: " + dup_string +
".")
if group_created:
group.store()
# Save the usp in the database if necessary and add them to the group
for pseudo, created in pseudo_and_created:
if created:
pseudo.store()
#aiidalogger.debug("New node {} created for file {}".format(
# pseudo.uuid, pseudo.filename))
else:
#aiidalogger.debug("Reusing node {} for file {}".format(
# pseudo.uuid, pseudo.filename))
pass
nodes_new = [
pseduo for pseduo, created in pseudo_and_created if created is True
]
nodes_add = [pseduo for pseduo, created in pseudo_and_created]
group.add_nodes(nodes_add)
return nfiles, len(nodes_new)
def upload_psf_family(folder, group_label, group_description, stop_if_existing=True):
"""
Upload a set of PSF files in a given group.
:param folder: a path containing all PSF files to be added.
Only files ending in .PSF (case-insensitive) are considered.
:param group_label: the name of the group to create. If it exists and is
non-empty, a UniquenessError is raised.
:param group_description: a string to be set as the group description.
Overwrites previous descriptions, if the group was existing.
:param stop_if_existing: if True, check for the md5 of the files and,
if the file already exists in the DB, raises a MultipleObjectsError.
If False, simply adds the existing PsfData node to the group.
"""
import os
from aiida import orm
from aiida.common import AIIDA_LOGGER as aiidalogger
from aiida.common.exceptions import UniquenessError
from aiida.orm.querybuilder import QueryBuilder
from aiida_siesta.groups.pseudos import PsfFamily
message = ( #pylint: disable=invalid-name
'This function has been deprecated and will be removed in `v2.0.0`. ' +
'`upload_psf_family` is substitued by `fam.create_from_folder` ' +
'where `fam` is an instance of the families classes in `aiida_pseudo.groups.family`.'
)
warnings.warn(message, AiidaSiestaDeprecationWarning)
if not os.path.isdir(folder):
raise ValueError("folder must be a directory")
# only files, and only those ending with .psf or .PSF;
# go to the real file if it is a symlink
files = [
os.path.realpath(os.path.join(folder, i))
for i in os.listdir(folder)
if os.path.isfile(os.path.join(folder, i)) and i.lower().endswith('.psf')
]
nfiles = len(files)
automatic_user = orm.User.objects.get_default()
group, group_created = PsfFamily.objects.get_or_create(label=group_label, user=automatic_user)
if group.user.email != automatic_user.email:
raise UniquenessError(
"There is already a PsfFamily group with name {}"
", but it belongs to user {}, therefore you "
"cannot modify it".format(group_label, group.user.email)
)
# Always update description, even if the group already existed
group.description = group_description
# NOTE: GROUP SAVED ONLY AFTER CHECKS OF UNICITY
pseudo_and_created = []
for afile in files:
md5sum = md5_file(afile)
qb = QueryBuilder()
qb.append(PsfData, filters={'attributes.md5': {'==': md5sum}})
existing_psf = qb.first()
#existing_psf = PsfData.query(dbattributes__key="md5",
# dbattributes__tval = md5sum)
if existing_psf is None:
# return the psfdata instances, not stored
pseudo, created = PsfData.get_or_create(afile, use_first=True, store_psf=False)
# to check whether only one psf per element exists
# NOTE: actually, created has the meaning of "to_be_created"
pseudo_and_created.append((pseudo, created))
else:
if stop_if_existing:
raise ValueError(
"A PSF with identical MD5 to "
" {} cannot be added with stop_if_existing"
"".format(afile)
)
existing_psf = existing_psf[0]
pseudo_and_created.append((existing_psf, False))
# check whether pseudo are unique per element
elements = [(i[0].element, i[0].md5sum) for i in pseudo_and_created]
# If group already exists, check also that I am not inserting more than
# once the same element
if not group_created:
for aiida_n in group.nodes:
# Skip non-pseudos
if not isinstance(aiida_n, PsfData):
continue
elements.append((aiida_n.element, aiida_n.md5sum))
elements = set(elements) # Discard elements with the same MD5, that would
# not be stored twice
elements_names = [e[0] for e in elements]
if not len(elements_names) == len(set(elements_names)):
duplicates = {x for x in elements_names if elements_names.count(x) > 1}
duplicates_string = ", ".join(i for i in duplicates)
raise UniquenessError("More than one PSF found for the elements: " + duplicates_string + ".")
# At this point, save the group, if still unstored
if group_created:
group.store()
# save the psf in the database, and add them to group
for pseudo, created in pseudo_and_created:
if created:
pseudo.store()
aiidalogger.debug("New node {} created for file {}".format(pseudo.uuid, pseudo.filename))
else:
aiidalogger.debug("Reusing node {} for file {}".format(pseudo.uuid, pseudo.filename))
# Add elements to the group all togetehr
group.add_nodes([pseudo for pseudo, created in pseudo_and_created])
nuploaded = len([_ for _, created in pseudo_and_created if created])
return nfiles, nuploaded
