def cmd_install(version, functional, protocol, traceback): """Install a configuration of the SSSP.""" # pylint: disable=too-many-locals import requests import tempfile from aiida.common import exceptions from aiida.common.files import md5_file from aiida.orm import QueryBuilder from aiida_sssp import __version__ from aiida_sssp.groups import SsspFamily label = '{}/{}/{}/{}'.format('SSSP', version, functional, protocol) description = 'SSSP v{} {} {} installed with aiida-sssp v{}'.format(version, functional, protocol, __version__) try: QueryBuilder().append(SsspFamily, filters={'label': label}).limit(1).one() except exceptions.NotExistent: pass else: echo.echo_critical('SSSP {} {} {} is already installed: {}'.format(version, functional, protocol, label)) try: url_base = os.path.join(URL_BASE, URL_MAPPING[(version, functional, protocol)]) except KeyError: echo.echo_critical('No SSSP available for {} {} {}'.format(version, functional, protocol)) with tempfile.TemporaryDirectory() as dirpath: url_archive = url_base + '.tar.gz' url_metadata = url_base + '.json' filepath_archive = os.path.join(dirpath, 'archive.tar.gz') filepath_metadata = os.path.join(dirpath, 'metadata.json') with attempt('downloading selected pseudo potentials archive... ', include_traceback=traceback): response = requests.get(url_archive) response.raise_for_status() with open(filepath_archive, 'wb') as handle: handle.write(response.content) handle.flush() description += '\nArchive pseudos md5: {}'.format(md5_file(filepath_archive)) with attempt('downloading selected pseudo potentials metadata... ', include_traceback=traceback): response = requests.get(url_metadata) response.raise_for_status() with open(filepath_metadata, 'wb') as handle: handle.write(response.content) handle.flush() description += '\nPseudo metadata md5: {}'.format(md5_file(filepath_metadata)) with attempt('unpacking archive and parsing pseudos... ', include_traceback=traceback): family = create_family_from_archive(label, filepath_archive, filepath_metadata) family.description = description echo.echo_success('installed `{}` containing {} pseudo potentials'.format(label, family.count()))
def get_or_create(cls, filepath, use_first=False, store_upf=True): """Get the `UpfData` with the same md5 of the given file, or create it if it does not yet exist. :param filepath: an absolute filepath on disk :param use_first: if False (default), raise an exception if more than one potential is found. If it is True, instead, use the first available pseudopotential. :param store_upf: boolean, if false, the `UpfData` if created will not be stored. :return: tuple of `UpfData` and boolean indicating whether it was created. """ import os from aiida.common.files import md5_file if not os.path.isabs(filepath): raise ValueError('filepath must be an absolute path') pseudos = cls.from_md5(md5_file(filepath)) if not pseudos: instance = cls(file=filepath) if store_upf: instance.store() return (instance, True) if len(pseudos) > 1: if use_first: return (pseudos[0], False) raise ValueError( 'More than one copy of a pseudopotential with the same MD5 has been found in the DB. pks={}' .format(','.join([str(i.pk) for i in pseudos]))) return (pseudos[0], False)
def set_file(self, file, filename=None): """Store the file in the repository and parse it to set the `element` and `md5` attributes. :param file: filepath or filelike object of the UPF potential file to store. Hint: Pass io.BytesIO(b"my string") to construct the file directly from a string. :param filename: specify filename to use (defaults to name of provided file). """ # pylint: disable=redefined-builtin from aiida.common.exceptions import ParsingError from aiida.common.files import md5_file, md5_from_filelike parsed_data = parse_upf(file) try: md5sum = md5_file(file) except TypeError: md5sum = md5_from_filelike(file) try: element = parsed_data['element'] except KeyError: raise ParsingError( "No 'element' parsed in the UPF file {}; unable to store". format(self.filename)) super(UpfData, self).set_file(file, filename=filename) self.set_attribute('element', str(element)) self.set_attribute('md5', md5sum)
def set_file(self, file): """Store the file in the repository and parse it to set the `element` and `md5` attributes. :param file: filepath or filelike object of the UPF potential file to store. """ # pylint: disable=redefined-builtin from aiida.common.exceptions import ParsingError from aiida.common.files import md5_file, md5_from_filelike parsed_data = parse_upf(file) try: md5sum = md5_file(file) except TypeError: md5sum = md5_from_filelike(file) try: element = parsed_data['element'] except KeyError: raise ParsingError( "No 'element' parsed in the UPF file {}; unable to store". format(self.filename)) super(UpfData, self).set_file(file) self.set_attribute('element', str(element)) self.set_attribute('md5', md5sum)
def _validate(self): from aiida.common import ValidationError super(UspData, self)._validate() # Check again, in case things changes usp_abspath = str(self._abs_path) if not usp_abspath: raise ValidationError("No valid usp file was passed") parsed_element = get_usp_element(usp_abspath) md5 = md5_file(usp_abspath) attr_element = self.element if attr_element is None: raise ValidationError("No element is set") attr_md5 = self.get_attribute('md5', None) if self.md5sum is None: raise ValidationError("attribute 'md5' not set.") if md5 != attr_md5: raise ValidationError( "Mismatch between store md5 and actual md5 value") # Warn if the parsed elemnt (if any) is not matching the attribute if attr_element != parsed_element and parsed_element is not None: raise ValidationError("Attribute 'element' says '{}' but '{}' was " "parsed from file name instead.".format( attr_element, parsed_element))
def set_file(self, file, filename=None): """ Extract element and compute the md5hash """ filename = Path(file).name try: element = get_usp_element(file) except KeyError: element = None else: # Only set the element if it is not there if self.element is None: if element is not None: self.set_element(element) else: warnings.warn( "Cannot extract element form the usp/recpot file {}." "Please set it manually.".format(file)) else: # The element is already set, no need to process further pass md5sum = md5_file(file) self.set_attribute('md5', md5sum) super(UspData, self).set_file(file, filename)
def store(self): """Write the current config to file. .. note:: if the configuration file already exists on disk and its contents differ from those in memory, a backup of the original file on disk will be created before overwriting it. :return: self """ import tempfile from aiida.common.files import md5_from_filelike, md5_file # If the filepath of this configuration does not yet exist, simply write it. if not os.path.isfile(self.filepath): with io.open(self.filepath, 'wb') as handle: self._write(handle) return self # Otherwise, we write the content to a temporary file and compare its md5 checksum with the current config on # disk. When the checksums differ, we first create a backup and only then overwrite the existing file. with tempfile.NamedTemporaryFile() as handle: self._write(handle) handle.seek(0) if md5_from_filelike(handle) != md5_file(self.filepath): self._backup(self.filepath) shutil.copy(handle.name, self.filepath) return self
def store(self): """Write the current config to file. .. note:: if the configuration file already exists on disk and its contents differ from those in memory, a backup of the original file on disk will be created before overwriting it. :return: self """ from aiida.common.files import md5_from_filelike, md5_file from .settings import DEFAULT_CONFIG_INDENT_SIZE # If the filepath of this configuration does not yet exist, simply write it. if not os.path.isfile(self.filepath): self._atomic_write() return self # Otherwise, we write the content to a temporary file and compare its md5 checksum with the current config on # disk. When the checksums differ, we first create a backup and only then overwrite the existing file. with tempfile.NamedTemporaryFile() as handle: json.dump(self.dictionary, handle, indent=DEFAULT_CONFIG_INDENT_SIZE) handle.seek(0) if md5_from_filelike(handle) != md5_file(self.filepath): self._backup(self.filepath) self._atomic_write() return self
def get_or_create(cls, filename, use_first=False, store_psf=True): """ Pass the same parameter of the init; if a file with the same md5 is found, that PsfData is returned. :param filename: an absolute filename on disk :param use_first: if False (default), raise an exception if more than \ one potential is found.\ If it is True, instead, use the first available pseudopotential. :param bool store_psf: If false, the PsfData objects are not stored in the database. default=True. :return (psf, created): where psf is the PsfData object, and create is either\ True if the object was created, or False if the object was retrieved\ from the DB. """ import os message = ( #pylint: disable=invalid-name 'This method has been deprecated and will be removed in `v2.0.0`. Support on psf pseudos and ' + 'corresponding families is moved to the aiida_pseudo package. Use the `get_or_create` '+ 'method of `aiida_pseudo.data.pseudo.psf.PsfData`.' ) warnings.warn(message, AiidaSiestaDeprecationWarning) if not os.path.abspath(filename): raise ValueError("filename must be an absolute path") md5 = md5_file(filename) pseudos = cls.from_md5(md5) if not pseudos: instance = cls(file=filename) if store_psf: instance.store() return (instance, True) if len(pseudos) > 1: if use_first: return (pseudos[0], False) raise ValueError( "More than one copy of a pseudopotential " "with the same MD5 has been found in the " "DB. pks={}".format(",".join([str(i.pk) for i in pseudos])) ) return (pseudos[0], False)
def set_file(self, file, filename=None): """ I pre-parse the file to store the attributes. """ from aiida.common.exceptions import ParsingError parsed_data = parse_psml(file) md5sum = md5_file(file) try: element = parsed_data['element'] except KeyError: raise ParsingError( "No 'element' parsed in the PSML file: unable to store") super(PsmlData, self).set_file(file) self.set_attribute('element', str(element)) self.set_attribute('md5', md5sum)
def get_or_create(cls, filename, element=None, use_first=False, store_usp=True): """ Same ase init. Check md5 in the db, it is found return a UspData. Otherwise will store the data into the db :return (usp, created) """ import aiida.common.utils import os # Convert the filename to an absolute path filename = str(filename) if filename != os.path.abspath(filename): raise ValueError("filename must be an absolute path") md5 = md5_file(filename) # Check if we have got the file already pseudos = cls.from_md5(md5) if len(pseudos) == 0: # No existing pseudopotential file is in the database instance = cls(file=filename) # If we there is an element given then I set it if element is not None: instance.set_element(element) # Store the usp if requested if store_usp is True: instance.store() return (instance, True) else: if len(pseudos) > 1: if use_first: return (pseudos[0], False) else: pks = ", ".join([str(i.pk) for i in pseudos]) raise ValueError("More than one copy of a pseudopotential" " found. pks={}".format(pks)) else: return (pseudos[0], False)
def get_or_create(cls, filename, use_first=False, store_psml=True): """ Pass the same parameter of the init; if a file with the same md5 is found, that PsmlData is returned. :param filename: an absolute filename on disk :param use_first: if False (default), raise an exception if more than \ one potential is found.\ If it is True, instead, use the first available pseudopotential. :param bool store_psml: If false, the PsmlData objects are not stored in the database. default=True. :return (psml, created): where psml is the PsmlData object, and create is either\ True if the object was created, or False if the object was retrieved\ from the DB. """ import aiida.common.utils import os if not os.path.abspath(filename): raise ValueError("filename must be an absolute path") md5 = md5_file(filename) pseudos = cls.from_md5(md5) if len(pseudos) == 0: if store_psml: instance = cls(file=filename).store() return (instance, True) else: instance = cls(file=filename) return (instance, True) else: if len(pseudos) > 1: if use_first: return (pseudos[0], False) else: raise ValueError("More than one copy of a pseudopotential " "with the same MD5 has been found in the " "DB. pks={}".format(",".join( [str(i.pk) for i in pseudos]))) else: return (pseudos[0], False)
def get_or_create(cls, filename, use_first=False, store_cif=True): """ Pass the same parameter of the init; if a file with the same md5 is found, that CifData is returned. :param filename: an absolute filename on disk :param use_first: if False (default), raise an exception if more than \ one CIF file is found.\ If it is True, instead, use the first available CIF file. :param bool store_cif: If false, the CifData objects are not stored in the database. default=True. :return (cif, created): where cif is the CifData object, and create is either\ True if the object was created, or False if the object was retrieved\ from the DB. """ import os from aiida.common.files import md5_file if not os.path.abspath(filename): raise ValueError('filename must be an absolute path') md5 = md5_file(filename) cifs = cls.from_md5(md5) if not cifs: if store_cif: instance = cls(file=filename).store() return (instance, True) instance = cls(file=filename) return (instance, True) if len(cifs) > 1: if use_first: return (cifs[0], False) raise ValueError( 'More than one copy of a CIF file ' 'with the same MD5 has been found in ' 'the DB. pks={}'.format(','.join([str(i.pk) for i in cifs])) ) return cifs[0], False
def set_file(self, filename): """ I pre-parse the file to store the attributes. """ from aiida.common.exceptions import ParsingError import aiida.common.utils # print("Called set_file","type of filename:",type(filename)) parsed_data = parse_psml(filename) md5sum = md5_file(filename) try: element = parsed_data['element'] except KeyError: raise ParsingError("No 'element' parsed in the PSML file {};" " unable to store".format(self.filename)) super(PsmlData, self).set_file(filename) self.set_attribute('element', str(element)) self.set_attribute('md5', md5sum)
def upload_upf_family(folder, group_label, group_description, stop_if_existing=True): """Upload a set of UPF files in a given group. :param folder: a path containing all UPF files to be added. Only files ending in .UPF (case-insensitive) are considered. :param group_label: the name of the group to create. If it exists and is non-empty, a UniquenessError is raised. :param group_description: string to be set as the group description. Overwrites previous descriptions. :param stop_if_existing: if True, check for the md5 of the files and, if the file already exists in the DB, raises a MultipleObjectsError. If False, simply adds the existing UPFData node to the group. """ # pylint: disable=too-many-locals,too-many-branches import os from aiida import orm from aiida.common import AIIDA_LOGGER from aiida.common.exceptions import UniquenessError from aiida.common.files import md5_file if not os.path.isdir(folder): raise ValueError('folder must be a directory') # only files, and only those ending with .upf or .UPF; # go to the real file if it is a symlink filenames = [ os.path.realpath(os.path.join(folder, i)) for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i)) and i.lower().endswith('.upf') ] nfiles = len(filenames) automatic_user = orm.User.objects.get_default() group, group_created = orm.Group.objects.get_or_create( label=group_label, type_string=UPFGROUP_TYPE, user=automatic_user) if group.user.email != automatic_user.email: raise UniquenessError( 'There is already a UpfFamily group with label {}' ', but it belongs to user {}, therefore you ' 'cannot modify it'.format(group_label, group.user.email)) # Always update description, even if the group already existed group.description = group_description # NOTE: GROUP SAVED ONLY AFTER CHECKS OF UNICITY pseudo_and_created = [] for filename in filenames: md5sum = md5_file(filename) builder = orm.QueryBuilder() builder.append(UpfData, filters={'attributes.md5': {'==': md5sum}}) existing_upf = builder.first() if existing_upf is None: # return the upfdata instances, not stored pseudo, created = UpfData.get_or_create(filename, use_first=True, store_upf=False) # to check whether only one upf per element exists # NOTE: actually, created has the meaning of "to_be_created" pseudo_and_created.append((pseudo, created)) else: if stop_if_existing: raise ValueError('A UPF with identical MD5 to ' ' {} cannot be added with stop_if_existing' ''.format(filename)) existing_upf = existing_upf[0] pseudo_and_created.append((existing_upf, False)) # check whether pseudo are unique per element elements = [(i[0].element, i[0].md5sum) for i in pseudo_and_created] # If group already exists, check also that I am not inserting more than # once the same element if not group_created: for aiida_n in group.nodes: # Skip non-pseudos if not isinstance(aiida_n, UpfData): continue elements.append((aiida_n.element, aiida_n.md5sum)) elements = set(elements) # Discard elements with the same MD5, that would # not be stored twice elements_names = [e[0] for e in elements] if not len(elements_names) == len(set(elements_names)): duplicates = {x for x in elements_names if elements_names.count(x) > 1} duplicates_string = ', '.join(i for i in duplicates) raise UniquenessError('More than one UPF found for the elements: ' + duplicates_string + '.') # At this point, save the group, if still unstored if group_created: group.store() # save the upf in the database, and add them to group for pseudo, created in pseudo_and_created: if created: pseudo.store() AIIDA_LOGGER.debug('New node {} created for file {}'.format( pseudo.uuid, pseudo.filename)) else: AIIDA_LOGGER.debug('Reusing node {} for file {}'.format( pseudo.uuid, pseudo.filename)) # Add elements to the group all togetehr group.add_nodes([pseudo for pseudo, created in pseudo_and_created]) nuploaded = len([_ for _, created in pseudo_and_created if created]) return nfiles, nuploaded
def upload_usp_family(folder, group_label, group_description, stop_if_existing=True): """ Upload a set of usp/recpot files in a give group :param folder: a path containing all UPF files to be added. Only files ending in .usp/.recpot are considered. :param group_label: the name of the group to create. If it exists and is non-empty, a UniquenessError is raised. :param group_description: a string to be set as the group description. Overwrites previous descriptions, if the group was existing. :param stop_if_existing: if True, check for the md5 of the files and, if the file already exists in the DB, raises a MultipleObjectsError. If False, simply adds the existing UPFData node to the group. """ import os import aiida.common #from aiida.common import aiidalogger from aiida.common import UniquenessError, NotExistent from aiida.orm.querybuilder import QueryBuilder from .otfg import OTFGGroup files = [ os.path.realpath(os.path.join(folder, i)) for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i)) and ( i.lower().endswith('.usp') or i.lower().endswith('recpot') or i.lower().endswith('.uspcc')) ] nfiles = len(files) try: group = OTFGGroup.get(label=group_label) group_created = False except NotExistent: group = OTFGGroup(label=group_label, ) group_created = True # Update the descript even if the group already existed group.description = group_description pseudo_and_created = [] # A list of records (UspData, created) for f in files: md5sum = md5_file(f) qb = QueryBuilder() qb.append(UspData, filters={'attributes.md5': {'==': md5sum}}) existing_usp = qb.first() # Add the file if it is in the database if existing_usp is None: pseudo, created = UspData.get_or_create(f, use_first=True, store_usp=False) pseudo_and_created.append((pseudo, created)) # The same file is there already else: if stop_if_existing: raise ValueError("A usp/recpot with identical MD5 to" " {} cannot be added with stop_if_existing" "".format(f)) existing_usp = existing_usp[0] pseudo_and_created.append((existing_usp, False)) # Check for unique per element elements = [(i[0].element, i[0].md5sum) for i in pseudo_and_created] # Check if we will duplicate after insertion if not group_created: for aiida_n in group.nodes: if not isinstance(aiida_n, UspData): continue elements.append((aiida_n.element, aiida_n.md5sum)) # Discard duplicated pairs elements = set(elements) elements_names = [e[0] for e in elements] # Check the uniqueness of the complete group if not len(elements_names) == len(set(elements_names)): duplicates = set( [x for x in elements_names if elements_names.count(x) > 1]) dup_string = ", ".join(duplicates) raise UniquenessError( "More than one usp/recpot found for the elements: " + dup_string + ".") if group_created: group.store() # Save the usp in the database if necessary and add them to the group for pseudo, created in pseudo_and_created: if created: pseudo.store() #aiidalogger.debug("New node {} created for file {}".format( # pseudo.uuid, pseudo.filename)) else: #aiidalogger.debug("Reusing node {} for file {}".format( # pseudo.uuid, pseudo.filename)) pass nodes_new = [ pseduo for pseduo, created in pseudo_and_created if created is True ] nodes_add = [pseduo for pseduo, created in pseudo_and_created] group.add_nodes(nodes_add) return nfiles, len(nodes_new)
def upload_psf_family(folder, group_label, group_description, stop_if_existing=True): """ Upload a set of PSF files in a given group. :param folder: a path containing all PSF files to be added. Only files ending in .PSF (case-insensitive) are considered. :param group_label: the name of the group to create. If it exists and is non-empty, a UniquenessError is raised. :param group_description: a string to be set as the group description. Overwrites previous descriptions, if the group was existing. :param stop_if_existing: if True, check for the md5 of the files and, if the file already exists in the DB, raises a MultipleObjectsError. If False, simply adds the existing PsfData node to the group. """ import os from aiida import orm from aiida.common import AIIDA_LOGGER as aiidalogger from aiida.common.exceptions import UniquenessError from aiida.orm.querybuilder import QueryBuilder from aiida_siesta.groups.pseudos import PsfFamily message = ( #pylint: disable=invalid-name 'This function has been deprecated and will be removed in `v2.0.0`. ' + '`upload_psf_family` is substitued by `fam.create_from_folder` ' + 'where `fam` is an instance of the families classes in `aiida_pseudo.groups.family`.' ) warnings.warn(message, AiidaSiestaDeprecationWarning) if not os.path.isdir(folder): raise ValueError("folder must be a directory") # only files, and only those ending with .psf or .PSF; # go to the real file if it is a symlink files = [ os.path.realpath(os.path.join(folder, i)) for i in os.listdir(folder) if os.path.isfile(os.path.join(folder, i)) and i.lower().endswith('.psf') ] nfiles = len(files) automatic_user = orm.User.objects.get_default() group, group_created = PsfFamily.objects.get_or_create(label=group_label, user=automatic_user) if group.user.email != automatic_user.email: raise UniquenessError( "There is already a PsfFamily group with name {}" ", but it belongs to user {}, therefore you " "cannot modify it".format(group_label, group.user.email) ) # Always update description, even if the group already existed group.description = group_description # NOTE: GROUP SAVED ONLY AFTER CHECKS OF UNICITY pseudo_and_created = [] for afile in files: md5sum = md5_file(afile) qb = QueryBuilder() qb.append(PsfData, filters={'attributes.md5': {'==': md5sum}}) existing_psf = qb.first() #existing_psf = PsfData.query(dbattributes__key="md5", # dbattributes__tval = md5sum) if existing_psf is None: # return the psfdata instances, not stored pseudo, created = PsfData.get_or_create(afile, use_first=True, store_psf=False) # to check whether only one psf per element exists # NOTE: actually, created has the meaning of "to_be_created" pseudo_and_created.append((pseudo, created)) else: if stop_if_existing: raise ValueError( "A PSF with identical MD5 to " " {} cannot be added with stop_if_existing" "".format(afile) ) existing_psf = existing_psf[0] pseudo_and_created.append((existing_psf, False)) # check whether pseudo are unique per element elements = [(i[0].element, i[0].md5sum) for i in pseudo_and_created] # If group already exists, check also that I am not inserting more than # once the same element if not group_created: for aiida_n in group.nodes: # Skip non-pseudos if not isinstance(aiida_n, PsfData): continue elements.append((aiida_n.element, aiida_n.md5sum)) elements = set(elements) # Discard elements with the same MD5, that would # not be stored twice elements_names = [e[0] for e in elements] if not len(elements_names) == len(set(elements_names)): duplicates = {x for x in elements_names if elements_names.count(x) > 1} duplicates_string = ", ".join(i for i in duplicates) raise UniquenessError("More than one PSF found for the elements: " + duplicates_string + ".") # At this point, save the group, if still unstored if group_created: group.store() # save the psf in the database, and add them to group for pseudo, created in pseudo_and_created: if created: pseudo.store() aiidalogger.debug("New node {} created for file {}".format(pseudo.uuid, pseudo.filename)) else: aiidalogger.debug("Reusing node {} for file {}".format(pseudo.uuid, pseudo.filename)) # Add elements to the group all togetehr group.add_nodes([pseudo for pseudo, created in pseudo_and_created]) nuploaded = len([_ for _, created in pseudo_and_created if created]) return nfiles, nuploaded