def _collect_files(base, path=''): """ Recursively collects files from the tree, starting at a given path. """ from aiida.common.folders import Folder from aiida.common.utils import md5_file, sha1_file import os def get_filename(file_dict): return file_dict['name'] if os.path.isdir(os.path.join(base, path)): folder = Folder(os.path.join(base, path)) files_now = [] if path != '': if not path.endswith(os.sep): path = "{}{}".format(path, os.sep) if path != '': files_now.append({ 'name': path, 'type': 'folder', }) for f in folder.get_content_list(): files = _collect_files(base, path=os.path.join(path, f)) files_now.extend(files) return sorted(files_now, key=get_filename) elif path == '.aiida/calcinfo.json': files = [] with open(os.path.join(base, path)) as f: files.append({ 'name': path, 'contents': f.read(), 'md5': md5_file(os.path.join(base, path)), 'sha1': sha1_file(os.path.join(base, path)), 'type': 'file', }) import json with open(os.path.join(base, path)) as f: calcinfo = json.load(f) if 'local_copy_list' in calcinfo: for local_copy in calcinfo['local_copy_list']: with open(local_copy[0]) as f: files.append({ 'name': os.path.normpath(local_copy[1]), 'contents': f.read(), 'md5': md5_file(local_copy[0]), 'sha1': sha1_file(local_copy[0]), 'type': 'file', }) return files else: with open(os.path.join(base, path)) as f: return [{ 'name': path, 'contents': f.read(), 'md5': md5_file(os.path.join(base, path)), 'sha1': sha1_file(os.path.join(base, path)), 'type': 'file', }]
def potcar(self, value): name = 'POTCAR' self.folder.insert_path(value, 'path/'+name) attr_dict = pcparser.parse_potcar(value) self._set_attr('md5', md5_file(value)) for k, v in attr_dict.iteritems(): self._set_attr(k, v)
def _collect_files(base, path=''): """ Recursively collects files from the tree, starting at a given path. """ from aiida.common.folders import Folder from aiida.common.utils import md5_file,sha1_file import os if os.path.isdir(os.path.join(base,path)): folder = Folder(os.path.join(base,path)) files_now = [] if path != '': if not path.endswith(os.sep): path = "{}{}".format(path,os.sep) if path != '': files_now.append({ 'name': path, 'type': 'folder', }) for f in sorted(folder.get_content_list()): files = _collect_files(base,path=os.path.join(path,f)) files_now.extend(files) return files_now else: with open(os.path.join(base,path)) as f: return [{ 'name': path, 'contents': f.read(), 'md5': md5_file(os.path.join(base,path)), 'sha1': sha1_file(os.path.join(base,path)), 'type': 'file', }]
def extract_cif(infile, folder, nodes_export_subfolder="nodes", aiida_export_subfolder="aiida", silent=False): """ Extract the nodes to be imported from a TCOD CIF file. TCOD CIFs, exported by AiiDA, may contain an importable subset of AiiDA database, which can be imported. This function prepares SandboxFolder with files required for import. :param infile: file path :param folder: a SandboxFolder, used to extract the file tree :param nodes_export_subfolder: name of the subfolder for AiiDA nodes :param aiida_export_subfolder: name of the subfolder for AiiDA data inside the TCOD CIF internal file tree :param silent: suppress debug print """ import os import urllib2 import CifFile from aiida.common.exceptions import ValidationError from aiida.common.utils import md5_file, sha1_file from aiida.tools.dbexporters.tcod import decode_textfield values = CifFile.ReadCif(infile) values = values[values.keys()[0]] # taking the first datablock in CIF for i in range(0,len(values['_tcod_file_id'])-1): name = values['_tcod_file_name'][i] if not name.startswith(aiida_export_subfolder+os.sep): continue dest_path = os.path.relpath(name,aiida_export_subfolder) if name.endswith(os.sep): if not os.path.exists(folder.get_abs_path(dest_path)): folder.get_subfolder(folder.get_abs_path(dest_path),create=True) continue contents = values['_tcod_file_contents'][i] if contents == '?' or contents == '.': uri = values['_tcod_file_uri'][i] if uri is not None and uri != '?' and uri != '.': contents = urllib2.urlopen(uri).read() encoding = values['_tcod_file_content_encoding'][i] if encoding == '.': encoding = None contents = decode_textfield(contents,encoding) if os.path.dirname(dest_path) != '': folder.get_subfolder(os.path.dirname(dest_path)+os.sep,create=True) with open(folder.get_abs_path(dest_path),'w') as f: f.write(contents) f.flush() md5 = values['_tcod_file_md5sum'][i] if md5 is not None: if md5_file(folder.get_abs_path(dest_path)) != md5: raise ValidationError("MD5 sum for extracted file '{}' is " "different from given in the CIF " "file".format(dest_path)) sha1 = values['_tcod_file_sha1sum'][i] if sha1 is not None: if sha1_file(folder.get_abs_path(dest_path)) != sha1: raise ValidationError("SHA1 sum for extracted file '{}' is " "different from given in the CIF " "file".format(dest_path))
def get_or_create(cls, pawpath, store=False, psctr=None): if os.path.isdir(pawpath): pp = os.path.join(pawpath, 'POTCAR') isdir = True else: pp = pawpath isdir = False md5new = md5_file(pp) paw = cls.query(dbattributes__key='md5', dbattributes__tval=md5new).first() created = False if not paw: if isdir: paw = cls.from_folder(pawpath) else: paw = cls.from_potcar(pawpath, ctrpath=psctr) if store: paw.store_all() created = True return paw, created
def __init__(self, **kwargs): """ Initializes the Workflow super class, store the instance in the DB and in case stores the starting parameters. If initialized with an uuid the Workflow is loaded from the DB, if not a new workflow is generated and added to the DB following the stack frameworks. This means that only modules inside aiida.workflows are allowed to implements the workflow super calls and be stored. The caller names, modules and files are retrieved from the stack. :param uuid: a string with the uuid of the object to be loaded. :param params: a dictionary of storable objects to initialize the specific workflow :raise: NotExistent: if there is no entry of the desired workflow kind with the given uuid. """ from aiida.backends.djsite.db.models import DbWorkflow self._to_be_stored = True self._logger = logger.getChild(self.__class__.__name__) uuid = kwargs.pop('uuid', None) if uuid is not None: self._to_be_stored = False if kwargs: raise ValueError("If you pass a UUID, you cannot pass any further parameter") try: self._dbworkflowinstance = DbWorkflow.objects.get(uuid=uuid) # self.logger.info("Workflow found in the database, now retrieved") self._repo_folder = RepositoryFolder(section=self._section_name, uuid=self.uuid) except ObjectDoesNotExist: raise NotExistent("No entry with the UUID {} found".format(uuid)) else: # ATTENTION: Do not move this code outside or encapsulate it in a function import inspect stack = inspect.stack() # cur_fr = inspect.currentframe() #call_fr = inspect.getouterframes(cur_fr, 2) # Get all the caller data caller_frame = stack[1][0] caller_file = stack[1][1] caller_funct = stack[1][3] caller_module = inspect.getmodule(caller_frame) caller_module_class = caller_frame.f_locals.get('self', None).__class__ if not caller_funct == "__init__": raise SystemError("A workflow must implement the __init__ class explicitly") # Test if the launcher is another workflow # print "caller_module", caller_module # print "caller_module_class", caller_module_class # print "caller_file", caller_file # print "caller_funct", caller_funct # Accept only the aiida.workflows packages if caller_module == None or not caller_module.__name__.startswith("aiida.workflows"): raise SystemError("The superclass can't be called directly") self.caller_module = caller_module.__name__ self.caller_module_class = caller_module_class.__name__ self.caller_file = caller_file self.caller_funct = caller_funct self._temp_folder = SandboxFolder() self.current_folder.insert_path(self.caller_file, self.caller_module_class) # self.store() # Test if there are parameters as input params = kwargs.pop('params', None) if params is not None: if type(params) is dict: self.set_params(params) # This stores the MD5 as well, to test in case the workflow has # been modified after the launch self._dbworkflowinstance = DbWorkflow(user=get_automatic_user(), module=self.caller_module, module_class=self.caller_module_class, script_path=self.caller_file, script_md5=md5_file(self.caller_file)) self.attach_calc_lazy_storage = {} self.attach_subwf_lazy_storage = {}