def _collect_files(base, path=''):
"""
Recursively collects files from the tree, starting at a given path.
"""
from aiida.common.folders import Folder
from aiida.common.utils import md5_file, sha1_file
import os
def get_filename(file_dict):
return file_dict['name']
if os.path.isdir(os.path.join(base, path)):
folder = Folder(os.path.join(base, path))
files_now = []
if path != '':
if not path.endswith(os.sep):
path = "{}{}".format(path, os.sep)
if path != '':
files_now.append({
'name': path,
'type': 'folder',
})
for f in folder.get_content_list():
files = _collect_files(base, path=os.path.join(path, f))
files_now.extend(files)
return sorted(files_now, key=get_filename)
elif path == '.aiida/calcinfo.json':
files = []
with open(os.path.join(base, path)) as f:
files.append({
'name': path,
'contents': f.read(),
'md5': md5_file(os.path.join(base, path)),
'sha1': sha1_file(os.path.join(base, path)),
'type': 'file',
})
import json
with open(os.path.join(base, path)) as f:
calcinfo = json.load(f)
if 'local_copy_list' in calcinfo:
for local_copy in calcinfo['local_copy_list']:
with open(local_copy[0]) as f:
files.append({
'name': os.path.normpath(local_copy[1]),
'contents': f.read(),
'md5': md5_file(local_copy[0]),
'sha1': sha1_file(local_copy[0]),
'type': 'file',
})
return files
else:
with open(os.path.join(base, path)) as f:
return [{
'name': path,
'contents': f.read(),
'md5': md5_file(os.path.join(base, path)),
'sha1': sha1_file(os.path.join(base, path)),
'type': 'file',
}]
def _collect_files(base, path=''):
"""
Recursively collects files from the tree, starting at a given path.
"""
from aiida.common.folders import Folder
from aiida.common.utils import md5_file,sha1_file
import os
if os.path.isdir(os.path.join(base,path)):
folder = Folder(os.path.join(base,path))
files_now = []
if path != '':
if not path.endswith(os.sep):
path = "{}{}".format(path,os.sep)
if path != '':
files_now.append({
'name': path,
'type': 'folder',
})
for f in sorted(folder.get_content_list()):
files = _collect_files(base,path=os.path.join(path,f))
files_now.extend(files)
return files_now
else:
with open(os.path.join(base,path)) as f:
return [{
'name': path,
'contents': f.read(),
'md5': md5_file(os.path.join(base,path)),
'sha1': sha1_file(os.path.join(base,path)),
'type': 'file',
}]
def extract_cif(infile, folder, nodes_export_subfolder="nodes",
aiida_export_subfolder="aiida", silent=False):
"""
Extract the nodes to be imported from a TCOD CIF file. TCOD CIFs,
exported by AiiDA, may contain an importable subset of AiiDA database,
which can be imported. This function prepares SandboxFolder with files
required for import.
:param infile: file path
:param folder: a SandboxFolder, used to extract the file tree
:param nodes_export_subfolder: name of the subfolder for AiiDA nodes
:param aiida_export_subfolder: name of the subfolder for AiiDA data
inside the TCOD CIF internal file tree
:param silent: suppress debug print
"""
import os
import urllib2
import CifFile
from aiida.common.exceptions import ValidationError
from aiida.common.utils import md5_file, sha1_file
from aiida.tools.dbexporters.tcod import decode_textfield
values = CifFile.ReadCif(infile)
values = values[values.keys()[0]] # taking the first datablock in CIF
for i in range(0,len(values['_tcod_file_id'])-1):
name = values['_tcod_file_name'][i]
if not name.startswith(aiida_export_subfolder+os.sep):
continue
dest_path = os.path.relpath(name,aiida_export_subfolder)
if name.endswith(os.sep):
if not os.path.exists(folder.get_abs_path(dest_path)):
folder.get_subfolder(folder.get_abs_path(dest_path),create=True)
continue
contents = values['_tcod_file_contents'][i]
if contents == '?' or contents == '.':
uri = values['_tcod_file_uri'][i]
if uri is not None and uri != '?' and uri != '.':
contents = urllib2.urlopen(uri).read()
encoding = values['_tcod_file_content_encoding'][i]
if encoding == '.':
encoding = None
contents = decode_textfield(contents,encoding)
if os.path.dirname(dest_path) != '':
folder.get_subfolder(os.path.dirname(dest_path)+os.sep,create=True)
with open(folder.get_abs_path(dest_path),'w') as f:
f.write(contents)
f.flush()
md5 = values['_tcod_file_md5sum'][i]
if md5 is not None:
if md5_file(folder.get_abs_path(dest_path)) != md5:
raise ValidationError("MD5 sum for extracted file '{}' is "
"different from given in the CIF "
"file".format(dest_path))
sha1 = values['_tcod_file_sha1sum'][i]
if sha1 is not None:
if sha1_file(folder.get_abs_path(dest_path)) != sha1:
raise ValidationError("SHA1 sum for extracted file '{}' is "
"different from given in the CIF "
"file".format(dest_path))