def create_collections(self, config_builder):
location = SetupParser.get("type")
for collection_type in self.COLLECTION_TYPES:
# Dont do anything if already set
if collection_type in self.collections and self.collections[collection_type]: continue
# Check if we are running LOCAL we should not have any base collections
if location == "LOCAL" and collection_type in self.base_collections:
print("The base_collection of type %s was specified but you are trying to run a LOCAL experiment.\n"
"Using COMPS collection with LOCAL experiments is not supported. The collection will be ignored..." % collection_type)
if location == "HPC":
# If we already have the master collection set -> set it as collection for every types
# Bypass the python type for now
if self.master_collection and collection_type != self.PYTHON:
self.collections[collection_type] = self.master_collection
continue
if collection_type not in self.base_collections:
base_id = SetupParser.get('base_collection_id_%s' % collection_type, None)
if base_id: self.set_base_collection(collection_type, base_id)
base_collection = self.base_collections.get(collection_type, None)
if not base_collection:
files = self._gather_files(config_builder, collection_type)
if files: self.collections[collection_type] = AssetCollection(local_files=files)
else:
self.collections[collection_type] = base_collection
# If there are manually added files -> add them now
if self.experiment_files:
self.collections[self.LOCAL] = AssetCollection(local_files=self.experiment_files)
def set_base_collection(self, collection_type, collection):
# Make sure we have the good inputs
if collection_type not in self.COLLECTION_TYPES and collection_type != self.MASTER:
raise self.InvalidCollection("Collection type %s is not supported..." % collection_type)
if not collection:
raise self.InvalidCollection("No collection provided in set_input_collection.")
# If the collection given is not already an AssetCollection -> retrieve
if not isinstance(collection, COMPSAssetCollection):
collection_id = collection
with SetupParser.TemporarySetup('HPC'):
collection = get_asset_collection(collection_id)
if not collection:
raise self.InvalidCollection("The input collection '%s' provided could not be found on COMPS." % collection_id)
if collection_type == self.MASTER:
self.master_collection = AssetCollection(base_collection=collection)
for t in self.SETUP_MAPPING: self.SETUP_MAPPING[t] = ""
else:
self.base_collections[collection_type] = AssetCollection(base_collection=collection)
# For the DLL filter out the exe as we usually have exe+dll in the same collection
if collection_type == self.DLL:
self.base_collections[self.DLL].asset_files_to_use = [a for a in self.base_collections[self.DLL].asset_files_to_use if not a.file_name.endswith('exe')]
def test_properly_label_local_and_remote_files_and_verify_file_data(self):
"""
This tests whether the code properly recognizes local/remote file usage (and precedence during conflicts)
as well as verifies local vs. COMPS AssetCollection files via file count and md5.
:return:
"""
# all files are remote/in an existing asset collection
new_collection = AssetCollection(base_collection=get_asset_collection(
self.existing_collection.collection_id))
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
self.assertEqual(f.is_local, False)
# all files remote, not necessarily in an existing asset collection
new_collection = AssetCollection(
remote_files=self.existing_COMPS_asset_files)
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
self.assertEqual(f.is_local, False)
# all files are local
new_collection = AssetCollection(base_collection=None,
local_files=self.local_files)
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
self.assertEqual(f.is_local, True)
# mix of local and existing remote files in a COMPS AssetCollection.
# local_files should be preferred in case of conflicts
new_collection = AssetCollection(
base_collection=get_asset_collection(
self.existing_collection.collection_id),
local_files=self.local_files_plus_minus)
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
if self.REMOTE_ONLY in f.file_name: # we removed this file from self.local_files_pluS_minus, so it is remote only now.
self.assertEqual(f.is_local, False)
elif self.LOCAL_ONLY in f.file_name:
self.assertEqual(f.is_local, True)
# finally, verify that the resultant remote COMPSAssetCollection has the same files + MD5s of the files we requested.
new_collection.prepare(location='HPC')
remote_asset_files = get_asset_collection(
new_collection.collection_id).assets
new_asset_files = sorted(new_collection.asset_files_to_use,
key=lambda x: x.file_name)
remote_asset_files = sorted(remote_asset_files,
key=lambda x: x.file_name)
self.assertEqual(len(new_collection.asset_files_to_use),
len(remote_asset_files))
for i in range(0, len(new_asset_files)):
self.assertEqual(new_asset_files[i].file_name,
remote_asset_files[i].file_name)
self.assertEqual(new_asset_files[i].relative_path,
remote_asset_files[i].relative_path)
self.assertEqual(new_asset_files[i].md5_checksum,
remote_asset_files[i].md5_checksum)
def prepare(self, config_builder):
"""
Calls prepare() on all unprepared contained AssetCollection objects.
:return: Nothing
"""
location = SetupParser.get("type")
self.collections = {}
self.create_collections(config_builder)
for collection in self.collections.values():
if not collection.prepared:
collection.prepare(location=location)
# Sort the collections to first gather the assets from base collections and finish with the locally generated
# Use a set to remove duplicates
sorted_collections = sorted(set(self.collections.values()),
key=lambda x: x.base_collection is None)
# Gather the collection_ids from the above collections now that they have been prepared/uploaded (as needed)
# and generate a 'super AssetCollection' containing all file references.
asset_files = {}
for collection in sorted_collections:
if location == 'LOCAL':
for asset in collection.asset_files_to_use:
asset_files[(asset.file_name, asset.relative_path)] = asset
else:
if collection.collection_id is not None: # None means "no files in this collection"
# Add the assets to the asset_files
# Make sure we have a key (file_name, path) to make sure we override assets
for asset in collection.comps_collection.assets:
asset_files[(asset.file_name,
asset.relative_path)] = asset
# Delete collections that are None (no files)
self.collections = {
cid: collection
for cid, collection in self.collections.items()
if collection.collection_id
}
logger.debug("Creating master collection with %d files" %
len(asset_files))
if len(asset_files) != 0:
self.master_collection = AssetCollection(
remote_files=asset_files.values())
self.master_collection.prepare(location=location)
self.prepared = True
def setUp(self):
SetupParser._uninit()
current_dir = os.path.dirname(os.path.realpath(__file__))
SetupParser.init(selected_block=self.SELECTED_BLOCK,
setup_file=os.path.join(current_dir, 'input',
'am_simtools.ini'))
COMPS_login(SetupParser.get('server_endpoint'))
self.existing_collection = AssetCollection(
base_collection=get_asset_collection(self.EXISTING_COLLECTION_ID))
self.existing_collection.prepare(location='HPC')
self.existing_COMPS_asset_files = self.existing_collection.asset_files_to_use
# a FileList object
dir = os.path.join(self.INPUT_DIR, 'files')
files = [os.path.join(dir, f) for f in os.listdir(dir)]
root = os.path.dirname(os.path.dirname(files[0]))
files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in files
]
self.local_files = FileList(root=root, files_in_root=files)
# take one away
files = [
os.path.join(f.relative_path, f.file_name)
for f in self.local_files.files
if f.file_name != 'file1_REMOTE_ONLY'
]
# add some more
dir = os.path.join(self.INPUT_DIR, 'additional_files')
additional_files = [os.path.join(dir, f) for f in os.listdir(dir)]
additional_files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in additional_files
]
files += additional_files
self.local_files_plus_minus = FileList(root=root, files_in_root=files)
def test_can_handle_empty_collections(self):
"""
Tests if an empty file list causes problems (should not)
"""
files = FileList(root=os.getcwd(), files_in_root=[])
collection = AssetCollection(local_files=files)
self.assertEqual(len(collection.asset_files_to_use), 0)
collection.prepare(location='LOCAL')
self.assertEqual(collection.collection_id, 'LOCAL')
collection.prepared = False
collection.prepare(location='HPC')
self.assertEqual(collection.collection_id, None)
class SimulationAssets(object):
"""
This class represents a set of AssetCollection objects that together define all files needed by a simulation.
"""
class InvalidCollection(Exception): pass
class NotPrepared(Exception): pass
EXE = 'exe'
DLL = 'dll'
INPUT = 'input'
LOCAL = 'local'
MASTER = 'master'
PYTHON = 'python'
COLLECTION_TYPES = [DLL, EXE, INPUT, PYTHON]
SETUP_MAPPING = {DLL: 'dll_root', EXE: 'exe_path', INPUT: 'input_root', PYTHON: 'python_path'}
def __init__(self):
self.collections = {}
self.base_collections = {}
self.experiment_files = FileList()
self.prepared = False
self.master_collection = None
self.paths = {}
def _get_path(self, path_type):
return self.paths.get(path_type, None) or SetupParser.get(self.SETUP_MAPPING[path_type])
def _set_path(self, path_type, path, is_dir=False):
if not os.path.exists(path) or (is_dir and not os.path.isdir(path)):
raise Exception("The path specified in {} does not exist ({})".format(self.SETUP_MAPPING[path_type], path))
# Remove the base collection
self.base_collections[path_type] = None
# Set the path
self.paths[path_type] = path
@property
def exe_name(self):
return os.path.basename(self.exe_path)
@property
def exe_path(self):
if self.master_collection:
for f in self.master_collection.asset_files_to_use:
if f.file_name.endswith("exe"):
return f.file_name
# If we do have a EXE collection, the exe_path should come from there
if self.EXE in self.base_collections and self.base_collections[self.EXE]:
for f in self.base_collections[self.EXE].asset_files_to_use:
if f.file_name.endswith("exe"):
return f.file_name
raise Exception("The asset collection ({}) used for executable does not contain any executable..."
.format(str(self.base_collections[self.EXE].base_collection.id)))
return self._get_path(self.EXE)
@exe_path.setter
def exe_path(self, exe_path):
self._set_path(self.EXE, exe_path)
@property
def input_root(self):
return self._get_path(self.INPUT)
@input_root.setter
def input_root(self, input_root):
self._set_path(self.INPUT, input_root, is_dir=True)
@property
def dll_root(self):
return self._get_path(self.DLL)
@dll_root.setter
def dll_root(self, dll_root):
self._set_path(self.DLL, dll_root, is_dir=True)
@property
def python_path(self):
return self._get_path(self.PYTHON)
@python_path.setter
def python_path(self, python_path):
self._set_path(self.PYTHON, python_path, is_dir=True)
def __contains__(self, item):
for col in self.collections.values():
if item in col: return True
return False
@property
def collection_id(self):
if not self.prepared or not self.master_collection:
raise self.NotPrepared("Cannot query asset collection id if collection is not prepared.")
return self.master_collection.collection_id
def set_base_collection(self, collection_type, collection):
# Make sure we have the good inputs
if collection_type not in self.COLLECTION_TYPES and collection_type != self.MASTER:
raise self.InvalidCollection("Collection type %s is not supported..." % collection_type)
if not collection:
raise self.InvalidCollection("No collection provided in set_input_collection.")
# If the collection given is not already an AssetCollection -> retrieve
if not isinstance(collection, COMPSAssetCollection):
collection_id = collection
with SetupParser.TemporarySetup('HPC'):
collection = get_asset_collection(collection_id)
if not collection:
raise self.InvalidCollection("The input collection '%s' provided could not be found on COMPS." % collection_id)
if collection_type == self.MASTER:
self.master_collection = AssetCollection(base_collection=collection)
for t in self.SETUP_MAPPING: self.SETUP_MAPPING[t] = ""
else:
self.base_collections[collection_type] = AssetCollection(base_collection=collection)
# For the DLL filter out the exe as we usually have exe+dll in the same collection
if collection_type == self.DLL:
self.base_collections[self.DLL].asset_files_to_use = [a for a in self.base_collections[self.DLL].asset_files_to_use if not a.file_name.endswith('exe')]
def prepare(self, config_builder):
"""
Calls prepare() on all unprepared contained AssetCollection objects.
:return: Nothing
"""
location = SetupParser.get("type")
self.collections = {}
self.create_collections(config_builder)
for collection in self.collections.values():
if not collection.prepared:
collection.prepare(location=location)
# Sort the collections to first gather the assets from base collections and finish with the locally generated
# Use a set to remove duplicates
sorted_collections = sorted(set(self.collections.values()), key=lambda x: x.base_collection is None)
# Gather the collection_ids from the above collections now that they have been prepared/uploaded (as needed)
# and generate a 'super AssetCollection' containing all file references.
asset_files = {}
for collection in sorted_collections:
if location == 'LOCAL':
for asset in collection.asset_files_to_use:
asset_files[(asset.file_name, asset.relative_path)] = asset
else:
if collection.collection_id is not None: # None means "no files in this collection"
# Add the assets to the asset_files
# Make sure we have a key (file_name, path) to make sure we override assets
for asset in collection.comps_collection.assets:
asset_files[(asset.file_name, asset.relative_path)] = asset
# Delete collections that are None (no files)
self.collections = {cid:collection for cid, collection in self.collections.items() if collection.collection_id}
logger.debug("Creating master collection with %d files" % len(asset_files))
self.master_collection = AssetCollection(remote_files=asset_files.values())
self.master_collection.prepare(location=location)
self.prepared = True
def create_collections(self, config_builder):
location = SetupParser.get("type")
for collection_type in self.COLLECTION_TYPES:
# Dont do anything if already set
if collection_type in self.collections and self.collections[collection_type]: continue
# Check if we are running LOCAL we should not have any base collections
if location == "LOCAL" and collection_type in self.base_collections:
print("The base_collection of type %s was specified but you are trying to run a LOCAL experiment.\n"
"Using COMPS collection with LOCAL experiments is not supported. The collection will be ignored..." % collection_type)
if location == "HPC":
# If we already have the master collection set -> set it as collection for every types
# Bypass the python type for now
if self.master_collection and collection_type != self.PYTHON:
self.collections[collection_type] = self.master_collection
continue
if collection_type not in self.base_collections:
base_id = SetupParser.get('base_collection_id_%s' % collection_type, None)
if base_id: self.set_base_collection(collection_type, base_id)
base_collection = self.base_collections.get(collection_type, None)
if not base_collection:
files = self._gather_files(config_builder, collection_type)
if files: self.collections[collection_type] = AssetCollection(local_files=files)
else:
self.collections[collection_type] = base_collection
# If there are manually added files -> add them now
if self.experiment_files:
self.collections[self.LOCAL] = AssetCollection(local_files=self.experiment_files)
def _gather_files(self, config_builder, collection_type):
"""
Identifies local files associated with the given collection_type.
:param config_builder: A ConfigBuilder object associated with this process
:param collection_type: one of cls.COLLECTION_TYPES
:return: A FileList object
"""
file_list = None
if collection_type == self.EXE:
exe_path = self.exe_path
file_list = FileList(root=os.path.dirname(exe_path),
files_in_root=[os.path.basename(exe_path)],
ignore_missing=config_builder.ignore_missing)
elif collection_type == self.INPUT:
# returns a Hash with some items that need filtering through
input_files = config_builder.get_input_file_paths()
if input_files:
file_list = FileList(root=self.input_root,
files_in_root=input_files,
ignore_missing=config_builder.ignore_missing)
elif collection_type == self.DLL:
dll_relative_paths = config_builder.get_dll_paths_for_asset_manager()
if dll_relative_paths:
file_list = FileList(root=self.dll_root,
files_in_root=dll_relative_paths,
ignore_missing=config_builder.ignore_missing)
elif collection_type == self.PYTHON:
file_list = FileList(root=self.python_path, relative_path="python") if self.python_path else None
else:
raise Exception("Unknown asset classification: %s" % collection_type)
return file_list
from simtools.Utilities.COMPSUtilities import get_asset_collection
if __name__ == "__main__":
# Initialize an HPC setup
SetupParser.init('HPC')
# Create a FileList, this will contain all the files we want to add to the collection
fl = FileList()
# Lets add the custom_collection folder and have it recursively browsed
# If recursive is False, only the file present in the directory will be added.
# With recursive enabled, the sub directory are also scanned for files and added to the list.
fl.add_path('inputs/custom_collection', recursive=True)
# Create an asset collection and pass this file list
ac = AssetCollection(local_files=fl)
# Prepare the collection
# This will create the collection in COMPS and send the missing files
ac.prepare('HPC')
# Our collection is created -> the id is:
print("The collection ID is: %s " % ac.collection_id)
# Now create another collection based on the one we just created and add some arbitrary files
# First we need to create the filelist of Local files to use
local_fl = FileList()
local_fl.add_file('inputs/docs/some_doc.txt')
# THen lets create an AssetCollection but this time we will give set the base_collection to be the one we created earlier
# base_collection needs a COMPSAssetCollection instance so we can retrieve this by doing:
def test_default_collection_usage_properly_sets_the_AssetCollection(self):
collection = AssetCollection(
base_collection=get_asset_collection(self.DEFAULT_COLLECTION_NAME))
self.assertEqual(str(collection.base_collection.id),
self.DEFAULT_COLLECTION_ID)
class TestAssetCollection(unittest.TestCase):
LOCAL_ONLY = 'LOCAL_ONLY'
REMOTE_ONLY = 'REMOTE_ONLY'
EXISTING_COLLECTION_ID = '2129c37c-324a-e711-80c1-f0921c167860'
INPUT_DIR = os.path.join(os.path.dirname(__file__), 'input',
'AssetCollection')
SELECTED_BLOCK = 'AssetCollection'
def setUp(self):
SetupParser._uninit()
current_dir = os.path.dirname(os.path.realpath(__file__))
SetupParser.init(selected_block=self.SELECTED_BLOCK,
setup_file=os.path.join(current_dir, 'input',
'am_simtools.ini'))
COMPS_login(SetupParser.get('server_endpoint'))
self.existing_collection = AssetCollection(
base_collection=get_asset_collection(self.EXISTING_COLLECTION_ID))
self.existing_collection.prepare(location='HPC')
self.existing_COMPS_asset_files = self.existing_collection.asset_files_to_use
# a FileList object
dir = os.path.join(self.INPUT_DIR, 'files')
files = [os.path.join(dir, f) for f in os.listdir(dir)]
root = os.path.dirname(os.path.dirname(files[0]))
files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in files
]
self.local_files = FileList(root=root, files_in_root=files)
# take one away
files = [
os.path.join(f.relative_path, f.file_name)
for f in self.local_files.files
if f.file_name != 'file1_REMOTE_ONLY'
]
# add some more
dir = os.path.join(self.INPUT_DIR, 'additional_files')
additional_files = [os.path.join(dir, f) for f in os.listdir(dir)]
additional_files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in additional_files
]
files += additional_files
self.local_files_plus_minus = FileList(root=root, files_in_root=files)
def tearDown(self):
SetupParser._uninit()
def test_file_invalid_configurations(self):
kwargs = {}
self.assertRaises(AssetCollection.InvalidConfiguration,
AssetCollection, **kwargs)
kwargs = {'base_collection': 'abc', 'remote_files': 'def'}
self.assertRaises(AssetCollection.InvalidConfiguration,
AssetCollection, **kwargs)
def test_can_handle_empty_collections(self):
"""
Tests if an empty file list causes problems (should not)
"""
files = FileList(root=os.getcwd(), files_in_root=[])
collection = AssetCollection(local_files=files)
self.assertEqual(len(collection.asset_files_to_use), 0)
collection.prepare(location='LOCAL')
self.assertEqual(collection.collection_id, 'LOCAL')
collection.prepared = False
collection.prepare(location='HPC')
self.assertEqual(collection.collection_id, None)
def test_properly_label_local_and_remote_files_and_verify_file_data(self):
"""
This tests whether the code properly recognizes local/remote file usage (and precedence during conflicts)
as well as verifies local vs. COMPS AssetCollection files via file count and md5.
:return:
"""
# all files are remote/in an existing asset collection
new_collection = AssetCollection(base_collection=get_asset_collection(
self.existing_collection.collection_id))
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
self.assertEqual(f.is_local, False)
# all files remote, not necessarily in an existing asset collection
new_collection = AssetCollection(
remote_files=self.existing_COMPS_asset_files)
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
self.assertEqual(f.is_local, False)
# all files are local
new_collection = AssetCollection(base_collection=None,
local_files=self.local_files)
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
self.assertEqual(f.is_local, True)
# mix of local and existing remote files in a COMPS AssetCollection.
# local_files should be preferred in case of conflicts
new_collection = AssetCollection(
base_collection=get_asset_collection(
self.existing_collection.collection_id),
local_files=self.local_files_plus_minus)
for f in new_collection.asset_files_to_use:
self.assertTrue(hasattr(f, 'is_local'))
if self.REMOTE_ONLY in f.file_name: # we removed this file from self.local_files_pluS_minus, so it is remote only now.
self.assertEqual(f.is_local, False)
elif self.LOCAL_ONLY in f.file_name:
self.assertEqual(f.is_local, True)
# finally, verify that the resultant remote COMPSAssetCollection has the same files + MD5s of the files we requested.
new_collection.prepare(location='HPC')
remote_asset_files = get_asset_collection(
new_collection.collection_id).assets
new_asset_files = sorted(new_collection.asset_files_to_use,
key=lambda x: x.file_name)
remote_asset_files = sorted(remote_asset_files,
key=lambda x: x.file_name)
self.assertEqual(len(new_collection.asset_files_to_use),
len(remote_asset_files))
for i in range(0, len(new_asset_files)):
self.assertEqual(new_asset_files[i].file_name,
remote_asset_files[i].file_name)
self.assertEqual(new_asset_files[i].relative_path,
remote_asset_files[i].relative_path)
self.assertEqual(new_asset_files[i].md5_checksum,
remote_asset_files[i].md5_checksum)
def test_prepare_existing_collection(self):
self.existing_collection.prepare(location='HPC')
self.assertTrue(self.existing_collection.prepared)
self.assertEqual(str(self.existing_collection.collection_id),
self.EXISTING_COLLECTION_ID)
def test_prepare_new_collection(self):
import tempfile
import random
with tempfile.NamedTemporaryFile(mode='w+') as new_file:
new_file.write('hello world! %s' % random.random())
asset_file = COMPSAssetCollectionFile(file_name=os.path.basename(
new_file.name),
relative_path='.')
self.existing_collection.asset_files_to_use.append(asset_file)
self.existing_collection.prepare(location='HPC')
self.assertTrue(self.existing_collection.prepared)
self.assertTrue(self.existing_collection.collection_id is not None)
self.assertTrue(self.existing_collection.collection_id !=
self.EXISTING_COLLECTION_ID) # a new collection
DEFAULT_COLLECTION_NAME = 'EMOD 2.10'
DEFAULT_COLLECTION_ID = 'c2d9468d-2b4a-e711-80c1-f0921c167860'
def test_asset_collection_id_for_tag(self):
tag_name = 'Name'
asset_collections = [
{ # single match case
'Name': self.DEFAULT_COLLECTION_NAME,
'id': self.DEFAULT_COLLECTION_ID
},
{ # 0 match case
'Name': 'To be, or not to be: that is the question',
'id': None
},
{ # multiple match case
'Name': 'EMOD',
'id': None
}
]
for collection in asset_collections:
key_tag = 'Name'
c = get_asset_collection_by_tag(tag_name=key_tag,
tag_value=collection[key_tag])
id = c.id if c else None
id = str(id) if id else id # convert UUID to string if not None
self.assertEqual(id, collection['id'])
# This verifies that a well-known default collection name is converted to the expected asset collection id
def test_default_collection_usage_properly_sets_the_AssetCollection(self):
collection = AssetCollection(
base_collection=get_asset_collection(self.DEFAULT_COLLECTION_NAME))
self.assertEqual(str(collection.base_collection.id),
self.DEFAULT_COLLECTION_ID)