def test_categories_keys(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): # add a couple test Treants to collection col = collection(testtreant, testtreant2) col.categories.add({'age': 42, 'bark': 'smooth'}) t1 = dtr.Treant('maple') t2 = dtr.Treant('sequoia') t1.categories.add({'age': 'seedling', 'bark': 'rough', 'type': 'deciduous'}) t2.categories.add({'age': 'adult', 'bark': 'rough', 'type': 'evergreen', 'nickname': 'redwood'}) col += collection(t1, t2) for k in col.categories.keys(scope='all'): for member in col: assert k in member.categories for k in col.categories.keys(scope='any'): for member in col: if k == 'nickname': if member.name == 'maple': assert k not in member.categories elif member.name == 'sequoia': assert k in member.categories elif k == 'type': if (member.name != 'maple' and member.name != 'sequoia'): assert k not in member.categories else: assert k in member.categories
def filled_collection(self, tmpdir): # returns (a bundle of [t1, t2, t3], then individal references to each) with tmpdir.as_cwd(): t1 = dtr.Treant('larry') t2 = dtr.Treant('curly') t3 = dtr.Treant('moe') b = dtr.Bundle((t1, t2, t3)) return b, (t1, t2, t3)
def test_tags_filter(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): maple = dtr.Treant('maple') pine = dtr.Treant('pine') maple.tags.add({'tree', 'new jersey', 'deciduous'}) pine.tags.add({'tree', 'new york', 'evergreen'}) col = collection(maple, pine) tags = col.tags maple_bund = dtr.Bundle(maple) pine_bund = dtr.Bundle(pine) assert len(tags.any) == 5 # filter using single tags assert tags.filter('tree') == col assert tags.filter({'tree'}) == dtr.Bundle() assert tags.filter('deciduous') == maple_bund assert tags.filter('evergreen') == pine_bund assert tags.filter('new jersey') == maple_bund assert tags.filter('new york') == pine_bund # filter Treants that DON'T have a given tag assert tags.filter({'new york'}) == maple_bund assert tags.filter({'deciduous'}) == pine_bund # filter Treants containing all of the tags assert tags.filter(['deciduous', 'tree']) == maple_bund assert tags.filter(['evergreen', 'tree']) == pine_bund assert tags.filter(['deciduous', 'new york']) == dtr.Bundle() # filter Treants containing any of the tags tags assert tags.filter(('evergreen', 'tree')) == col assert tags.filter(('deciduous', 'new york')) == col assert tags.filter(('evergreen', 'new york')) == pine_bund # filter Treants that exclude any of the provided tags assert tags.filter({'deciduous', 'new york'}) == col assert tags.filter({'deciduous', 'new jersey'}) == pine_bund assert tags.filter({'evergreen', 'tree'}) == maple_bund # complex logic tests # give those that are evergreen or in NY AND also not deciduous selection = [('evergreen', 'new york'), {'deciduous'}] assert tags.filter(selection) == pine_bund # give those that are evergreen or in NY AND also not a tree selection = [('evergreen', 'new york'), {'tree'}] assert tags.filter(selection) == dtr.Bundle() # give a tree that's in NJ OR anything that's not evergreen selection = (['tree', 'new jersey'], {'evergreen'}) assert tags.filter(selection) == maple_bund # cannot be a tree in NJ, AND must also be deciduous # I.e., give all deciduous things that aren't trees in NJ selection = [{'tree', 'new jersey'}, 'deciduous'] assert tags.filter(selection) == dtr.Bundle()
def test_tags_fuzzy(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): t1 = dtr.Treant('maple') t2 = dtr.Treant('pine') t1.tags.add({'tree', 'new jersey', 'deciduous'}) t2.tags.add({'tree', 'new york', 'evergreen'}) col = collection(t1, t2) tags = col.tags assert len(tags.any) == 5 all_tree1 = tags.fuzzy('tree', threshold=80, scope='all') all_tree2 = tags.fuzzy('tree') assert all_tree1 == all_tree2 assert all_tree2 == ('tree',) any_deciduous = tags.fuzzy('deciduous', scope='any') assert any_deciduous == ('deciduous',) all_evergreen = tags.fuzzy('evergreen') assert all_evergreen == () # check that fuzzy matching is independent of threshold when # exact tag is present in all members all_tree_strict = tags.fuzzy('tree', threshold=99) assert all_tree_strict == ('tree',) all_tree_tolerant = tags.fuzzy('tree', threshold=0) assert all_tree_tolerant == ('tree',) # check that fuzzy matching will give differing tags when # members have similar tag names ('new') and the threshold is # varied all_ny = tags.fuzzy('new york') assert all_ny == () any_ny_strict = tags.fuzzy('new york', scope='any') assert any_ny_strict == ('new york',) any_ny_tol = tags.fuzzy('new york', threshold=50, scope='any') assert set(any_ny_tol) == {'new york', 'new jersey'} # check fuzzy matching for multiple tags (scope='all') new_ever = ['new', 'evergreen'] all_mul_strict = tags.fuzzy(new_ever, threshold=80) assert all_mul_strict == () all_mul_tol = tags.fuzzy(new_ever, threshold=30) assert all_mul_tol == ('tree',) # check fuzzy matching for multiple tags (scope='any') new_tree = ['new', 'tree'] any_mul_stric = tags.fuzzy(new_tree, threshold=90, scope='any') assert any_mul_stric == ('tree',) any_mul_tol = tags.fuzzy(new_tree, threshold=80, scope='any') assert set(any_mul_tol) == {'new york', 'new jersey', 'tree'} nj_decid = ['new jersey', 'decid'] any_mul_njdec = tags.fuzzy(nj_decid, threshold=80, scope='any') assert set(any_mul_njdec) == {'new jersey', 'deciduous'}
def test_tags_getitem(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): t1 = dtr.Treant('maple') t2 = dtr.Treant('pine') t1.tags.add({'tree', 'new jersey', 'deciduous'}) t2.tags.add({'tree', 'new york', 'evergreen'}) col = collection(t1, t2) tags = col.tags assert len(tags.any) == 5 # test single tags assert tags['tree'] == [True, True] assert tags['deciduous'] == [True, False] assert tags['evergreen'] == [False, True] assert tags['new jersey'] == [True, False] assert tags['new york'] == [False, True] assert tags[{'tree'}] == [False, False] assert tags[{'deciduous'}] == [False, True] assert tags[{'evergreen'}] == [True, False] assert tags[{'new jersey'}] == [False, True] assert tags[{'new york'}] == [True, False] # test for Treants with ALL the given tags assert tags[['tree', 'deciduous']] == [True, False] assert tags[['tree', 'evergreen']] == [False, True] assert tags[['new jersey', 'evergreen']] == [False, False] # test for Treants with ANY of the given tags assert tags[('tree', 'deciduous')] == [True, True] assert tags[('deciduous', 'evergreen')] == [True, True] assert tags[('new york', 'evergreen')] == [False, True] # test for Treants without at least one of the given tags assert tags[{'deciduous', 'evergreen'}] == [True, True] assert tags[{'tree', 'deciduous'}] == [False, True] assert tags[{'tree', 'new york', 'evergreen'}] == [True, False] # complex logic tests # give those that are evergreen or in NY AND also not deciduous selection = [('evergreen', 'new york'), {'deciduous'}] assert tags[selection] == [False, True] # give those that are evergreen or in NY AND also not a tree selection = [('evergreen', 'new york'), {'tree'}] assert tags[selection] == [False, False] # give a tree that's in NJ OR anything that's not evergreen selection = (['tree', 'new jersey'], {'evergreen'}) assert tags[selection] == [True, False] # cannot be a tree in NJ, AND must also be deciduous # I.e., give all deciduous things that aren't trees in NJ selection = [{'tree', 'new jersey'}, 'deciduous'] assert tags[selection] == [False, False]
def test_parents(self, collection, tmpdir): with tmpdir.as_cwd(): t1 = dtr.Treant('free-associate/lark') t2 = dtr.Treant('free-associate/hark') t3 = dtr.Treant('characters/linus') col = collection(t1, t2, t3) assert len(col.parents()) == 2 assert 'free-associate' in col.parents().names assert 'linus' not in col.parents().names assert 'characters' in col.parents().names
def test_init_regenerate_via_name(self, tags, categories, tmpdir): # test regenerating a Treant from its directory with tmpdir.as_cwd(): t = dtr.Treant('this') t2 = dtr.Treant('this', tags=tags, categories=categories) if tags is not None: for tag in tags: assert tag in t2.tags if categories is not None: for cat, val in categories.items(): assert cat in t2.categories assert t2.categories[cat] == val
def test_tags_all(self, collection, tmpdir): with tmpdir.as_cwd(): moe = dtr.Treant('moe', tags=['smartest', 'mean', 'stooge']) larry = dtr.Treant('larry', tags=['weird', 'means well', 'stooge']) curly = dtr.Treant('curly', tags=['dumb', 'nyuk-nyuk', 'stooge']) col = collection(moe, larry, curly) assert len(col.tags.all) == 1 assert 'stooge' in col.tags.all
def test_member_attributes(self, collection, tmpdir): """Get member names and abspaths""" with tmpdir.as_cwd(): t1 = dtr.Treant('bigger') t2 = dtr.Treant('faster') t3 = dtr.Treant('stronger') col = collection(t1, t2, t3) names = [treant.name for treant in [t1, t2, t3]] assert col.names == names abspaths = [treant.abspath for treant in [t1, t2, t3]] assert col.abspaths == abspaths
def test_map(self, collection, tmpdir): with tmpdir.as_cwd(): t1 = dtr.Treant('lark') t2 = dtr.Treant('hark') t3 = dtr.Treant('linus') col = collection(t1, t2, t3) comp = [cont.name for cont in col] assert col.map(do_stuff) == comp assert col.map(do_stuff, processes=2) == comp assert col.map(return_nothing) is None assert col.map(return_nothing, processes=2) is None
def test_add_categories(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): # add a couple test Treants to collection col = collection(testtreant, testtreant2) assert len(col.categories) == 0 # add 'age' and 'bark' as categories of this collection col.categories.add({'age': 42}, bark='smooth') assert len(col.categories) == 2 for member in col: assert member.categories['age'] == 42 assert member.categories['bark'] == 'smooth' for key in ['age', 'bark']: assert key in col.categories.any t1 = dtr.Treant('hickory') t1.categories.add(bark='shaggy', species='ovata') col += collection(t1) assert len(col.categories) == 1 assert len(col.categories.all) == 1 assert len(col.categories.any) == 3 col.categories.add(location='USA') assert len(col.categories) == 2 assert len(col.categories.all) == 2 assert len(col.categories.any) == 4 for member in col: assert member.categories['location'] == 'USA'
def test_tags_setting(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): col = collection(testtreant, testtreant2) assert len(col.tags) == 0 # add as list col.tags = ['broiled', 'not baked'] assert len(col.tags) == 2 for tag in ('broiled', 'not baked'): assert tag in col.tags col.tags.clear() # add as set col.tags = {'broiled', 'not baked'} assert len(col.tags) == 2 for tag in ('broiled', 'not baked'): assert tag in col.tags col.tags.clear() # add as Tags t = dtr.Treant('mark twain') t.tags.add('literature', 'quotables') col.tags = t.tags assert len(col.tags) == 2 for tag in ('literature', 'quotables'): assert tag in col.tags
def test_tags_any(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): moe = dtr.Treant('moe', tags=['smartest', 'mean', 'stooge']) larry = dtr.Treant('larry', tags=['weird', 'means well', 'stooge']) curly = dtr.Treant('curly', tags=['dumb', 'nyuk-nyuk', 'stooge']) col = collection(moe, larry, curly) assert len(col.tags.any) == 7 for tag in ('smartest', 'mean', 'weird', 'means well', 'dumb', 'nyuk-nyuk', 'stooge'): assert tag in col.tags.any
def filled_bundle(self, tmpdir): with tmpdir.as_cwd(): t1 = dtr.Treant('one') t2 = dtr.Treant('two') t3 = dtr.Treant('three') t4 = dtr.Treant('four') t1.tags.add('odd', 'one') t2.tags.add('even', 'two') t3.tags.add('odd', 'three') t4.tags.add('even', 'four') t1.categories.add(is_even=False, value=1) t2.categories.add(is_even=True, value=2) t3.categories.add(is_even=False, value=3) t4.categories.add(is_even=True, value=4) return [t1, t2, t3, t4], dtr.Bundle([t1, t2, t3, t4])
def test_categories_setitem(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): # add a couple test Treants to collection col = collection(testtreant, testtreant2) # add 'age' and 'bark' as categories of this collection col.categories.add({'age': 42, 'bark': 'smooth'}) t1 = dtr.Treant('maple') t2 = dtr.Treant('sequoia') t1.categories.add({'age': 'seedling', 'bark': 'rough', 'type': 'deciduous'}) t2.categories.add({'age': 'adult', 'bark': 'rough', 'type': 'evergreen', 'nickname': 'redwood'}) col += collection(t1, t2) # test setting a category when all members have it for value in col.categories['age']: assert value in [42, 42, 'seedling', 'adult'] col.categories['age'] = 'old' for value in col.categories['age']: assert value in ['old', 'old', 'old', 'old'] # test setting a new category (no members have it) assert 'location' not in col.categories.any col.categories['location'] = 'USA' for value in col.categories['location']: assert value in ['USA', 'USA', 'USA', 'USA'] # test setting a category that only some members have assert 'nickname' in col.categories.any assert 'nickname' not in col.categories.all col.categories['nickname'] = 'friend' for value in col.categories['nickname']: assert value in ['friend', 'friend', 'friend', 'friend'] # test setting values for individual members assert 'favorite ice cream' not in col.categories ice_creams = ['rocky road', 'americone dream', 'moose tracks', 'vanilla'] col.categories['favorite ice cream'] = ice_creams for member, ice_cream in zip(col, ice_creams): assert member.categories['favorite ice cream'] == ice_cream
def test_get_members(self, collection, tmpdir): """Access members with indexing and slicing""" with tmpdir.as_cwd(): t1 = dtr.Treant('larry') t2 = dtr.Treant('curly') t3 = dtr.Treant('moe') col = collection([[[t1, [t2, [t3]]]]]) assert col[1] == t2 t4 = dtr.treants.Treant('shemp') col = col + t4 for member in (t1, t2, t3): assert member in col[:3] assert t4 not in col[:3] assert t4 == col[-1]
def test_convert(old_treant, tmpdir): treant_folder = str(tmpdir) with open(path.join(treant_folder, 'Treant-uuid.json'), 'w') as fh: json.dump(old_treant, fh) convert(treant_folder) treant = dtr.Treant(treant_folder) assert treant.tags == old_treant['tags'] assert treant.categories == old_treant['categories']
def sim(tmpdir_factory): folder = tmpdir_factory.mktemp("simulation") return dtr.Treant( str(folder), categories={ "nodes": 42, "host": "draco", "gpu": False, "module": "namd/11" }, )
def write_benchmark( engine, base_directory, template, nodes, gpu, module, name, relative_path, job_name, host, time, ): """Generate a benchmark folder with the respective Sim object.""" # Create the `dtr.Treant` object sim = dtr.Treant(base_directory["{}/".format(nodes)]) # Do MD engine specific things. Here we also format the name. name = engine.prepare_benchmark(name=name, relative_path=relative_path, sim=sim) if job_name is None: job_name = name # Add categories to the `Sim` object sim.categories = { "module": module, "gpu": gpu, "nodes": nodes, "host": host, "time": time, "name": name, "started": False, } # Add some time buffer to the requested time. Otherwise the queuing system # kills the job before the benchmark is finished formatted_time = "{:02d}:{:02d}:00".format(*divmod(time + 5, 60)) # Create benchmark job script script = template.render( name=name, job_name=job_name, gpu=gpu, module=module, mdengine=engine.NAME, n_nodes=nodes, time=time, formatted_time=formatted_time, ) # Write the actual job script that is going to be submitted to the cluster with open(sim["bench.job"].relpath, "w") as fh: fh.write(script)
def test_categories_values(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): # add a couple test Treants to collection col = collection(testtreant, testtreant2) col.categories.add({'age': 'young', 'bark': 'smooth'}) t1 = dtr.Treant('maple') t2 = dtr.Treant('sequoia') t1.categories.add({'age': 'seedling', 'bark': 'rough', 'type': 'deciduous'}) t2.categories.add({'age': 'adult', 'bark': 'rough', 'type': 'evergreen', 'nickname': 'redwood'}) col += collection(t1, t2) for scope in ('all', 'any'): for i, v in enumerate( col.categories.values(scope=scope)): assert v == col.categories[ col.categories.keys(scope=scope)[i]]
def sim_old(tmpdir_factory): folder = tmpdir_factory.mktemp("simulation") return dtr.Treant( str(folder), categories={ "nodes": 42, "host": "draco", "gpu": False, "version": "5.1.4" }, )
def test_categories_remove(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): t1 = dtr.Treant('maple') t2 = dtr.Treant('sequoia') col = collection(t1, t2) col.categories.add({'age': 'sprout'}, bark='rough') # add a couple test Treants to collection col += collection(testtreant, testtreant2) assert len(col.categories) == 0 assert len(col.categories.any) == 2 # add 'USA', ensure 'location', 'age', 'bark' is a category in # at least one of the members col.categories.add(location='USA') assert len(col.categories) == 1 for key in ['location', 'age', 'bark']: assert key in col.categories.any # ensure 'age' and 'bark' are each not categories for all # members in collection assert 'age' not in col.categories assert 'bark' not in col.categories # remove 'bark', test for any instance of 'bark' in the # collection col.categories.remove('bark') assert len(col.categories) == 1 for key in ['location', 'age']: assert key in col.categories.any assert 'bark' not in col.categories.any # remove 'age', test that 'age' is not a category for any # member in collection col.categories.remove('age') for member in col: assert 'age' not in member.categories # test that 'age' is not a category of this collection assert 'age' not in col.categories.any
def test_init_generate(self, tags, categories, tmpdir): # test combinations of tags and categories # when generating from scratch with tmpdir.as_cwd(): t = dtr.Treant('babs', tags=tags, categories=categories) if tags is not None: for tag in tags: assert tag in t.tags if categories is not None: for cat, val in categories.items(): assert cat in t.categories assert t.categories[cat] == val
def test_categories_any(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): # add a couple test Treants to collection col = collection(testtreant, testtreant2) # add 'age' and 'bark' as categories of this collection col.categories.add({'age': 42}, bark='smooth') assert len(col.categories.any) == 2 # add categories to 'hickory' Treant, then add to collection t1 = dtr.Treant('hickory') t1.categories.add(bark='shaggy', species='ovata') col += collection(t1) # check the contents of 'bark', ensure 'age' and 'species' are # not shared categories of the collection every_category = col.categories.any assert len(every_category) == 3 assert every_category['age'] == [42, 42, None] assert every_category['bark'] == ['smooth', 'smooth', 'shaggy'] assert every_category['species'] == [None, None, 'ovata'] # add 'location' category to collection col.categories.add(location='USA') every_category = col.categories.any # ensure all members have 'USA' for their 'location' assert len(every_category) == 4 assert every_category['age'] == [42, 42, None] assert every_category['bark'] == ['smooth', 'smooth', 'shaggy'] assert every_category['species'] == [None, None, 'ovata'] assert every_category['location'] == ['USA', 'USA', 'USA'] # add 'sprout' to 'age' category of 'hickory' Treant t1.categories['age'] = 'sprout' every_category = col.categories.any # check 'age' is category for 'hickory' and is 'sprout' assert len(every_category) == 4 assert every_category['age'] == [42, 42, 'sprout'] assert every_category['bark'] == ['smooth', 'smooth', 'shaggy'] assert every_category['species'] == [None, None, 'ovata'] assert every_category['location'] == ['USA', 'USA', 'USA'] # add 'location' category to collection col.categories.remove('bark') every_category = col.categories.any # check that only 'location' is a shared category assert len(every_category) == 3 assert every_category['age'] == [42, 42, 'sprout'] assert every_category['species'] == [None, None, 'ovata'] assert every_category['location'] == ['USA', 'USA', 'USA'] assert 'bark' not in every_category
def convert(folder): treants = glob(path.join(folder, 'Treant*')) if len(treants) == 0: warnings.warn("No treant found in folder: {}".format(folder)) return elif len(treants) > 1: warnings.warn("Multiple treants found in folder: {}\n" " Skipping conversion".format(folder)) return json_file = treants[0] with open(json_file) as fh: old = json.load(fh) datreant.Treant(folder, categories=old['categories'], tags=old['tags'])
def test_categories_getitem(self, collection, testtreant, testtreant2, tmpdir): with tmpdir.as_cwd(): # add a couple test Treants to collection col = collection((testtreant, testtreant2)) # add 'age' and 'bark' as categories of this collection col.categories.add({'age': 42, 'bark': 'smooth'}) t1 = dtr.Treant('maple') t2 = dtr.Treant('sequoia') t1.categories.add({'age': 'seedling', 'bark': 'rough', 'type': 'deciduous'}) t2.categories.add({'age': 'adult', 'bark': 'rough', 'type': 'evergreen', 'nickname': 'redwood'}) col += collection((t1, t2)) assert len(col.categories) == 2 assert len(col.categories.any) == 4 # test values for each category in the collection age_list = [42, 42, 'seedling', 'adult'] assert age_list == col.categories['age'] bark_list = ['smooth', 'smooth', 'rough', 'rough'] assert bark_list == col.categories['bark'] type_list = [None, None, 'deciduous', 'evergreen'] assert type_list == col.categories['type'] nick_list = [None, None, None, 'redwood'] assert nick_list == col.categories['nickname'] # test list of keys as input cat_list = [age_list, type_list] assert cat_list == col.categories[['age', 'type']] # test set of keys as input cat_set = {'bark': bark_list, 'nickname': nick_list} assert cat_set == col.categories[{'bark', 'nickname'}]
def test_discover_depth(tmpdir): """Check that using `depth` parameter gives expected result.""" with tmpdir.as_cwd(): ghosts = ('something/inky', 'something/else/blinky', 'pinky', 'something/clyde') for name in ghosts: dtr.Treant(name) assert len(discover('.', depth=0)) == 0 assert len(discover('pinky', depth=0)) == 1 assert len(discover('.', depth=1)) == 1 assert len(discover('.', depth=2)) == 3 assert len(discover('.', depth=3)) == 4
def test_discover(tmpdir): with tmpdir.as_cwd(): ghosts = ('inky', 'blinky', 'pinky', 'clyde') for name in ghosts: dtr.Treant( 'a/very/deep/directory/structure/that/just/keeps/going/' + name) b = discover('.') assert len(b) == 4 for name in ghosts: assert name in b.names
def test_prepare_benchmark(engine, input_name, extensions, tmpdir): """Test that the preparation functions of all supported MD engines work.""" with tmpdir.as_cwd(): for ext in extensions: filename = "md.{}".format(ext) if input_name.startswith("../"): filename = "../md.{}".format(ext) open(filename, "a").close() relative_path, filename = os.path.split(input_name) sim = dtr.Treant("./{}".format(engine)) name = engine.prepare_benchmark( name=filename, relative_path=relative_path, sim=sim ) assert name == "md" for ext in extensions: assert os.path.exists("./{}/md.{}".format(engine, ext))
def test_discover_treantdepth(tmpdir): """Check that using `treantdepth` parameter gives expected result.""" with tmpdir.as_cwd(): ghosts = ('inky', 'inky/blinky', 'pinky', 'inky/blinky/clyde') for name in ghosts: dtr.Treant(name) assert len(discover('.', treantdepth=0)) == 2 assert len(discover('pinky', treantdepth=0)) == 1 assert len(discover('inky', treantdepth=0)) == 1 assert len(discover('.', treantdepth=1)) == 3 assert len(discover('.', treantdepth=2)) == 4 assert len(discover('inky', treantdepth=1)) == 2 assert len(discover('inky/blinky', treantdepth=1)) == 2 assert len(discover('inky/blinky', treantdepth=1)) == 2