class TestDidMetaClient(unittest.TestCase): def setUp(self): self.did_client = DIDClient() self.tmp_scope = 'mock' self.session = get_session() self.json_implemented = JSONDidMeta().json_implemented(self.session) def tearDown(self): self.session.commit() # pylint: disable=no-member def test_set_metadata(self): """ META (CLIENTS) : Adds a fully set json column to a did, updates if some keys present """ tmp_name = 'name_%s' % generate_uuid() self.did_client.add_did(scope=self.tmp_scope, name=tmp_name, type="DATASET") # Test JSON case if self.json_implemented: # data1 = ["key1": "value_" + str(generate_uuid()), "key2": "value_" + str(generate_uuid()), "key3": "value_" + str(generate_uuid())] value1 = "value_" + str(generate_uuid()) value2 = "value_" + str(generate_uuid()) value3 = "value_" + str(generate_uuid()) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key1", value=value1) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key2", value=value2) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key3", value=value3) metadata = self.did_client.get_metadata(scope=self.tmp_scope, name=tmp_name, plugin="JSON") assert len(metadata) == 3 assert metadata['key1'] == value1 assert metadata['key2'] == value2 assert metadata['key3'] == value3 # Test DID_COLUMNS case self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key='project', value='data12_12TeV') assert self.did_client.get_metadata( scope=self.tmp_scope, name=tmp_name)['project'] == 'data12_12TeV' def test_delete_metadata(self): """ META (CLIENTS) : Deletes metadata key """ tmp_name = 'name_%s' % generate_uuid() self.did_client.add_did(scope=self.tmp_scope, name=tmp_name, type="DATASET") # Test JSON case if self.json_implemented: value1 = "value_" + str(generate_uuid()) value2 = "value_" + str(generate_uuid()) value3 = "value_" + str(generate_uuid()) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key1", value=value1) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key2", value=value2) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key3", value=value3) self.did_client.delete_metadata(scope=self.tmp_scope, name=tmp_name, key='key2') metadata = self.did_client.get_metadata(scope=self.tmp_scope, name=tmp_name, plugin="JSON") assert len(metadata) == 2 assert metadata['key1'] == value1 assert metadata['key3'] == value3 with pytest.raises(KeyNotFound): self.did_client.delete_metadata(scope=self.tmp_scope, name=tmp_name, key="key9") def test_get_metadata(self): """ META (CLIENTS) : Gets all metadata for the given did """ tmp_name = 'name_%s' % generate_uuid() self.did_client.add_did(scope=self.tmp_scope, name=tmp_name, type="DATASET") # Test JSON case if self.json_implemented: value1 = "value_" + str(generate_uuid()) value2 = "value_" + str(generate_uuid()) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key1", value=value1) self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key="key2", value=value2) metadata = self.did_client.get_metadata(scope=self.tmp_scope, name=tmp_name, plugin="JSON") assert len(metadata) == 2 assert metadata['key1'] == value1 assert metadata['key2'] == value2 # Test DID_COLUMNS case self.did_client.set_metadata(scope=self.tmp_scope, name=tmp_name, key='project', value='data12_14TeV') assert self.did_client.get_metadata( scope=self.tmp_scope, name=tmp_name)['project'] == 'data12_14TeV' # Test Mixed case if self.json_implemented: all_metadata = self.did_client.get_metadata(scope=self.tmp_scope, name=tmp_name, plugin="ALL") assert all_metadata['key1'] == value1 assert all_metadata['key2'] == value2 assert all_metadata['project'] == "data12_14TeV" def test_list_dids_extended(self): """ META (CLIENTS) : Get all dids matching the values of the provided metadata keys """ # Test did Columns use case dsns = [] tmp_scope = 'mock' tmp_dsn1 = 'dsn_%s' % generate_uuid() dsns.append(tmp_dsn1) dataset_meta = { 'project': 'data12_8TeV', 'run_number': 400000, 'stream_name': 'physics_CosmicCalo', 'prod_step': 'merge', 'datatype': 'NTUP_TRIG', 'version': 'f392_m920', } self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn1, meta=dataset_meta) tmp_dsn2 = 'dsn_%s' % generate_uuid() dsns.append(tmp_dsn2) dataset_meta['run_number'] = 400001 self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn2, meta=dataset_meta) tmp_dsn3 = 'dsn_%s' % generate_uuid() dsns.append(tmp_dsn3) dataset_meta['stream_name'] = 'physics_Egamma' dataset_meta['datatype'] = 'NTUP_SMWZ' self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn3, meta=dataset_meta) dids = self.did_client.list_dids_extended(tmp_scope, { 'project': 'data12_8TeV', 'version': 'f392_m920' }) results = [] for d in dids: results.append(d) for dsn in dsns: assert dsn in results dsns.remove(tmp_dsn1) dids = self.did_client.list_dids_extended(tmp_scope, { 'project': 'data12_8TeV', 'run_number': 400001 }) results = [] for d in dids: results.append(d) for dsn in dsns: assert dsn in results dsns.remove(tmp_dsn2) dids = self.did_client.list_dids_extended( tmp_scope, { 'project': 'data12_8TeV', 'stream_name': 'physics_Egamma', 'datatype': 'NTUP_SMWZ' }) results = [] for d in dids: results.append(d) for dsn in dsns: assert dsn in results # Test JSON use case if self.json_implemented: did1 = 'name_%s' % generate_uuid() did2 = 'name_%s' % generate_uuid() did3 = 'name_%s' % generate_uuid() did4 = 'name_%s' % generate_uuid() key1 = 'key_1_%s' % generate_uuid() key2 = 'key_2_%s' % generate_uuid() key3 = 'key_3_%s' % generate_uuid() value1 = 'value_1_%s' % generate_uuid() value2 = 'value_2_%s' % generate_uuid() value3 = 'value_3_%s' % generate_uuid() value_not_1 = 'value_not_1_%s' % generate_uuid() value_not_2 = 'value_not_1_%s' % generate_uuid() value_unique = 'value_unique_%s' % generate_uuid() self.did_client.add_did(scope=tmp_scope, name=did1, type="DATASET") self.did_client.add_did(scope=tmp_scope, name=did2, type="DATASET") self.did_client.add_did(scope=tmp_scope, name=did3, type="DATASET") self.did_client.add_did(scope=tmp_scope, name=did4, type="DATASET") self.did_client.set_metadata(scope=tmp_scope, name=did1, key=key1, value=value1) self.did_client.set_metadata(scope=tmp_scope, name=did1, key=key2, value=value2) self.did_client.set_metadata(scope=tmp_scope, name=did2, key=key1, value=value1) self.did_client.set_metadata(scope=tmp_scope, name=did2, key=key2, value=value_not_2) self.did_client.set_metadata(scope=tmp_scope, name=did2, key=key3, value=value3) self.did_client.set_metadata(scope=tmp_scope, name=did3, key=key1, value=value_not_1) self.did_client.set_metadata(scope=tmp_scope, name=did3, key=key2, value=value2) self.did_client.set_metadata(scope=tmp_scope, name=did3, key=key3, value=value3) self.did_client.set_metadata(scope=tmp_scope, name=did4, key=key1, value=value1) self.did_client.set_metadata(scope=tmp_scope, name=did4, key=key2, value=value2) self.did_client.set_metadata(scope=tmp_scope, name=did4, key=key3, value=value_unique) # Key not there dids = self.did_client.list_dids_extended(tmp_scope, {'key45': 'value'}) results = [] for d in dids: results.append(d) assert len(results) == 0 # Value not there dids = self.did_client.list_dids_extended( tmp_scope, {key1: 'value_not_there'}) results = [] for d in dids: results.append(d) assert len(results) == 0 # key1 = value1 dids = self.did_client.list_dids_extended(tmp_scope, {key1: value1}) results = [] for d in dids: results.append(d) assert len(results) == 3 assert did1 in results assert did2 in results assert did4 in results # key1, key2 dids = self.did_client.list_dids_extended(tmp_scope, { key1: value1, key2: value2 }) results = [] for d in dids: results.append(d) assert len(results) == 2 assert did1 in results assert did4 in results # key1, key2, key 3 dids = self.did_client.list_dids_extended(tmp_scope, { key1: value1, key2: value2, key3: value3 }) results = [] for d in dids: results.append(d) assert len(results) == 0 # key3 = unique value dids = self.did_client.list_dids_extended(tmp_scope, {key3: value_unique}) results = [] for d in dids: results.append(d) assert len(results) == 1 assert did4 in results
# In[6]: today = str(time.strftime('%Y%m%d')) set_meta = didc.set_metadata(scope=Default_Scope, name=name_file, key='night', value=today, recursive=False) print(set_meta) # In[7]: get_meta = didc.get_metadata(scope=Default_Scope, name=name_file, plugin='ALL') for x in get_meta: print(x, ':', get_meta[str(x)]) # In[9]: list_associated_meta = didc.list_dids_extended(scope=Default_Scope, filters={'night': today}, type='all', long=False, recursive=False) print(list(list_associated_meta)) # In[ ]: