def test_apply_filter(self):
"""Test that apply filters function can be used to strip items out of the data"""
data = [{
'a': '11',
'b': '21',
'c': '31'
}, {
'a': '12',
'b': '22',
'c': '32'
}]
result = gran.apply_filters(
[gran.drop_fields('a'),
gran.drop_fields('b')], data)
expected = [{'c': '31'}, {'c': '32'}]
self.assertEqual(expected, result, "drop fields")
data = [{
'meta': {
'1': 'one',
'2': 'two'
},
'umm': {
'two': '2',
'one': '1'
}
}]
result = gran.apply_filters([gran.all_fields, gran.meta_fields], data)
expected = [{'1': 'one', '2': 'two'}]
self.assertEqual(expected, result, "All and meta fields")
result = gran.apply_filters([gran.all_fields, gran.umm_fields], data)
expected = [{'two': '2', 'one': '1'}]
self.assertEqual(expected, result, "All and umm fields")
data = [{
'meta': {
'concept-id': 'g1',
'2': 'two'
},
'umm': {
'concept-id': 'g2',
'one': '1'
}
}]
result = gran.apply_filters([gran.all_fields, gran.concept_id_fields],
data)
expected = [{'concept-id': 'g1'}]
self.assertEqual(expected, result, "find concept id from meta")
data = [{'concept-id': 'g2', 'one': '1'}]
result = gran.apply_filters([gran.all_fields, gran.concept_id_fields],
data)
expected = [{'concept-id': 'g2'}]
self.assertEqual(expected, result, "find concept id from dict")
def test_compound_search_gran(self, urlopen_mock):
"""
Assuming that _collection_samples works and CMR work, test _granule_samples.
Do this By hard coding the granule search and making sure that the remaining
code correctly returns the granule information.
"""
# Setup
recorded_file = tutil.resolve_full_path(
'../data/cmr/search/combo_gran_result.json')
urlopen_mock.return_value = valid_cmr_response(recorded_file)
# Inputs
found_collections = [
'C179003030-ORNL_DAAC', 'C179002914-ORNL_DAAC',
'C1000000000-ORNL_DAAC', 'C1536961538-ORNL_DAAC',
'C179126725-ORNL_DAAC', 'C179003380-ORNL_DAAC',
'C179130805-ORNL_DAAC', 'C179003657-ORNL_DAAC',
'C1227811476-ORNL_DAAC', 'C179130785-ORNL_DAAC'
]
filters = [
gran.granule_core_fields,
gran.drop_fields('GranuleUR'),
gran.drop_fields('revision-id'),
gran.drop_fields('native-id')
]
limit = 1
config = {}
# Run test
found_granules = gran._granule_samples(found_collections, filters,
limit, config)
expected = [{
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}]
self.assertEqual(expected, found_granules,
"Compound Search - Granual Compound")
def test_apply_filter(self):
"""Test that apply filters function can be used to strip items out of the data"""
data = [{
'a': '11',
'b': '21',
'c': '31'
}, {
'a': '12',
'b': '22',
'c': '32'
}]
result = gran.apply_filters(
[gran.drop_fields('a'),
gran.drop_fields('b')], data)
expected = [{'c': '31'}, {'c': '32'}]
self.assertEqual(expected, result)
def test_search(self, urlopen_mock):
"""
def search(query=None, filters=None, limit=None, options=None):
"""
# Setup
urlopen_mock.return_value = valid_cmr_response(
tutil.resolve_full_path(
'../data/cmr/search/one_granule_cmr_result.json'))
# Basic
full_result = gran.search({'provider': 'SEDAC'}, limit=1)
self.assertEqual(1, len(full_result))
# Unfiltered Test
unfiltered_result = gran.search({'provider': 'SEDAC'},
filters=[gran.all_fields],
limit=1)
self.assertEqual(full_result, unfiltered_result)
# Meta filter Test
meta_results = gran.search({'provider': 'SEDAC'},
filters=[gran.meta_fields],
limit=1)
expected = [{
'concept-type': 'granule',
'concept-id': 'G1527288030-SEDAC',
'revision-id': 2,
'native-id': 'urbanspatial-hist-urban-pop-3700bc-ad2000-xlsx.xlsx',
'provider-id': 'SEDAC',
'format': 'application/echo10+xml',
'revision-date': '2020-08-11T13:54:58.567Z'
}]
self.assertEqual(expected, meta_results)
# UMM filter Test
umm_results = gran.search({'provider': 'SEDAC'},
filters=[gran.umm_fields],
limit=1)
expected = 'urbanspatial-hist-urban-pop-3700bc-ad2000-xlsx.xlsx'
self.assertEqual(expected, umm_results[0]['GranuleUR'])
self.assertEqual(11, len(umm_results[0].keys()))
# Collection ID Filter Test
cid_results = gran.search({'provider': 'SEDAC'},
filters=[gran.concept_id_fields],
limit=1)
expected = [{'concept-id': 'G1527288030-SEDAC'}]
self.assertEqual(expected, cid_results)
# Drop Filter Test
drop_results = gran.search({'provider': 'SEDAC'},
filters=[
gran.meta_fields,
gran.drop_fields('concept-type'),
gran.drop_fields('revision-date'),
gran.drop_fields('format')
],
limit=1)
expected = [{'concept-id': 'G1527288030-SEDAC'}]
meta_count = len(meta_results[0].keys()) #from test above
drop_count = len(drop_results[0].keys())
self.assertEqual(3, meta_count - drop_count)
#IDs Filter Test
ids_results = gran.search({'provider': 'SEDAC'},
filters=[gran.granule_core_fields],
limit=1)
expected = [{
'concept-id':
'G1527288030-SEDAC',
'revision-id':
2,
'native-id':
'urbanspatial-hist-urban-pop-3700bc-ad2000-xlsx.xlsx',
'GranuleUR':
'urbanspatial-hist-urban-pop-3700bc-ad2000-xlsx.xlsx'
}]
self.assertEqual(expected, ids_results)
def test_compound_search(self, urlopen_mock, coll_mock):
"""
Do a full test of the compound search, assuming that the component pieces
work. Assume the response of the collection samples and hard code the CMR
response for the granule search.
"""
coll_mock.return_value = [
'C179003030-ORNL_DAAC', 'C179002914-ORNL_DAAC',
'C1000000000-ORNL_DAAC', 'C1536961538-ORNL_DAAC',
'C179126725-ORNL_DAAC', 'C179003380-ORNL_DAAC',
'C179130805-ORNL_DAAC', 'C179003657-ORNL_DAAC',
'C1227811476-ORNL_DAAC', 'C179130785-ORNL_DAAC'
]
# Setup
recorded_file = tutil.resolve_full_path(
'../data/cmr/search/combo_gran_result.json')
urlopen_mock.return_value = valid_cmr_response(recorded_file)
collection_query = {'provider': 'GHRC_CLOUD'}
filters = [
gran.granule_core_fields,
gran.drop_fields('GranuleUR'),
gran.drop_fields('revision-id'),
gran.drop_fields('native-id')
]
# cut down the function parameters to assume all the parts that will not change
runner = partial(gran.sample_by_collections,
collection_query,
filters=filters)
# pylint: disable=C0301 # lambda must be on one line
tester = lambda expected, limit, msg: self.assertEqual(
expected, runner(limits=limit), msg)
expected = [{
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}, {
'concept-id': 'G1527288030-SEDAC'
}]
tester(expected, 10, "at most 10, default collection, using int")
tester(expected, [10, 1], "at most 10, one collection, array")
tester(expected[:5], {
"granule": 5,
"collection": 1
}, "at most five, using dictionary")
tester(expected, {
"granule": 1,
"collection": None
}, "defaulting with dictionary")