class ApiTest(unittest.TestCase):
"""Tests ConfigManager and local/remote config access."""
def setUp(self):
self.api = Api() # url='http://api.eoss.cloud'
def testCreateConfig(self):
"""
Create simple config object from string
"""
ds = self.api.get_dataset('LC81920272016240LGN00')
self.assertEqual(len(ds), 1)
ds = self.api.get_dataset('LE71010172003151EDC00')
self.assertEqual(len(ds), 1)
ds = self.api.get_dataset(
'S2A_OPER_PRD_MSIL1C_PDMC_20160806T202847_R142_V20160805T192909_20160805T192909'
)
self.assertEqual(len(ds), 21)
def testCatalogSearch(self):
aoi_nw = (-91.5175095, 16.8333384)
aoi_se = (-91.3617268, 16.8135385)
aoi_ne = (aoi_se[0], aoi_nw[1])
aoi_sw = (aoi_nw[0], aoi_se[1])
aoi = [aoi_nw, aoi_ne, aoi_se, aoi_sw, aoi_nw]
# Object representation
results = self.api.search_dataset(aoi,
100,
parse('2015-01-01'),
parse('2015-03-01'),
'landsat8',
full_objects=True)
self.assertEqual(len(results), 3)
for item in results:
self.assertTrue(type(item).__name__ == 'Catalog_Dataset')
results = self.api.search_dataset(aoi,
100,
parse('2015-01-01'),
parse('2015-03-01'),
'landsat8',
full_objects=False)
# JSON representation
self.assertEqual(len(results), 3)
for item in results:
self.assertTrue(type(item) == dict)
def main(sensor, start_date, days, api_endpoint):
api = Api(api_endpoint)
logger.info('Checking consistencty for %s between %s + %s' % (sensor, start_date, days))
aoi_nw = (-180, 90)
aoi_se = (180, -90)
aoi_ne = (aoi_se[0], aoi_nw[1])
aoi_sw = (aoi_nw[0], aoi_se[1])
aoi = [aoi_nw, aoi_ne, aoi_se, aoi_sw, aoi_nw]
wrong_urls = list()
for delta_day in range(1, days):
start_time = time.time()
start_date_date = parse(start_date)+ datetime.timedelta(days=delta_day)
end_date_date = start_date_date + datetime.timedelta(days=1)
logger.info('Checking consistencty for %s between %s and %s' % (sensor, start_date_date.isoformat(), end_date_date.isoformat()))
# Object representation
results = api.search_dataset(aoi, 100, start_date_date, end_date_date, sensor, full_objects=False)
url_resources = list()
missing_urls = list()
missing_types = list()
for r in results:
if r['resources']['s3public']['zip'] != None:
url_resources.append(r['resources']['s3public']['zip'])
else:
missing_urls.append('%s:%s' % (r['tile_identifier'], r['entity_id']))
missing_types.append('zip')
if r['resources']['metadata']!= None:
url_resources.append(r['resources']['metadata'])
else:
missing_urls.append('%s:%s' % (r['tile_identifier'], r['entity_id']))
missing_types.append('metadata')
if r['resources']['quicklook'] != None:
url_resources.append(r['resources']['quicklook'])
else:
missing_urls.append('%s:%s' % (r['tile_identifier'], r['entity_id']))
missing_types.append('quicklook')
logger.info('total scans: %d' %len(url_resources))
logger.info('already missed resources: %d' %len(missing_urls))
if False:
for counter, res in enumerate(url_resources):
req = requests.head(res)
if req.status_code != requests.codes.ok:
print res, req.status_code
missing_urls.append(res)
print res
if (counter % 25) == 0:
print counter
else:
counter = 0
for url_parts in chunks(url_resources, 500):
counter+=1
rs = (grequests.head(u) for u in url_parts)
res = grequests.map(rs)
for req in res:
if req is not None:
if req.status_code != requests.codes.ok:
wrong_urls.append(res)
missing_types.append('zip_registered')
else:
print req.url, req
if len(wrong_urls) > 0:
for item in wrong_urls:
print item
for req in item:
if req.status_code != requests.codes.ok:
append_data('/tmp/wrong_urls.txt', req.url)
if len(missing_urls) > 0:
append_data('/tmp/missing_urls.txt', missing_urls)
if len(missing_types) > 0:
for type in ['zip_registered', 'quicklook', 'metadata', 'zip']:
logger.info('%d:%s' % (operator.countOf(missing_types, type), type))
logger.info('wrong resources resources: %d' % len(wrong_urls))
logger.info('Executed in %f secs.' % (time.time()-start_time))
print 'Wrong URLs:', wrong_urls