def import_from_file_s2(in_csv, block_size):
import pprint
n, m = (0, block_size)
api = Api()
for n in range(0, count_lines(in_csv), m):
print 'Range: <%d:%d>' % (n, n + m)
datasets = sentinel_harvester(in_csv, n, m)
out = api.create_dataset(datasets)
pprint.pprint(out)
def import_from_sentinel_catalog(sensor, start_date, api_url):
import numpy
api = Api(api_url)
max_cloud_ratio = 1.0
ag_season_start = dateutil.parser.parse(start_date)
ag_season_end = ag_season_start + datetime.timedelta(days=1)
for lon in numpy.arange(-180, 180, 9):
for lat in numpy.arange(-90, 90, 9):
lon_end = lon + 9
lat_end = lat + 9
aoi_se = (lon_end, lat)
aoi_nw = (lon, lat_end)
aoi_ne = (aoi_se[0], aoi_nw[1])
aoi_sw = (aoi_nw[0], aoi_se[1])
aoi = [aoi_nw, aoi_ne, aoi_se, aoi_sw, aoi_nw]
cat = SentinelCatalog()
datasets = cat.find(sensor, aoi, ag_season_start, ag_season_end,
max_cloud_ratio)
if datasets != None:
ds_found = list()
ds_missing = list()
for counter, ds in enumerate(datasets):
catalog_ds = api.get_dataset(ds.entity_id)
if catalog_ds is None or len(catalog_ds) == 0:
ds_missing.append(ds)
elif len(catalog_ds) == 1:
ds_found.append(catalog_ds)
else:
print 'More in catalog found: %s (%d)' % (
ds.entity_id, len(catalog_ds))
if (counter % 25) == 0:
print counter, len(datasets)
print 'already registered: ', len(ds_found), len(datasets)
print 'missing: ', len(ds_missing), len(datasets)
for counter, ds_obj in enumerate(ds_missing):
new_ds = api.create_dataset(ds_obj)
if not new_ds is None:
print new_ds
if (counter % 25) == 0:
print counter, len(ds_missing)
else:
print 'No data found in catalog for %s from %s to %s' % (
sensor, ag_season_start.strftime("%Y-%m-%d"),
ag_season_end.strftime("%Y-%m-%d"))
def import_from_pipe_ls(lines):
datasets = landsat_harvester_line(lines)
api = Api()
skipped = list()
registered = list()
for c, ds in enumerate(datasets):
try:
out = api.create_dataset(ds)
if not 'already' in str(out):
registered.append(c)
else:
skipped.append(c)
except Exception, e:
print e
def import_from_landsat_catalog(sensor, start_date, api_url):
api = Api(api_url)
max_cloud_ratio = 1.0
ag_season_start = dateutil.parser.parse(start_date)
ag_season_end = ag_season_start + datetime.timedelta(days=1)
aoi_se = (180, -90)
aoi_nw = (-180, 90)
aoi_ne = (aoi_se[0], aoi_nw[1])
aoi_sw = (aoi_nw[0], aoi_se[1])
aoi = [aoi_nw, aoi_ne, aoi_se, aoi_sw, aoi_nw]
cat = USGSCatalog()
# "LANDSAT_8", "LANDSAT_ETM_SLC_OFF", "LANDSAT_ETM"
datasets = cat.find(sensor, aoi, ag_season_start, ag_season_end,
max_cloud_ratio)
if datasets != None:
ds_found = list()
ds_missing = list()
for counter, ds in enumerate(datasets):
catalog_ds = api.get_dataset(ds.entity_id)
if catalog_ds is None or len(catalog_ds) == 0:
ds_missing.append(ds)
elif len(catalog_ds) == 1:
ds_found.append(catalog_ds)
else:
print 'More in catalog found: %s (%d)' % (ds.entity_id,
len(catalog_ds))
if (counter % 25) == 0:
print counter, len(datasets)
print 'already registered: ', len(ds_found), len(datasets)
print 'missing: ', len(ds_missing), len(datasets)
for counter, ds_obj in enumerate(ds_missing):
new_ds = api.create_dataset(ds_obj)
if not new_ds is None:
print new_ds
if (counter % 25) == 0:
print counter, len(ds_missing)
else:
print 'No data found in catalog for sentinel from %s to %s' % (
ag_season_start.strftime("%Y-%m-%d"),
ag_season_end.strftime("%Y-%m-%d"))
def import_from_file_ls(in_csv):
datasets = landsat_harvester(in_csv)
api = Api()
skipped = list()
registered = list()
for c, ds in enumerate(datasets):
try:
out = api.create_dataset(ds)
if not 'title' in str(out):
registered.append(c)
else:
skipped.append(c)
except Exception, e:
print e
if c % 100 == 0:
print c
print 'skipped:', skipped
print 'registered:', registered
skipped = list()
registered = list()
def update_catalog(queue_name, api_endpoint):
api = Api(api_endpoint)
sqs = boto3.resource('sqs')
if queue_name not in get_all_queues():
raise Exception('Queue %s does not exist in %s' %
(queue_name, get_all_queues()))
queue = sqs.get_queue_by_name(QueueName=queue_name)
should_break = False
counter = 1
while not should_break:
if int(queue.attributes.get('ApproximateNumberOfMessages')) == 0:
time_interval = 60
else:
time_interval = 2
logger.info('Getting messages from SQS: %s (%d sec. interval)' %
(queue_name, time_interval))
messages_to_delete = list()
for message_obj in queue.receive_messages(
MaxNumberOfMessages=MAX_MESSAGES,
WaitTimeSeconds=10,
VisibilityTimeout=60,
):
messages_to_delete = list()
notification = ujson.loads(message_obj.body)
message = ujson.loads(notification[u'Message'])
if get_message_type(message) == 'landsat':
for rec in message[u'Records']:
s3 = extract_s3_structure(rec)
s3['metadata'] = os.path.join(s3['s3_http'], s3['s3_path'],
s3['entity_id'] + '_MTL.txt')
s3['metadata_json'] = os.path.join(
s3['s3_http'], s3['s3_path'],
s3['entity_id'] + '_MTL.json')
s3['quicklook'] = os.path.join(
s3['s3_http'], s3['s3_path'],
s3['entity_id'] + '_thumb_large.jpg')
req = requests.get(s3['metadata_json'])
try:
obj = parse_l1_metadata_file(req.json(), s3)
new_ds = api.create_dataset(obj)
if not new_ds is None:
print new_ds
counter += 1
messages_to_delete.append({
'Id':
message_obj.message_id,
'ReceiptHandle':
message_obj.receipt_handle
})
except ValueError:
logger.exception(
'ERROR: metadata location structure corrupted',
extra={'metadata_response': req.text})
except Exception, e:
logging.exception('General Error ooccured',
extra={'request_url': req.url})
should_break = True
finally:
if len(messages_to_delete) > 0:
messages_to_delete = remove_messages_from_queue(
queue, messages_to_delete)
def import_from_pipe_s2(lines):
import pprint
api = Api()
datasets = sentinel_harvester_line(lines)
out = api.create_dataset(datasets)
pprint.pprint(out)