def authenticate_via_api_key(api_key: str) -> User:
log = logging.getLogger(__name__)
log.info('Users service auth api key', action='service users auth api key')
if not PATTERN_API_KEY.match(api_key):
log.error('Cannot verify malformed API key: "%s"', api_key)
raise MalformedAPIKey()
log.debug('Checking "%s"', api_key)
conn = db.get_connection()
try:
row = db.users.select_user_by_api_key(conn, api_key=api_key).fetchone()
except db.DatabaseError as err:
log.error('Database query for API key "%s" failed', api_key)
db.print_diagnostics(err)
raise
finally:
conn.close()
if not row:
log.error('Unauthorized API key "%s"', api_key)
raise Unauthorized('CoastLine API key is not active')
return User(
user_id=row['user_id'],
api_key=row['api_key'],
name=row['user_name'],
created_on=row['created_on'],
)
def _save_execution_error(job_id: str,
execution_step: str,
error_message: str,
status: str = piazza.STATUS_ERROR):
log = logging.getLogger(__name__)
log.debug('<%s> updating database record', job_id)
conn = db.get_connection()
transaction = conn.begin()
try:
db.jobs.update_status(
conn,
job_id=job_id,
status=status,
)
db.jobs.insert_job_failure(
conn,
job_id=job_id,
execution_step=execution_step,
error_message=error_message,
)
transaction.commit()
except db.DatabaseError as err:
transaction.rollback()
log.error('<%s> database update failed', job_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
def get_detections(job_id: str) -> str:
"""
Returns a potentially massive stringified GeoJSON feature collection containing all detections
for a given job.
"""
log = logging.getLogger(__name__)
log.info('Job service get detections', action='service job get detections')
conn = db.get_connection()
log.info('Packaging detections for <job:%s>', job_id)
try:
if not db.jobs.exists(conn, job_id=job_id):
raise NotFound(job_id)
geojson = db.jobs.select_detections(conn, job_id=job_id).scalar()
except db.DatabaseError as err:
log.error('Could not package detections for <job:%s>', job_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
log.debug('Packaging complete: %d bytes for <job:%s>', len(geojson),
job_id)
return geojson
def get(user_id: str, job_id: str) -> Job:
log = logging.getLogger(__name__)
log.info('Job service get', action='service job get', actor=user_id)
conn = db.get_connection()
try:
row = db.jobs.select_job(conn, job_id=job_id).fetchone()
if not row:
raise NotFound(job_id)
# Add job to user's tracked jobs list
db.jobs.insert_job_user(conn, job_id=job_id, user_id=user_id)
except db.DatabaseError as err:
log.error('Could not get <job:%s> for user "%s"', job_id, user_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
return Job(
algorithm_name=row['algorithm_name'],
algorithm_version=row['algorithm_version'],
created_by=row['created_by'],
created_on=row['created_on'],
geometry=json.loads(row['geometry']),
job_id=row['job_id'],
name=row['name'],
scene_time_of_collect=row['captured_on'],
scene_sensor_name=row['sensor_name'],
scene_id=row['scene_id'],
status=row['status'],
tide=row['tide'],
tide_min_24h=row['tide_min_24h'],
tide_max_24h=row['tide_max_24h'],
)
def get_all() -> List[ProductLine]:
log = logging.getLogger(__name__)
log.info('Productline service get all',
action='service productline get all')
conn = db.get_connection()
try:
cursor = db.productlines.select_all(conn)
except db.DatabaseError as err:
log.error('Could not list productlines')
db.print_diagnostics(err)
raise
finally:
conn.close()
productlines = []
for row in cursor.fetchall():
productlines.append(
ProductLine(
productline_id=row['productline_id'],
algorithm_name=row['algorithm_name'],
bbox=json.loads(row['bbox']),
category=row['category'],
created_by=row['created_by'],
created_on=row['created_on'],
max_cloud_cover=row['max_cloud_cover'],
name=row['name'],
owned_by=row['owned_by'],
spatial_filter_id=row['spatial_filter_id'],
start_on=row['start_on'],
stop_on=row['stop_on'],
))
return productlines
def _create_user(user_id, user_name) -> User:
log = logging.getLogger(__name__)
api_key = uuid.uuid4().hex
log.info('Creating user account for "%s"',
user_id,
actor=user_id,
action='create account')
conn = db.get_connection()
try:
db.users.insert_user(
conn,
user_id=user_id,
user_name=user_name,
api_key=api_key,
)
except db.DatabaseError as err:
log.error('Could not save user account "%s" to database', user_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
return User(
user_id=user_id,
name=user_name,
api_key=api_key,
created_on=datetime.utcnow(),
)
def get_by_scene(scene_id: str) -> List[Job]:
log = logging.getLogger(__name__)
log.info('Job service get by scene', action=' service job get by scene')
conn = db.get_connection()
try:
cursor = db.jobs.select_jobs_for_scene(conn, scene_id=scene_id)
except db.DatabaseError as err:
log.error('Could not list jobs for <scene:%s>', scene_id)
db.print_diagnostics(err)
raise err
finally:
conn.close()
jobs = []
for row in cursor.fetchall():
jobs.append(
Job(
algorithm_name=row['algorithm_name'],
algorithm_version=row['algorithm_version'],
created_by=row['created_by'],
created_on=row['created_on'],
geometry=json.loads(row['geometry']),
job_id=row['job_id'],
name=row['name'],
scene_time_of_collect=row['captured_on'],
scene_sensor_name=row['sensor_name'],
scene_id=row['scene_id'],
status=row['status'],
tide=row['tide'],
tide_min_24h=row['tide_min_24h'],
tide_max_24h=row['tide_max_24h'],
))
return jobs
def forget(user_id: str, job_id: str) -> None:
log = logging.getLogger(__name__)
log.info('Job service forget',
action=' service job forget',
actor=user_id)
conn = db.get_connection()
try:
if not db.jobs.exists(conn, job_id=job_id):
raise NotFound(job_id)
db.jobs.delete_job_user(conn, job_id=job_id, user_id=user_id)
except db.DatabaseError as err:
log.error('Could not forget <job:%s> for user "%s"', job_id, user_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
def _link_to_job(productline_id: str, job_id: str):
log = logging.getLogger(__name__)
log.info('<%s> Linking to job <%s>', productline_id, job_id)
conn = db.get_connection()
try:
db.productlines.insert_productline_job(
conn,
job_id=job_id,
productline_id=productline_id,
)
except db.DatabaseError as err:
log.error('Cannot link job and productline')
db.print_diagnostics(err)
raise
finally:
conn.close()
def create_productline(*, algorithm_id: str, bbox: tuple, category: str,
max_cloud_cover: int, name: str, spatial_filter_id: str,
start_on: date, stop_on: date,
user_id: str) -> ProductLine:
log = logging.getLogger(__name__)
log.info('Productline service create productline',
action='service productline create productline')
algorithm = services.algorithms.get(algorithm_id)
productline_id = _create_id()
log.info('Creating product line <%s>', productline_id)
conn = db.get_connection()
try:
db.productlines.insert_productline(
conn,
productline_id=productline_id,
algorithm_id=algorithm_id,
algorithm_name=algorithm.name,
bbox=bbox,
category=category,
max_cloud_cover=max_cloud_cover,
name=name,
spatial_filter_id=spatial_filter_id,
start_on=start_on,
stop_on=stop_on,
user_id=user_id,
)
except db.DatabaseError as err:
log.error('Could not insert product line record')
db.print_diagnostics(err)
raise
finally:
conn.close()
return ProductLine(
productline_id=productline_id,
algorithm_name=algorithm.name,
bbox=_to_geometry(bbox),
category=category,
created_by=user_id,
created_on=datetime.utcnow(),
max_cloud_cover=max_cloud_cover,
name=name,
owned_by=user_id,
spatial_filter_id=spatial_filter_id,
start_on=start_on,
stop_on=stop_on,
)
def _find_existing_job_id_for_scene(scene_id: str, algorithm_id: str) -> str:
log = logging.getLogger(__name__)
log.debug('Searching for existing jobs for scene <%s> and algorithm <%s>',
scene_id, algorithm_id)
conn = db.get_connection()
try:
job_id = db.jobs.select_jobs_for_inputs(
conn,
scene_id=scene_id,
algorithm_id=algorithm_id,
).scalar()
except db.DatabaseError as err:
log.error('Job query failed')
db.print_diagnostics(err)
raise
finally:
conn.close()
return job_id
def _save_to_database(scene: Scene):
log = logging.getLogger(__name__)
conn = db.get_connection()
try:
db.scenes.insert(
conn,
scene_id=scene.id,
captured_on=scene.capture_date,
catalog_uri=scene.uri,
cloud_cover=scene.cloud_cover,
geometry=scene.geometry,
resolution=scene.resolution,
sensor_name=scene.sensor_name,
)
except db.DatabaseError as err:
log.error('Could not save scene `%s` to database', scene.id)
db.print_diagnostics(err)
raise
finally:
conn.close()
def _run_cycle(self):
conn = db.get_connection()
try:
rows = db.jobs.select_outstanding_jobs(conn).fetchall()
except db.DatabaseError as err:
self._log.error('Could not list running jobs')
db.print_diagnostics(err)
raise
finally:
conn.close()
if not rows:
self._log.info('Nothing to do; next run at %s',
(datetime.utcnow() +
self._interval).strftime(FORMAT_TIME))
else:
self._log.info('Begin cycle for %d records', len(rows))
for i, row in enumerate(rows, start=1):
self._updater(row['job_id'], row['age'], i)
self._log.info('Cycle complete; next run at %s',
(datetime.utcnow() +
self._interval).strftime(FORMAT_TIME))
def delete_productline(user_id: str, productline_id: str) -> None:
log = logging.getLogger(__name__)
log.info('Productline service delete productline',
action='service productline delete productline')
conn = db.get_connection()
try:
productline = db.productlines.select_productline(
conn, productline_id=productline_id).fetchone()
if not productline:
raise NotFound(productline_id)
if user_id != productline['owned_by']:
raise PermissionError('only the owner can delete this productline')
db.productlines.delete_productline(conn, productline_id=productline_id)
except db.DatabaseError as err:
log.error('Could not delete productline <%s>', productline_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
def get_by_id(user_id: str) -> User:
log = logging.getLogger(__name__)
log.debug('Searching database for user "%s"', user_id)
conn = db.get_connection()
try:
row = db.users.select_user(conn, user_id=user_id).fetchone()
except db.DatabaseError as err:
log.error('Database query for user ID "%s" failed', user_id)
db.print_diagnostics(err)
raise
finally:
conn.close()
if not row:
return None
return User(
user_id=row['user_id'],
api_key=row['api_key'],
name=row['user_name'],
created_on=row['created_on'],
)
def _updater(self, job_id: str, age: timedelta, index: int):
log = self._log
job_ttl = self._job_ttl
# Get latest status
try:
status = piazza.get_status(job_id)
except piazza.Unauthorized:
log.error('<%03d/%s> credentials rejected during polling!', index,
job_id)
return
except (piazza.ServerError, piazza.Error) as err:
if isinstance(err, piazza.ServerError) and err.status_code == 404:
log.warning('<%03d/%s> Job not found', index, job_id)
_save_execution_error(job_id, STEP_POLLING, 'Job not found')
return
else:
log.error('<%03d/%s> call to Piazza failed: %s', index, job_id,
err.message)
return
# Emit console feedback
log.info('<%03d/%s> polled (%s; age=%s)', index, job_id, status.status,
age)
# Determine appropriate action by status
if status.status in (piazza.STATUS_SUBMITTED, piazza.STATUS_PENDING):
if age > job_ttl:
log.warning(
'<%03d/%s> appears to have stalled and will no longer be tracked',
index, job_id)
_save_execution_error(job_id,
STEP_QUEUED,
'Submission wait time exceeded',
status=STATUS_TIMED_OUT)
return
conn = db.get_connection()
try:
db.jobs.update_status(conn,
job_id=job_id,
status=status.status)
except db.DatabaseError as err:
log.error('<%03d/%s> Could not save status to database', index,
job_id)
db.print_diagnostics(err)
return
finally:
conn.close()
elif status.status == piazza.STATUS_RUNNING:
if age > job_ttl:
log.warning(
'<%03d/%s> appears to have stalled and will no longer be tracked',
index, job_id)
_save_execution_error(job_id,
STEP_PROCESSING,
'Processing time exceeded',
status=STATUS_TIMED_OUT)
return
conn = db.get_connection()
try:
db.jobs.update_status(conn,
job_id=job_id,
status=status.status)
except db.DatabaseError as err:
log.error('<%03d/%s> Could not save status to database', index,
job_id)
db.print_diagnostics(err)
return
finally:
conn.close()
elif status.status == piazza.STATUS_SUCCESS:
log.info('<%03d/%s> Resolving detections data ID (via <%s>)',
index, job_id, status.data_id)
try:
detections_data_id = _resolve_detections_data_id(
status.data_id)
except PostprocessingError as err:
log.error('<%03d/%s> Could not resolve detections data ID: %s',
index, job_id, err)
_save_execution_error(job_id, STEP_RESOLVE, str(err))
return
log.info('<%03d/%s> Fetching detections from Piazza', index,
job_id)
try:
geojson = piazza.get_file(detections_data_id).text
except piazza.ServerError as err:
log.error('<%03d/%s> Could not fetch data ID <%s>: %s', index,
job_id, detections_data_id, err)
_save_execution_error(
job_id, STEP_COLLECT_GEOJSON,
'Could not retrieve GeoJSON from Piazza')
return
log.info('<%03d/%s> Saving detections to database (%0.1fMB)',
index, job_id,
len(geojson) / 1024000)
conn = db.get_connection()
transaction = conn.begin()
try:
db.jobs.insert_detection(conn,
job_id=job_id,
feature_collection=geojson)
db.jobs.update_status(
conn,
job_id=job_id,
status=piazza.STATUS_SUCCESS,
)
transaction.commit()
except db.DatabaseError as err:
transaction.rollback()
transaction.close()
log.error(
'<%03d/%s> Could not save status and detections to database',
index, job_id)
db.print_diagnostics(err)
_save_execution_error(job_id, STEP_COLLECT_GEOJSON,
'Could not insert GeoJSON to database')
return
finally:
conn.close()
elif status.status in (piazza.STATUS_ERROR, piazza.STATUS_FAIL):
# FIXME -- use heuristics to generate a more descriptive error message
_save_execution_error(job_id, STEP_ALGORITHM,
'Job failed during algorithm execution')
elif status.status == piazza.STATUS_CANCELLED:
_save_execution_error(job_id,
STEP_ALGORITHM,
'Job was cancelled',
status=piazza.STATUS_CANCELLED)
def create(user_id: str, scene_id: str, service_id: str, job_name: str,
planet_api_key: str) -> Job:
log = logging.getLogger(__name__)
log.info('Job service create', action='service job create', actor=user_id)
# Fetch prerequisites
try:
algorithm = algorithms.get(service_id)
scene = scenes.get(scene_id, planet_api_key)
scenes.activate(scene, planet_api_key, user_id)
except (algorithms.NotFound, algorithms.ValidationError,
scenes.MalformedSceneID, scenes.CatalogError, scenes.NotFound,
scenes.NotPermitted, scenes.ValidationError) as err:
log.error('Preprocessing error: %s', err)
raise PreprocessingError(err)
# Determine GeoTIFF URLs.
if scene.platform in ('rapideye', 'planetscope'):
geotiff_filenames = ['multispectral.TIF']
geotiff_urls = [scenes.create_download_url(scene.id, planet_api_key)]
elif scene.platform == 'landsat':
geotiff_filenames = ['coastal.TIF', 'swir1.TIF']
geotiff_urls = [scene.geotiff_coastal, scene.geotiff_swir1]
else:
raise PreprocessingError(message='Unexpected platform')
# Dispatch to Piazza
try:
log.info('Dispatching <scene:%s> to <algo:%s>', scene_id,
algorithm.name)
cli_cmd = _create_algorithm_cli_cmd(algorithm.interface,
geotiff_filenames, scene.platform)
job_id = piazza.execute(
algorithm.service_id, {
'body': {
'content':
json.dumps({
'cmd': cli_cmd,
'inExtFiles': geotiff_urls,
'inExtNames': geotiff_filenames,
'outGeoJson': ['shoreline.geojson'],
'userID': user_id,
}),
'type':
'body',
'mimeType':
'application/json',
},
})
except piazza.Error as err:
log.error('Could not execute via Piazza: %s', err)
raise
# Record the data
log.debug('Saving job record <%s>', job_id)
conn = db.get_connection()
transaction = conn.begin()
try:
db.jobs.insert_job(
conn,
algorithm_id=algorithm.service_id,
algorithm_name=algorithm.name,
algorithm_version=algorithm.version,
job_id=job_id,
name=job_name,
scene_id=scene_id,
status=piazza.STATUS_PENDING,
user_id=user_id,
tide=scene.tide,
tide_min_24h=scene.tide_min,
tide_max_24h=scene.tide_max,
)
db.jobs.insert_job_user(
conn,
job_id=job_id,
user_id=user_id,
)
transaction.commit()
except db.DatabaseError as err:
transaction.rollback()
log.error('Could not save job to database')
db.print_diagnostics(err)
raise
finally:
conn.close()
return Job(
algorithm_name=algorithm.name,
algorithm_version=algorithm.version,
created_by=user_id,
created_on=datetime.utcnow(),
geometry=scene.geometry,
job_id=job_id,
name=job_name,
scene_time_of_collect=scene.capture_date,
scene_sensor_name=scene.sensor_name,
scene_id=scene_id,
status=piazza.STATUS_PENDING,
tide=scene.tide,
tide_min_24h=scene.tide_min,
tide_max_24h=scene.tide_max,
)