def receive_messages(queue_url,
visibility_timeout_s=600,
logger=None,
cfg=None):
"""
Generates successive messages from reading the queue. The caller
is responsible for deleting or returning each message to the queue
Parameters
----------
queue_url : string
The URL of the queue to receive messages on
visibility_timeout_s : int
The number of seconds to wait for a received message to be deleted
before it is returned to the queue
cfg : harmony.util.Config
The configuration values for this runtime environment.
Yields
------
receiptHandle, body : string, string
A tuple of the receipt handle, used to delete or update messages,
and the contents of the message
"""
# The implementation of this function has been moved to the
# harmony.aws module.
if cfg is None:
cfg = config()
if logger is None:
logger = build_logger(cfg)
touch_health_check_file(cfg.health_check_path)
return aws.receive_messages(cfg, queue_url, visibility_timeout_s, logger)
def run_cli(parser, args, AdapterClass, cfg=None):
"""
Runs the Harmony CLI invocation captured by the given args
Parameters
----------
parser : argparse.ArgumentParser
The parser being used to parse CLI arguments, used to provide CLI argument errors
args : Namespace
Argument values parsed from the command line, presumably via ArgumentParser.parse_args
AdapterClass : class
The BaseHarmonyAdapter subclass to use to handle service invocations
cfg : harmony.util.Config
A configuration instance for this service
"""
if cfg is None:
cfg = config()
if args.harmony_wrap_stdout:
setup_stdout_log_formatting(cfg)
if args.harmony_action == 'invoke':
start_time = datetime.datetime.now()
if not bool(args.harmony_input):
parser.error(
'--harmony-input must be provided for --harmony-action=invoke')
elif not bool(args.harmony_sources):
successful = _invoke_deprecated(AdapterClass, args.harmony_input,
cfg)
if not successful:
raise Exception('Service operation failed')
else:
try:
adapter = _build_adapter(AdapterClass, args.harmony_input,
args.harmony_sources,
args.harmony_data_location, cfg)
adapter.logger.info(f'timing.{cfg.app_name}.start')
_invoke(adapter, args.harmony_metadata_dir)
finally:
time_diff = datetime.datetime.now() - start_time
duration_ms = int(round(time_diff.total_seconds() * 1000))
duration_logger = build_logger(cfg)
extra_fields = {
'user': adapter.message.user,
'requestId': adapter.message.requestId,
'durationMs': duration_ms
}
duration_logger.info(f'timing.{cfg.app_name}.end',
extra=extra_fields)
if args.harmony_action == 'start':
if not bool(args.harmony_queue_url):
parser.error(
'--harmony-queue-url must be provided for --harmony-action=start'
)
else:
return _start(AdapterClass, args.harmony_queue_url,
args.harmony_visibility_timeout, cfg)
def stage(local_filename,
remote_filename,
mime,
logger=None,
location=None,
cfg=None):
"""
Stages the given local filename, including directory path, to an S3 location with the given
filename and mime-type
Requires the following environment variables:
AWS_DEFAULT_REGION: The AWS region in which the S3 client is operating
Parameters
----------
local_filename : string
A path and filename to the local file that should be staged
remote_filename : string
The basename to give to the remote file
mime : string
The mime type to apply to the staged file for use when it is served, e.g. "application/x-netcdf4"
location : string
The S3 prefix URL under which to place the output file. If not provided, STAGING_BUCKET and
STAGING_PATH must be set in the environment
logger : logging
The logger to use
cfg : harmony.util.Config
The configuration values for this runtime environment.
Returns
-------
url : string
An s3:// URL to the staged file
"""
# The implementation of this function has been moved to the
# harmony.aws module.
if cfg is None:
cfg = config()
if logger is None:
logger = build_logger(cfg)
return aws.stage(cfg, local_filename, remote_filename, mime, logger,
location)
def download(config, url: str, access_token: str, data, destination_file):
"""Downloads the given url using the provided EDL user access token
and writes it to the provided file-like object.
Exception cases:
1. No user access token
2. Invalid user access token
3. Unable to authenticate the user with Earthdata Login
a. User credentials (could happen even after token validation
b. Application credentials
4. Error response when downloading
5. Data requires EULA acceptance by user
6. If fallback authentication enabled, the application credentials are
invalid, or do not have permission to download the data.
Parameters
----------
config : harmony.util.Config
The configuration for the current runtime environment.
url : str
The url for the resource to download
access_token : str
A shared EDL access token created from the user's access token
and the app identity.
data : dict or Tuple[str, str]
Optional parameter for additional data to send to the server
when making an HTTP POST request. These data will be URL
encoded to a query string containing a series of `key=value`
pairs, separated by ampersands. If None (the default), the
request will be sent with an HTTP GET request.
destination_file : file-like
The destination file where the data will be written. Must be
a file-like object opened for binary write.
Returns
-------
requests.Response with the download result
Side-effects
------------
Will write to provided destination_file
"""
response = None
logger = build_logger(config)
start_time = datetime.datetime.now()
logger.info(f'timing.download.start {url}')
if data is not None:
logger.info('Query parameters supplied, will use POST method.')
data = urlencode(data).encode('utf-8')
if access_token is not None and _valid(
config.oauth_host, config.oauth_client_id, access_token):
response = _download(config, url, access_token, data)
if response is None or not response.ok:
if config.fallback_authn_enabled:
msg = (
'No valid user access token in request or EDL OAuth authentication failed.'
'Fallback authentication enabled: retrying with Basic auth.')
logger.warning(msg)
response = _download_with_fallback_authn(config, url, data)
if response.ok:
time_diff = datetime.datetime.now() - start_time
duration_ms = int(round(time_diff.total_seconds() * 1000))
destination_file.write(response.content)
file_size = sys.getsizeof(response.content)
duration_logger = build_logger(config)
_log_download_performance(duration_logger, url, duration_ms, file_size)
return response
if _is_eula_error(response.content):
msg = _eula_error_message(response.content)
logger.info(f'{msg} due to: {response.content}')
raise ForbiddenException(msg)
if response.status_code in (401, 403):
msg = f'Forbidden: Unable to download {url}'
logger.info(f'{msg} due to: {response.content}')
raise ForbiddenException(msg)
if response.status_code == 500:
logger.info(f'Unable to download (500) due to: {response.content}')
raise Exception('Unable to download.')
logger.info(
f'Unable to download (unknown error) due to: {response.content}')
raise Exception('Unable to download: unknown error.')
def download(url,
destination_dir,
logger=None,
access_token=None,
data=None,
cfg=None):
"""
Downloads the given URL to the given destination directory, using the basename of the URL
as the filename in the destination directory. Supports http://, https:// and s3:// schemes.
When using the s3:// scheme, will run against us-west-2 unless the "AWS_DEFAULT_REGION"
environment variable is set.
When using http:// or https:// schemes, the access_token will be used for authentication.
Parameters
----------
url : string
The URL to fetch
destination_dir : string
The directory in which to place the downloaded file
logger : Logger
A logger to which the function will write, if provided
access_token :
The Earthdata Login token of the caller to use for downloads
data : dict or Tuple[str, str]
Optional parameter for additional data to
send to the server when making a HTTP POST request through
urllib.get.urlopen. These data will be URL encoded to a query string
containing a series of `key=value` pairs, separated by ampersands. If
None (the default), urllib.get.urlopen will use the GET
method.
cfg : harmony.util.Config
The configuration values for this runtime environment.
Returns
-------
destination : string
The filename, including directory, of the downloaded file
"""
if cfg is None:
cfg = config()
if logger is None:
logger = build_logger(cfg)
if _is_file_url(url):
return _url_as_filename(url)
source = http.localhost_url(url, cfg.localstack_host)
destination_path = _filename(destination_dir, url)
if destination_path.exists():
return str(destination_path)
destination_path = str(destination_path)
with open(destination_path, 'wb') as destination_file:
if aws.is_s3(source):
aws.download(cfg, source, destination_file)
elif http.is_http(source):
http.download(cfg, source, access_token, data, destination_file)
else:
msg = f'Unable to download a url of unknown type: {url}'
logger.error(msg)
raise Exception(msg)
return destination_path