def try_download_head(bucket, filename):
client = get_data_dl_s3_client()
# Check for range request
range_header = get_range_header_val()
try:
if not range_header:
download = client.get_object(Bucket=bucket, Key=filename)
else:
log.info("Downloading range {0}".format(range_header))
download = client.get_object(Bucket=bucket,
Key=filename,
Range=range_header)
except ClientError as e:
log.warning("Could get head for s3://{0}/{1}: {2}".format(
bucket, filename, e))
template_vars = {
'contentstring': 'File not found',
'title': 'File not found'
}
headers = {}
return make_html_response(template_vars, headers, 404, 'error.html')
log.debug(download)
#return 'Finish this thing'
response_headers = {'Content-Type': download['ContentType']}
for header in download['ResponseMetadata']['HTTPHeaders']:
name = header_map[header] if header in header_map else header
value = download['ResponseMetadata']['HTTPHeaders'][
header] if header != 'server' else 'egress'
log.debug("setting header {0} to {1}.".format(name, value))
response_headers[name] = value
# Try Redirecting to HEAD. There should be a better way.
cookievars = get_cookie_vars(app.current_request.headers)
if 'urs-user-id' in cookievars:
user_id = cookievars['urs-user-id']
else:
user_id = 'Unknown'
# Generate URL
creds = get_role_creds(user_id=user_id)
client = get_data_dl_s3_client()
bucket_region = client.get_bucket_location(
Bucket=bucket)['LocationConstraint']
bucket_region = 'us-east-1' if not bucket_region else bucket_region
presigned_url = get_presigned_url(creds, bucket, filename, bucket_region,
24 * 3600, user_id, 'HEAD')
s3_host = urlparse(presigned_url).netloc
# Return a redirect to a HEAD
log.debug("Presigned HEAD URL host was {0}".format(s3_host))
return make_redriect(presigned_url, {}, 303)
def get_data_dl_s3_client():
user_id = get_jwt_field(get_cookie_vars(app.current_request.headers),
'urs-user-id')
session = get_role_session(user_id=user_id)
params = {}
params['config'] = bc_Config(**get_bcconfig(user_id))
client = session.client('s3', **params)
return client
def dynamic_url():
log.debug('attempting to GET a thing')
restore_bucket_vars()
if 'proxy' in app.current_request.uri_params:
path, bucket, filename = process_varargs(app.current_request.uri_params['proxy'], b_map)
log.debug('path, bucket, filename: {}'.format(( path, bucket, filename)))
if not bucket:
template_vars = {'contentstring': 'File not found', 'title': 'File not found'}
headers = {}
return make_html_response(template_vars, headers, 404, 'error.html')
else:
path, bucket, filename = (None, None, None)
cookievars = get_cookie_vars(app.current_request.headers)
user_profile = None
if cookievars:
log.debug('cookievars: {}'.format(cookievars))
if os.getenv('JWT_COOKIENAME','asf-urs') in cookievars:
# this means our cookie is a jwt and we don't need to go digging in the session db
user_profile = cookievars[os.getenv('JWT_COOKIENAME','asf-urs')]
else:
log.warning('jwt cookie not found, falling back to old style')
user_profile = get_session(cookievars['urs-user-id'], cookievars['urs-access-token'])
# Check for public bucket
if check_public_bucket(bucket, public_buckets, b_map):
log.debug("Accessing public bucket {0}".format(path))
elif not user_profile:
return do_auth_and_return(app.current_request.context)
# Check that the bucket is either NOT private, or user belongs to that group
private_check = check_private_bucket(bucket, private_buckets, b_map)
log.debug('private check: {}'.format(private_check))
u_in_g, new_user_profile = user_in_group(private_check, cookievars, user_profile, False)
if new_user_profile and new_user_profile != user_profile:
log.debug("Profile was mutated from {0} => {1}".format(user_profile,new_user_profile))
user_profile = new_user_profile
log.debug('user_in_group: {}'.format(u_in_g))
if private_check and not u_in_g:
template_vars = {'contentstring': 'This data is not currently available.', 'title': 'Could not access data'}
headers = {}
return make_html_response(template_vars, headers, 403, 'error.html')
if not filename:
log.warning("Request was made to directory listing instead of object: {0}".format(path))
template_vars = {'contentstring': 'Request does not appear to be valid.', 'title': 'Request Not Serviceable'}
headers = {}
return make_html_response(template_vars, headers, 404, 'error.html')
return try_download_from_bucket(bucket, filename, user_profile)
def get_data_dl_s3_client():
cookievars = get_cookie_vars(app.current_request.headers)
if cookievars:
user_id = cookievars['urs-user-id']
else:
user_id = None
session = get_role_session(user_id=user_id)
params = {}
BCCONFIG = {'user_agent': "Egress App for userid={0}".format(user_id)}
if os.getenv('S3_SIGNATURE_VERSION'):
BCCONFIG['signature_version'] = os.getenv('S3_SIGNATURE_VERSION')
params['config'] = bc_Config(**BCCONFIG)
client = session.client('s3', **params)
return client
def __call__(self, event, context):
resource_path = event.get('requestContext', {}).get('resourcePath')
log_context(route=resource_path, request_id=context.aws_request_id)
# get_jwt_field() below generates log messages, so the above log_context() sets the
# vars for it to use while it's doing the username lookup
userid = get_jwt_field(get_cookie_vars(event.get('headers', {}) or {}),
'urs-user-id')
log_context(user_id=userid)
resp = super().__call__(event, context)
resp['headers'].update({'x-request-id': context.aws_request_id})
log_context(user_id=None, route=None, request_id=None)
return resp
def root():
user_profile = False
template_vars = {'title': 'Welcome'}
cookievars = get_cookie_vars(app.current_request.headers)
if cookievars:
user_profile = get_session(cookievars['urs-user-id'], cookievars['urs-access-token'])
if user_profile:
if os.getenv('MATURITY', '') == 'DEV':
template_vars['profile'] = user_profile
else:
template_vars['URS_URL'] = get_urs_url(app.current_request.context)
headers = {'Content-Type': 'text/html'}
return make_html_response(template_vars, headers, 200, 'root.html')
def logout():
cookievars = get_cookie_vars(app.current_request.headers)
template_vars = {'title': 'Logged Out', 'URS_URL': get_urs_url(app.current_request.context)}
if cookievars:
user_id = cookievars['urs-user-id']
urs_access_token = cookievars['urs-access-token']
delete_session(user_id, urs_access_token)
template_vars['contentstring'] = 'You are logged out.'
else:
template_vars['contentstring'] = 'No active login found.'
headers = {
'Content-Type': 'text/html',
}
headers.update(make_set_cookie_headers('deleted', 'deleted', 'Thu, 01 Jan 1970 00:00:00 GMT', os.getenv('COOKIE_DOMAIN', '')))
return make_html_response(template_vars, headers, 200, 'root.html')
def root():
user_profile = False
template_vars = {'title': 'Welcome'}
cookievars = get_cookie_vars(app.current_request.headers)
if cookievars:
if os.getenv('JWT_COOKIENAME', 'asf-urs') in cookievars:
# We have a JWT cookie
user_profile = cookievars[os.getenv('JWT_COOKIENAME', 'asf-urs')]
if user_profile:
if os.getenv('MATURITY') == 'DEV':
template_vars['profile'] = user_profile
else:
template_vars['URS_URL'] = get_urs_url(app.current_request.context)
headers = {'Content-Type': 'text/html'}
return make_html_response(template_vars, headers, 200, 'root.html')
def logout():
cookievars = get_cookie_vars(app.current_request.headers)
template_vars = {
'title': 'Logged Out',
'URS_URL': get_urs_url(app.current_request.context)
}
if JWT_COOKIE_NAME in cookievars:
template_vars['contentstring'] = 'You are logged out.'
else:
template_vars['contentstring'] = 'No active login found.'
headers = {
'Content-Type': 'text/html',
}
headers.update(
make_set_cookie_headers_jwt({}, 'Thu, 01 Jan 1970 00:00:00 GMT',
os.getenv('COOKIE_DOMAIN', '')))
return make_html_response(template_vars, headers, 200, 'root.html')
def root():
user_profile = False
template_vars = {'title': 'Welcome'}
cookievars = get_cookie_vars(app.current_request.headers)
if cookievars:
if os.getenv('JWT_COOKIENAME','asf-urs') in cookievars:
# this means our cookie is a jwt and we don't need to go digging in the session db
user_profile = cookievars[os.getenv('JWT_COOKIENAME','asf-urs')]
else:
log.warning('jwt cookie not found, falling back to old style')
user_profile = get_session(cookievars['urs-user-id'], cookievars['urs-access-token'])
if user_profile:
if os.getenv('MATURITY', '') == 'DEV':
template_vars['profile'] = user_profile
else:
template_vars['URS_URL'] = get_urs_url(app.current_request.context)
headers = {'Content-Type': 'text/html'}
return make_html_response(template_vars, headers, 200, 'root.html')
def dynamic_url():
t = [time.time()]
custom_headers = {}
log.debug('attempting to GET a thing')
restore_bucket_vars()
log.debug(f'b_map: {b_map}')
t.append(time.time())
if 'proxy' in app.current_request.uri_params:
path, bucket, filename, custom_headers = process_request(
app.current_request.uri_params['proxy'], b_map)
log.debug('path, bucket, filename, custom_headers: {}'.format(
(path, bucket, filename, custom_headers)))
if not bucket:
template_vars = {
'contentstring': 'File not found',
'title': 'File not found',
'requestid': get_request_id(),
}
headers = {}
return make_html_response(template_vars, headers, 404,
'error.html')
else:
path, bucket, filename = (None, None, None)
cookievars = get_cookie_vars(app.current_request.headers)
user_profile = None
if cookievars:
log.debug('cookievars: {}'.format(cookievars))
if JWT_COOKIE_NAME in cookievars:
# this means our cookie is a jwt and we don't need to go digging in the session db
user_profile = cookievars[JWT_COOKIE_NAME]
else:
log.warning('jwt cookie not found')
# Not kicking user out just yet. We might be dealing with a public bucket
t.append(time.time()) # 2
# Check for public bucket
pub_bucket = check_public_bucket(bucket, b_map, filename)
t.append(time.time()) # 3
if pub_bucket:
log.debug("Accessing public bucket {0}".format(path))
elif not user_profile:
if 'Authorization' in app.current_request.headers and app.current_request.headers[
'Authorization'].split()[0].lower() == 'bearer':
# we will deal with "bearer" auth here. "Basic" auth will be handled by do_auth_and_return()
log.debug('we got an Authorization header. {}'.format(
app.current_request.headers['Authorization']))
token = app.current_request.headers['Authorization'].split()[1]
action, data = handle_auth_bearer_header(token)
if action == 'return':
# Not a successful event.
return data
user_profile = data
user_id = user_profile['uid']
log_context(user_id=user_id)
log.debug(f'User {user_id} has user profile: {user_profile}')
jwt_payload = user_profile_2_jwt_payload(user_id, token,
user_profile)
log.debug(f"Encoding JWT_PAYLOAD: {jwt_payload}")
custom_headers.update(
make_set_cookie_headers_jwt(jwt_payload, '',
os.getenv('COOKIE_DOMAIN', '')))
else:
return do_auth_and_return(app.current_request.context)
t.append(time.time()) # 4
# Check that the bucket is either NOT private, or user belongs to that group
private_check = check_private_bucket(
bucket, b_map, filename) # NOTE: Is an optimization attempt worth it
# if we're asking for a public file and we
# omit this check?
log.debug('private check: {}'.format(private_check))
t.append(time.time()) # 5
u_in_g, new_user_profile = user_in_group(private_check, cookievars,
user_profile, False)
t.append(time.time()) # 6
new_jwt_cookie_headers = {}
if new_user_profile:
log.debug(
f"We got new profile from user_in_group() {new_user_profile}")
user_profile = new_user_profile
jwt_cookie_payload = user_profile_2_jwt_payload(
get_jwt_field(cookievars, 'urs-user-id'),
get_jwt_field(cookievars, 'urs-access-token'), user_profile)
new_jwt_cookie_headers.update(
make_set_cookie_headers_jwt(jwt_cookie_payload, '',
os.getenv('COOKIE_DOMAIN', '')))
log.debug('user_in_group: {}'.format(u_in_g))
if private_check and not u_in_g:
template_vars = {
'contentstring': 'This data is not currently available.',
'title': 'Could not access data',
'requestid': get_request_id(),
}
return make_html_response(template_vars, new_jwt_cookie_headers, 403,
'error.html')
if not filename: # Maybe this belongs up above, right after setting the filename var?
log.warning(
"Request was made to directory listing instead of object: {0}".
format(path))
template_vars = {
'contentstring': 'Request does not appear to be valid.',
'title': 'Request Not Serviceable',
'requestid': get_request_id(),
}
return make_html_response(template_vars, new_jwt_cookie_headers, 404,
'error.html')
custom_headers.update(new_jwt_cookie_headers)
log.debug(
f'custom headers before try download from bucket: {custom_headers}')
t.append(time.time()) # 7
log.debug('timing for dynamic_url()')
log.debug('ET for restore_bucket_vars(): {}s'.format(t[1] - t[0]))
log.debug('ET for check_public_bucket(): {}s'.format(t[3] - t[2]))
log.debug('ET for possible auth header handling: {}s'.format(t[4] - t[3]))
log.debug('ET for user_in_group(): {}s'.format(t[6] - t[5]))
log.debug('ET for total: {}s'.format(t[7] - t[0]))
return try_download_from_bucket(bucket, filename, user_profile,
custom_headers)
def try_download_head(bucket, filename):
t = [time.time()]
client = get_data_dl_s3_client()
t.append(time.time())
# Check for range request
range_header = get_range_header_val()
try:
if not range_header:
download = client.get_object(Bucket=bucket, Key=filename)
else:
log.info("Downloading range {0}".format(range_header))
download = client.get_object(Bucket=bucket,
Key=filename,
Range=range_header)
t.append(time.time())
except ClientError as e:
log.warning("Could not get head for s3://{0}/{1}: {2}".format(
bucket, filename, e))
# cumulus uses this log message for metrics purposes.
template_vars = {
'contentstring': 'File not found',
'title': 'File not found',
'requestid': get_request_id(),
}
headers = {}
cumulus_log_message(
'failure', 404, 'HEAD', {
'reason': 'Could not find requested data',
's3': f'{bucket}/{filename}'
})
return make_html_response(template_vars, headers, 404, 'error.html')
log.debug(download)
response_headers = {'Content-Type': download['ContentType']}
for header in download['ResponseMetadata']['HTTPHeaders']:
name = header_map[header] if header in header_map else header
value = download['ResponseMetadata']['HTTPHeaders'][
header] if header != 'server' else 'egress'
log.debug("setting header {0} to {1}.".format(name, value))
response_headers[name] = value
# Try Redirecting to HEAD. There should be a better way.
user_id = get_jwt_field(get_cookie_vars(app.current_request.headers),
'urs-user-id')
log_context(user_id=user_id)
# Generate URL
t.append(time.time())
creds, offset = get_role_creds(user_id=user_id)
url_lifespan = 3600 - offset
bucket_region = client.get_bucket_location(
Bucket=bucket)['LocationConstraint']
bucket_region = 'us-east-1' if not bucket_region else bucket_region
t.append(time.time())
presigned_url = get_presigned_url(creds, bucket, filename, bucket_region,
url_lifespan, user_id, 'HEAD')
t.append(time.time())
s3_host = urlparse(presigned_url).netloc
# Return a redirect to a HEAD
log.debug("Presigned HEAD URL host was {0}".format(s3_host))
log.debug('timing for try_download_head()')
log.debug('ET for get_data_dl_s3_client(): {}s'.format(t[1] - t[0]))
log.debug('ET for client.get_object(): {}s'.format(t[2] - t[1]))
log.debug('ET for get_role_creds(): {}s'.format(t[4] - t[3]))
log.debug('ET for get_presigned_url(): {}s'.format(t[5] - t[4]))
return make_redirect(presigned_url, {}, 303)
def get_data_dl_s3_client():
user_id = get_jwt_field(get_cookie_vars(app.current_request.headers),
'urs-user-id')
return get_bc_config_client(user_id)
