def run(*, email, authorizing_agent=None, update=False):
check_true(email, "A non-null email must be given.")
check_true('@' in email, "Email address missing '@'.")
vapp = make_dev_vapp(remote_user=authorizing_agent or email)
me_result = vapp.get("/me", status=307).json
PRINT("Recognized as user:"******"Allocated access_key_id, secret_access_key for %s: %s, %s" %
(email, access_key_id, secret_access_key))
else:
keyfile = os.path.expanduser("~/.cgap-keys.json")
backupfile = keyfile + ".BAK"
try:
with open(keyfile, 'r') as old_keys_fp:
keys = json.load(old_keys_fp)
except FileNotFoundError:
keys = {}
keys[LOCAL_DEV_ENV_NAME]['key'] = access_key_id
keys[LOCAL_DEV_ENV_NAME]['secret'] = secret_access_key
optionally(os.remove)(backupfile)
saved = optionally(os.rename)(keyfile, backupfile)
optionally(os.chmod)(backupfile, 0o600)
with open(keyfile, 'w') as new_keys_fp:
json.dump(keys, new_keys_fp, indent=4)
os.chmod(keyfile, 0o600)
PRINT("Allocated access keys for %s." % email)
extra = " (Old file saved as: %s)" % backupfile if not isinstance(
saved, Exception) else ""
PRINT("Wrote: %s%s" % (keyfile, extra))
def permits(self, context, principals, permission):
principals = local_principals(context, principals)
result = self.wrapped_policy.permits(context, principals, permission)
if DEBUG_PERMISSIONS:
PRINT("LocalRolesAuthorizationPolicy.permits")
PRINT(" permission=", permission)
PRINT(" principals=", principals)
PRINT("LocalRolesAuthorizationPolicy.permits returning", result)
return result
def execute_prearranged_upload(path, upload_credentials, auth=None):
"""
This performs a file upload using special credentials received from ff_utils.patch_metadata.
:param path: the name of a local file to upload
:param upload_credentials: a dictionary of credentials to be used for the upload,
containing the keys 'AccessKeyId', 'SecretAccessKey', 'SessionToken', and 'upload_url'.
:param auth: auth info in the form of a dictionary containing 'key', 'secret', and 'server',
and possibly other useful information such as an encryption key id.
"""
if DEBUG_PROTOCOL: # pragma: no cover
PRINT(
f"Upload credentials contain {conjoined_list(list(upload_credentials.keys()))}."
)
try:
s3_encrypt_key_id = get_s3_encrypt_key_id(
upload_credentials=upload_credentials, auth=auth)
extra_env = dict(
AWS_ACCESS_KEY_ID=upload_credentials['AccessKeyId'],
AWS_SECRET_ACCESS_KEY=upload_credentials['SecretAccessKey'],
AWS_SECURITY_TOKEN=upload_credentials['SessionToken'])
env = dict(os.environ, **extra_env)
except Exception as e:
raise ValueError("Upload specification is not in good form. %s: %s" %
(e.__class__.__name__, e))
start = time.time()
try:
source = path
target = upload_credentials['upload_url']
show("Going to upload %s to %s." % (source, target))
command = ['aws', 's3', 'cp']
if s3_encrypt_key_id:
command = command + [
'--sse', 'aws:kms', '--sse-kms-key-id', s3_encrypt_key_id
]
command = command + ['--only-show-errors', source, target]
if DEBUG_PROTOCOL: # pragma: no cover
PRINT(f"Executing: {command}")
PRINT(f" ==> {' '.join(command)}")
PRINT(
f"Environment variables include {conjoined_list(list(extra_env.keys()))}."
)
options = {}
if running_on_windows_native():
options = {"shell": True}
subprocess.check_call(command, env=env, **options)
except subprocess.CalledProcessError as e:
raise RuntimeError("Upload failed with exit code %d" % e.returncode)
else:
end = time.time()
duration = end - start
show("Uploaded in %.2f seconds" % duration)
def show(*args, with_time=False):
"""
Prints its args space-separated, as 'print' would, possibly with an hh:mm:ss timestamp prepended.
:param args: an object to be printed
:param with_time: a boolean specifying whether to prepend a timestamp
"""
if with_time:
hh_mm_ss = str(datetime.datetime.now().strftime("%H:%M:%S"))
PRINT(hh_mm_ss, *args)
else:
PRINT(*args)
def principals_allowed_by_permission(self, context, permission):
principals = self.wrapped_policy.principals_allowed_by_permission(
context, permission)
result = merged_local_principals(context, principals)
if DEBUG_PERMISSIONS:
PRINT(
"LocalRolesAuthorizationPolicy.principals_allowed_by_permission"
)
PRINT(" permission=", permission)
PRINT(" principals=", principals)
PRINT(
"LocalRolesAuthorizationPolicy.principals_allowed_by_permission returning",
result)
return result
def get_s3_encrypt_key_id(*, upload_credentials, auth):
if 's3_encrypt_key_id' in upload_credentials:
s3_encrypt_key_id = upload_credentials.get('s3_encrypt_key_id')
if DEBUG_PROTOCOL: # pragma: no cover
PRINT(
f"Extracted s3_encrypt_key_id from upload_credentials: {s3_encrypt_key_id}"
)
else:
if DEBUG_PROTOCOL: # pragma: no cover
PRINT(f"No s3_encrypt_key_id entry found in upload_credentials.")
PRINT(f"Fetching s3_encrypt_key_id from health page.")
s3_encrypt_key_id = get_s3_encrypt_key_id_from_health_page(auth)
if DEBUG_PROTOCOL: # pragma: no cover
PRINT(f" =id=> {s3_encrypt_key_id!r}")
return s3_encrypt_key_id
def main():
parser = argparse.ArgumentParser( # noqa - PyCharm wrongly thinks the formatter_class is specified wrong here.
description="Configure Kibana Index",
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('email',
help='An email address to get access keys for')
# I don't know a case where this actually ends up being useful, since anyone can allocate their own access keys.
# But I put it here just in case to illustrate how it would work. -kmp 29-Mar-2021
# parser.add_argument('--authorizing-agent', help='Whose account to authorize this (not usually needed)')
parser.add_argument("--update",
help='Whether to update ~/.cgap-keys.json',
action='store_true')
args = parser.parse_args()
try:
run(
email=args.email,
# authorizing_agent=args.authorizing_agent,
update=args.update)
exit(0)
except Exception as e:
PRINT("FAILED (%s): %s" % (full_class_name(e), e))
exit(1)
def local_principals(context, principals):
""" The idea behind this is to process __ac_local_roles__ (and a boolean __ac_local_roles_block__
to disable) and add local principals. This only works if you're in correct context, though,
which does not seem to be the case.
"""
local_principals = set()
block = False
for location in lineage(context):
if block:
break
block = getattr(location, '__ac_local_roles_block__', False)
local_roles = getattr(location, '__ac_local_roles__', None)
if local_roles and callable(local_roles):
local_roles = local_roles()
if not local_roles:
continue
for principal in principals:
try:
roles = local_roles[principal]
except KeyError:
pass
else:
if not is_nonstr_iter(roles):
roles = [roles]
local_principals.update(roles)
if not local_principals:
return principals
local_principals.update(principals)
if DEBUG_PERMISSIONS:
PRINT("local_principals")
PRINT(" context.collection=", context.collection)
PRINT(" context.__acl__()=", context.__acl__())
PRINT(" context.collection.__ac_local_roles_()=",
context.__ac_local_roles__())
PRINT("local_principals returning", local_principals)
return local_principals
def merged_local_principals(context, principals):
# XXX Possibly limit to prefix like 'role.'
set_principals = frozenset(principals)
local_principals = set()
block = False
for location in lineage(context):
if block:
break
block = getattr(location, '__ac_local_roles_block__', False)
local_roles = getattr(location, '__ac_local_roles__', None)
if local_roles and callable(local_roles):
local_roles = local_roles()
if not local_roles:
continue
for principal, roles in local_roles.items():
if not is_nonstr_iter(roles):
roles = [roles]
if not set_principals.isdisjoint(roles):
local_principals.add(principal)
if not local_principals:
return principals
local_principals.update(principals)
local_principals = list(local_principals)
if DEBUG_PERMISSIONS:
PRINT("merged_local_principals")
PRINT(" context.collection=", context.collection)
PRINT(" context.__acl__()=", context.__acl__())
PRINT(" context.collection.__ac_local_roles_()=",
context.__ac_local_roles__())
PRINT("merged_local_principals returning", local_principals)
return local_principals
def test_documentation():
with io.open(_DCICUTILS_DOC_FILE) as fp:
line_number = 0
current_module = None
automodules_seen = 0
prev_line = None
problems = []
expected_modules = {remove_suffix(".py", os.path.basename(file)) for file in _DCICUTILS_FILES} - SKIP_MODULES
documented_modules = set()
for line in fp:
line_number += 1 # We count the first line as line 1
line = line.strip()
if _SUBSUBSECTION_LINE.match(line):
if current_module and automodules_seen == 0:
problems.append(f"Line {line_number}: Missing automodule declaration for section {current_module}.")
current_module = prev_line
automodules_seen = 0
elif _SECTION_OR_SUBSECTION_LINE.match(line):
current_module = None
automodules_seen = 0
else:
matched = _AUTOMODULE_LINE.match(line)
if matched:
automodule_module = matched.group(1)
if not current_module:
problems.append(f"Line {line_number}: Unexpected automodule declaration"
f" outside of module section.")
else:
documented_modules.add(automodule_module)
if automodules_seen == 1:
# If fewer than 1 seen, no issue.
# If more than 1 seen, we already warned, so don't duplicate.
# So really only the n == 1 case matters to us.
problems.append(f"Line {line_number}: More than one automodule"
f" in section {current_module}?")
if automodule_module != current_module:
problems.append(f"Line {line_number}: Unexpected automodule declaration"
f" for section {current_module}: {automodule_module}.")
automodules_seen += 1
prev_line = line
undocumented_modules = expected_modules - documented_modules
if undocumented_modules:
problems.append(there_are(sorted(undocumented_modules), kind="undocumented module", punctuate=True))
if problems:
for n, problem in enumerate(problems, start=1):
PRINT(f"PROBLEM {n}: {problem}")
message = there_are(problems, kind="problem", tense='past', show=False,
context=f"found in the readthedocs declaration file, {_DCICUTILS_DOC_FILE!r}")
raise AssertionError(message)
def _discover_es_health_from_boto3_eb_metadata(envname):
try:
eb_client = boto3.client('elasticbeanstalk', region_name=REGION)
# Calling describe_beanstalk_environments is pretty much the same as doing eb_client.describe_environments(...)
# except it's robust against AWS throttling us for calling it too often.
envs_from_eb = describe_beanstalk_environments(
eb_client, EnvironmentNames=[envname])['Environments']
for env in envs_from_eb:
PRINT(f"Checking {env.get('EnvironmentName')} for {envname}...")
if env.get('EnvironmentName') == envname:
cname = env.get('CNAME')
# TODO: It would be nice if we were using https: for everything. -kmp 14-Aug-2020
res = requests.get("http://%s/health?format=json" % cname)
health_json = res.json()
return health_json
except Exception as e:
raise RuntimeError(
"Unable to discover elasticsearch info for %s:\n%s: %s" %
(envname, e.__class__.__name__, e))
def _discover_es_url_from_boto3_eb_metadata(integrated_envname):
try:
discovered_health_json_from_eb = _discover_es_health_from_boto3_eb_metadata(
integrated_envname)
assert discovered_health_json_from_eb, f"No health page for {integrated_envname} was discovered."
PRINT(
f"In _discover_es_url_from_boto3_eb_metadata,"
f"discovered_health_json_from_eb={json.dumps(discovered_health_json_from_eb, indent=2)}"
)
time.sleep(1) # Reduce throttling risk
ff_health_json = get_health_page(ff_env=integrated_envname)
# Consistency check that both utilities are returning the same info.
assert discovered_health_json_from_eb[
'beanstalk_env'] == ff_health_json['beanstalk_env']
assert discovered_health_json_from_eb[
'elasticsearch'] == ff_health_json['elasticsearch']
assert discovered_health_json_from_eb['namespace'] == ff_health_json[
'namespace']
# Not all health pages have a namespace. Production ones may not.
# But they are not good environments for us to use for testing.
discovered_namespace = discovered_health_json_from_eb['namespace']
# We _think_ these are always the same, but maybe not. Perhaps worth noting if/when they diverge.
assert discovered_namespace == integrated_envname, (
f"While doing ES URL discovery for integrated envname {integrated_envname},"
f" the namespace, {discovered_namespace}, discovered on the health page"
f" does not match the integrated envname.")
# This should be all we actually need:
return discovered_health_json_from_eb['elasticsearch']
except Exception as e:
# Errors sometimes happen when running tests with the orchestration credentials.
PRINT("********************************************")
PRINT("** ERROR DURING ELASTICSEARCH DISCOVERY **")
PRINT("** Make sure you have legacy credentials **")
PRINT("** enabled while running these tests. **")
PRINT("********************************************")
PRINT(f"{e.__class__.__name__}: {e}")
raise RuntimeError(
f"Failed to discover ES URL for {integrated_envname}.")
def processing_context(self):
self.log.info("Processing {submission_id} as {ingestion_type}.".format(
submission_id=self.submission_id,
ingestion_type=self.ingestion_type))
submission_id = self.submission_id
manifest_key = "%s/manifest.json" % submission_id
response = self.s3_client.get_object(Bucket=self.bucket,
Key=manifest_key)
manifest = json.load(response['Body'])
self.object_name = object_name = manifest['object_name']
self.parameters = parameters = manifest['parameters']
email = manifest['email']
debuglog(submission_id, "object_name:", object_name)
debuglog(submission_id, "parameters:", parameters)
started_key = "%s/started.txt" % submission_id
create_empty_s3_file(self.s3_client,
bucket=self.bucket,
key=started_key)
# PyCharm thinks this is unused. -kmp 26-Jul-2020
# data_stream = submission.s3_client.get_object(Bucket=submission.bucket, Key="%s/manifest.json" % submission_id)['Body']
resolution = {
"data_key": object_name,
"manifest_key": manifest_key,
"started_key": started_key,
}
try:
other_keys = {}
if email:
other_keys['submitted_by'] = email
self.patch_item(submission_id=submission_id,
object_name=object_name,
parameters=parameters,
processing_status={"state": "processing"},
**other_keys)
self.resolution = resolution
yield resolution
if not self.is_done():
self.succeed()
self.patch_item(processing_status={
"state": "done",
"outcome": self.outcome,
"progress": "complete"
},
**self.other_details)
except Exception as e:
resolution[
"traceback_key"] = traceback_key = "%s/traceback.txt" % submission_id
with s3_output_stream(self.s3_client,
bucket=self.bucket,
key=traceback_key) as fp:
traceback.print_exc(file=fp)
resolution["error_type"] = e.__class__.__name__
resolution["error_message"] = str(e)
self.patch_item(errors=["%s: %s" % (e.__class__.__name__, e)],
processing_status={
"state": "done",
"outcome": "error",
"progress": "incomplete"
})
with s3_output_stream(self.s3_client,
bucket=self.bucket,
key="%s/resolution.json" % submission_id) as fp:
PRINT(json.dumps(resolution, indent=2), file=fp)
def groupfinder(login, request):
if '.' not in login:
if DEBUG_PERMISSIONS:
PRINT("groupfinder sees no '.' in %s, returning None" % login)
return None
namespace, localname = login.split('.', 1)
user = None
collections = request.registry[COLLECTIONS]
""" At least part of this stanza seems mainly for testing purposes
should the testing bits be refactored elsewhere???
20-09-08 changed permission model requires import of Authenticated
is that kosher
"""
# TODO (C4-332): Consolidate permissions all in one perms.py file once this all stabilizes.
if namespace == 'remoteuser':
# These names are used in testing or special service situations to force the permissions result
# to known values without any need to go through lookup of any particular user and process
# their groups or project_roles.
synthetic_result = None
if localname in ['EMBED', 'INDEXER']:
synthetic_result = []
elif localname in ['TEST', 'IMPORT', 'UPGRADE', 'INGESTION']:
synthetic_result = ['group.admin']
elif localname in ['TEST_SUBMITTER']:
synthetic_result = ['group.submitter']
elif localname in ['TEST_AUTHENTICATED']:
synthetic_result = [Authenticated]
if synthetic_result is not None:
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "returning synthetic result:",
synthetic_result)
return synthetic_result
# Note that the above 'if' has no final 'else', and the remainder of cases,
# having the form remoteuser.<username>, are processed in the next 'if' below.
if namespace in ('mailto', 'remoteuser', 'auth0'):
users = collections.by_item_type['user']
try:
user = users[localname]
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "found user", localname)
except KeyError:
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "failed to find user",
localname)
return None
elif namespace == 'accesskey':
access_keys = collections.by_item_type['access_key']
try:
access_key = access_keys[localname]
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "found access key", localname)
except KeyError:
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "failed to find access key",
localname)
return None
access_key_status = access_key.properties.get('status')
if access_key_status in ('deleted', 'revoked'):
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "found", access_key_status,
"access key", localname)
return None
userid = access_key.properties['user']
user = collections.by_item_type['user'][userid]
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "decoded access key", localname,
"as user", user)
if user is None:
PRINT("groupfinder for", login, "returning None because user is None")
return None
user_properties = user.properties
if user_properties.get('status') in ('deleted'):
if DEBUG_PERMISSIONS:
PRINT(
"groupfinder for %s found user %s, but that user has status deleted."
% (login, user))
return None
principals = ['userid.%s' % user.uuid]
if DEBUG_PERMISSIONS:
PRINT("groupfinder starting with principals", principals)
def add_principal(principal):
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "adding", principal,
"to principals.")
principals.append(principal)
# first pass implementation uses project to give view access only - will need to be
# be modified when different user roles can provide different levels of access
# and users can belong to different project
# project_roles is a list of embedded objects with 'project' property required
project_roles = user_properties.get('project_roles', [])
if DEBUG_PERMISSIONS:
PRINT("groupfind for", login, "found project roles:",
json.dumps(project_roles))
if project_roles:
add_principal('group.project_editor')
for pr in project_roles:
add_principal('editor_for.{}'.format(pr.get('project')))
for group in user_properties.get('groups', []):
add_principal('group.%s' % group)
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "returning principals",
json.dumps(principals, indent=2))
return principals
def add_principal(principal):
if DEBUG_PERMISSIONS:
PRINT("groupfinder for", login, "adding", principal,
"to principals.")
principals.append(principal)
def _post_submission(server, keypair, ingestion_filename, creation_post_data,
submission_post_data):
""" This takes care of managing the compatibility step of using either the old or new ingestion protocol.
OLD PROTOCOL: Post directly to /submit_for_ingestion
NEW PROTOCOL: Create an IngestionSubmission and then use /ingestion-submissions/<guid>/submit_for_ingestion
:param server: the name of the server as a URL
:param keypair: a tuple which is a keypair (key_id, secret_key)
:param ingestion_filename: the bundle filename to be submitted
:param creation_post_data: data to become part of the post data for the creation
:param submission_post_data: data to become part of the post data for the ingestion
:return: the results of the ingestion call (whether by the one-step or two-step process)
"""
def post_files_data():
return {"datafile": io.open(ingestion_filename, 'rb')}
old_style_submission_url = url_path_join(server, "submit_for_ingestion")
old_style_post_data = dict(creation_post_data, **submission_post_data)
response = requests.post(old_style_submission_url,
auth=keypair,
data=old_style_post_data,
headers={'Content-type': 'application/json'},
files=post_files_data())
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("old_style_submission_url=", old_style_submission_url)
PRINT("old_style_post_data=", json.dumps(old_style_post_data,
indent=2))
PRINT("keypair=", keypair)
PRINT("response=", response)
if response.status_code == 404:
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("Retrying with new protocol.")
creation_post_headers = {
'Content-type': 'application/json',
'Accept': 'application/json',
}
creation_post_url = url_path_join(server, "IngestionSubmission")
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("creation_post_data=",
json.dumps(creation_post_data, indent=2))
PRINT("creation_post_url=", creation_post_url)
creation_response = requests.post(creation_post_url,
auth=keypair,
headers=creation_post_headers,
json=creation_post_data
# data=json.dumps(creation_post_data)
)
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("headers:", creation_response.request.headers)
creation_response.raise_for_status()
[submission] = creation_response.json()['@graph']
submission_id = submission['@id']
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("server=", server, "submission_id=", submission_id)
new_style_submission_url = url_path_join(server, submission_id,
"submit_for_ingestion")
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("submitting new_style_submission_url=",
new_style_submission_url)
response = requests.post(new_style_submission_url,
auth=keypair,
data=submission_post_data,
files=post_files_data())
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("response received for submission post:", response)
PRINT("response.content:", response.content)
else:
if DEBUG_PROTOCOL: # pragma: no cover
PRINT("Old style protocol worked.")
return response