def index_document(self):
indexd_server = config.get("INDEXD") or config["BASE_URL"] + "/index"
url = indexd_server + "/index/"
try:
res = requests.get(url + self.file_id)
except Exception as e:
logger.error(
"failed to reach indexd at {0}: {1}".format(url + self.file_id, e)
)
raise UnavailableError("Fail to reach id service to find data location")
if res.status_code == 200:
try:
json_response = res.json()
if "urls" not in json_response:
logger.error(
"URLs are not included in response from "
"indexd: {}".format(url + self.file_id)
)
raise InternalError("URLs and metadata not found")
return res.json()
except Exception as e:
logger.error(
"indexd response missing JSON field {}".format(url + self.file_id)
)
raise InternalError("internal error from indexd: {}".format(e))
elif res.status_code == 404:
logger.error(
"Not Found. indexd could not find {}: {}".format(
url + self.file_id, res.text
)
)
raise NotFound("No indexed document found with id {}".format(self.file_id))
else:
raise UnavailableError(res.text)
def get_index_document(file_id):
indexd_server = (current_app.config.get('INDEXD')
or current_app.config['BASE_URL'] + '/index')
url = indexd_server + '/index/'
try:
res = requests.get(url + file_id)
except Exception as e:
current_app.logger.error("failed to reach indexd at {0}: {1}".format(
url + file_id, e))
raise UnavailableError(
"Fail to reach id service to find data location")
if res.status_code == 200:
try:
json_response = res.json()
if 'urls' not in json_response or 'metadata' not in json_response:
current_app.logger.error(
'URLs and metadata are not included in response from indexd: {}'
.format(url + file_id))
raise InternalError('URLs and metadata not found')
return res.json()
except Exception as e:
flask.current_app.logger.error(
'indexd response missing JSON field {}'.format(url + file_id))
raise InternalError('internal error from indexd: {}'.format(e))
elif res.status_code == 404:
flask.current_app.logger.error(
'indexd did not find find {}; {}'.format(url + file_id, res.text))
raise NotFound("Can't find a location for the data")
else:
raise UnavailableError(res.text)
def get_bucket_region(self, bucket, config):
try:
if "aws_access_key_id" in config:
self.s3_client = client("s3", **config)
response = self.s3_client.get_bucket_location(Bucket=bucket)
region = response.get("LocationConstraint")
except Boto3Error as ex:
self.logger.exception(ex)
raise InternalError("Fail to get bucket region: {}".format(ex))
except Exception as ex:
self.logger.exception(ex)
raise UnavailableError("Fail to reach AWS: {}".format(ex))
if region is None:
return "us-east-1"
return region
def assume_role(self, role_arn, duration_seconds, config=None):
try:
if config and config.has_key("aws_access_key_id"):
self.sts_client = client("sts", **config)
session_name_postfix = uuid.uuid4()
return self.sts_client.assume_role(
RoleArn=role_arn,
DurationSeconds=duration_seconds,
RoleSessionName="{}-{}".format("gen3", session_name_postfix),
)
except Boto3Error as ex:
self.logger.exception(ex)
raise InternalError("Fail to assume role: {}".format(ex.message))
except Exception as ex:
self.logger.exception(ex)
raise UnavailableError("Fail to reach AWS: {}".format(ex.message))
def assume_role(self, role_arn, duration_seconds, config=None):
assert (
duration_seconds
), 'assume_role() cannot be called without "duration_seconds" parameter; please check your "expires_in" parameters'
try:
if config and "aws_access_key_id" in config:
self.sts_client = client("sts", **config)
session_name_postfix = uuid.uuid4()
return self.sts_client.assume_role(
RoleArn=role_arn,
DurationSeconds=duration_seconds,
RoleSessionName="{}-{}".format("gen3", session_name_postfix),
)
except Boto3Error as ex:
self.logger.exception(ex)
raise InternalError("Fail to assume role: {}".format(ex))
except Exception as ex:
self.logger.exception(ex)
raise UnavailableError("Fail to reach AWS: {}".format(ex))
def raise_unavailable(gpg_email):
raise UnavailableError(
"Error: Google unable to delete proxy group {}. Aborting".format(
gpg_email))
def raise_unavailable(sae):
raise UnavailableError(
"Error: Google unable to delete service account {}. Aborting".
format(sae))