def add_to_params(self, parameters, value):
if value is None:
return
if parameters.get('parameterValues', None) is not None:
raise Exception(
"Only parameter-values or parameter-values-uri is allowed")
parameter_object = {}
# break string into = point
for argument in value:
try:
argument_components = argument.split('=', 1)
key = argument_components[0]
value = argument_components[1]
if key in parameter_object:
parameter_object[key] = [parameter_object[key], value]
else:
parameter_object[key] = value
except IndexError:
raise ParameterDefinitionError(
"Invalid inline parameter format: %s" % argument)
parsed = {'values': parameter_object}
parameter_values = translator.definition_to_parameter_values(parsed)
parameters['parameterValues'] = parameter_values
def deploy(config_file='config.yaml'):
path_to_config_file = os.path.join(os.getcwd(), config_file)
cfg = read_cfg(path_to_config_file)
profile = cfg.get('aws_profile')
access_key_id = cfg.get('aws_access_key_id')
secret_access_key = cfg.get('aws_secret_access_key')
region = cfg.get('region')
client = get_client(access_key_id, secret_access_key, region, profile)
name = cfg.get('name')
unique_id = cfg.get('unique_id')
description = cfg.get('description', '')
create_response = client.create_pipeline(name, unique_id, description)
pipeline_id = create_response.get('pipelineId')
parameter_objects = translator.definition_to_api_parameters(
read_json_file(cfg.get('parameter_objects')))
parameter_values = translator.definition_to_parameter_values(
read_json_file(cfg.get('parameter_values')))
pipeline_definition = translator.definition_to_api_objects(
read_json_file(cfg.get('pipeline_definition')))
return client.put_pipeline_definition(pipeline_id, pipeline_definition,
parameter_objects, parameter_values)
def define_data_pipeline(self, client, pipe_id, emr_core_instances):
import awscli.customizations.datapipeline.translator as trans
base = self.get_package_path()
if emr_core_instances != 0:
definition_file = base + 'yaetos/definition.json' # see syntax in datapipeline-dg.pdf p285 # to add in there: /*"AdditionalMasterSecurityGroups": "#{}", /* To add later to match EMR mode */
else:
definition_file = base + 'yaetos/definition_standalone_cluster.json'
# TODO: have 1 json for both to avoid having to track duplication.
definition = json.load(
open(definition_file,
'r')) # Note: Data Pipeline doesn't support emr-6.0.0 yet.
pipelineObjects = trans.definition_to_api_objects(definition)
parameterObjects = trans.definition_to_api_parameters(definition)
parameterValues = trans.definition_to_parameter_values(definition)
parameterValues = self.update_params(parameterValues)
logger.info('Filled pipeline with data from ' + definition_file)
response = client.put_pipeline_definition(
pipelineId=pipe_id,
pipelineObjects=pipelineObjects,
parameterObjects=parameterObjects,
parameterValues=parameterValues)
logger.info('put_pipeline_definition response: ' + str(response))
return parameterValues
def add_to_params(self, parameters, value):
if value is None:
return
if parameters.get('parameterValues', None) is not None:
raise Exception(
"Only parameter-values or parameter-values-uri is allowed"
)
parameter_object = {}
# break string into = point
for argument in value:
try:
argument_components = argument.split('=', 1)
key = argument_components[0]
value = argument_components[1]
if key in parameter_object:
parameter_object[key] = [parameter_object[key], value]
else:
parameter_object[key] = value
except IndexError:
raise ParameterDefinitionError(
"Invalid inline parameter format: %s" % argument
)
parsed = {'values': parameter_object}
parameter_values = translator.definition_to_parameter_values(parsed)
parameters['parameterValues'] = parameter_values
def set_pipeline_definition(self):
"""Translates the json definition and puts it on created pipeline
Returns:
dict: the response of the Boto3 command
"""
if not self.pipeline_id:
self.get_pipeline_id()
json_def = self.datapipeline_data['json_definition']
try:
pipelineobjects = translator.definition_to_api_objects(json_def)
parameterobjects = translator.definition_to_api_parameters(
json_def)
parametervalues = translator.definition_to_parameter_values(
json_def)
except translator.PipelineDefinitionError as error:
LOG.warning(error)
raise DataPipelineDefinitionError
response = self.client.put_pipeline_definition(
pipelineId=self.pipeline_id,
pipelineObjects=pipelineobjects,
parameterObjects=parameterobjects,
parameterValues=parametervalues)
LOG.debug(response)
LOG.info("Successfully applied pipeline definition")
return response
def to_new_definition_objects(new_definition, new_parameter_values):
if new_parameter_values is not None:
new_definition['values'] = new_parameter_values
return dict(
pipelineObjects=translator.definition_to_api_objects(new_definition),
parameterObjects=translator.definition_to_api_parameters(
new_definition),
parameterValues=translator.definition_to_parameter_values(
new_definition))
def activate_pipeline(module, dp, pipeline_id, parameters):
try:
req = dict(pipelineId=pipeline_id)
if parameters is not None:
req['parameterValues'] = translator.definition_to_parameter_values(
{'values': parameters})
dp.activate_pipeline(**req)
except boto.exception.BotoServerError, err:
module.fail_json(changed=False, msg=boto_exception(err))
def test_df_to_api_with_parameter_values(self):
definition = self.load_def("""{"values": {
"myMaximumRetries":"2",
"myOccurrenceNumber":"3"
}
}""")
actual = translator.definition_to_parameter_values(definition)
api = [{"id": "myMaximumRetries", "stringValue": "2"},
{"id": "myOccurrenceNumber", "stringValue": "3"}]
self.assertEqual(actual, api)
def add_to_params(self, parameters, value):
if value is None:
return
if parameters.get("parameterValues", None) is not None:
raise Exception("Only parameter-values or parameter-values-uri is allowed")
parsed = json.loads(value)
parameter_values = translator.definition_to_parameter_values(parsed)
parameters["parameterValues"] = parameter_values
def add_to_params(self, parameters, value):
if value is None:
return
if parameters.get('parameterValues', None) is not None:
raise Exception(
"Only parameter-values or parameter-values-uri is allowed")
parsed = json.loads(value)
parameter_values = translator.definition_to_parameter_values(parsed)
parameters['parameterValues'] = parameter_values
def add_to_params(self, parameters, value):
if value is None:
return
parsed = json.loads(value)
api_objects = translator.definition_to_api_objects(parsed)
parameter_objects = translator.definition_to_api_parameters(parsed)
parameter_values = translator.definition_to_parameter_values(parsed)
parameters["pipelineObjects"] = api_objects
# Use Parameter objects and values from def if not already provided
if "parameterObjects" not in parameters and parameter_objects is not None:
parameters["parameterObjects"] = parameter_objects
if "parameterValues" not in parameters and parameter_values is not None:
parameters["parameterValues"] = parameter_values
def add_to_params(self, parameters, value):
if value is None:
return
parsed = json.loads(value)
api_objects = translator.definition_to_api_objects(parsed)
parameter_objects = translator.definition_to_api_parameters(parsed)
parameter_values = translator.definition_to_parameter_values(parsed)
parameters['pipelineObjects'] = api_objects
# Use Parameter objects and values from def if not already provided
if 'parameterObjects' not in parameters \
and parameter_objects is not None:
parameters['parameterObjects'] = parameter_objects
if 'parameterValues' not in parameters \
and parameter_values is not None:
parameters['parameterValues'] = parameter_values
def test_df_to_api_with_parameter_values(self):
definition = self.load_def("""{"values": {
"myMaximumRetries":"2",
"myOccurrenceNumber":"3"
}
}""")
actual = translator.definition_to_parameter_values(definition)
api = [{
"id": "myMaximumRetries",
"stringValue": "2"
}, {
"id": "myOccurrenceNumber",
"stringValue": "3"
}]
self.assertEqual(actual, api)
def test_df_to_api_with_parameter_values_missing(self):
definition = self.load_def("""{"objects": [
{
"id" : "S3ToS3Copy",
"type" : "CopyActivity",
"schedule" : { "ref" : "CopyPeriod" },
"input" : { "ref" : "InputData" },
"output" : { "ref" : "OutputData" }
}
]}""")
try:
values = translator.definition_to_parameter_values(definition)
error = "Absence of parameter values should not display anything"
self.assertIsNone(values, error)
except PipelineDefinitionError:
self.fail("No exception should be thrown in absence of values")
def test_df_to_api_with_parameter_values_missing(self):
definition = self.load_def("""{"objects": [
{
"id" : "S3ToS3Copy",
"type" : "CopyActivity",
"schedule" : { "ref" : "CopyPeriod" },
"input" : { "ref" : "InputData" },
"output" : { "ref" : "OutputData" }
}
]}""")
try:
values = translator.definition_to_parameter_values(definition)
error = "Absence of parameter values should not display anything"
self.assertIsNone(values, error)
except PipelineDefinitionError:
self.fail("No exception should be thrown in absence of values")
def run_aws_data_pipeline(self):
self.s3_ops(self.session)
self.push_secrets(creds_or_file=self.app_args['connection_file']
) # TODO: fix privileges to get creds in dev env
# DataPipeline ops
import awscli.customizations.datapipeline.translator as trans
client = self.session.client('datapipeline')
pipe_id = self.create_date_pipeline(client)
definition_file = eu.LOCAL_APP_FOLDER + 'core/definition.json' # see syntax in datapipeline-dg.pdf p285 # to add in there: /*"AdditionalMasterSecurityGroups": "#{}", /* To add later to match EMR mode */
definition = json.load(
open(definition_file,
'r')) # Note: Data Pipeline doesn't support emr-6.0.0 yet.
pipelineObjects = trans.definition_to_api_objects(definition)
parameterObjects = trans.definition_to_api_parameters(definition)
parameterValues = trans.definition_to_parameter_values(definition)
parameterValues = self.update_params(parameterValues)
logger.info('Filled pipeline with data from ' + definition_file)
response = client.put_pipeline_definition(
pipelineId=pipe_id,
pipelineObjects=pipelineObjects,
parameterObjects=parameterObjects,
parameterValues=parameterValues)
logger.info('put_pipeline_definition response: ' + str(response))
response = client.activate_pipeline(
pipelineId=pipe_id,
parameterValues=
parameterValues, # optional. If set, need to specify all params as per json.
# startTimestamp=datetime(2018, 12, 1) # optional
)
logger.info('activate_pipeline response: ' + str(response))
logger.info('Activated pipeline ' + pipe_id)
