def add_to_params(self, parameters, value): if value is None: return if parameters.get('parameterValues', None) is not None: raise Exception( "Only parameter-values or parameter-values-uri is allowed") parameter_object = {} # break string into = point for argument in value: try: argument_components = argument.split('=', 1) key = argument_components[0] value = argument_components[1] if key in parameter_object: parameter_object[key] = [parameter_object[key], value] else: parameter_object[key] = value except IndexError: raise ParameterDefinitionError( "Invalid inline parameter format: %s" % argument) parsed = {'values': parameter_object} parameter_values = translator.definition_to_parameter_values(parsed) parameters['parameterValues'] = parameter_values
def deploy(config_file='config.yaml'): path_to_config_file = os.path.join(os.getcwd(), config_file) cfg = read_cfg(path_to_config_file) profile = cfg.get('aws_profile') access_key_id = cfg.get('aws_access_key_id') secret_access_key = cfg.get('aws_secret_access_key') region = cfg.get('region') client = get_client(access_key_id, secret_access_key, region, profile) name = cfg.get('name') unique_id = cfg.get('unique_id') description = cfg.get('description', '') create_response = client.create_pipeline(name, unique_id, description) pipeline_id = create_response.get('pipelineId') parameter_objects = translator.definition_to_api_parameters( read_json_file(cfg.get('parameter_objects'))) parameter_values = translator.definition_to_parameter_values( read_json_file(cfg.get('parameter_values'))) pipeline_definition = translator.definition_to_api_objects( read_json_file(cfg.get('pipeline_definition'))) return client.put_pipeline_definition(pipeline_id, pipeline_definition, parameter_objects, parameter_values)
def define_data_pipeline(self, client, pipe_id, emr_core_instances): import awscli.customizations.datapipeline.translator as trans base = self.get_package_path() if emr_core_instances != 0: definition_file = base + 'yaetos/definition.json' # see syntax in datapipeline-dg.pdf p285 # to add in there: /*"AdditionalMasterSecurityGroups": "#{}", /* To add later to match EMR mode */ else: definition_file = base + 'yaetos/definition_standalone_cluster.json' # TODO: have 1 json for both to avoid having to track duplication. definition = json.load( open(definition_file, 'r')) # Note: Data Pipeline doesn't support emr-6.0.0 yet. pipelineObjects = trans.definition_to_api_objects(definition) parameterObjects = trans.definition_to_api_parameters(definition) parameterValues = trans.definition_to_parameter_values(definition) parameterValues = self.update_params(parameterValues) logger.info('Filled pipeline with data from ' + definition_file) response = client.put_pipeline_definition( pipelineId=pipe_id, pipelineObjects=pipelineObjects, parameterObjects=parameterObjects, parameterValues=parameterValues) logger.info('put_pipeline_definition response: ' + str(response)) return parameterValues
def add_to_params(self, parameters, value): if value is None: return if parameters.get('parameterValues', None) is not None: raise Exception( "Only parameter-values or parameter-values-uri is allowed" ) parameter_object = {} # break string into = point for argument in value: try: argument_components = argument.split('=', 1) key = argument_components[0] value = argument_components[1] if key in parameter_object: parameter_object[key] = [parameter_object[key], value] else: parameter_object[key] = value except IndexError: raise ParameterDefinitionError( "Invalid inline parameter format: %s" % argument ) parsed = {'values': parameter_object} parameter_values = translator.definition_to_parameter_values(parsed) parameters['parameterValues'] = parameter_values
def set_pipeline_definition(self): """Translates the json definition and puts it on created pipeline Returns: dict: the response of the Boto3 command """ if not self.pipeline_id: self.get_pipeline_id() json_def = self.datapipeline_data['json_definition'] try: pipelineobjects = translator.definition_to_api_objects(json_def) parameterobjects = translator.definition_to_api_parameters( json_def) parametervalues = translator.definition_to_parameter_values( json_def) except translator.PipelineDefinitionError as error: LOG.warning(error) raise DataPipelineDefinitionError response = self.client.put_pipeline_definition( pipelineId=self.pipeline_id, pipelineObjects=pipelineobjects, parameterObjects=parameterobjects, parameterValues=parametervalues) LOG.debug(response) LOG.info("Successfully applied pipeline definition") return response
def to_new_definition_objects(new_definition, new_parameter_values): if new_parameter_values is not None: new_definition['values'] = new_parameter_values return dict( pipelineObjects=translator.definition_to_api_objects(new_definition), parameterObjects=translator.definition_to_api_parameters( new_definition), parameterValues=translator.definition_to_parameter_values( new_definition))
def activate_pipeline(module, dp, pipeline_id, parameters): try: req = dict(pipelineId=pipeline_id) if parameters is not None: req['parameterValues'] = translator.definition_to_parameter_values( {'values': parameters}) dp.activate_pipeline(**req) except boto.exception.BotoServerError, err: module.fail_json(changed=False, msg=boto_exception(err))
def test_df_to_api_with_parameter_values(self): definition = self.load_def("""{"values": { "myMaximumRetries":"2", "myOccurrenceNumber":"3" } }""") actual = translator.definition_to_parameter_values(definition) api = [{"id": "myMaximumRetries", "stringValue": "2"}, {"id": "myOccurrenceNumber", "stringValue": "3"}] self.assertEqual(actual, api)
def add_to_params(self, parameters, value): if value is None: return if parameters.get("parameterValues", None) is not None: raise Exception("Only parameter-values or parameter-values-uri is allowed") parsed = json.loads(value) parameter_values = translator.definition_to_parameter_values(parsed) parameters["parameterValues"] = parameter_values
def add_to_params(self, parameters, value): if value is None: return if parameters.get('parameterValues', None) is not None: raise Exception( "Only parameter-values or parameter-values-uri is allowed") parsed = json.loads(value) parameter_values = translator.definition_to_parameter_values(parsed) parameters['parameterValues'] = parameter_values
def add_to_params(self, parameters, value): if value is None: return parsed = json.loads(value) api_objects = translator.definition_to_api_objects(parsed) parameter_objects = translator.definition_to_api_parameters(parsed) parameter_values = translator.definition_to_parameter_values(parsed) parameters["pipelineObjects"] = api_objects # Use Parameter objects and values from def if not already provided if "parameterObjects" not in parameters and parameter_objects is not None: parameters["parameterObjects"] = parameter_objects if "parameterValues" not in parameters and parameter_values is not None: parameters["parameterValues"] = parameter_values
def add_to_params(self, parameters, value): if value is None: return parsed = json.loads(value) api_objects = translator.definition_to_api_objects(parsed) parameter_objects = translator.definition_to_api_parameters(parsed) parameter_values = translator.definition_to_parameter_values(parsed) parameters['pipelineObjects'] = api_objects # Use Parameter objects and values from def if not already provided if 'parameterObjects' not in parameters \ and parameter_objects is not None: parameters['parameterObjects'] = parameter_objects if 'parameterValues' not in parameters \ and parameter_values is not None: parameters['parameterValues'] = parameter_values
def test_df_to_api_with_parameter_values(self): definition = self.load_def("""{"values": { "myMaximumRetries":"2", "myOccurrenceNumber":"3" } }""") actual = translator.definition_to_parameter_values(definition) api = [{ "id": "myMaximumRetries", "stringValue": "2" }, { "id": "myOccurrenceNumber", "stringValue": "3" }] self.assertEqual(actual, api)
def test_df_to_api_with_parameter_values_missing(self): definition = self.load_def("""{"objects": [ { "id" : "S3ToS3Copy", "type" : "CopyActivity", "schedule" : { "ref" : "CopyPeriod" }, "input" : { "ref" : "InputData" }, "output" : { "ref" : "OutputData" } } ]}""") try: values = translator.definition_to_parameter_values(definition) error = "Absence of parameter values should not display anything" self.assertIsNone(values, error) except PipelineDefinitionError: self.fail("No exception should be thrown in absence of values")
def run_aws_data_pipeline(self): self.s3_ops(self.session) self.push_secrets(creds_or_file=self.app_args['connection_file'] ) # TODO: fix privileges to get creds in dev env # DataPipeline ops import awscli.customizations.datapipeline.translator as trans client = self.session.client('datapipeline') pipe_id = self.create_date_pipeline(client) definition_file = eu.LOCAL_APP_FOLDER + 'core/definition.json' # see syntax in datapipeline-dg.pdf p285 # to add in there: /*"AdditionalMasterSecurityGroups": "#{}", /* To add later to match EMR mode */ definition = json.load( open(definition_file, 'r')) # Note: Data Pipeline doesn't support emr-6.0.0 yet. pipelineObjects = trans.definition_to_api_objects(definition) parameterObjects = trans.definition_to_api_parameters(definition) parameterValues = trans.definition_to_parameter_values(definition) parameterValues = self.update_params(parameterValues) logger.info('Filled pipeline with data from ' + definition_file) response = client.put_pipeline_definition( pipelineId=pipe_id, pipelineObjects=pipelineObjects, parameterObjects=parameterObjects, parameterValues=parameterValues) logger.info('put_pipeline_definition response: ' + str(response)) response = client.activate_pipeline( pipelineId=pipe_id, parameterValues= parameterValues, # optional. If set, need to specify all params as per json. # startTimestamp=datetime(2018, 12, 1) # optional ) logger.info('activate_pipeline response: ' + str(response)) logger.info('Activated pipeline ' + pipe_id)