def _test(self, artman_yaml, artifact_name, expected_legacy_config): artifact_config = loader.load_artifact_config(os.path.join( self.TESTDATA, artman_yaml), artifact_name, '/tmp/input') actual_legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, '/tmp/input', '/tmp/output') with io.open(os.path.join( self.TESTDATA, expected_legacy_config), 'r') as yaml_file: expected_legacy_config_dict = yaml.load(yaml_file) self.assertDictEqual( expected_legacy_config_dict, actual_legacy_config_dict)
def _test(self, artman_yaml, artifact_name, expected_legacy_config): artifact_config = loader.load_artifact_config( os.path.join(self.TESTDATA, artman_yaml), artifact_name, '/tmp/input') actual_legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, '/tmp/input', '/tmp/output') with io.open(os.path.join(self.TESTDATA, expected_legacy_config), 'r') as yaml_file: expected_legacy_config_dict = yaml.load(yaml_file) self.assertDictEqual(expected_legacy_config_dict, actual_legacy_config_dict)
def _test(self, artman_yaml, artifact_name, expected_legacy_config): artifact_config = loader.load_artifact_config(os.path.join( self.TESTDATA, artman_yaml), artifact_name) actual_legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, '/tmp/input', '/tmp/output') with io.open(os.path.join( self.TESTDATA, expected_legacy_config), 'r') as yaml_file: expected_legacy_config_dict = yaml.load(yaml_file) self.assertDictEqual( expected_legacy_config_dict, actual_legacy_config_dict, 'Actual yaml is:\n{}\nExpected yaml is:\n{}\n'.format( yaml.dump(actual_legacy_config_dict, default_flow_style=False), yaml.dump(expected_legacy_config_dict, default_flow_style=False)))
def normalize_flags(flags, user_config): """Combine the argparse flags and user configuration together. Args: flags (argparse.Namespace): The flags parsed from sys.argv user_config (dict): The user configuration taken from ~/.artman/config.yaml. Returns: tuple (str, dict): 2-tuple containing: - pipeline name - pipeline arguments """ if flags.root_dir: flags.root_dir = os.path.abspath(flags.root_dir) flags.config = os.path.join(flags.root_dir, flags.config) else: flags.root_dir = os.getcwd() flags.config = os.path.abspath(flags.config) root_dir = flags.root_dir flags.output_dir = os.path.abspath(flags.output_dir) pipeline_args = {} # Determine logging verbosity and then set up logging. verbosity = INFO if getattr(flags, 'verbosity', None): verbosity = getattr(flags, 'verbosity') setup_logging(verbosity) # Save local paths, if applicable. # This allows the user to override the path to api-client-staging or # toolkit on his or her machine. pipeline_args['root_dir'] = root_dir pipeline_args['toolkit_path'] = user_config.local.toolkit pipeline_args['generator_args'] = flags.generator_args artman_config_path = flags.config if not os.path.isfile(artman_config_path): logger.error( 'Artman config file `%s` doesn\'t exist.' % artman_config_path) sys.exit(96) try: artifact_config = loader.load_artifact_config( artman_config_path, flags.artifact_name, flags.aspect) except ValueError as ve: logger.error('Artifact config loading failed with `%s`' % ve) sys.exit(96) legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, root_dir, flags.output_dir) logger.debug('Below is the legacy config after conversion:\n%s' % pprint.pformat(legacy_config_dict)) language = Artifact.Language.Name( artifact_config.language).lower() # Set the pipeline artifact_type = artifact_config.type pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type) pipeline_args['aspect'] = Artifact.Aspect.Name(artifact_config.aspect) if artifact_type == Artifact.GAPIC_ONLY: pipeline_name = 'GapicOnlyClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC: pipeline_name = 'GapicClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.DISCOGAPIC: pipeline_name = 'DiscoGapicClientPipeline' pipeline_args['language'] = language pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.GRPC: pipeline_name = 'GrpcClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC_CONFIG: pipeline_name = 'GapicConfigPipeline' elif artifact_type == Artifact.DISCOGAPIC_CONFIG: pipeline_name = 'DiscoGapicConfigPipeline' pipeline_args['discovery_doc'] = artifact_config.discovery_doc if os.path.abspath(flags.output_dir) != os.path.abspath(DEFAULT_OUTPUT_DIR): logger.warning("`output_dir` is ignored in DiscoGapicConfigGen. " + "Yamls are saved at the path specified by `gapic_yaml`.") pipeline_args['output_dir'] = tempfile.mkdtemp() elif artifact_type == Artifact.PROTOBUF: pipeline_name = 'ProtoClientPipeline' pipeline_args['language'] = language else: raise ValueError('Unrecognized artifact.') # Parse out the full configuration. config_args = config_util.load_config_spec(legacy_config_dict, language) config_args.update(pipeline_args) pipeline_args = config_args # Print out the final arguments to stdout, to help the user with # possible debugging. pipeline_args_repr = yaml.dump( pipeline_args, block_seq_indent=2, default_flow_style=False, indent=2, ) logger.info('Final args:') for line in pipeline_args_repr.split('\n'): if 'token' in line: index = line.index(':') line = line[:index + 2] + '<< REDACTED >>' logger.info(' {0}'.format(line)) # Return the final arguments. return pipeline_name, pipeline_args
def normalize_flags(flags, user_config): """Combine the argparse flags and user configuration together. Args: flags (argparse.Namespace): The flags parsed from sys.argv user_config (dict): The user configuration taken from ~/.artman/config.yaml. Returns: tuple (str, dict): 2-tuple containing: - pipeline name - pipeline arguments """ if flags.root_dir: flags.root_dir = os.path.abspath(flags.root_dir) flags.config = os.path.join(flags.root_dir, flags.config) else: flags.root_dir = os.getcwd() flags.config = os.path.abspath(flags.config) root_dir = flags.root_dir flags.output_dir = os.path.abspath(flags.output_dir) pipeline_args = {} # Determine logging verbosity and then set up logging. verbosity = INFO if getattr(flags, 'verbosity', None): verbosity = getattr(flags, 'verbosity') setup_logging(verbosity) # Save local paths, if applicable. # This allows the user to override the path to api-client-staging or # toolkit on his or her machine. pipeline_args['root_dir'] = root_dir pipeline_args['toolkit'] = user_config.local.toolkit if flags.subcommand == 'publish' and flags.local_repo_dir: if not flags.dry_run: logger.error('`--dry-run` flag must be passed when ' '`--local-repo-dir` is specified') sys.exit(96) flags.local_repo_dir = os.path.abspath(flags.local_repo_dir) pipeline_args['local_repo_dir'] = flags.local_repo_dir artman_config_path = flags.config if not os.path.isfile(artman_config_path): logger.error( 'Artman config file `%s` doesn\'t exist.' % artman_config_path) sys.exit(96) try: artifact_config = loader.load_artifact_config( artman_config_path, flags.artifact_name) except ValueError as ve: logger.error('Artifact config loading failed with `%s`' % ve) sys.exit(96) # If we were given just an API or BATCH, then expand it into the --config # syntax. shared_config_name = 'common.yaml' if artifact_config.language in (Artifact.RUBY, Artifact.NODEJS,): shared_config_name = 'doc.yaml' legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, root_dir, flags.output_dir) logger.debug('Below is the legacy config after conversion:\n%s' % pprint.pformat(legacy_config_dict)) tmp_legacy_config_yaml = '%s.tmp' % artman_config_path with io.open(tmp_legacy_config_yaml, 'w') as outfile: yaml.dump(legacy_config_dict, outfile, default_flow_style=False) config = ','.join([ '{artman_config_path}', '{googleapis}/gapic/lang/{shared_config_name}', ]).format( artman_config_path=tmp_legacy_config_yaml, googleapis=root_dir, shared_config_name=shared_config_name, ) language = Artifact.Language.Name( artifact_config.language).lower() # Set the pipeline artifact_type = artifact_config.type pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type) if artifact_type == Artifact.GAPIC_ONLY: pipeline_name = 'GapicOnlyClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC: pipeline_name = 'GapicClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.DISCOGAPIC: pipeline_name = 'DiscoGapicClientPipeline' pipeline_args['language'] = language pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.GRPC: pipeline_name = 'GrpcClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC_CONFIG: pipeline_name = 'GapicConfigPipeline' elif artifact_type == Artifact.DISCOGAPIC_CONFIG: pipeline_name = 'DiscoGapicConfigPipeline' pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.PROTOBUF: pipeline_name = 'ProtoClientPipeline' pipeline_args['language'] = language else: raise ValueError('Unrecognized artifact.') # Parse out the full configuration. # Note: the var replacement is still needed because they are still being # used in some shared/common config yamls. config_sections = ['common'] for config_spec in config.split(','): config_args = config_util.load_config_spec( config_spec=config_spec, config_sections=config_sections, repl_vars={ 'GOOGLEAPIS': root_dir, 'DISCOVERY_ARTIFACT_MANAGER': root_dir, 'TOOLKIT': user_config.local.toolkit }, language=language, ) pipeline_args.update(config_args) # Setup publishing related config if needed. if flags.subcommand == 'generate': pipeline_args['publish'] = 'noop' elif flags.subcommand == 'publish': publishing_config = _get_publishing_config(artifact_config, flags.target) if publishing_config.type == Artifact.PublishTarget.GITHUB: if flags.dry_run: pipeline_args['publish'] = 'local' else: pipeline_args['publish'] = 'github' pipeline_args['github'] = support.parse_github_credentials( argv_flags=flags, github_config=user_config.github) repos = pipeline_args.pop('git_repos') pipeline_args['git_repo'] = support.select_git_repo( repos, publishing_config.name) else: logger.error( 'Publishing type `%s` is not supported yet.' % Artifact.PublishTarget.Type.Name(publishing_config.type)) sys.exit(96) # Print out the final arguments to stdout, to help the user with # possible debugging. pipeline_args_repr = yaml.dump( pipeline_args, block_seq_indent=2, default_flow_style=False, indent=2, ) logger.info('Final args:') for line in pipeline_args_repr.split('\n'): if 'token' in line: index = line.index(':') line = line[:index + 2] + '<< REDACTED >>' logger.info(' {0}'.format(line)) # Clean up the tmp legacy artman config. os.remove(tmp_legacy_config_yaml) # Return the final arguments. return pipeline_name, pipeline_args
def normalize_flags(flags, user_config): """Combine the argparse flags and user configuration together. Args: flags (argparse.Namespace): The flags parsed from sys.argv user_config (dict): The user configuration taken from ~/.artman/config.yaml. Returns: tuple (str, dict): 2-tuple containing: - pipeline name - pipeline arguments """ if flags.root_dir: flags.root_dir = os.path.abspath(flags.root_dir) flags.config = os.path.join(flags.root_dir, flags.config) else: flags.root_dir = os.getcwd() flags.config = os.path.abspath(flags.config) root_dir = flags.root_dir flags.output_dir = os.path.abspath(flags.output_dir) pipeline_args = {} # Determine logging verbosity and then set up logging. verbosity = INFO if getattr(flags, 'verbosity', None): verbosity = getattr(flags, 'verbosity') setup_logging(verbosity) # Save local paths, if applicable. # This allows the user to override the path to api-client-staging or # toolkit on his or her machine. pipeline_args['root_dir'] = root_dir pipeline_args['toolkit_path'] = user_config.local.toolkit if flags.subcommand == 'publish' and flags.local_repo_dir: if not flags.dry_run: logger.error('`--dry-run` flag must be passed when ' '`--local-repo-dir` is specified') sys.exit(96) flags.local_repo_dir = os.path.abspath(flags.local_repo_dir) pipeline_args['local_repo_dir'] = flags.local_repo_dir artman_config_path = flags.config if not os.path.isfile(artman_config_path): logger.error( 'Artman config file `%s` doesn\'t exist.' % artman_config_path) sys.exit(96) try: artifact_config = loader.load_artifact_config( artman_config_path, flags.artifact_name, flags.aspect) except ValueError as ve: logger.error('Artifact config loading failed with `%s`' % ve) sys.exit(96) legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, root_dir, flags.output_dir) logger.debug('Below is the legacy config after conversion:\n%s' % pprint.pformat(legacy_config_dict)) language = Artifact.Language.Name( artifact_config.language).lower() # Set the pipeline artifact_type = artifact_config.type pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type) pipeline_args['aspect'] = Artifact.Aspect.Name(artifact_config.aspect) if artifact_type == Artifact.GAPIC_ONLY: pipeline_name = 'GapicOnlyClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC: pipeline_name = 'GapicClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.DISCOGAPIC: pipeline_name = 'DiscoGapicClientPipeline' pipeline_args['language'] = language pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.GRPC: pipeline_name = 'GrpcClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC_CONFIG: pipeline_name = 'GapicConfigPipeline' elif artifact_type == Artifact.DISCOGAPIC_CONFIG: pipeline_name = 'DiscoGapicConfigPipeline' pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.PROTOBUF: pipeline_name = 'ProtoClientPipeline' pipeline_args['language'] = language else: raise ValueError('Unrecognized artifact.') # Parse out the full configuration. config_args = config_util.load_config_spec(legacy_config_dict, language) pipeline_args.update(config_args) # Setup publishing related config if needed. if flags.subcommand == 'generate': pipeline_args['publish'] = 'noop' elif flags.subcommand == 'publish': publishing_config = _get_publishing_config(artifact_config, flags.target) if publishing_config.type == Artifact.PublishTarget.GITHUB: if flags.dry_run: pipeline_args['publish'] = 'local' else: pipeline_args['publish'] = 'github' pipeline_args['github'] = support.parse_github_credentials( argv_flags=flags, github_config=user_config.github) repos = pipeline_args.pop('git_repos') pipeline_args['git_repo'] = support.select_git_repo( repos, publishing_config.name) else: logger.error( 'Publishing type `%s` is not supported yet.' % Artifact.PublishTarget.Type.Name(publishing_config.type)) sys.exit(96) # Print out the final arguments to stdout, to help the user with # possible debugging. pipeline_args_repr = yaml.dump( pipeline_args, block_seq_indent=2, default_flow_style=False, indent=2, ) logger.info('Final args:') for line in pipeline_args_repr.split('\n'): if 'token' in line: index = line.index(':') line = line[:index + 2] + '<< REDACTED >>' logger.info(' {0}'.format(line)) # Return the final arguments. return pipeline_name, pipeline_args
def normalize_flags(flags, user_config): """Combine the argparse flags and user configuration together. Args: flags (argparse.Namespace): The flags parsed from sys.argv user_config (dict): The user configuration taken from ~/.artman/config.yaml. Returns: tuple (str, dict): 2-tuple containing: - pipeline name - pipeline arguments """ if flags.root_dir: flags.root_dir = os.path.abspath(flags.root_dir) flags.config = os.path.join(flags.root_dir, flags.config) else: flags.root_dir = os.getcwd() flags.config = os.path.abspath(flags.config) root_dir = flags.root_dir flags.output_dir = os.path.abspath(flags.output_dir) pipeline_args = {} # Determine logging verbosity and then set up logging. verbosity = INFO if getattr(flags, 'verbosity', None): verbosity = getattr(flags, 'verbosity') setup_logging(verbosity) # Save local paths, if applicable. # This allows the user to override the path to api-client-staging or # toolkit on his or her machine. pipeline_args['root_dir'] = root_dir # TODO two args reference the same concept - clean this up pipeline_args['toolkit'] = user_config.local.toolkit pipeline_args['toolkit_path'] = user_config.local.toolkit if flags.subcommand == 'publish' and flags.local_repo_dir: if not flags.dry_run: logger.error('`--dry-run` flag must be passed when ' '`--local-repo-dir` is specified') sys.exit(96) flags.local_repo_dir = os.path.abspath(flags.local_repo_dir) pipeline_args['local_repo_dir'] = flags.local_repo_dir artman_config_path = flags.config if not os.path.isfile(artman_config_path): logger.error('Artman config file `%s` doesn\'t exist.' % artman_config_path) sys.exit(96) try: artifact_config = loader.load_artifact_config(artman_config_path, flags.artifact_name) except ValueError as ve: logger.error('Artifact config loading failed with `%s`' % ve) sys.exit(96) legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, root_dir, flags.output_dir) logger.debug('Below is the legacy config after conversion:\n%s' % pprint.pformat(legacy_config_dict)) language = Artifact.Language.Name(artifact_config.language).lower() # Set the pipeline artifact_type = artifact_config.type pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type) if artifact_type == Artifact.GAPIC_ONLY: pipeline_name = 'GapicOnlyClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC: pipeline_name = 'GapicClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.DISCOGAPIC: pipeline_name = 'DiscoGapicClientPipeline' pipeline_args['language'] = language pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.GRPC: pipeline_name = 'GrpcClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC_CONFIG: pipeline_name = 'GapicConfigPipeline' elif artifact_type == Artifact.DISCOGAPIC_CONFIG: pipeline_name = 'DiscoGapicConfigPipeline' pipeline_args['discovery_doc'] = artifact_config.discovery_doc elif artifact_type == Artifact.PROTOBUF: pipeline_name = 'ProtoClientPipeline' pipeline_args['language'] = language else: raise ValueError('Unrecognized artifact.') # Parse out the full configuration. config_args = config_util.load_config_spec(legacy_config_dict, language) pipeline_args.update(config_args) # Setup publishing related config if needed. if flags.subcommand == 'generate': pipeline_args['publish'] = 'noop' elif flags.subcommand == 'publish': publishing_config = _get_publishing_config(artifact_config, flags.target) if publishing_config.type == Artifact.PublishTarget.GITHUB: if flags.dry_run: pipeline_args['publish'] = 'local' else: pipeline_args['publish'] = 'github' pipeline_args['github'] = support.parse_github_credentials( argv_flags=flags, github_config=user_config.github) repos = pipeline_args.pop('git_repos') pipeline_args['git_repo'] = support.select_git_repo( repos, publishing_config.name) else: logger.error( 'Publishing type `%s` is not supported yet.' % Artifact.PublishTarget.Type.Name(publishing_config.type)) sys.exit(96) # Print out the final arguments to stdout, to help the user with # possible debugging. pipeline_args_repr = yaml.dump( pipeline_args, block_seq_indent=2, default_flow_style=False, indent=2, ) logger.info('Final args:') for line in pipeline_args_repr.split('\n'): if 'token' in line: index = line.index(':') line = line[:index + 2] + '<< REDACTED >>' logger.info(' {0}'.format(line)) # Return the final arguments. return pipeline_name, pipeline_args
def normalize_flags(flags, user_config): """Combine the argparse flags and user configuration together. Args: flags (argparse.Namespace): The flags parsed from sys.argv user_config (dict): The user configuration taken from ~/.artman/config.yaml. Returns: tuple (str, dict): 2-tuple containing: - pipeline name - pipeline arguments """ flags.input_dir = os.path.abspath(flags.input_dir) flags.output_dir = os.path.abspath(flags.output_dir) flags.config = os.path.abspath(flags.config) pipeline_args = {} # Determine logging verbosity and then set up logging. verbosity = support.resolve('verbosity', user_config, flags, default=INFO) setup_logging(verbosity) # Save local paths, if applicable. # This allows the user to override the path to api-client-staging or # toolkit on his or her machine. pipeline_args['local_paths'] = support.parse_local_paths( user_config, flags.input_dir) artman_config_path = flags.config if not os.path.isfile(artman_config_path): logger.error( 'Artman config file `%s` doesn\'t exist.' % artman_config_path) sys.exit(96) try: artifact_config = loader.load_artifact_config( artman_config_path, flags.artifact_name, flags.input_dir) except ValueError as ve: logger.error('Artifact config loading failed with `%s`' % ve) sys.exit(96) # If we were given just an API or BATCH, then expand it into the --config # syntax. shared_config_name = 'common.yaml' if artifact_config.language in (Artifact.RUBY, Artifact.NODEJS,): shared_config_name = 'doc.yaml' legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, flags.input_dir, flags.output_dir) logger.debug('Below is the legacy config after conversion:\n%s' % pprint.pformat(legacy_config_dict)) tmp_legacy_config_yaml = '%s.tmp' % artman_config_path with io.open(tmp_legacy_config_yaml, 'w') as outfile: yaml.dump(legacy_config_dict, outfile, default_flow_style=False) googleapis = os.path.realpath( os.path.expanduser( pipeline_args['local_paths']['googleapis'], )) config = ','.join([ '{artman_config_path}', '{googleapis}/gapic/lang/{shared_config_name}', ]).format( artman_config_path=tmp_legacy_config_yaml, googleapis=googleapis, shared_config_name=shared_config_name, ) language = Artifact.Language.Name( artifact_config.language).lower() # Set the pipeline as well as package_type and packaging artifact_type = artifact_config.type if artifact_type in (Artifact.GAPIC, Artifact.GAPIC_ONLY): pipeline_name = 'GapicClientPipeline' pipeline_args['language'] = language elif artifact_type in (Artifact.GRPC, Artifact.GRPC_COMMON): pipeline_name = 'GrpcClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC_CONFIG: pipeline_name = 'GapicConfigPipeline' else: raise ValueError('Unrecognized artifact.') # Parse out the full configuration. # Note: the var replacement is still needed because they are still being # used in some shared/common config yamls. config_sections = ['common'] for config_spec in config.split(','): config_args = config_util.load_config_spec( config_spec=config_spec, config_sections=config_sections, repl_vars={ k.upper(): v for k, v in pipeline_args['local_paths'].items() }, language=language, ) pipeline_args.update(config_args) # Setup publishing related config if needed. if flags.subcommand == 'generate': pipeline_args['publish'] = 'noop' elif flags.subcommand == 'publish': publishing_config = _get_publishing_config(artifact_config, flags.target) if publishing_config.type == Artifact.PublishTarget.GITHUB: pipeline_args['publish'] = 'local' if flags.dry_run else 'github' pipeline_args['github'] = support.parse_github_credentials( argv_flags=flags, config=user_config.get('github', {}), ) repos = pipeline_args.pop('git_repos') pipeline_args['git_repo'] = support.select_git_repo( repos, publishing_config.name) else: logger.error( 'Publishing type `%s` is not supported yet.' % Artifact.PublishTarget.Type.Name(publishing_config.type)) sys.exit(96) # Print out the final arguments to stdout, to help the user with # possible debugging. pipeline_args_repr = yaml.dump( pipeline_args, block_seq_indent=2, default_flow_style=False, indent=2, ) logger.info('Final args:') for line in pipeline_args_repr.split('\n'): if 'token' in line: index = line.index(':') line = line[:index + 2] + '<< REDACTED >>' logger.info(' {0}'.format(line)) # Clean up the tmp legacy artman config. os.remove(tmp_legacy_config_yaml) # Return the final arguments. return pipeline_name, pipeline_args
def normalize_flags(flags, user_config): """Combine the argparse flags and user configuration together. Args: flags (argparse.Namespace): The flags parsed from sys.argv user_config (dict): The user configuration taken from ~/.artman/config.yaml. Returns: tuple (str, dict): 2-tuple containing: - pipeline name - pipeline arguments """ if flags.input_dir: flags.input_dir = os.path.abspath(flags.input_dir) flags.output_dir = os.path.abspath(flags.output_dir) flags.config = os.path.abspath(flags.config) pipeline_args = {} # Determine logging verbosity and then set up logging. verbosity = support.resolve('verbosity', user_config, flags, default=INFO) setup_logging(verbosity) # Save local paths, if applicable. # This allows the user to override the path to api-client-staging or # toolkit on his or her machine. pipeline_args['local_paths'] = support.parse_local_paths( user_config, flags.input_dir) # Save the input directory back to flags if it was not explicitly set. if not flags.input_dir: flags.input_dir = pipeline_args['local_paths']['googleapis'] artman_config_path = flags.config if not os.path.isfile(artman_config_path): logger.error( 'Artman config file `%s` doesn\'t exist.' % artman_config_path) sys.exit(96) try: artifact_config = loader.load_artifact_config( artman_config_path, flags.artifact_name, flags.input_dir) except ValueError as ve: logger.error('Artifact config loading failed with `%s`' % ve) sys.exit(96) # If we were given just an API or BATCH, then expand it into the --config # syntax. shared_config_name = 'common.yaml' if artifact_config.language in (Artifact.RUBY, Artifact.NODEJS,): shared_config_name = 'doc.yaml' legacy_config_dict = converter.convert_to_legacy_config_dict( artifact_config, flags.input_dir, flags.output_dir) logger.debug('Below is the legacy config after conversion:\n%s' % pprint.pformat(legacy_config_dict)) tmp_legacy_config_yaml = '%s.tmp' % artman_config_path with io.open(tmp_legacy_config_yaml, 'w') as outfile: yaml.dump(legacy_config_dict, outfile, default_flow_style=False) googleapis = os.path.realpath( os.path.expanduser( pipeline_args['local_paths']['googleapis'], )) config = ','.join([ '{artman_config_path}', '{googleapis}/gapic/lang/{shared_config_name}', ]).format( artman_config_path=tmp_legacy_config_yaml, googleapis=googleapis, shared_config_name=shared_config_name, ) language = Artifact.Language.Name( artifact_config.language).lower() # Set the pipeline as well as package_type and packaging artifact_type = artifact_config.type if artifact_type in (Artifact.GAPIC, Artifact.GAPIC_ONLY): pipeline_name = 'GapicClientPipeline' pipeline_args['language'] = language elif artifact_type in (Artifact.GRPC, Artifact.GRPC_COMMON): pipeline_name = 'GrpcClientPipeline' pipeline_args['language'] = language elif artifact_type == Artifact.GAPIC_CONFIG: pipeline_name = 'GapicConfigPipeline' else: raise ValueError('Unrecognized artifact.') # Parse out the full configuration. # Note: the var replacement is still needed because they are still being # used in some shared/common config yamls. config_sections = ['common'] for config_spec in config.split(','): config_args = config_util.load_config_spec( config_spec=config_spec, config_sections=config_sections, repl_vars={ k.upper(): v for k, v in pipeline_args['local_paths'].items() }, language=language, ) pipeline_args.update(config_args) # Setup publishing related config if needed. if flags.subcommand == 'generate': pipeline_args['publish'] = 'noop' elif flags.subcommand == 'publish': publishing_config = _get_publishing_config(artifact_config, flags.target) if publishing_config.type == Artifact.PublishTarget.GITHUB: pipeline_args['publish'] = 'local' if flags.dry_run else 'github' pipeline_args['github'] = support.parse_github_credentials( argv_flags=flags, config=user_config.get('github', {}), ) repos = pipeline_args.pop('git_repos') pipeline_args['git_repo'] = support.select_git_repo( repos, publishing_config.name) else: logger.error( 'Publishing type `%s` is not supported yet.' % Artifact.PublishTarget.Type.Name(publishing_config.type)) sys.exit(96) # Print out the final arguments to stdout, to help the user with # possible debugging. pipeline_args_repr = yaml.dump( pipeline_args, block_seq_indent=2, default_flow_style=False, indent=2, ) logger.info('Final args:') for line in pipeline_args_repr.split('\n'): if 'token' in line: index = line.index(':') line = line[:index + 2] + '<< REDACTED >>' logger.info(' {0}'.format(line)) # Clean up the tmp legacy artman config. os.remove(tmp_legacy_config_yaml) # Return the final arguments. return pipeline_name, pipeline_args