def _pull_and_execute_tasks(task_client, queue_name):
pull_task_response = _pull_task(task_client, queue_name)
tasks = pull_task_response.get('tasks', [])
if not tasks:
# Sleep for 30 seconds if there is no tasks returned.
logger.debug('There is no pending task. Sleep for 10 seconds.')
time.sleep(10)
for task in tasks:
task_id, tmp_root, artman_user_config, log_file_path = _prepare_dir()
log_file_handler = None
try:
log_file_handler = _setup_logger(log_file_path)
logger.info('Starting to execute task %s' % task)
if int(task['taskStatus']['attemptDispatchCount']) > MAX_ATTEMPTS:
logger.info('Delete task which exceeds max attempts.')
_delete_task(task_client, task)
continue
_execute_task(artman_user_config, task)
_ack_task(task_client, task)
logger.info('Task execution finished')
except Exception as e:
logger.error('\n'.join(traceback.format_tb(sys.exc_info()[2])))
_cancel_task_lease(task_client, task)
finally:
logger.info('Cleanup tmp directory %s' % tmp_root)
# Use task id as log name
_write_to_cloud_logging(task_id, log_file_path)
_cleanup(tmp_root, log_file_handler)
def execute(self,
src_proto_path,
import_proto_path,
output_dir,
api_name,
api_version,
organization_name,
toolkit_path,
desc_proto_path=None,
excluded_proto_path=[]):
desc_proto_path = desc_proto_path or []
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_path,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_path
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(api_name, api_version,
organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
# DescGen don't use _group_by_dirname right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
self.exec_command(
['protoc'] + protoc_utils.protoc_header_params(
header_proto_path, toolkit_path) +
protoc_utils.protoc_desc_params(output_dir, desc_out_file) +
desc_protos)
return os.path.join(output_dir, desc_out_file)
def _pull_and_execute_tasks(task_client, queue_name):
pull_task_response = _pull_task(task_client, queue_name)
tasks = pull_task_response.get('tasks', [])
if not tasks:
# Sleep for 30 seconds if there is no tasks returned.
logger.debug('There is no pending task. Sleep for 10 seconds.')
time.sleep(10)
for task in tasks:
task_id, tmp_root, artman_user_config, log_file_path = _prepare_dir()
log_file_handler = None
try:
log_file_handler = _setup_logger(log_file_path)
logger.info('Starting to execute task %s' % task)
if int(task['taskStatus']['attemptDispatchCount']) > MAX_ATTEMPTS:
logger.info('Delete task which exceeds max attempts.')
_delete_task(task_client, task)
continue
_execute_task(artman_user_config, task)
_ack_task(task_client, task)
logger.info('Task execution finished')
except Exception as e:
logger.error('\n'.join(traceback.format_tb(sys.exc_info()[2])))
_cancel_task_lease(task_client, task)
finally:
logger.info('Cleanup tmp directory %s' % tmp_root)
# Use task id as log name
_write_to_cloud_logging(task_id, log_file_path)
_cleanup(tmp_root, log_file_handler)
def _find_protobuf_path(toolkit_path):
"""Fetch and locate protobuf source"""
global _protobuf_path
if not _protobuf_path:
logger.debug('Searching for latest protobuf source')
_protobuf_path = task_utils.get_java_tool_path(toolkit_path, 'protobufJavaDir')
return _protobuf_path
def _find_protobuf_path(toolkit_path):
"""Fetch and locate protobuf source"""
global _protobuf_path
if not _protobuf_path:
logger.debug('Searching for latest protobuf source')
_protobuf_path = task_utils.get_java_tool_path(toolkit_path, 'protobufJavaDir')
return _protobuf_path
def execute(self,
src_proto_path,
import_proto_path,
output_dir,
api_name,
api_version,
organization_name,
toolkit_path,
root_dir,
excluded_proto_path=[],
proto_deps=[]):
desc_proto_paths = []
for dep in proto_deps:
if 'proto_path' in dep and dep['proto_path']:
desc_proto_paths.append(
os.path.join(root_dir, dep['proto_path']))
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_paths,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_paths
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(api_name, api_version,
organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
# DescGen doesn't use group protos by package right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
self.exec_command(
['protoc'] + protoc_utils.protoc_header_params(
header_proto_path, toolkit_path) +
protoc_utils.protoc_desc_params(output_dir, desc_out_file) +
desc_protos)
return os.path.join(output_dir, desc_out_file)
def execute(self, src_proto_path, import_proto_path, output_dir,
api_name, api_version, organization_name, toolkit_path,
root_dir, excluded_proto_path=[], proto_deps=[]):
desc_proto_paths = []
for dep in proto_deps:
if 'proto_path' in dep and dep['proto_path']:
desc_proto_paths.append(os.path.join(root_dir, dep['proto_path']))
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_paths,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_paths
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(
api_name, api_version, organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
# DescGen don't use _group_by_dirname right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
self.exec_command(
['protoc'] +
protoc_utils.protoc_header_params(header_proto_path, toolkit_path) +
protoc_utils.protoc_desc_params(output_dir, desc_out_file) +
desc_protos)
return os.path.join(output_dir, desc_out_file)
def _run_artman_in_docker(flags):
"""Executes artman command.
Args:
root_dir: The input directory that will be mounted to artman docker
container as local googleapis directory.
Returns:
The output directory with artman-generated files.
"""
ARTMAN_CONTAINER_NAME = 'artman-docker'
root_dir = flags.root_dir
output_dir = flags.output_dir
artman_config_dirname = os.path.dirname(flags.config)
docker_image = flags.image
inner_artman_cmd_str = ' '.join(sys.argv[1:])
# Because artman now supports setting root dir in either command line or
# user config, make sure `--root-dir` flag gets explicitly passed to the
# artman command running inside Artman Docker container.
if '--root-dir' not in inner_artman_cmd_str:
inner_artman_cmd_str = '--root-dir %s %s' % (
root_dir, inner_artman_cmd_str)
# TODO(ethanbao): Such folder to folder mounting won't work on windows.
base_cmd = [
'docker', 'run', '--name', ARTMAN_CONTAINER_NAME, '--rm', '-i', '-t',
'-e', 'HOST_USER_ID=%s' % os.getuid(),
'-e', 'HOST_GROUP_ID=%s' % os.getgid(),
'-e', '%s=True' % RUNNING_IN_ARTMAN_DOCKER_TOKEN,
'-v', '%s:%s' % (root_dir, root_dir),
'-v', '%s:%s' % (output_dir, output_dir),
'-v', '%s:%s' % (artman_config_dirname, artman_config_dirname),
'-w', root_dir
]
base_cmd.extend([docker_image, '/bin/bash', '-c'])
inner_artman_debug_cmd_str = inner_artman_cmd_str
# Because debug_cmd is run inside the Docker image, we want to
# make sure --local is set
if '--local' not in inner_artman_debug_cmd_str:
inner_artman_debug_cmd_str = '--local %s' % inner_artman_debug_cmd_str
debug_cmd = list(base_cmd)
debug_cmd.append('"artman %s; bash"' % inner_artman_debug_cmd_str)
cmd = base_cmd
cmd.append('artman --local %s' % (inner_artman_cmd_str))
try:
output = subprocess.check_output(cmd)
logger.info(output.decode('utf8'))
return output_dir
except subprocess.CalledProcessError as e:
logger.error(e.output.decode('utf8'))
logger.error(
'Artman execution failed. For additional logging, re-run the '
'command with the "--verbose" flag')
sys.exit(32)
finally:
logger.debug('For further inspection inside docker container, run `%s`'
% ' '.join(debug_cmd))
def _run_artman_in_docker(flags):
"""Executes artman command.
Args:
root_dir: The input directory that will be mounted to artman docker
container as local googleapis directory.
Returns:
The output directory with artman-generated files.
"""
ARTMAN_CONTAINER_NAME = 'artman-docker'
root_dir = flags.root_dir
output_dir = flags.output_dir
artman_config_dirname = os.path.dirname(flags.config)
docker_image = flags.image
inner_artman_cmd_str = ' '.join(["'" + arg + "'" for arg in sys.argv[1:]])
# Because artman now supports setting root dir in either command line or
# user config, make sure `--root-dir` flag gets explicitly passed to the
# artman command running inside Artman Docker container.
if '--root-dir' not in inner_artman_cmd_str:
inner_artman_cmd_str = '--root-dir %s %s' % (
root_dir, inner_artman_cmd_str)
# TODO(ethanbao): Such folder to folder mounting won't work on windows.
base_cmd = [
'docker', 'run', '--name', ARTMAN_CONTAINER_NAME, '--rm', '-i', '-t',
'-e', 'HOST_USER_ID=%s' % os.getuid(),
'-e', 'HOST_GROUP_ID=%s' % os.getgid(),
'-e', '%s=True' % RUNNING_IN_ARTMAN_DOCKER_TOKEN,
'-v', '%s:%s' % (root_dir, root_dir),
'-v', '%s:%s' % (output_dir, output_dir),
'-v', '%s:%s' % (artman_config_dirname, artman_config_dirname),
'-w', root_dir
]
base_cmd.extend([docker_image, '/bin/bash', '-c'])
inner_artman_debug_cmd_str = inner_artman_cmd_str
# Because debug_cmd is run inside the Docker image, we want to
# make sure --local is set
if '--local' not in inner_artman_debug_cmd_str:
inner_artman_debug_cmd_str = '--local %s' % inner_artman_debug_cmd_str
debug_cmd = list(base_cmd)
debug_cmd.append('"artman %s; bash"' % inner_artman_debug_cmd_str)
cmd = base_cmd
cmd.append('artman --local %s' % (inner_artman_cmd_str))
try:
output = subprocess.check_output(cmd)
logger.info(output.decode('utf8'))
return output_dir
except subprocess.CalledProcessError as e:
logger.error(e.output.decode('utf8'))
logger.error(
'Artman execution failed. For additional logging, re-run the '
'command with the "--verbose" flag')
sys.exit(32)
finally:
logger.debug('For further inspection inside docker container, run `%s`'
% ' '.join(debug_cmd))
def execute(self, gapic_code_dir, toolkit_path):
logger.debug('Formatting files in %s.' %
os.path.abspath(gapic_code_dir))
path = task_utils.get_java_tool_path(toolkit_path, 'googleJavaFormatJar')
targetFiles = []
for root, dirs, files in os.walk(gapic_code_dir):
for filename in files:
if filename.endswith('.java'):
targetFile = os.path.abspath(os.path.join(root, filename))
targetFiles.append(targetFile)
self.exec_command(
['java', '-jar', path, '--replace'] + targetFiles)
def execute(self, gapic_code_dir):
logger.debug('Formatting files in %s.' %
os.path.abspath(gapic_code_dir))
targetFiles = []
for root, dirs, files in os.walk(gapic_code_dir):
for filename in files:
if filename.endswith('.py'):
targetFile = os.path.abspath(os.path.join(root, filename))
targetFiles.append(targetFile)
# yapf returns code 2 when it formats, so we can't use `check_call`.
exit_code = subprocess.call(['yapf', '-i'] + targetFiles)
if exit_code not in [0, 2]:
raise subprocess.CalledProcessError(exit_code, 'yapf')
def _run_artman_in_docker(flags):
"""Executes artman command.
Args:
input_dir: The input directory that will be mounted to artman docker
container as local googleapis directory.
Returns:
The output directory with artman-generated files.
"""
ARTMAN_CONTAINER_NAME = 'artman-docker'
input_dir = flags.input_dir
output_dir = flags.output_dir
artman_config_dirname = os.path.dirname(flags.config)
user_config = os.path.join(os.path.expanduser('~'), '.artman')
docker_image = flags.image
inner_artman_cmd_str = ' '.join(sys.argv[1:])
# TODO(ethanbao): Such folder to folder mounting won't work on windows.
base_cmd = [
'docker', 'run', '--name', ARTMAN_CONTAINER_NAME, '--rm', '-i', '-t',
'-e',
'HOST_USER_ID=%s' % os.getuid(), '-e',
'HOST_GROUP_ID=%s' % os.getgid(), '-e',
'%s=True' % RUNNING_IN_ARTMAN_DOCKER_TOKEN, '-v',
'%s:%s' % (input_dir, input_dir), '-v',
'%s:%s' % (output_dir, output_dir), '-v',
'%s:%s' % (artman_config_dirname, artman_config_dirname), '-v',
'%s:/home/.artman' % user_config, '-w', input_dir, docker_image,
'/bin/bash', '-c'
]
debug_cmd = list(base_cmd)
debug_cmd.append('"artman2 %s; bash"' % inner_artman_cmd_str)
cmd = base_cmd
cmd.append('artman2 --local %s' % (inner_artman_cmd_str))
try:
output = subprocess.check_output(cmd)
logger.info(output.decode('utf8'))
return output_dir
except subprocess.CalledProcessError as e:
logger.error(e.output.decode('utf8'))
logger.error(
'Artman execution failed. For additional logging, re-run the '
'command with the "--verbose" flag')
raise
finally:
logger.debug(
'For further inspection inside docker container, run `%s`' %
' '.join(debug_cmd))
def _run_artman_in_docker(flags):
"""Executes artman command.
Args:
root_dir: The input directory that will be mounted to artman docker
container as local googleapis directory.
Returns:
The output directory with artman-generated files.
"""
ARTMAN_CONTAINER_NAME = 'artman-docker'
root_dir = flags.root_dir
output_dir = flags.output_dir
artman_config_dirname = os.path.dirname(flags.config)
user_config = os.path.join(os.path.expanduser('~'), '.artman')
docker_image = flags.image
inner_artman_cmd_str = ' '.join(sys.argv[1:])
# TODO(ethanbao): Such folder to folder mounting won't work on windows.
base_cmd = [
'docker', 'run', '--name', ARTMAN_CONTAINER_NAME, '--rm', '-i', '-t',
'-e', 'HOST_USER_ID=%s' % os.getuid(),
'-e', 'HOST_GROUP_ID=%s' % os.getgid(),
'-e', '%s=True' % RUNNING_IN_ARTMAN_DOCKER_TOKEN,
'-v', '%s:%s' % (root_dir, root_dir),
'-v', '%s:%s' % (output_dir, output_dir),
'-v', '%s:%s' % (artman_config_dirname, artman_config_dirname),
'-v', '%s:/home/.artman' % user_config,
'-w', root_dir,
docker_image, '/bin/bash', '-c'
]
debug_cmd = list(base_cmd)
debug_cmd.append('"artman %s; bash"' % inner_artman_cmd_str)
cmd = base_cmd
cmd.append('artman --local %s' % (inner_artman_cmd_str))
try:
output = subprocess.check_output(cmd)
logger.info(output.decode('utf8'))
return output_dir
except subprocess.CalledProcessError as e:
logger.error(e.output.decode('utf8'))
logger.error(
'Artman execution failed. For additional logging, re-run the '
'command with the "--verbose" flag')
raise
finally:
logger.debug('For further inspection inside docker container, run `%s`'
% ' '.join(debug_cmd))
def execute(self, gapic_code_dir):
abs_code_dir = os.path.abspath(gapic_code_dir)
logger.debug('Formatting file using php-cs-fixer in %s.' % abs_code_dir)
subprocess.call(['php-cs-fixer', 'fix',
'--rules=@Symfony,-phpdoc_annotation_without_dot',
gapic_code_dir])
# We require a second call to php-cs-fixer because instances of @type
# have been converted to @var. We cannot disable this conversion in
# the first call without affecting other aspects of the formatting.
subprocess.call(['php-cs-fixer',
'fix',
'--rules={"phpdoc_no_alias_tag" : {"replacements" : '
'{"var" : "type"}}}',
gapic_code_dir])
logger.debug('Formatting file using phpcbf in %s.' % abs_code_dir)
subprocess.call(['phpcbf', '--standard=PSR2', '--no-patch',
gapic_code_dir])
def execute(self,
src_proto_path,
import_proto_path,
output_dir,
api_name,
api_version,
organization_name,
toolkit_path,
root_dir,
excluded_proto_path=[],
proto_deps=[],
language='python'):
desc_proto_paths = []
for dep in proto_deps:
if 'proto_path' in dep and dep['proto_path']:
desc_proto_paths.append(
os.path.join(root_dir, dep['proto_path']))
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_paths,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_paths
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(api_name, api_version,
organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
proto_params = protoc_utils.PROTO_PARAMS_MAP[language]
proto_compiler_command = proto_params.proto_compiler_command
logger.debug(
'Using protoc command: {0}'.format(proto_compiler_command))
# DescGen doesn't use group protos by package right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
(common_resources_includes, common_resources_paths) = \
protoc_utils.protoc_common_resources_params(root_dir)
params = proto_params.proto_compiler_command + \
common_resources_includes + \
protoc_utils.protoc_header_params(header_proto_path, toolkit_path) + \
protoc_utils.protoc_desc_params(output_dir, desc_out_file) + \
common_resources_paths + \
desc_protos
self.exec_command(params)
return os.path.join(output_dir, desc_out_file)
def normalize_flags(flags, user_config):
"""Combine the argparse flags and user configuration together.
Args:
flags (argparse.Namespace): The flags parsed from sys.argv
user_config (dict): The user configuration taken from
~/.artman/config.yaml.
Returns:
tuple (str, dict): 2-tuple containing:
- pipeline name
- pipeline arguments
"""
if flags.root_dir:
flags.root_dir = os.path.abspath(flags.root_dir)
flags.config = os.path.join(flags.root_dir, flags.config)
else:
flags.root_dir = os.getcwd()
flags.config = os.path.abspath(flags.config)
root_dir = flags.root_dir
flags.output_dir = os.path.abspath(flags.output_dir)
pipeline_args = {}
# Determine logging verbosity and then set up logging.
verbosity = INFO
if getattr(flags, 'verbosity', None):
verbosity = getattr(flags, 'verbosity')
setup_logging(verbosity)
# Save local paths, if applicable.
# This allows the user to override the path to api-client-staging or
# toolkit on his or her machine.
pipeline_args['root_dir'] = root_dir
pipeline_args['toolkit'] = user_config.local.toolkit
if flags.subcommand == 'publish' and flags.local_repo_dir:
if not flags.dry_run:
logger.error('`--dry-run` flag must be passed when '
'`--local-repo-dir` is specified')
sys.exit(96)
flags.local_repo_dir = os.path.abspath(flags.local_repo_dir)
pipeline_args['local_repo_dir'] = flags.local_repo_dir
artman_config_path = flags.config
if not os.path.isfile(artman_config_path):
logger.error(
'Artman config file `%s` doesn\'t exist.' % artman_config_path)
sys.exit(96)
try:
artifact_config = loader.load_artifact_config(
artman_config_path, flags.artifact_name)
except ValueError as ve:
logger.error('Artifact config loading failed with `%s`' % ve)
sys.exit(96)
# If we were given just an API or BATCH, then expand it into the --config
# syntax.
shared_config_name = 'common.yaml'
if artifact_config.language in (Artifact.RUBY, Artifact.NODEJS,):
shared_config_name = 'doc.yaml'
legacy_config_dict = converter.convert_to_legacy_config_dict(
artifact_config, root_dir, flags.output_dir)
logger.debug('Below is the legacy config after conversion:\n%s' %
pprint.pformat(legacy_config_dict))
tmp_legacy_config_yaml = '%s.tmp' % artman_config_path
with io.open(tmp_legacy_config_yaml, 'w') as outfile:
yaml.dump(legacy_config_dict, outfile, default_flow_style=False)
config = ','.join([
'{artman_config_path}',
'{googleapis}/gapic/lang/{shared_config_name}',
]).format(
artman_config_path=tmp_legacy_config_yaml,
googleapis=root_dir,
shared_config_name=shared_config_name,
)
language = Artifact.Language.Name(
artifact_config.language).lower()
# Set the pipeline
artifact_type = artifact_config.type
pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type)
if artifact_type == Artifact.GAPIC_ONLY:
pipeline_name = 'GapicOnlyClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC:
pipeline_name = 'GapicClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.DISCOGAPIC:
pipeline_name = 'DiscoGapicClientPipeline'
pipeline_args['language'] = language
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.GRPC:
pipeline_name = 'GrpcClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC_CONFIG:
pipeline_name = 'GapicConfigPipeline'
elif artifact_type == Artifact.DISCOGAPIC_CONFIG:
pipeline_name = 'DiscoGapicConfigPipeline'
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.PROTOBUF:
pipeline_name = 'ProtoClientPipeline'
pipeline_args['language'] = language
else:
raise ValueError('Unrecognized artifact.')
# Parse out the full configuration.
# Note: the var replacement is still needed because they are still being
# used in some shared/common config yamls.
config_sections = ['common']
for config_spec in config.split(','):
config_args = config_util.load_config_spec(
config_spec=config_spec,
config_sections=config_sections,
repl_vars={
'GOOGLEAPIS': root_dir,
'DISCOVERY_ARTIFACT_MANAGER': root_dir,
'TOOLKIT': user_config.local.toolkit
},
language=language, )
pipeline_args.update(config_args)
# Setup publishing related config if needed.
if flags.subcommand == 'generate':
pipeline_args['publish'] = 'noop'
elif flags.subcommand == 'publish':
publishing_config = _get_publishing_config(artifact_config,
flags.target)
if publishing_config.type == Artifact.PublishTarget.GITHUB:
if flags.dry_run:
pipeline_args['publish'] = 'local'
else:
pipeline_args['publish'] = 'github'
pipeline_args['github'] = support.parse_github_credentials(
argv_flags=flags,
github_config=user_config.github)
repos = pipeline_args.pop('git_repos')
pipeline_args['git_repo'] = support.select_git_repo(
repos, publishing_config.name)
else:
logger.error(
'Publishing type `%s` is not supported yet.' %
Artifact.PublishTarget.Type.Name(publishing_config.type))
sys.exit(96)
# Print out the final arguments to stdout, to help the user with
# possible debugging.
pipeline_args_repr = yaml.dump(
pipeline_args,
block_seq_indent=2,
default_flow_style=False,
indent=2, )
logger.info('Final args:')
for line in pipeline_args_repr.split('\n'):
if 'token' in line:
index = line.index(':')
line = line[:index + 2] + '<< REDACTED >>'
logger.info(' {0}'.format(line))
# Clean up the tmp legacy artman config.
os.remove(tmp_legacy_config_yaml)
# Return the final arguments.
return pipeline_name, pipeline_args
def normalize_flags(flags, user_config):
"""Combine the argparse flags and user configuration together.
Args:
flags (argparse.Namespace): The flags parsed from sys.argv
user_config (dict): The user configuration taken from
~/.artman/config.yaml.
Returns:
tuple (str, dict): 2-tuple containing:
- pipeline name
- pipeline arguments
"""
if flags.root_dir:
flags.root_dir = os.path.abspath(flags.root_dir)
flags.config = os.path.join(flags.root_dir, flags.config)
else:
flags.root_dir = os.getcwd()
flags.config = os.path.abspath(flags.config)
root_dir = flags.root_dir
flags.output_dir = os.path.abspath(flags.output_dir)
pipeline_args = {}
# Determine logging verbosity and then set up logging.
verbosity = INFO
if getattr(flags, 'verbosity', None):
verbosity = getattr(flags, 'verbosity')
setup_logging(verbosity)
# Save local paths, if applicable.
# This allows the user to override the path to api-client-staging or
# toolkit on his or her machine.
pipeline_args['root_dir'] = root_dir
pipeline_args['toolkit_path'] = user_config.local.toolkit
pipeline_args['generator_args'] = flags.generator_args
artman_config_path = flags.config
if not os.path.isfile(artman_config_path):
logger.error(
'Artman config file `%s` doesn\'t exist.' % artman_config_path)
sys.exit(96)
try:
artifact_config = loader.load_artifact_config(
artman_config_path, flags.artifact_name, flags.aspect)
except ValueError as ve:
logger.error('Artifact config loading failed with `%s`' % ve)
sys.exit(96)
legacy_config_dict = converter.convert_to_legacy_config_dict(
artifact_config, root_dir, flags.output_dir)
logger.debug('Below is the legacy config after conversion:\n%s' %
pprint.pformat(legacy_config_dict))
language = Artifact.Language.Name(
artifact_config.language).lower()
# Set the pipeline
artifact_type = artifact_config.type
pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type)
pipeline_args['aspect'] = Artifact.Aspect.Name(artifact_config.aspect)
if artifact_type == Artifact.GAPIC_ONLY:
pipeline_name = 'GapicOnlyClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC:
pipeline_name = 'GapicClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.DISCOGAPIC:
pipeline_name = 'DiscoGapicClientPipeline'
pipeline_args['language'] = language
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.GRPC:
pipeline_name = 'GrpcClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC_CONFIG:
pipeline_name = 'GapicConfigPipeline'
elif artifact_type == Artifact.DISCOGAPIC_CONFIG:
pipeline_name = 'DiscoGapicConfigPipeline'
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
if os.path.abspath(flags.output_dir) != os.path.abspath(DEFAULT_OUTPUT_DIR):
logger.warning("`output_dir` is ignored in DiscoGapicConfigGen. "
+ "Yamls are saved at the path specified by `gapic_yaml`.")
pipeline_args['output_dir'] = tempfile.mkdtemp()
elif artifact_type == Artifact.PROTOBUF:
pipeline_name = 'ProtoClientPipeline'
pipeline_args['language'] = language
else:
raise ValueError('Unrecognized artifact.')
# Parse out the full configuration.
config_args = config_util.load_config_spec(legacy_config_dict, language)
config_args.update(pipeline_args)
pipeline_args = config_args
# Print out the final arguments to stdout, to help the user with
# possible debugging.
pipeline_args_repr = yaml.dump(
pipeline_args,
block_seq_indent=2,
default_flow_style=False,
indent=2, )
logger.info('Final args:')
for line in pipeline_args_repr.split('\n'):
if 'token' in line:
index = line.index(':')
line = line[:index + 2] + '<< REDACTED >>'
logger.info(' {0}'.format(line))
# Return the final arguments.
return pipeline_name, pipeline_args
def normalize_flags(flags, user_config):
"""Combine the argparse flags and user configuration together.
Args:
flags (argparse.Namespace): The flags parsed from sys.argv
user_config (dict): The user configuration taken from
~/.artman/config.yaml.
Returns:
tuple (str, dict): 2-tuple containing:
- pipeline name
- pipeline arguments
"""
flags.input_dir = os.path.abspath(flags.input_dir)
flags.output_dir = os.path.abspath(flags.output_dir)
flags.config = os.path.abspath(flags.config)
pipeline_args = {}
# Determine logging verbosity and then set up logging.
verbosity = support.resolve('verbosity', user_config, flags, default=INFO)
setup_logging(verbosity)
# Save local paths, if applicable.
# This allows the user to override the path to api-client-staging or
# toolkit on his or her machine.
pipeline_args['local_paths'] = support.parse_local_paths(
user_config, flags.input_dir)
artman_config_path = flags.config
if not os.path.isfile(artman_config_path):
logger.error(
'Artman config file `%s` doesn\'t exist.' % artman_config_path)
sys.exit(96)
try:
artifact_config = loader.load_artifact_config(
artman_config_path, flags.artifact_name, flags.input_dir)
except ValueError as ve:
logger.error('Artifact config loading failed with `%s`' % ve)
sys.exit(96)
# If we were given just an API or BATCH, then expand it into the --config
# syntax.
shared_config_name = 'common.yaml'
if artifact_config.language in (Artifact.RUBY, Artifact.NODEJS,):
shared_config_name = 'doc.yaml'
legacy_config_dict = converter.convert_to_legacy_config_dict(
artifact_config, flags.input_dir, flags.output_dir)
logger.debug('Below is the legacy config after conversion:\n%s' %
pprint.pformat(legacy_config_dict))
tmp_legacy_config_yaml = '%s.tmp' % artman_config_path
with io.open(tmp_legacy_config_yaml, 'w') as outfile:
yaml.dump(legacy_config_dict, outfile, default_flow_style=False)
googleapis = os.path.realpath(
os.path.expanduser(
pipeline_args['local_paths']['googleapis'], ))
config = ','.join([
'{artman_config_path}',
'{googleapis}/gapic/lang/{shared_config_name}',
]).format(
artman_config_path=tmp_legacy_config_yaml,
googleapis=googleapis,
shared_config_name=shared_config_name, )
language = Artifact.Language.Name(
artifact_config.language).lower()
# Set the pipeline as well as package_type and packaging
artifact_type = artifact_config.type
if artifact_type in (Artifact.GAPIC, Artifact.GAPIC_ONLY):
pipeline_name = 'GapicClientPipeline'
pipeline_args['language'] = language
elif artifact_type in (Artifact.GRPC, Artifact.GRPC_COMMON):
pipeline_name = 'GrpcClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC_CONFIG:
pipeline_name = 'GapicConfigPipeline'
else:
raise ValueError('Unrecognized artifact.')
# Parse out the full configuration.
# Note: the var replacement is still needed because they are still being
# used in some shared/common config yamls.
config_sections = ['common']
for config_spec in config.split(','):
config_args = config_util.load_config_spec(
config_spec=config_spec,
config_sections=config_sections,
repl_vars={
k.upper(): v
for k, v in pipeline_args['local_paths'].items()
},
language=language, )
pipeline_args.update(config_args)
# Setup publishing related config if needed.
if flags.subcommand == 'generate':
pipeline_args['publish'] = 'noop'
elif flags.subcommand == 'publish':
publishing_config = _get_publishing_config(artifact_config,
flags.target)
if publishing_config.type == Artifact.PublishTarget.GITHUB:
pipeline_args['publish'] = 'local' if flags.dry_run else 'github'
pipeline_args['github'] = support.parse_github_credentials(
argv_flags=flags,
config=user_config.get('github', {}), )
repos = pipeline_args.pop('git_repos')
pipeline_args['git_repo'] = support.select_git_repo(
repos, publishing_config.name)
else:
logger.error(
'Publishing type `%s` is not supported yet.' %
Artifact.PublishTarget.Type.Name(publishing_config.type))
sys.exit(96)
# Print out the final arguments to stdout, to help the user with
# possible debugging.
pipeline_args_repr = yaml.dump(
pipeline_args,
block_seq_indent=2,
default_flow_style=False,
indent=2, )
logger.info('Final args:')
for line in pipeline_args_repr.split('\n'):
if 'token' in line:
index = line.index(':')
line = line[:index + 2] + '<< REDACTED >>'
logger.info(' {0}'.format(line))
# Clean up the tmp legacy artman config.
os.remove(tmp_legacy_config_yaml)
# Return the final arguments.
return pipeline_name, pipeline_args
def execute(self, gapic_code_dir):
logger.debug('Formatting files in %s.' %
os.path.abspath(gapic_code_dir))
self.exec_command(['gofmt', '-w', gapic_code_dir])
def normalize_flags(flags, user_config):
"""Combine the argparse flags and user configuration together.
Args:
flags (argparse.Namespace): The flags parsed from sys.argv
user_config (dict): The user configuration taken from
~/.artman/config.yaml.
Returns:
tuple (str, dict): 2-tuple containing:
- pipeline name
- pipeline arguments
"""
if flags.input_dir:
flags.input_dir = os.path.abspath(flags.input_dir)
flags.output_dir = os.path.abspath(flags.output_dir)
flags.config = os.path.abspath(flags.config)
pipeline_args = {}
# Determine logging verbosity and then set up logging.
verbosity = support.resolve('verbosity', user_config, flags, default=INFO)
setup_logging(verbosity)
# Save local paths, if applicable.
# This allows the user to override the path to api-client-staging or
# toolkit on his or her machine.
pipeline_args['local_paths'] = support.parse_local_paths(
user_config, flags.input_dir)
# Save the input directory back to flags if it was not explicitly set.
if not flags.input_dir:
flags.input_dir = pipeline_args['local_paths']['googleapis']
artman_config_path = flags.config
if not os.path.isfile(artman_config_path):
logger.error(
'Artman config file `%s` doesn\'t exist.' % artman_config_path)
sys.exit(96)
try:
artifact_config = loader.load_artifact_config(
artman_config_path, flags.artifact_name, flags.input_dir)
except ValueError as ve:
logger.error('Artifact config loading failed with `%s`' % ve)
sys.exit(96)
# If we were given just an API or BATCH, then expand it into the --config
# syntax.
shared_config_name = 'common.yaml'
if artifact_config.language in (Artifact.RUBY, Artifact.NODEJS,):
shared_config_name = 'doc.yaml'
legacy_config_dict = converter.convert_to_legacy_config_dict(
artifact_config, flags.input_dir, flags.output_dir)
logger.debug('Below is the legacy config after conversion:\n%s' %
pprint.pformat(legacy_config_dict))
tmp_legacy_config_yaml = '%s.tmp' % artman_config_path
with io.open(tmp_legacy_config_yaml, 'w') as outfile:
yaml.dump(legacy_config_dict, outfile, default_flow_style=False)
googleapis = os.path.realpath(
os.path.expanduser(
pipeline_args['local_paths']['googleapis'], ))
config = ','.join([
'{artman_config_path}',
'{googleapis}/gapic/lang/{shared_config_name}',
]).format(
artman_config_path=tmp_legacy_config_yaml,
googleapis=googleapis,
shared_config_name=shared_config_name,
)
language = Artifact.Language.Name(
artifact_config.language).lower()
# Set the pipeline as well as package_type and packaging
artifact_type = artifact_config.type
if artifact_type in (Artifact.GAPIC, Artifact.GAPIC_ONLY):
pipeline_name = 'GapicClientPipeline'
pipeline_args['language'] = language
elif artifact_type in (Artifact.GRPC, Artifact.GRPC_COMMON):
pipeline_name = 'GrpcClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC_CONFIG:
pipeline_name = 'GapicConfigPipeline'
else:
raise ValueError('Unrecognized artifact.')
# Parse out the full configuration.
# Note: the var replacement is still needed because they are still being
# used in some shared/common config yamls.
config_sections = ['common']
for config_spec in config.split(','):
config_args = config_util.load_config_spec(
config_spec=config_spec,
config_sections=config_sections,
repl_vars={
k.upper(): v
for k, v in pipeline_args['local_paths'].items()
},
language=language, )
pipeline_args.update(config_args)
# Setup publishing related config if needed.
if flags.subcommand == 'generate':
pipeline_args['publish'] = 'noop'
elif flags.subcommand == 'publish':
publishing_config = _get_publishing_config(artifact_config,
flags.target)
if publishing_config.type == Artifact.PublishTarget.GITHUB:
pipeline_args['publish'] = 'local' if flags.dry_run else 'github'
pipeline_args['github'] = support.parse_github_credentials(
argv_flags=flags,
config=user_config.get('github', {}), )
repos = pipeline_args.pop('git_repos')
pipeline_args['git_repo'] = support.select_git_repo(
repos, publishing_config.name)
else:
logger.error(
'Publishing type `%s` is not supported yet.' %
Artifact.PublishTarget.Type.Name(publishing_config.type))
sys.exit(96)
# Print out the final arguments to stdout, to help the user with
# possible debugging.
pipeline_args_repr = yaml.dump(
pipeline_args,
block_seq_indent=2,
default_flow_style=False,
indent=2, )
logger.info('Final args:')
for line in pipeline_args_repr.split('\n'):
if 'token' in line:
index = line.index(':')
line = line[:index + 2] + '<< REDACTED >>'
logger.info(' {0}'.format(line))
# Clean up the tmp legacy artman config.
os.remove(tmp_legacy_config_yaml)
# Return the final arguments.
return pipeline_name, pipeline_args
def normalize_flags(flags, user_config):
"""Combine the argparse flags and user configuration together.
Args:
flags (argparse.Namespace): The flags parsed from sys.argv
user_config (dict): The user configuration taken from
~/.artman/config.yaml.
Returns:
tuple (str, dict): 2-tuple containing:
- pipeline name
- pipeline arguments
"""
if flags.root_dir:
flags.root_dir = os.path.abspath(flags.root_dir)
flags.config = os.path.join(flags.root_dir, flags.config)
else:
flags.root_dir = os.getcwd()
flags.config = os.path.abspath(flags.config)
root_dir = flags.root_dir
flags.output_dir = os.path.abspath(flags.output_dir)
pipeline_args = {}
# Determine logging verbosity and then set up logging.
verbosity = INFO
if getattr(flags, 'verbosity', None):
verbosity = getattr(flags, 'verbosity')
setup_logging(verbosity)
# Save local paths, if applicable.
# This allows the user to override the path to api-client-staging or
# toolkit on his or her machine.
pipeline_args['root_dir'] = root_dir
pipeline_args['toolkit_path'] = user_config.local.toolkit
if flags.subcommand == 'publish' and flags.local_repo_dir:
if not flags.dry_run:
logger.error('`--dry-run` flag must be passed when '
'`--local-repo-dir` is specified')
sys.exit(96)
flags.local_repo_dir = os.path.abspath(flags.local_repo_dir)
pipeline_args['local_repo_dir'] = flags.local_repo_dir
artman_config_path = flags.config
if not os.path.isfile(artman_config_path):
logger.error(
'Artman config file `%s` doesn\'t exist.' % artman_config_path)
sys.exit(96)
try:
artifact_config = loader.load_artifact_config(
artman_config_path, flags.artifact_name, flags.aspect)
except ValueError as ve:
logger.error('Artifact config loading failed with `%s`' % ve)
sys.exit(96)
legacy_config_dict = converter.convert_to_legacy_config_dict(
artifact_config, root_dir, flags.output_dir)
logger.debug('Below is the legacy config after conversion:\n%s' %
pprint.pformat(legacy_config_dict))
language = Artifact.Language.Name(
artifact_config.language).lower()
# Set the pipeline
artifact_type = artifact_config.type
pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type)
pipeline_args['aspect'] = Artifact.Aspect.Name(artifact_config.aspect)
if artifact_type == Artifact.GAPIC_ONLY:
pipeline_name = 'GapicOnlyClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC:
pipeline_name = 'GapicClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.DISCOGAPIC:
pipeline_name = 'DiscoGapicClientPipeline'
pipeline_args['language'] = language
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.GRPC:
pipeline_name = 'GrpcClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC_CONFIG:
pipeline_name = 'GapicConfigPipeline'
elif artifact_type == Artifact.DISCOGAPIC_CONFIG:
pipeline_name = 'DiscoGapicConfigPipeline'
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.PROTOBUF:
pipeline_name = 'ProtoClientPipeline'
pipeline_args['language'] = language
else:
raise ValueError('Unrecognized artifact.')
# Parse out the full configuration.
config_args = config_util.load_config_spec(legacy_config_dict, language)
pipeline_args.update(config_args)
# Setup publishing related config if needed.
if flags.subcommand == 'generate':
pipeline_args['publish'] = 'noop'
elif flags.subcommand == 'publish':
publishing_config = _get_publishing_config(artifact_config,
flags.target)
if publishing_config.type == Artifact.PublishTarget.GITHUB:
if flags.dry_run:
pipeline_args['publish'] = 'local'
else:
pipeline_args['publish'] = 'github'
pipeline_args['github'] = support.parse_github_credentials(
argv_flags=flags,
github_config=user_config.github)
repos = pipeline_args.pop('git_repos')
pipeline_args['git_repo'] = support.select_git_repo(
repos, publishing_config.name)
else:
logger.error(
'Publishing type `%s` is not supported yet.' %
Artifact.PublishTarget.Type.Name(publishing_config.type))
sys.exit(96)
# Print out the final arguments to stdout, to help the user with
# possible debugging.
pipeline_args_repr = yaml.dump(
pipeline_args,
block_seq_indent=2,
default_flow_style=False,
indent=2, )
logger.info('Final args:')
for line in pipeline_args_repr.split('\n'):
if 'token' in line:
index = line.index(':')
line = line[:index + 2] + '<< REDACTED >>'
logger.info(' {0}'.format(line))
# Return the final arguments.
return pipeline_name, pipeline_args
def normalize_flags(flags, user_config):
"""Combine the argparse flags and user configuration together.
Args:
flags (argparse.Namespace): The flags parsed from sys.argv
user_config (dict): The user configuration taken from
~/.artman/config.yaml.
Returns:
tuple (str, dict): 2-tuple containing:
- pipeline name
- pipeline arguments
"""
if flags.root_dir:
flags.root_dir = os.path.abspath(flags.root_dir)
flags.config = os.path.join(flags.root_dir, flags.config)
else:
flags.root_dir = os.getcwd()
flags.config = os.path.abspath(flags.config)
root_dir = flags.root_dir
flags.output_dir = os.path.abspath(flags.output_dir)
pipeline_args = {}
# Determine logging verbosity and then set up logging.
verbosity = INFO
if getattr(flags, 'verbosity', None):
verbosity = getattr(flags, 'verbosity')
setup_logging(verbosity)
# Save local paths, if applicable.
# This allows the user to override the path to api-client-staging or
# toolkit on his or her machine.
pipeline_args['root_dir'] = root_dir
# TODO two args reference the same concept - clean this up
pipeline_args['toolkit'] = user_config.local.toolkit
pipeline_args['toolkit_path'] = user_config.local.toolkit
if flags.subcommand == 'publish' and flags.local_repo_dir:
if not flags.dry_run:
logger.error('`--dry-run` flag must be passed when '
'`--local-repo-dir` is specified')
sys.exit(96)
flags.local_repo_dir = os.path.abspath(flags.local_repo_dir)
pipeline_args['local_repo_dir'] = flags.local_repo_dir
artman_config_path = flags.config
if not os.path.isfile(artman_config_path):
logger.error('Artman config file `%s` doesn\'t exist.' %
artman_config_path)
sys.exit(96)
try:
artifact_config = loader.load_artifact_config(artman_config_path,
flags.artifact_name)
except ValueError as ve:
logger.error('Artifact config loading failed with `%s`' % ve)
sys.exit(96)
legacy_config_dict = converter.convert_to_legacy_config_dict(
artifact_config, root_dir, flags.output_dir)
logger.debug('Below is the legacy config after conversion:\n%s' %
pprint.pformat(legacy_config_dict))
language = Artifact.Language.Name(artifact_config.language).lower()
# Set the pipeline
artifact_type = artifact_config.type
pipeline_args['artifact_type'] = Artifact.Type.Name(artifact_type)
if artifact_type == Artifact.GAPIC_ONLY:
pipeline_name = 'GapicOnlyClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC:
pipeline_name = 'GapicClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.DISCOGAPIC:
pipeline_name = 'DiscoGapicClientPipeline'
pipeline_args['language'] = language
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.GRPC:
pipeline_name = 'GrpcClientPipeline'
pipeline_args['language'] = language
elif artifact_type == Artifact.GAPIC_CONFIG:
pipeline_name = 'GapicConfigPipeline'
elif artifact_type == Artifact.DISCOGAPIC_CONFIG:
pipeline_name = 'DiscoGapicConfigPipeline'
pipeline_args['discovery_doc'] = artifact_config.discovery_doc
elif artifact_type == Artifact.PROTOBUF:
pipeline_name = 'ProtoClientPipeline'
pipeline_args['language'] = language
else:
raise ValueError('Unrecognized artifact.')
# Parse out the full configuration.
config_args = config_util.load_config_spec(legacy_config_dict, language)
pipeline_args.update(config_args)
# Setup publishing related config if needed.
if flags.subcommand == 'generate':
pipeline_args['publish'] = 'noop'
elif flags.subcommand == 'publish':
publishing_config = _get_publishing_config(artifact_config,
flags.target)
if publishing_config.type == Artifact.PublishTarget.GITHUB:
if flags.dry_run:
pipeline_args['publish'] = 'local'
else:
pipeline_args['publish'] = 'github'
pipeline_args['github'] = support.parse_github_credentials(
argv_flags=flags, github_config=user_config.github)
repos = pipeline_args.pop('git_repos')
pipeline_args['git_repo'] = support.select_git_repo(
repos, publishing_config.name)
else:
logger.error(
'Publishing type `%s` is not supported yet.' %
Artifact.PublishTarget.Type.Name(publishing_config.type))
sys.exit(96)
# Print out the final arguments to stdout, to help the user with
# possible debugging.
pipeline_args_repr = yaml.dump(
pipeline_args,
block_seq_indent=2,
default_flow_style=False,
indent=2,
)
logger.info('Final args:')
for line in pipeline_args_repr.split('\n'):
if 'token' in line:
index = line.index(':')
line = line[:index + 2] + '<< REDACTED >>'
logger.info(' {0}'.format(line))
# Return the final arguments.
return pipeline_name, pipeline_args
