def execute(self,
src_proto_path,
import_proto_path,
output_dir,
api_name,
api_version,
organization_name,
toolkit_path,
desc_proto_path=None,
excluded_proto_path=[]):
desc_proto_path = desc_proto_path or []
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_path,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_path
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(api_name, api_version,
organization_name) + '.desc'
logger.info('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
# DescGen don't use _group_by_dirname right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
self.exec_command(
['protoc'] + protoc_utils.protoc_header_params(
header_proto_path, toolkit_path) +
protoc_utils.protoc_desc_params(output_dir, desc_out_file) +
desc_protos)
return os.path.join(output_dir, desc_out_file)
def test_find_protos_no_exclusion():
expected = [
'test/fake-repos/fake-proto/fake.proto',
'test/fake-repos/fake-proto/excluded/excluded.proto'
]
src_proto_paths = ['test/fake-repos/fake-proto']
assert list(protoc_utils.find_protos(src_proto_paths, [])) == expected
def execute(self,
src_proto_path,
import_proto_path,
output_dir,
api_name,
api_version,
organization_name,
toolkit_path,
root_dir,
excluded_proto_path=[],
proto_deps=[]):
desc_proto_paths = []
for dep in proto_deps:
if 'proto_path' in dep and dep['proto_path']:
desc_proto_paths.append(
os.path.join(root_dir, dep['proto_path']))
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_paths,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_paths
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(api_name, api_version,
organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
# DescGen doesn't use group protos by package right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
self.exec_command(
['protoc'] + protoc_utils.protoc_header_params(
header_proto_path, toolkit_path) +
protoc_utils.protoc_desc_params(output_dir, desc_out_file) +
desc_protos)
return os.path.join(output_dir, desc_out_file)
def execute(self, src_proto_path, import_proto_path, output_dir,
api_name, api_version, organization_name, toolkit_path,
root_dir, excluded_proto_path=[], proto_deps=[]):
desc_proto_paths = []
for dep in proto_deps:
if 'proto_path' in dep and dep['proto_path']:
desc_proto_paths.append(os.path.join(root_dir, dep['proto_path']))
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_paths,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_paths
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(
api_name, api_version, organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
# DescGen don't use _group_by_dirname right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
self.exec_command(
['protoc'] +
protoc_utils.protoc_header_params(header_proto_path, toolkit_path) +
protoc_utils.protoc_desc_params(output_dir, desc_out_file) +
desc_protos)
return os.path.join(output_dir, desc_out_file)
def test_find_protos_listing_filename():
expected = [
'test/fake-repos/fake-proto/fake.proto',
]
src_proto_paths = [
'test/fake-repos/fake-proto/fake.proto',
]
assert list(protoc_utils.find_protos(src_proto_paths, [])) == expected
def test_find_protos_listing_filename():
expected = [
'test/fake-repos/fake-proto/fake.proto',
]
src_proto_paths = [
'test/fake-repos/fake-proto/fake.proto',
]
assert list(protoc_utils.find_protos(src_proto_paths, [])) == expected
def _execute_proto_codegen(self,
language,
src_proto_path,
import_proto_path,
pkg_dir,
api_name,
api_version,
organization_name,
toolkit_path,
gapic_api_yaml,
gen_proto=False,
gen_grpc=False,
final_src_proto_path=None,
final_import_proto_path=None,
excluded_proto_path=[]):
gapic_api_yaml = gapic_api_yaml[0] if gapic_api_yaml else None
src_proto_path = final_src_proto_path or src_proto_path
import_proto_path = final_import_proto_path or import_proto_path
proto_params = protoc_utils.PROTO_PARAMS_MAP[language]
if gen_proto:
protoc_proto_params = protoc_utils.protoc_proto_params(
proto_params, pkg_dir, gapic_api_yaml, with_grpc=True)
else:
protoc_proto_params = []
if gen_grpc:
protoc_grpc_params = protoc_utils.protoc_grpc_params(
proto_params, pkg_dir, toolkit_path)
else:
protoc_grpc_params = []
# protoc-gen-go has some peculiarities:
# It can only compile one package per invocation. So, we need to split
# proto files by packages.
#
# The order of the input files affects comments and internal variables.
# While this doesn't affect the correctness of the result, we sort
# proto files for reproducibility.
#
# Other languages don't mind us doing this, so we just do it for
# everyone.
for (dirname, protos) in protoc_utils.group_by_dirname(
protoc_utils.find_protos(src_proto_path,
excluded_proto_path)).items():
# It is possible to get duplicate protos. De-dupe them.
protos = sorted(set(protos))
# Execute protoc.
self.exec_command(proto_params.proto_compiler_command +
protoc_utils.protoc_header_params(
import_proto_path +
src_proto_path, toolkit_path) +
protoc_proto_params + protoc_grpc_params +
protos)
return pkg_dir
def test_find_protos_no_exclusion():
expected = [
'test/fake-repos/fake-proto/fake.proto',
'test/fake-repos/fake-proto/excluded/excluded.proto'
]
src_proto_paths = [
'test/fake-repos/fake-proto'
]
assert list(protoc_utils.find_protos(src_proto_paths, [])) == expected
def execute(self, src_proto_path, proto_code_dir, excluded_proto_path=[]):
grpc_proto_dir = os.path.join(proto_code_dir, 'src', 'main', 'proto')
for proto_path in src_proto_path:
index = protoc_utils.find_google_dir_index(proto_path)
for src_proto_file in protoc_utils.find_protos(
[proto_path], excluded_proto_path):
relative_proto_file = src_proto_file[index:]
dst_proto_file = os.path.join(
grpc_proto_dir, relative_proto_file)
self.exec_command(
['mkdir', '-p', os.path.dirname(dst_proto_file)])
self.exec_command(['cp', src_proto_file, dst_proto_file])
def test_find_protos_with_exclusion():
expected = [
'test/fake-repos/fake-proto/fake.proto'
]
src_proto_paths = [
'test/fake-repos/fake-proto'
]
excluded_proto_paths = [
'test/fake-repos/fake-proto/excluded'
]
assert list(protoc_utils.find_protos(
src_proto_paths, excluded_proto_paths)) == expected
def test_find_protos_with_exclusion():
expected = [
'test/fake-repos/fake-proto/fake.proto'
]
src_proto_paths = [
'test/fake-repos/fake-proto'
]
excluded_proto_paths = [
'test/fake-repos/fake-proto/excluded'
]
assert list(protoc_utils.find_protos(
src_proto_paths, excluded_proto_paths)) == expected
def execute(self, src_proto_path, proto_code_dir, excluded_proto_path=[]):
grpc_proto_dir = os.path.join(proto_code_dir, 'src', 'main', 'proto')
for proto_path in src_proto_path:
index = protoc_utils.find_google_dir_index(proto_path)
for src_proto_file in protoc_utils.find_protos(
[proto_path], excluded_proto_path):
relative_proto_file = src_proto_file[index:]
dst_proto_file = os.path.join(
grpc_proto_dir, relative_proto_file)
self.exec_command(
['mkdir', '-p', os.path.dirname(dst_proto_file)])
self.exec_command(['cp', src_proto_file, dst_proto_file])
def execute(self, gapic_code_dir, src_proto_path, excluded_proto_path=[]):
final_output_dir = os.path.join(gapic_code_dir, 'protos')
for proto_path in src_proto_path:
index = protoc_utils.find_google_dir_index(proto_path)
for src_proto_file in protoc_utils.find_protos(
[proto_path], excluded_proto_path):
relative_proto_file = src_proto_file[index:]
dst_proto_file = os.path.join(
final_output_dir, relative_proto_file)
dst_proto_dir = os.path.dirname(dst_proto_file)
if not os.path.exists(dst_proto_dir):
self.exec_command(['mkdir', '-p', dst_proto_dir])
self.exec_command(['cp', src_proto_file, dst_proto_file])
def execute(self, gapic_code_dir, src_proto_path, excluded_proto_path=[]):
final_output_dir = os.path.join(gapic_code_dir, 'proto')
for proto_path in src_proto_path:
index = protoc_utils.find_google_dir_index(proto_path)
for src_proto_file in protoc_utils.find_protos(
[proto_path], excluded_proto_path):
relative_proto_file = src_proto_file[index:]
dst_proto_file = os.path.join(final_output_dir,
relative_proto_file)
dst_proto_dir = os.path.dirname(dst_proto_file)
if not os.path.exists(dst_proto_dir):
self.exec_command(['mkdir', '-p', dst_proto_dir])
self.exec_command(['cp', src_proto_file, dst_proto_file])
def _execute_proto_codegen(
self, language, src_proto_path, import_proto_path,
pkg_dir, api_name, api_version, organization_name,
toolkit_path, gapic_api_yaml, gen_proto=False, gen_grpc=False,
final_src_proto_path=None, final_import_proto_path=None,
excluded_proto_path=[]):
gapic_api_yaml = gapic_api_yaml[0] if gapic_api_yaml else None
src_proto_path = final_src_proto_path or src_proto_path
import_proto_path = final_import_proto_path or import_proto_path
proto_params = protoc_utils.PROTO_PARAMS_MAP[language]
if gen_proto:
protoc_proto_params = protoc_utils.protoc_proto_params(
proto_params, pkg_dir, gapic_api_yaml, with_grpc=True)
else:
protoc_proto_params = []
if gen_grpc:
protoc_grpc_params = protoc_utils.protoc_grpc_params(
proto_params, pkg_dir, toolkit_path)
else:
protoc_grpc_params = []
# protoc-gen-go has some peculiarities:
# It can only compile one package per invocation. So, we need to split
# proto files by packages.
#
# The order of the input files affects comments and internal variables.
# While this doesn't affect the correctness of the result, we sort
# proto files for reproducibility.
#
# Other languages don't mind us doing this, so we just do it for
# everyone.
for (dirname, protos) in protoc_utils.group_by_dirname(
protoc_utils.find_protos(src_proto_path, excluded_proto_path)).items():
# It is possible to get duplicate protos. De-dupe them.
protos = sorted(set(protos))
# Execute protoc.
self.exec_command(proto_params.proto_compiler_command +
protoc_utils.protoc_header_params(
import_proto_path + src_proto_path, toolkit_path) +
protoc_proto_params +
protoc_grpc_params +
protos)
return pkg_dir
def _execute_proto_codegen(self,
language,
src_proto_path,
import_proto_path,
pkg_dir,
api_name,
api_version,
organization_name,
toolkit_path,
gapic_api_yaml,
gen_proto=False,
gen_grpc=False,
final_src_proto_path=None,
final_import_proto_path=None,
excluded_proto_path=[]):
gapic_api_yaml = gapic_api_yaml[0] if gapic_api_yaml else None
src_proto_path = final_src_proto_path or src_proto_path
import_proto_path = final_import_proto_path or import_proto_path
proto_params = protoc_utils.PROTO_PARAMS_MAP[language]
if gen_proto:
protoc_proto_params = protoc_utils.protoc_proto_params(
proto_params, pkg_dir, gapic_api_yaml, with_grpc=True)
else:
protoc_proto_params = []
if gen_grpc:
protoc_grpc_params = protoc_utils.protoc_grpc_params(
proto_params, pkg_dir, toolkit_path)
else:
protoc_grpc_params = []
# protoc-gen-go must compile all protos in a package at the same
# time, and *only* the protos in that package. This doesn't break
# other languages, so we do it that way for all of them.
for (dirname, protos) in protoc_utils.group_by_dirname(
protoc_utils.find_protos(src_proto_path,
excluded_proto_path)).items():
self.exec_command(proto_params.proto_compiler_command +
protoc_utils.protoc_header_params(
import_proto_path +
src_proto_path, toolkit_path) +
protoc_proto_params + protoc_grpc_params +
protos)
return pkg_dir
def execute(self,
src_proto_path,
import_proto_path,
output_dir,
api_name,
api_version,
organization_name,
toolkit_path,
root_dir,
excluded_proto_path=[],
proto_deps=[],
language='python'):
desc_proto_paths = []
for dep in proto_deps:
if 'proto_path' in dep and dep['proto_path']:
desc_proto_paths.append(
os.path.join(root_dir, dep['proto_path']))
desc_protos = list(
protoc_utils.find_protos(src_proto_path + desc_proto_paths,
excluded_proto_path))
header_proto_path = import_proto_path + desc_proto_paths
header_proto_path.extend(src_proto_path)
desc_out_file = task_utils.api_full_name(api_name, api_version,
organization_name) + '.desc'
logger.debug('Compiling descriptors for {0}'.format(desc_protos))
self.exec_command(['mkdir', '-p', output_dir])
proto_params = protoc_utils.PROTO_PARAMS_MAP[language]
proto_compiler_command = proto_params.proto_compiler_command
logger.debug(
'Using protoc command: {0}'.format(proto_compiler_command))
# DescGen doesn't use group protos by package right now because
# - it doesn't have to
# - and multiple invocation will overwrite the desc_out_file
(common_resources_includes, common_resources_paths) = \
protoc_utils.protoc_common_resources_params(root_dir)
params = proto_params.proto_compiler_command + \
common_resources_includes + \
protoc_utils.protoc_header_params(header_proto_path, toolkit_path) + \
protoc_utils.protoc_desc_params(output_dir, desc_out_file) + \
common_resources_paths + \
desc_protos
self.exec_command(params)
return os.path.join(output_dir, desc_out_file)
def _copy_and_transform_directories(
self, src_directories, destination_directory, common_protos,
paths=None):
for path in src_directories:
protos = list(protoc_utils.find_protos([path], []))
for proto in protos:
src_base_dirs = self._extract_base_dirs(proto)
sub_new_src = os.path.join(
destination_directory,
self._transform(
src_base_dirs, os.path.sep, common_protos))
if paths is not None:
paths.add(sub_new_src)
dest = os.path.join(sub_new_src, os.path.basename(proto))
if not os.path.exists(dest):
self.exec_command(['mkdir', '-p', sub_new_src])
self._copy_proto(
proto, os.path.join(sub_new_src, dest), common_protos)
def _copy_and_transform_directories(self,
src_directories,
destination_directory,
common_protos,
paths=None):
for path in src_directories:
protos = list(protoc_utils.find_protos([path], []))
for proto in protos:
src_base_dirs = self._extract_base_dirs(proto)
sub_new_src = os.path.join(
destination_directory,
self._transform(src_base_dirs, os.path.sep, common_protos))
if paths is not None:
paths.add(sub_new_src)
dest = os.path.join(sub_new_src, os.path.basename(proto))
if not os.path.exists(dest):
self.exec_command(['mkdir', '-p', sub_new_src])
self._copy_proto(proto, os.path.join(sub_new_src, dest),
common_protos)
def execute(self, gapic_code_dir, src_proto_path, excluded_proto_path=[]):
final_output_dir = os.path.join(gapic_code_dir, 'protos')
src_dir = os.path.join(gapic_code_dir, 'src')
proto_files = []
for proto_path in src_proto_path:
index = protoc_utils.find_google_dir_index(proto_path)
for src_proto_file in protoc_utils.find_protos(
[proto_path], excluded_proto_path):
relative_proto_file = src_proto_file[index:]
proto_files.append(relative_proto_file)
dst_proto_file = os.path.join(final_output_dir,
relative_proto_file)
dst_proto_dir = os.path.dirname(dst_proto_file)
if not os.path.exists(dst_proto_dir):
self.exec_command(['mkdir', '-p', dst_proto_dir])
self.exec_command(['cp', src_proto_file, dst_proto_file])
# Execute compileProtos from Docker image (a part of from google-gax)
cwd = os.getcwd()
os.chdir(gapic_code_dir)
self.exec_command(['compileProtos', './src'])
os.chdir(cwd)
def _execute_proto_codegen(self,
language,
src_proto_path,
import_proto_path,
pkg_dir,
api_name,
api_version,
organization_name,
toolkit_path,
gapic_yaml,
root_dir,
gen_proto=False,
gen_grpc=False,
gen_common_resources=False,
final_src_proto_path=None,
final_import_proto_path=None,
excluded_proto_path=[],
language_out_override=None):
# Adding 17th parameter is a sin that I commit here just because
# refactoring of this code will never happen.
src_proto_path = final_src_proto_path or src_proto_path
import_proto_path = final_import_proto_path or import_proto_path
proto_params = protoc_utils.PROTO_PARAMS_MAP[language]
if gen_proto:
protoc_proto_params = protoc_utils.protoc_proto_params(
proto_params,
pkg_dir,
gapic_yaml,
with_grpc=True,
language_out_override=language_out_override)
else:
protoc_proto_params = []
if gen_grpc:
protoc_grpc_params = protoc_utils.protoc_grpc_params(
proto_params, pkg_dir, toolkit_path)
else:
protoc_grpc_params = []
if gen_common_resources:
(common_resources_includes, common_resources_paths) = \
protoc_utils.protoc_common_resources_params(root_dir)
protoc_plugin_params = protoc_utils.protoc_plugin_params(
proto_params, pkg_dir, gapic_yaml)
else:
(common_resources_includes, common_resources_paths) = ([], [])
protoc_plugin_params = []
if not protoc_proto_params \
and not protoc_grpc_params \
and not protoc_plugin_params:
return pkg_dir
# protoc-gen-go has some peculiarities:
# It can only compile one package per invocation. So, we need to split
# proto files by packages.
#
# The order of the input files affects comments and internal variables.
# While this doesn't affect the correctness of the result, we sort
# proto files for reproducibility.
# For other languages, we'll pass all proto files into the same protoc
# invocation (PHP especially needs that).
all_protos = protoc_utils.find_protos(src_proto_path,
excluded_proto_path)
if language == "go":
protos_map = protoc_utils.group_by_go_package(all_protos)
else:
protos_map = {"": all_protos}
for (dirname, protos) in protos_map.items():
# It is possible to get duplicate protos. De-dupe them.
protos = sorted(set(protos))
command_params = proto_params.proto_compiler_command + \
common_resources_includes + \
protoc_utils.protoc_header_params(
import_proto_path + src_proto_path, toolkit_path) + \
protoc_proto_params + \
protoc_grpc_params + \
protoc_plugin_params + \
common_resources_paths + \
protos
# Execute protoc.
self.exec_command(command_params)
return pkg_dir