def build_package_protos(package_root, strict_mode=False):
proto_files = []
inclusion_root = os.path.abspath(package_root)
for root, _, files in os.walk(inclusion_root):
for filename in files:
if filename.endswith('.proto'):
proto_files.append(
os.path.abspath(os.path.join(root, filename)))
well_known_protos_include = pkg_resources.resource_filename(
'grpc_tools', '_proto')
for proto_file in proto_files:
command = [
'grpc_tools.protoc',
'--proto_path={}'.format(inclusion_root),
'--proto_path={}'.format(well_known_protos_include),
'--python_out={}'.format(inclusion_root),
'--grpc_python_out={}'.format(inclusion_root),
] + [proto_file]
if protoc.main(command) != 0:
if strict_mode:
raise Exception('error: {} failed'.format(command))
else:
sys.stderr.write('warning: {} failed'.format(command))
def setUp(self):
same_proto_contents = pkgutil.get_data(
'tests.protoc_plugin.protos.invocation_testing', 'same.proto')
self.directory = tempfile.mkdtemp(suffix='same_common', dir='.')
self.proto_directory = os.path.join(self.directory, 'proto_path')
self.python_out_directory = os.path.join(self.directory, 'python_out')
self.grpc_python_out_directory = self.python_out_directory
os.makedirs(self.proto_directory)
os.makedirs(self.python_out_directory)
same_proto_file = os.path.join(self.proto_directory,
'same_common.proto')
open(same_proto_file, 'wb').write(
same_proto_contents.replace(
_COMMON_NAMESPACE,
b'package grpc_protoc_plugin.invocation_testing.same_common;'))
protoc_result = protoc.main([
'',
'--proto_path={}'.format(self.proto_directory),
'--python_out={}'.format(self.python_out_directory),
'--grpc_python_out={}'.format(self.grpc_python_out_directory),
same_proto_file,
])
if protoc_result != 0:
raise Exception("unexpected protoc error")
open(os.path.join(self.python_out_directory, '__init__.py'),
'w').write('')
self.pb2_import = 'same_common_pb2'
self.pb2_grpc_import = 'same_common_pb2_grpc'
self.should_find_services_in_pb2 = True
def run(self):
import grpc_tools.protoc as protoc
include_regex = re.compile(self.include)
exclude_regex = re.compile(self.exclude) if self.exclude else None
paths = []
for walk_root, directories, filenames in os.walk(PROTO_STEM):
for filename in filenames:
path = os.path.join(walk_root, filename)
if include_regex.match(path) and not (
exclude_regex and exclude_regex.match(path)):
paths.append(path)
# TODO(kpayson): It would be nice to do this in a batch command,
# but we currently have name conflicts in src/proto
for path in paths:
command = [
'grpc_tools.protoc',
'-I {}'.format(PROTO_STEM),
'--python_out={}'.format(PROTO_STEM),
'--grpc_python_out={}'.format(PROTO_STEM),
] + [path]
if protoc.main(command) != 0:
sys.stderr.write(
'warning: Command:\n{}\nFailed'.format(command))
# Generated proto directories dont include __init__.py, but
# these are needed for python package resolution
for walk_root, _, _ in os.walk(PROTO_STEM):
path = os.path.join(walk_root, '__init__.py')
open(path, 'a').close()
def generate_proto_files():
try:
import grpc_tools
except ImportError:
warnings.warn('Installing grpcio-tools is recommended for development.')
py_sdk_root = os.path.dirname(os.path.abspath(__file__))
common = os.path.join(py_sdk_root, '..', 'common')
proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS]
proto_files = sum(
[glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH)
out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
if out_files and not proto_files:
# We have out_files but no protos; assume they're up to date.
# This is actually the common case (e.g. installation from an sdist).
logging.info('No proto files; using existing generated files.')
return
elif not out_files and not proto_files:
if not os.path.exists(common):
raise RuntimeError(
'Not in apache git tree; unable to find proto definitions.')
else:
raise RuntimeError(
'No proto files found in %s.' % proto_dirs)
# Regenerate iff the proto files are newer.
elif not out_files or len(out_files) < len(proto_files) or (
min(os.path.getmtime(path) for path in out_files)
<= max(os.path.getmtime(path) for path in proto_files)):
try:
from grpc_tools import protoc
except ImportError:
# Use a subprocess to avoid messing with this process' path and imports.
# Note that this requires a separate module from setup.py for Windows:
# https://docs.python.org/2/library/multiprocessing.html#windows
p = multiprocessing.Process(
target=_install_grpcio_tools_and_generate_proto_files)
p.start()
p.join()
else:
logging.info('Regenerating out-of-date Python proto definitions.')
builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto')
args = (
[sys.executable] + # expecting to be called from command line
['--proto_path=%s' % builtin_protos] +
['--proto_path=%s' % d for d in proto_dirs] +
['--python_out=%s' % out_dir] +
['--grpc_python_out=%s' % out_dir] +
proto_files)
ret_code = protoc.main(args)
if ret_code:
raise RuntimeError(
'Protoc returned non-zero status (see logs for details): '
'%s' % ret_code)
def _protoc(proto_path, python_out, grpc_python_out_flag, grpc_python_out,
absolute_proto_file_names):
args = [
'',
'--proto_path={}'.format(proto_path),
]
if python_out is not None:
args.append('--python_out={}'.format(python_out))
if grpc_python_out is not None:
args.append('--grpc_python_out={}:{}'.format(grpc_python_out_flag,
grpc_python_out))
args.extend(absolute_proto_file_names)
return protoc.main(args)
def setUp(self):
services_proto_contents = pkgutil.get_data(
'tests.protoc_plugin.protos.invocation_testing.split_services',
'services.proto')
messages_proto_contents = pkgutil.get_data(
'tests.protoc_plugin.protos.invocation_testing.split_messages',
'messages.proto')
self.directory = tempfile.mkdtemp(suffix='split_separate', dir='.')
self.proto_directory = os.path.join(self.directory, 'proto_path')
self.python_out_directory = os.path.join(self.directory, 'python_out')
self.grpc_python_out_directory = os.path.join(self.directory,
'grpc_python_out')
os.makedirs(self.proto_directory)
os.makedirs(self.python_out_directory)
os.makedirs(self.grpc_python_out_directory)
services_proto_file = os.path.join(self.proto_directory,
'split_separate_services.proto')
messages_proto_file = os.path.join(self.proto_directory,
'split_separate_messages.proto')
open(services_proto_file, 'wb').write(
services_proto_contents.replace(
_MESSAGES_IMPORT, b'import "split_separate_messages.proto";')
.replace(
_SPLIT_NAMESPACE,
b'package grpc_protoc_plugin.invocation_testing.split_separate;'
))
open(messages_proto_file, 'wb').write(
messages_proto_contents.replace(
_SPLIT_NAMESPACE,
b'package grpc_protoc_plugin.invocation_testing.split_separate;'
))
protoc_result = protoc.main([
'',
'--proto_path={}'.format(self.proto_directory),
'--python_out={}'.format(self.python_out_directory),
'--grpc_python_out=grpc_2_0:{}'.format(
self.grpc_python_out_directory),
services_proto_file,
messages_proto_file,
])
if protoc_result != 0:
raise Exception("unexpected protoc error")
open(os.path.join(self.python_out_directory, '__init__.py'),
'w').write('')
self.pb2_import = 'split_separate_messages_pb2'
self.pb2_grpc_import = 'split_separate_services_pb2_grpc'
self.should_find_services_in_pb2 = False
def _protoc(self):
args = [
'',
'--proto_path={}'.format(self._proto_path),
'--python_out={}'.format(self._python_out),
'--grpc_python_out=grpc_1_0:{}'.format(self._python_out),
] + list(self._proto_file_names)
protoc_exit_code = protoc.main(args)
self.assertEqual(0, protoc_exit_code)
_packagify(self._python_out)
with _system_path([self._python_out]):
self._payload_pb2 = importlib.import_module(_PAYLOAD_PB2)
self._requests_pb2 = importlib.import_module(_REQUESTS_PB2)
self._responses_pb2 = importlib.import_module(_RESPONSES_PB2)
self._service_pb2 = importlib.import_module(_SERVICE_PB2)
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main((
'',
'-I../protos',
'--python_out=.',
'--grpc_python_out=.',
'../protos/prime.proto',
))
from grpc_tools import protoc
protoc.main(('', '-I./app/proto', '--python_out=./app/proto/dist',
'--grpc_python_out=./app/proto/dist', 'simple.proto'))
def generate_proto_files(force=False):
try:
import grpc_tools # pylint: disable=unused-variable
except ImportError:
warnings.warn('Installing grpcio-tools is recommended for development.')
py_sdk_root = os.path.dirname(os.path.abspath(__file__))
common = os.path.join(py_sdk_root, '..', 'common')
proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS]
proto_files = sum(
[glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH)
out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
if out_files and not proto_files and not force:
# We have out_files but no protos; assume they're up to date.
# This is actually the common case (e.g. installation from an sdist).
logging.info('No proto files; using existing generated files.')
return
elif not out_files and not proto_files:
if not os.path.exists(common):
raise RuntimeError(
'Not in apache git tree; unable to find proto definitions.')
else:
raise RuntimeError(
'No proto files found in %s.' % proto_dirs)
# Regenerate iff the proto files are newer.
elif force or not out_files or len(out_files) < len(proto_files) or (
min(os.path.getmtime(path) for path in out_files)
<= max(os.path.getmtime(path) for path in proto_files)):
try:
from grpc_tools import protoc
except ImportError:
if platform.system() == 'Windows':
# For Windows, grpcio-tools has to be installed manually.
raise RuntimeError(
'Cannot generate protos for Windows since grpcio-tools package is '
'not installed. Please install this package manually '
'using \'pip install grpcio-tools\'.')
# Use a subprocess to avoid messing with this process' path and imports.
# Note that this requires a separate module from setup.py for Windows:
# https://docs.python.org/2/library/multiprocessing.html#windows
p = multiprocessing.Process(
target=_install_grpcio_tools_and_generate_proto_files)
p.start()
p.join()
if p.exitcode:
raise ValueError("Proto generation failed (see log for details).")
else:
logging.info('Regenerating out-of-date Python proto definitions.')
builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto')
args = (
[sys.executable] + # expecting to be called from command line
['--proto_path=%s' % builtin_protos] +
['--proto_path=%s' % d for d in proto_dirs] +
['--python_out=%s' % out_dir] +
# TODO(robertwb): Remove the prefix once it's the default.
['--grpc_python_out=grpc_2_0:%s' % out_dir] +
proto_files)
ret_code = protoc.main(args)
if ret_code:
raise RuntimeError(
'Protoc returned non-zero status (see logs for details): '
'%s' % ret_code)
if sys.version_info[0] >= 3:
ret_code = subprocess.call(
["futurize", "--both-stages", "--write", "--verbose", "--no-diff",
out_dir])
if ret_code:
raise RuntimeError(
'Error applying futurize to generated protobuf python files.')
# -- Compile Protobufs -------------------------------------------------------
# Protobufs are not precompiled and need to be generated at Sphinx runtime.
# This is a hack and should be made into an extension.
import pkg_resources
from glob import glob
from grpc_tools import protoc
_proto_include = pkg_resources.resource_filename('grpc_tools', '_proto')
_project_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
_status_code = protoc.main([
f"-I{_project_dir}",
f"--python_out={_project_dir}",
f"--grpc_python_out={_project_dir}",
*glob(f"{_project_dir}/needlestack/apis/*.proto"),
f"-I{_proto_include}",
f"--proto_path={_proto_include}",
f"--proto_path={_project_dir}",
])
# -- Project information -----------------------------------------------------
project = 'Needlestack'
copyright = '2019, Cung Tran'
author = 'Cung Tran'
version = "0.1.0"
release = version
# -- General configuration ---------------------------------------------------
if not os.path.exists(proto_directory):
os.makedirs(proto_directory)
copy("file.proto", proto_directory)
if not os.path.exists(generated_code_directory):
os.makedirs(generated_code_directory)
try:
with open(generated_code_directory + '/__init__.py', "w+") as f:
f.write('')
except Exception:
pass
try:
protoc.main(('', '-I' + proto_directory,
'--python_out=' + generated_code_directory,
'--grpc_python_out=' + generated_code_directory,
os.path.join(proto_directory, 'file.proto')))
except Exception as e:
print(str(e))
competition_config = {"target": ["MAPE"]}
code = ''
competition = Competition(None,
name=name,
datastream_id=1,
initial_batch_size=20,
initial_training_time=20,
batch_size=5,
time_interval=5,
target_class="target",
start_date=format_time(now),
file_path="file.proto",
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main((
'',
'-I.',
'--python_out=.',
'--grpc_python_out=.',
'rpcDefinitions.proto',
))
#Parts:
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
import sys
import os
install_dir = os.path.join(os.environ['MESON_INSTALL_DESTDIR_PREFIX'], 'libexec', 'warpinator')
protoc.main((
'',
'-I%s' % sys.argv[1],
'--python_out=%s' % install_dir,
'--grpc_python_out=%s' % install_dir,
os.path.join(sys.argv[1], sys.argv[2])
))
# -*- coding:utf-8 -*-
from grpc_tools import protoc
protoc.main(
(
'',
'-I../proto_buffer_config',
'--python_out=../pb_api',
'--grpc_python_out=../pb_api',
'../proto_buffer_config/test.proto',
),
# (
# '',
# '-I./grpc_protos',
# '--python_out=./pb',
# '--grpc_python_out=./pb',
# './grpc_protos/user_info.proto',
# )
# python3 -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. helloworld.proto
)
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main((
'',
'-I../protos',
'--python_out=.',
'--grpc_python_out=.',
'../protos/ouroboros.proto',
))
def generate_proto_files(force=True, output_dir=DEFAULT_PYTHON_OUTPUT_PATH):
try:
import grpc_tools # noqa # pylint: disable=unused-import
except ImportError:
warnings.warn(
'Installing grpcio-tools is recommended for development.')
proto_dirs = [
os.path.join(PYFLINK_ROOT_PATH, path) for path in PROTO_PATHS
]
proto_files = sum(
[glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
out_dir = os.path.join(PYFLINK_ROOT_PATH, output_dir)
out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
if out_files and not proto_files and not force:
# We have out_files but no protos; assume they're up to date.
# This is actually the common case (e.g. installation from an sdist).
logging.info('No proto files; using existing generated files.')
return
elif not out_files and not proto_files:
raise RuntimeError('No proto files found in %s.' % proto_dirs)
# Regenerate iff the proto files or this file are newer.
elif force or not out_files or len(out_files) < len(proto_files) or (min(
os.path.getmtime(path) for path in out_files) <= max(
os.path.getmtime(path)
for path in proto_files + [os.path.realpath(__file__)])):
try:
from grpc_tools import protoc
except ImportError:
if platform.system() == 'Windows':
# For Windows, grpcio-tools has to be installed manually.
raise RuntimeError(
'Cannot generate protos for Windows since grpcio-tools package is '
'not installed. Please install this package manually '
'using \'pip install grpcio-tools\'.')
# Use a subprocess to avoid messing with this process' path and imports.
# Note that this requires a separate module from setup.py for Windows:
# https://docs.python.org/2/library/multiprocessing.html#windows
p = multiprocessing.Process(
target=_install_grpcio_tools_and_generate_proto_files(
force, output_dir))
p.start()
p.join()
if p.exitcode:
raise ValueError(
"Proto generation failed (see log for details).")
else:
_check_grpcio_tools_version()
logging.info('Regenerating out-of-date Python proto definitions.')
builtin_protos = pkg_resources.resource_filename(
'grpc_tools', '_proto')
args = (
[sys.executable] + # expecting to be called from command line
['--proto_path=%s' % builtin_protos] +
['--proto_path=%s' % d for d in proto_dirs] +
['--python_out=%s' % out_dir] + proto_files)
ret_code = protoc.main(args)
if ret_code:
raise RuntimeError(
'Protoc returned non-zero status (see logs for details): '
'%s' % ret_code)
for output_file in os.listdir(output_dir):
if output_file.endswith('_pb2.py'):
_add_license_header(output_dir, output_file)
#-*- coding:utf8 -*-
# Copyright (c) 2020 barriery
# Python release: 3.7.0
# Create time: 2020-04-04
from setuptools import setup, find_packages
from grpc_tools import protoc
# run bdware proto codegen
protoc.main((
'',
'-I.',
'--python_out=.',
'--grpc_python_out=.',
'./scheduletool/bdware/proto/schedule_service.proto',
))
protoc.main((
'',
'-I.',
'--python_out=.',
'--grpc_python_out=.',
'./scheduletool/buaacps/proto/entity.proto',
))
protoc.main((
'',
'-I.',
'--python_out=.',
'--grpc_python_out=.',
'./scheduletool/buaacps/proto/result.proto',
))
'''
Created on 2020. 5. 20.
@author: kwlee
'''
from grpc_tools import protoc
protoc.main((
'',
'-Iproto',
'--python_out=.',
'--grpc_python_out=.',
'proto/marmot_type.proto',
'proto/marmot_dataset.proto',
'proto/marmot_file.proto',
'proto/dric.proto',
))
if __name__ == '__main__':
pass
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main((
'',
'-I.',
'--python_out=.',
'--grpc_python_out=.',
'csci4220_hw3.proto',
))
'_proto/google/protobuf/descriptor.proto'),
f'{PROTO_DIR}/descriptor.proto')
copyfile(
'../../../../protobuf/io/casperlabs/casper/protocol/CasperMessage.proto',
f'{PROTO_DIR}/CasperMessage.proto')
os.system("""perl -p -i -e 's,^import \"google/protobuf/,import ",' proto/*""")
os.system("""perl -p -i -e 's,^import \"scalapb/,import ",' proto/*""")
google_proto = join(dirname(grpc_tools.__file__), '_proto')
protoc.main((
'',
f'-I./{PROTO_DIR}',
'-I' + google_proto,
'-I.',
'--python_out=.',
'--grpc_python_out=.',
f'{PROTO_DIR}/empty.proto',
))
protoc.main((
'',
f'-I./{PROTO_DIR}',
'-I' + google_proto,
'-I.',
'--python_out=.',
'--grpc_python_out=.',
f'{PROTO_DIR}/descriptor.proto',
))
from grpc_tools import protoc
protoc.main(
(
'',
'-I.',
'--python_out=.',
'FlightLog.proto',
)
)
from grpc_tools import protoc
import os
#ssslibrary = os.path.dirname(os.__file__)
library = "../../lib/python3.6/site-packages/"
protoc.main(
(
'',
'--proto_path=.',
'--python_out={}'.format(library),
'--grpc_python_out={}'.format(library),
'./addition.proto',
)
)
def generate_proto_files(force=False):
try:
import grpc_tools
except ImportError:
warnings.warn(
'Installing grpcio-tools is recommended for development.')
py_sdk_root = os.path.dirname(os.path.abspath(__file__))
common = os.path.join(py_sdk_root, '..', 'common')
proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS]
proto_files = sum(
[glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH)
out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
if out_files and not proto_files and not force:
# We have out_files but no protos; assume they're up to date.
# This is actually the common case (e.g. installation from an sdist).
logging.info('No proto files; using existing generated files.')
return
elif not out_files and not proto_files:
if not os.path.exists(common):
raise RuntimeError(
'Not in apache git tree; unable to find proto definitions.')
else:
raise RuntimeError('No proto files found in %s.' % proto_dirs)
# Regenerate iff the proto files are newer.
elif force or not out_files or len(out_files) < len(proto_files) or (min(
os.path.getmtime(path) for path in out_files) <= max(
os.path.getmtime(path) for path in proto_files)):
try:
from grpc_tools import protoc
except ImportError:
if platform.system() == 'Windows':
# For Windows, grpcio-tools has to be installed manually.
raise RuntimeError(
'Cannot generate protos for Windows since grpcio-tools package is '
'not installed. Please install this package manually '
'using \'pip install grpcio-tools\'.')
# Use a subprocess to avoid messing with this process' path and imports.
# Note that this requires a separate module from setup.py for Windows:
# https://docs.python.org/2/library/multiprocessing.html#windows
p = multiprocessing.Process(
target=_install_grpcio_tools_and_generate_proto_files)
p.start()
p.join()
if p.exitcode:
raise ValueError(
"Proto generation failed (see log for details).")
else:
logging.info('Regenerating out-of-date Python proto definitions.')
builtin_protos = pkg_resources.resource_filename(
'grpc_tools', '_proto')
args = (
[sys.executable] + # expecting to be called from command line
['--proto_path=%s' % builtin_protos] +
['--proto_path=%s' % d
for d in proto_dirs] + ['--python_out=%s' % out_dir] +
# TODO(robertwb): Remove the prefix once it's the default.
['--grpc_python_out=grpc_2_0:%s' % out_dir] + proto_files)
ret_code = protoc.main(args)
if ret_code:
raise RuntimeError(
'Protoc returned non-zero status (see logs for details): '
'%s' % ret_code)
from grpc_tools import protoc
import os
fnames = os.listdir('.')
files = []
for fname in fnames:
if fname.split('.')[-1] != 'proto':
continue
files.append(fname)
params = ["-I.", "--python_out=./"]
params.extend(files)
protoc.main(params)
"""
Runs protoc with the gRPC plugin to generate messages and gRPC stubs
"""
from grpc_tools import protoc
# this file contains definitions of SERVICE_NAME,
# RPC_AND_MESSAGE_NAMES and MESSAGE_FIELDS
# these are shared between client and server
from PROTO_DEFINITIONS import *
subDirectory = '.'
outcome = protoc.main((
'',
'--proto_path=.',
'--python_out=' + subDirectory,
'--grpc_python_out=' + subDirectory,
'./' + PROTO_FILE + '.proto',
))
print(outcome)
import os
import sys
import grpc_tools.protoc as protoc
from setuptools import setup
protoDir = os.path.split(os.path.abspath(sys.argv[0]))[0] + "/../proto/"
proto = protoDir + "service.proto"
if os.path.exists(protoDir):
protoc.main([
"grpc_tools.protoc",
"--proto_path=%s" % (protoDir, ),
"--python_out=.",
"--grpc_python_out=.",
proto])
setup(name='packt_beam_chapter6',
version='1.0',
description='Packt Building Big Data Pipelines using Apache Beam chapter 6',
py_modules=['service_pb2', 'service_pb2_grpc', 'utils'])
def competitions():
"""
Competitions route.
Implements two methods:
GET: retrieve the competitions
POST: create new competition:
Parameters:
- Competition name
- Datastream
- Description of the competition
- Competition settings: size of initial batch, initial training time, size of a regular batch,
time interval between the batches, name of the label column and the evaluation
metric, start and end date, time interval to send the predictions, .proto file
:return: Responses: {200: If method== "GET", return the list of the competitions, If method == "POST", confirm success, 500: Error}
"""
if request.method == 'GET':
status = request.args.get('status')
page = request.args.get('page')
step = request.args.get('step')
competitions = _COMPETITION_REPO.get_all_competitions(
status, int(page), int(step))
return jsonify(competitions)
if request.method == 'POST':
# Adding a competition
data = dict(request.form)
data = data['competition']
data = data[0]
data = json.loads(data)
# TODO: store in mongodb
competition_config = data['config']
print("Competition configuration: ", competition_config)
logging.debug("Competition config: {}".format(competition_config))
name = data['name']
datastream_id = data['datastream_id']
initial_batch_size = data['initial_batch_size']
initial_training_time = data['initial_training_time']
batch_size = data['batch_size']
time_interval = data['time_interval']
target_class = data['target_class']
logging.debug("Competition target: {}".format(target_class))
start_date = data['start_date'].replace('T', ' ').replace('Z', '')
end_date = data['end_date'].replace('T', ' ').replace('Z', '')
predictions_time_interval = data['predictions_time_interval']
description = data['description']
data_file = request.files['file']
file_name = data_file.filename
# Check if the file is one of the allowed types/extensions
if data_file and allowed_file(file_name):
# Make the filename safe, remove unsupported chars
extension = get_file_extension(file_name)
filename = secure_filename(str(name) + str(extension))
# Move the file form the temporal folder to
# the upload folder we setup
proto_directory = os.path.join(config['UPLOAD_REPO'],
config['COMPETITION_PROTO_REPO'],
name)
if not os.path.exists(proto_directory):
os.makedirs(proto_directory)
data_file.save(os.path.join(proto_directory, 'file.proto'))
generated_code_directory = os.path.join(
config['UPLOAD_REPO'], config['COMPETITION_GENERATED_CODE'],
name)
if not os.path.exists(generated_code_directory):
os.makedirs(generated_code_directory)
try:
with open(generated_code_directory + '/__init__.py',
"w+") as f:
f.write('')
except Exception:
pass
try:
protoc.main(('', '-I' + proto_directory,
'--python_out=' + generated_code_directory,
'--grpc_python_out=' + generated_code_directory,
os.path.join(proto_directory, 'file.proto')))
except Exception as e:
print(str(e))
code = ''
competition = Competition(
None,
name=name,
datastream_id=datastream_id,
initial_batch_size=initial_batch_size,
initial_training_time=initial_training_time,
batch_size=batch_size,
time_interval=time_interval,
target_class=target_class,
start_date=start_date,
file_path=file_name,
predictions_time_interval=predictions_time_interval,
end_date=end_date,
description=description,
code=code)
_COMPETITION_REPO.insert_one(competition)
code = code_generator(competition.competition_id)
_COMPETITION_REPO.set_competition_code(competition.competition_id,
code)
evaluation_measures = {
'competition_id': competition.competition_id,
'measures': competition_config
}
_MONGO_REPO.insert_document('evaluation_measures',
'evaluation_measures', evaluation_measures)
_SCHEDULER.schedule_competition(competition, competition_config)
return json.dumps({'success': True}), 200, {
'ContentType': 'application/json'
}
return json.dumps({'error': True}), 500, {
'ContentType': 'application/json'
}
"""
Runs protoc with the gRPC plugin to generate messages and gRPC stubs
"""
from grpc_tools import protoc
protoc.main((
'',
'--proto_path=.',
'--python_out=.',
'--grpc_python_out=.',
'./hellorepeatedworld.proto',
))
from grpc_tools import protoc
protoc.main(('', '-I../protos', '--python_out=.', '--grpc_python_out=.',
'../protos/store.proto'))
# Copyright 2018 Dgraph Labs, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
import os
from grpc_tools import protoc
dirpath = os.path.dirname(os.path.realpath(__file__))
protopath = os.path.realpath(os.path.join(dirpath, '../pydgraph/proto'))
protoc.main((
'',
'-I' + protopath,
'--python_out=' + protopath,
'--grpc_python_out=' + protopath,
os.path.join(protopath, 'api.proto'),
))
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main((
'',
'-I../../protos',
'--python_out=.',
'--grpc_python_out=.',
'../../protos/route_guide.proto',
))
import subprocess
from grpc_tools import protoc
subprocess.check_output(
'cp -f ../../server/src/magic_style/rpc/anime_style/anime_style.proto ./rpc',
shell=True)
protoc.main(('', '-I./', '--python_out=.', '--grpc_python_out=.',
'./rpc/anime_style.proto'))
#coding:utf-8
from grpc_tools import protoc
protoc.main((
'.',
'-I./protos',
'--python_out=./python/',
'--grpc_python_out=./python/',
'./protos/IUserService.proto',
'./protos/IActivityService.proto',
'./protos/IAppVersionService.proto',
'./protos/ICouponRecordService.proto',
'./protos/IMessageService.proto',
'./protos/IOrderService.proto',
'./protos/IProductService.proto',
'./protos/IIndexService.proto',
))
import glob
from grpc_tools import protoc
for f in glob.glob('ds\\proto\\*.proto'):
print(f)
protoc.main((
'',
'-I./ds/proto',
'--python_out=./ds/proto',
'--grpc_python_out=./ds/proto',
f,
))
from grpc_tools import protoc
#grpc_tools.protoc -I./protos --python_out=. --grpc_python_out=. ./protos/example.proto
protoc.main((
'',
'-I./protos',
'--python_out=.',
'--grpc_python_out=.',
'./protos/pingpong.proto',
))
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
import os
files = os.listdir('.')
proto_files = []
for x in files:
if ".proto" in x:
proto_files.append(x)
for x in proto_files:
protoc.main((
'',
'-I.',
'--python_out=.',
'--grpc_python_out=.',
x,
))
from grpc_tools import protoc
protoc.main(('', '-I../protos', '--python_out=.', '--grpc_python_out=.',
'../protos/command_line.proto'))
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Generates protocol messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main(
(
'',
'-I../../protos',
'--python_out=.',
'--grpc_python_out=.',
'../../protos/helloworld.proto',
)
)
protoc.main(
(
'',
'-I../../protos',
'--python_out=.',
'--grpc_python_out=.',
'../../protos/route_guide.proto',
)
)
def generate_proto_files(force=False, log=None):
try:
import grpc_tools # pylint: disable=unused-import
except ImportError:
warnings.warn(
'Installing grpcio-tools is recommended for development.')
if log is None:
logging.basicConfig()
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
py_sdk_root = os.path.dirname(os.path.abspath(__file__))
common = os.path.join(py_sdk_root, '..', 'common')
proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS]
proto_files = sum(
[glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], [])
out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH)
out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))]
if out_files and not proto_files and not force:
# We have out_files but no protos; assume they're up to date.
# This is actually the common case (e.g. installation from an sdist).
log.info('No proto files; using existing generated files.')
return
elif not out_files and not proto_files:
if not os.path.exists(common):
raise RuntimeError(
'Not in apache git tree; unable to find proto definitions.')
else:
raise RuntimeError('No proto files found in %s.' % proto_dirs)
if force:
regenerate = 'forced'
elif not out_files:
regenerate = 'no output files'
elif len(out_files) < len(proto_files):
regenerate = 'not enough output files'
elif (min(os.path.getmtime(path) for path in out_files) <= max(
os.path.getmtime(path)
for path in proto_files + [os.path.realpath(__file__)])):
regenerate = 'output files are out-of-date'
elif len(out_files) > len(proto_files):
regenerate = 'output files without corresponding .proto files'
# too many output files: probably due to switching between git branches.
# remove them so they don't trigger constant regeneration.
for out_file in out_files:
os.remove(out_file)
else:
regenerate = None
if regenerate:
try:
from grpc_tools import protoc
except ImportError:
if platform.system() == 'Windows':
# For Windows, grpcio-tools has to be installed manually.
raise RuntimeError(
'Cannot generate protos for Windows since grpcio-tools package is '
'not installed. Please install this package manually '
'using \'pip install grpcio-tools\'.')
# Use a subprocess to avoid messing with this process' path and imports.
# Note that this requires a separate module from setup.py for Windows:
# https://docs.python.org/2/library/multiprocessing.html#windows
p = multiprocessing.Process(
target=_install_grpcio_tools_and_generate_proto_files,
kwargs={'force': force})
p.start()
p.join()
if p.exitcode:
raise ValueError(
"Proto generation failed (see log for details).")
else:
log.info('Regenerating Python proto definitions (%s).' %
regenerate)
builtin_protos = pkg_resources.resource_filename(
'grpc_tools', '_proto')
protoc_gen_mypy = _find_protoc_gen_mypy()
log.info('Found protoc_gen_mypy at %s' % protoc_gen_mypy)
args = (
[sys.executable] + # expecting to be called from command line
['--proto_path=%s' % builtin_protos] +
['--proto_path=%s' % d
for d in proto_dirs] + ['--python_out=%s' % out_dir] +
['--plugin=protoc-gen-mypy=%s' % protoc_gen_mypy] +
['--mypy_out=%s' % out_dir] +
# TODO(robertwb): Remove the prefix once it's the default.
['--grpc_python_out=grpc_2_0:%s' % out_dir] + proto_files)
ret_code = protoc.main(args)
if ret_code:
raise RuntimeError(
'Protoc returned non-zero status (see logs for details): '
'%s' % ret_code)
# copy resource files
for path in MODEL_RESOURCES:
shutil.copy2(os.path.join(py_sdk_root, path), out_dir)
ret_code = subprocess.call(
["futurize", "--both-stages", "--write", "--no-diff", out_dir])
if ret_code:
raise RuntimeError(
'Error applying futurize to generated protobuf python files.'
)
generate_urn_files(log, out_dir)
else:
log.info('Skipping proto regeneration: all files up to date')
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
from grpc_tools import protoc
protoc.main((
'',
'-I./protos',
'--python_out=python_grpc_mutual_tls_auth',
'--grpc_python_out=python_grpc_mutual_tls_auth',
'./protos/mutual_tls_auth.proto',
))
