def build_package_protos(package_root, strict_mode=False): proto_files = [] inclusion_root = os.path.abspath(package_root) for root, _, files in os.walk(inclusion_root): for filename in files: if filename.endswith('.proto'): proto_files.append( os.path.abspath(os.path.join(root, filename))) well_known_protos_include = pkg_resources.resource_filename( 'grpc_tools', '_proto') for proto_file in proto_files: command = [ 'grpc_tools.protoc', '--proto_path={}'.format(inclusion_root), '--proto_path={}'.format(well_known_protos_include), '--python_out={}'.format(inclusion_root), '--grpc_python_out={}'.format(inclusion_root), ] + [proto_file] if protoc.main(command) != 0: if strict_mode: raise Exception('error: {} failed'.format(command)) else: sys.stderr.write('warning: {} failed'.format(command))
def setUp(self): same_proto_contents = pkgutil.get_data( 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') self.directory = tempfile.mkdtemp(suffix='same_common', dir='.') self.proto_directory = os.path.join(self.directory, 'proto_path') self.python_out_directory = os.path.join(self.directory, 'python_out') self.grpc_python_out_directory = self.python_out_directory os.makedirs(self.proto_directory) os.makedirs(self.python_out_directory) same_proto_file = os.path.join(self.proto_directory, 'same_common.proto') open(same_proto_file, 'wb').write( same_proto_contents.replace( _COMMON_NAMESPACE, b'package grpc_protoc_plugin.invocation_testing.same_common;')) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), '--python_out={}'.format(self.python_out_directory), '--grpc_python_out={}'.format(self.grpc_python_out_directory), same_proto_file, ]) if protoc_result != 0: raise Exception("unexpected protoc error") open(os.path.join(self.python_out_directory, '__init__.py'), 'w').write('') self.pb2_import = 'same_common_pb2' self.pb2_grpc_import = 'same_common_pb2_grpc' self.should_find_services_in_pb2 = True
def run(self): import grpc_tools.protoc as protoc include_regex = re.compile(self.include) exclude_regex = re.compile(self.exclude) if self.exclude else None paths = [] for walk_root, directories, filenames in os.walk(PROTO_STEM): for filename in filenames: path = os.path.join(walk_root, filename) if include_regex.match(path) and not ( exclude_regex and exclude_regex.match(path)): paths.append(path) # TODO(kpayson): It would be nice to do this in a batch command, # but we currently have name conflicts in src/proto for path in paths: command = [ 'grpc_tools.protoc', '-I {}'.format(PROTO_STEM), '--python_out={}'.format(PROTO_STEM), '--grpc_python_out={}'.format(PROTO_STEM), ] + [path] if protoc.main(command) != 0: sys.stderr.write( 'warning: Command:\n{}\nFailed'.format(command)) # Generated proto directories dont include __init__.py, but # these are needed for python package resolution for walk_root, _, _ in os.walk(PROTO_STEM): path = os.path.join(walk_root, '__init__.py') open(path, 'a').close()
def generate_proto_files(): try: import grpc_tools except ImportError: warnings.warn('Installing grpcio-tools is recommended for development.') py_sdk_root = os.path.dirname(os.path.abspath(__file__)) common = os.path.join(py_sdk_root, '..', 'common') proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS] proto_files = sum( [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH) out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] if out_files and not proto_files: # We have out_files but no protos; assume they're up to date. # This is actually the common case (e.g. installation from an sdist). logging.info('No proto files; using existing generated files.') return elif not out_files and not proto_files: if not os.path.exists(common): raise RuntimeError( 'Not in apache git tree; unable to find proto definitions.') else: raise RuntimeError( 'No proto files found in %s.' % proto_dirs) # Regenerate iff the proto files are newer. elif not out_files or len(out_files) < len(proto_files) or ( min(os.path.getmtime(path) for path in out_files) <= max(os.path.getmtime(path) for path in proto_files)): try: from grpc_tools import protoc except ImportError: # Use a subprocess to avoid messing with this process' path and imports. # Note that this requires a separate module from setup.py for Windows: # https://docs.python.org/2/library/multiprocessing.html#windows p = multiprocessing.Process( target=_install_grpcio_tools_and_generate_proto_files) p.start() p.join() else: logging.info('Regenerating out-of-date Python proto definitions.') builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto') args = ( [sys.executable] + # expecting to be called from command line ['--proto_path=%s' % builtin_protos] + ['--proto_path=%s' % d for d in proto_dirs] + ['--python_out=%s' % out_dir] + ['--grpc_python_out=%s' % out_dir] + proto_files) ret_code = protoc.main(args) if ret_code: raise RuntimeError( 'Protoc returned non-zero status (see logs for details): ' '%s' % ret_code)
def _protoc(proto_path, python_out, grpc_python_out_flag, grpc_python_out, absolute_proto_file_names): args = [ '', '--proto_path={}'.format(proto_path), ] if python_out is not None: args.append('--python_out={}'.format(python_out)) if grpc_python_out is not None: args.append('--grpc_python_out={}:{}'.format(grpc_python_out_flag, grpc_python_out)) args.extend(absolute_proto_file_names) return protoc.main(args)
def setUp(self): services_proto_contents = pkgutil.get_data( 'tests.protoc_plugin.protos.invocation_testing.split_services', 'services.proto') messages_proto_contents = pkgutil.get_data( 'tests.protoc_plugin.protos.invocation_testing.split_messages', 'messages.proto') self.directory = tempfile.mkdtemp(suffix='split_separate', dir='.') self.proto_directory = os.path.join(self.directory, 'proto_path') self.python_out_directory = os.path.join(self.directory, 'python_out') self.grpc_python_out_directory = os.path.join(self.directory, 'grpc_python_out') os.makedirs(self.proto_directory) os.makedirs(self.python_out_directory) os.makedirs(self.grpc_python_out_directory) services_proto_file = os.path.join(self.proto_directory, 'split_separate_services.proto') messages_proto_file = os.path.join(self.proto_directory, 'split_separate_messages.proto') open(services_proto_file, 'wb').write( services_proto_contents.replace( _MESSAGES_IMPORT, b'import "split_separate_messages.proto";') .replace( _SPLIT_NAMESPACE, b'package grpc_protoc_plugin.invocation_testing.split_separate;' )) open(messages_proto_file, 'wb').write( messages_proto_contents.replace( _SPLIT_NAMESPACE, b'package grpc_protoc_plugin.invocation_testing.split_separate;' )) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), '--python_out={}'.format(self.python_out_directory), '--grpc_python_out=grpc_2_0:{}'.format( self.grpc_python_out_directory), services_proto_file, messages_proto_file, ]) if protoc_result != 0: raise Exception("unexpected protoc error") open(os.path.join(self.python_out_directory, '__init__.py'), 'w').write('') self.pb2_import = 'split_separate_messages_pb2' self.pb2_grpc_import = 'split_separate_services_pb2_grpc' self.should_find_services_in_pb2 = False
def _protoc(self): args = [ '', '--proto_path={}'.format(self._proto_path), '--python_out={}'.format(self._python_out), '--grpc_python_out=grpc_1_0:{}'.format(self._python_out), ] + list(self._proto_file_names) protoc_exit_code = protoc.main(args) self.assertEqual(0, protoc_exit_code) _packagify(self._python_out) with _system_path([self._python_out]): self._payload_pb2 = importlib.import_module(_PAYLOAD_PB2) self._requests_pb2 = importlib.import_module(_REQUESTS_PB2) self._responses_pb2 = importlib.import_module(_RESPONSES_PB2) self._service_pb2 = importlib.import_module(_SERVICE_PB2)
# Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc protoc.main(( '', '-I../protos', '--python_out=.', '--grpc_python_out=.', '../protos/prime.proto', ))
from grpc_tools import protoc protoc.main(('', '-I./app/proto', '--python_out=./app/proto/dist', '--grpc_python_out=./app/proto/dist', 'simple.proto'))
def generate_proto_files(force=False): try: import grpc_tools # pylint: disable=unused-variable except ImportError: warnings.warn('Installing grpcio-tools is recommended for development.') py_sdk_root = os.path.dirname(os.path.abspath(__file__)) common = os.path.join(py_sdk_root, '..', 'common') proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS] proto_files = sum( [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH) out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] if out_files and not proto_files and not force: # We have out_files but no protos; assume they're up to date. # This is actually the common case (e.g. installation from an sdist). logging.info('No proto files; using existing generated files.') return elif not out_files and not proto_files: if not os.path.exists(common): raise RuntimeError( 'Not in apache git tree; unable to find proto definitions.') else: raise RuntimeError( 'No proto files found in %s.' % proto_dirs) # Regenerate iff the proto files are newer. elif force or not out_files or len(out_files) < len(proto_files) or ( min(os.path.getmtime(path) for path in out_files) <= max(os.path.getmtime(path) for path in proto_files)): try: from grpc_tools import protoc except ImportError: if platform.system() == 'Windows': # For Windows, grpcio-tools has to be installed manually. raise RuntimeError( 'Cannot generate protos for Windows since grpcio-tools package is ' 'not installed. Please install this package manually ' 'using \'pip install grpcio-tools\'.') # Use a subprocess to avoid messing with this process' path and imports. # Note that this requires a separate module from setup.py for Windows: # https://docs.python.org/2/library/multiprocessing.html#windows p = multiprocessing.Process( target=_install_grpcio_tools_and_generate_proto_files) p.start() p.join() if p.exitcode: raise ValueError("Proto generation failed (see log for details).") else: logging.info('Regenerating out-of-date Python proto definitions.') builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto') args = ( [sys.executable] + # expecting to be called from command line ['--proto_path=%s' % builtin_protos] + ['--proto_path=%s' % d for d in proto_dirs] + ['--python_out=%s' % out_dir] + # TODO(robertwb): Remove the prefix once it's the default. ['--grpc_python_out=grpc_2_0:%s' % out_dir] + proto_files) ret_code = protoc.main(args) if ret_code: raise RuntimeError( 'Protoc returned non-zero status (see logs for details): ' '%s' % ret_code) if sys.version_info[0] >= 3: ret_code = subprocess.call( ["futurize", "--both-stages", "--write", "--verbose", "--no-diff", out_dir]) if ret_code: raise RuntimeError( 'Error applying futurize to generated protobuf python files.')
# -- Compile Protobufs ------------------------------------------------------- # Protobufs are not precompiled and need to be generated at Sphinx runtime. # This is a hack and should be made into an extension. import pkg_resources from glob import glob from grpc_tools import protoc _proto_include = pkg_resources.resource_filename('grpc_tools', '_proto') _project_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) _status_code = protoc.main([ f"-I{_project_dir}", f"--python_out={_project_dir}", f"--grpc_python_out={_project_dir}", *glob(f"{_project_dir}/needlestack/apis/*.proto"), f"-I{_proto_include}", f"--proto_path={_proto_include}", f"--proto_path={_project_dir}", ]) # -- Project information ----------------------------------------------------- project = 'Needlestack' copyright = '2019, Cung Tran' author = 'Cung Tran' version = "0.1.0" release = version # -- General configuration ---------------------------------------------------
if not os.path.exists(proto_directory): os.makedirs(proto_directory) copy("file.proto", proto_directory) if not os.path.exists(generated_code_directory): os.makedirs(generated_code_directory) try: with open(generated_code_directory + '/__init__.py', "w+") as f: f.write('') except Exception: pass try: protoc.main(('', '-I' + proto_directory, '--python_out=' + generated_code_directory, '--grpc_python_out=' + generated_code_directory, os.path.join(proto_directory, 'file.proto'))) except Exception as e: print(str(e)) competition_config = {"target": ["MAPE"]} code = '' competition = Competition(None, name=name, datastream_id=1, initial_batch_size=20, initial_training_time=20, batch_size=5, time_interval=5, target_class="target", start_date=format_time(now), file_path="file.proto",
# Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc protoc.main(( '', '-I.', '--python_out=.', '--grpc_python_out=.', 'rpcDefinitions.proto', ))
#Parts: # Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc import sys import os install_dir = os.path.join(os.environ['MESON_INSTALL_DESTDIR_PREFIX'], 'libexec', 'warpinator') protoc.main(( '', '-I%s' % sys.argv[1], '--python_out=%s' % install_dir, '--grpc_python_out=%s' % install_dir, os.path.join(sys.argv[1], sys.argv[2]) ))
# -*- coding:utf-8 -*- from grpc_tools import protoc protoc.main( ( '', '-I../proto_buffer_config', '--python_out=../pb_api', '--grpc_python_out=../pb_api', '../proto_buffer_config/test.proto', ), # ( # '', # '-I./grpc_protos', # '--python_out=./pb', # '--grpc_python_out=./pb', # './grpc_protos/user_info.proto', # ) # python3 -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. helloworld.proto )
# Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc protoc.main(( '', '-I../protos', '--python_out=.', '--grpc_python_out=.', '../protos/ouroboros.proto', ))
def generate_proto_files(force=True, output_dir=DEFAULT_PYTHON_OUTPUT_PATH): try: import grpc_tools # noqa # pylint: disable=unused-import except ImportError: warnings.warn( 'Installing grpcio-tools is recommended for development.') proto_dirs = [ os.path.join(PYFLINK_ROOT_PATH, path) for path in PROTO_PATHS ] proto_files = sum( [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) out_dir = os.path.join(PYFLINK_ROOT_PATH, output_dir) out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] if out_files and not proto_files and not force: # We have out_files but no protos; assume they're up to date. # This is actually the common case (e.g. installation from an sdist). logging.info('No proto files; using existing generated files.') return elif not out_files and not proto_files: raise RuntimeError('No proto files found in %s.' % proto_dirs) # Regenerate iff the proto files or this file are newer. elif force or not out_files or len(out_files) < len(proto_files) or (min( os.path.getmtime(path) for path in out_files) <= max( os.path.getmtime(path) for path in proto_files + [os.path.realpath(__file__)])): try: from grpc_tools import protoc except ImportError: if platform.system() == 'Windows': # For Windows, grpcio-tools has to be installed manually. raise RuntimeError( 'Cannot generate protos for Windows since grpcio-tools package is ' 'not installed. Please install this package manually ' 'using \'pip install grpcio-tools\'.') # Use a subprocess to avoid messing with this process' path and imports. # Note that this requires a separate module from setup.py for Windows: # https://docs.python.org/2/library/multiprocessing.html#windows p = multiprocessing.Process( target=_install_grpcio_tools_and_generate_proto_files( force, output_dir)) p.start() p.join() if p.exitcode: raise ValueError( "Proto generation failed (see log for details).") else: _check_grpcio_tools_version() logging.info('Regenerating out-of-date Python proto definitions.') builtin_protos = pkg_resources.resource_filename( 'grpc_tools', '_proto') args = ( [sys.executable] + # expecting to be called from command line ['--proto_path=%s' % builtin_protos] + ['--proto_path=%s' % d for d in proto_dirs] + ['--python_out=%s' % out_dir] + proto_files) ret_code = protoc.main(args) if ret_code: raise RuntimeError( 'Protoc returned non-zero status (see logs for details): ' '%s' % ret_code) for output_file in os.listdir(output_dir): if output_file.endswith('_pb2.py'): _add_license_header(output_dir, output_file)
#-*- coding:utf8 -*- # Copyright (c) 2020 barriery # Python release: 3.7.0 # Create time: 2020-04-04 from setuptools import setup, find_packages from grpc_tools import protoc # run bdware proto codegen protoc.main(( '', '-I.', '--python_out=.', '--grpc_python_out=.', './scheduletool/bdware/proto/schedule_service.proto', )) protoc.main(( '', '-I.', '--python_out=.', '--grpc_python_out=.', './scheduletool/buaacps/proto/entity.proto', )) protoc.main(( '', '-I.', '--python_out=.', '--grpc_python_out=.', './scheduletool/buaacps/proto/result.proto', ))
''' Created on 2020. 5. 20. @author: kwlee ''' from grpc_tools import protoc protoc.main(( '', '-Iproto', '--python_out=.', '--grpc_python_out=.', 'proto/marmot_type.proto', 'proto/marmot_dataset.proto', 'proto/marmot_file.proto', 'proto/dric.proto', )) if __name__ == '__main__': pass
# Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc protoc.main(( '', '-I.', '--python_out=.', '--grpc_python_out=.', 'csci4220_hw3.proto', ))
'_proto/google/protobuf/descriptor.proto'), f'{PROTO_DIR}/descriptor.proto') copyfile( '../../../../protobuf/io/casperlabs/casper/protocol/CasperMessage.proto', f'{PROTO_DIR}/CasperMessage.proto') os.system("""perl -p -i -e 's,^import \"google/protobuf/,import ",' proto/*""") os.system("""perl -p -i -e 's,^import \"scalapb/,import ",' proto/*""") google_proto = join(dirname(grpc_tools.__file__), '_proto') protoc.main(( '', f'-I./{PROTO_DIR}', '-I' + google_proto, '-I.', '--python_out=.', '--grpc_python_out=.', f'{PROTO_DIR}/empty.proto', )) protoc.main(( '', f'-I./{PROTO_DIR}', '-I' + google_proto, '-I.', '--python_out=.', '--grpc_python_out=.', f'{PROTO_DIR}/descriptor.proto', ))
from grpc_tools import protoc protoc.main( ( '', '-I.', '--python_out=.', 'FlightLog.proto', ) )
from grpc_tools import protoc import os #ssslibrary = os.path.dirname(os.__file__) library = "../../lib/python3.6/site-packages/" protoc.main( ( '', '--proto_path=.', '--python_out={}'.format(library), '--grpc_python_out={}'.format(library), './addition.proto', ) )
def generate_proto_files(force=False): try: import grpc_tools except ImportError: warnings.warn( 'Installing grpcio-tools is recommended for development.') py_sdk_root = os.path.dirname(os.path.abspath(__file__)) common = os.path.join(py_sdk_root, '..', 'common') proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS] proto_files = sum( [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH) out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] if out_files and not proto_files and not force: # We have out_files but no protos; assume they're up to date. # This is actually the common case (e.g. installation from an sdist). logging.info('No proto files; using existing generated files.') return elif not out_files and not proto_files: if not os.path.exists(common): raise RuntimeError( 'Not in apache git tree; unable to find proto definitions.') else: raise RuntimeError('No proto files found in %s.' % proto_dirs) # Regenerate iff the proto files are newer. elif force or not out_files or len(out_files) < len(proto_files) or (min( os.path.getmtime(path) for path in out_files) <= max( os.path.getmtime(path) for path in proto_files)): try: from grpc_tools import protoc except ImportError: if platform.system() == 'Windows': # For Windows, grpcio-tools has to be installed manually. raise RuntimeError( 'Cannot generate protos for Windows since grpcio-tools package is ' 'not installed. Please install this package manually ' 'using \'pip install grpcio-tools\'.') # Use a subprocess to avoid messing with this process' path and imports. # Note that this requires a separate module from setup.py for Windows: # https://docs.python.org/2/library/multiprocessing.html#windows p = multiprocessing.Process( target=_install_grpcio_tools_and_generate_proto_files) p.start() p.join() if p.exitcode: raise ValueError( "Proto generation failed (see log for details).") else: logging.info('Regenerating out-of-date Python proto definitions.') builtin_protos = pkg_resources.resource_filename( 'grpc_tools', '_proto') args = ( [sys.executable] + # expecting to be called from command line ['--proto_path=%s' % builtin_protos] + ['--proto_path=%s' % d for d in proto_dirs] + ['--python_out=%s' % out_dir] + # TODO(robertwb): Remove the prefix once it's the default. ['--grpc_python_out=grpc_2_0:%s' % out_dir] + proto_files) ret_code = protoc.main(args) if ret_code: raise RuntimeError( 'Protoc returned non-zero status (see logs for details): ' '%s' % ret_code)
from grpc_tools import protoc import os fnames = os.listdir('.') files = [] for fname in fnames: if fname.split('.')[-1] != 'proto': continue files.append(fname) params = ["-I.", "--python_out=./"] params.extend(files) protoc.main(params)
""" Runs protoc with the gRPC plugin to generate messages and gRPC stubs """ from grpc_tools import protoc # this file contains definitions of SERVICE_NAME, # RPC_AND_MESSAGE_NAMES and MESSAGE_FIELDS # these are shared between client and server from PROTO_DEFINITIONS import * subDirectory = '.' outcome = protoc.main(( '', '--proto_path=.', '--python_out=' + subDirectory, '--grpc_python_out=' + subDirectory, './' + PROTO_FILE + '.proto', )) print(outcome)
import os import sys import grpc_tools.protoc as protoc from setuptools import setup protoDir = os.path.split(os.path.abspath(sys.argv[0]))[0] + "/../proto/" proto = protoDir + "service.proto" if os.path.exists(protoDir): protoc.main([ "grpc_tools.protoc", "--proto_path=%s" % (protoDir, ), "--python_out=.", "--grpc_python_out=.", proto]) setup(name='packt_beam_chapter6', version='1.0', description='Packt Building Big Data Pipelines using Apache Beam chapter 6', py_modules=['service_pb2', 'service_pb2_grpc', 'utils'])
def competitions(): """ Competitions route. Implements two methods: GET: retrieve the competitions POST: create new competition: Parameters: - Competition name - Datastream - Description of the competition - Competition settings: size of initial batch, initial training time, size of a regular batch, time interval between the batches, name of the label column and the evaluation metric, start and end date, time interval to send the predictions, .proto file :return: Responses: {200: If method== "GET", return the list of the competitions, If method == "POST", confirm success, 500: Error} """ if request.method == 'GET': status = request.args.get('status') page = request.args.get('page') step = request.args.get('step') competitions = _COMPETITION_REPO.get_all_competitions( status, int(page), int(step)) return jsonify(competitions) if request.method == 'POST': # Adding a competition data = dict(request.form) data = data['competition'] data = data[0] data = json.loads(data) # TODO: store in mongodb competition_config = data['config'] print("Competition configuration: ", competition_config) logging.debug("Competition config: {}".format(competition_config)) name = data['name'] datastream_id = data['datastream_id'] initial_batch_size = data['initial_batch_size'] initial_training_time = data['initial_training_time'] batch_size = data['batch_size'] time_interval = data['time_interval'] target_class = data['target_class'] logging.debug("Competition target: {}".format(target_class)) start_date = data['start_date'].replace('T', ' ').replace('Z', '') end_date = data['end_date'].replace('T', ' ').replace('Z', '') predictions_time_interval = data['predictions_time_interval'] description = data['description'] data_file = request.files['file'] file_name = data_file.filename # Check if the file is one of the allowed types/extensions if data_file and allowed_file(file_name): # Make the filename safe, remove unsupported chars extension = get_file_extension(file_name) filename = secure_filename(str(name) + str(extension)) # Move the file form the temporal folder to # the upload folder we setup proto_directory = os.path.join(config['UPLOAD_REPO'], config['COMPETITION_PROTO_REPO'], name) if not os.path.exists(proto_directory): os.makedirs(proto_directory) data_file.save(os.path.join(proto_directory, 'file.proto')) generated_code_directory = os.path.join( config['UPLOAD_REPO'], config['COMPETITION_GENERATED_CODE'], name) if not os.path.exists(generated_code_directory): os.makedirs(generated_code_directory) try: with open(generated_code_directory + '/__init__.py', "w+") as f: f.write('') except Exception: pass try: protoc.main(('', '-I' + proto_directory, '--python_out=' + generated_code_directory, '--grpc_python_out=' + generated_code_directory, os.path.join(proto_directory, 'file.proto'))) except Exception as e: print(str(e)) code = '' competition = Competition( None, name=name, datastream_id=datastream_id, initial_batch_size=initial_batch_size, initial_training_time=initial_training_time, batch_size=batch_size, time_interval=time_interval, target_class=target_class, start_date=start_date, file_path=file_name, predictions_time_interval=predictions_time_interval, end_date=end_date, description=description, code=code) _COMPETITION_REPO.insert_one(competition) code = code_generator(competition.competition_id) _COMPETITION_REPO.set_competition_code(competition.competition_id, code) evaluation_measures = { 'competition_id': competition.competition_id, 'measures': competition_config } _MONGO_REPO.insert_document('evaluation_measures', 'evaluation_measures', evaluation_measures) _SCHEDULER.schedule_competition(competition, competition_config) return json.dumps({'success': True}), 200, { 'ContentType': 'application/json' } return json.dumps({'error': True}), 500, { 'ContentType': 'application/json' }
""" Runs protoc with the gRPC plugin to generate messages and gRPC stubs """ from grpc_tools import protoc protoc.main(( '', '--proto_path=.', '--python_out=.', '--grpc_python_out=.', './hellorepeatedworld.proto', ))
from grpc_tools import protoc protoc.main(('', '-I../protos', '--python_out=.', '--grpc_python_out=.', '../protos/store.proto'))
# Copyright 2018 Dgraph Labs, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" import os from grpc_tools import protoc dirpath = os.path.dirname(os.path.realpath(__file__)) protopath = os.path.realpath(os.path.join(dirpath, '../pydgraph/proto')) protoc.main(( '', '-I' + protopath, '--python_out=' + protopath, '--grpc_python_out=' + protopath, os.path.join(protopath, 'api.proto'), ))
# Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc protoc.main(( '', '-I../../protos', '--python_out=.', '--grpc_python_out=.', '../../protos/route_guide.proto', ))
import subprocess from grpc_tools import protoc subprocess.check_output( 'cp -f ../../server/src/magic_style/rpc/anime_style/anime_style.proto ./rpc', shell=True) protoc.main(('', '-I./', '--python_out=.', '--grpc_python_out=.', './rpc/anime_style.proto'))
#coding:utf-8 from grpc_tools import protoc protoc.main(( '.', '-I./protos', '--python_out=./python/', '--grpc_python_out=./python/', './protos/IUserService.proto', './protos/IActivityService.proto', './protos/IAppVersionService.proto', './protos/ICouponRecordService.proto', './protos/IMessageService.proto', './protos/IOrderService.proto', './protos/IProductService.proto', './protos/IIndexService.proto', ))
import glob from grpc_tools import protoc for f in glob.glob('ds\\proto\\*.proto'): print(f) protoc.main(( '', '-I./ds/proto', '--python_out=./ds/proto', '--grpc_python_out=./ds/proto', f, ))
from grpc_tools import protoc #grpc_tools.protoc -I./protos --python_out=. --grpc_python_out=. ./protos/example.proto protoc.main(( '', '-I./protos', '--python_out=.', '--grpc_python_out=.', './protos/pingpong.proto', ))
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc import os files = os.listdir('.') proto_files = [] for x in files: if ".proto" in x: proto_files.append(x) for x in proto_files: protoc.main(( '', '-I.', '--python_out=.', '--grpc_python_out=.', x, ))
from grpc_tools import protoc protoc.main(('', '-I../protos', '--python_out=.', '--grpc_python_out=.', '../protos/command_line.proto'))
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Generates protocol messages and gRPC stubs.""" from grpc_tools import protoc protoc.main( ( '', '-I../../protos', '--python_out=.', '--grpc_python_out=.', '../../protos/helloworld.proto', ) ) protoc.main( ( '', '-I../../protos', '--python_out=.', '--grpc_python_out=.', '../../protos/route_guide.proto', ) )
def generate_proto_files(force=False, log=None): try: import grpc_tools # pylint: disable=unused-import except ImportError: warnings.warn( 'Installing grpcio-tools is recommended for development.') if log is None: logging.basicConfig() log = logging.getLogger(__name__) log.setLevel(logging.INFO) py_sdk_root = os.path.dirname(os.path.abspath(__file__)) common = os.path.join(py_sdk_root, '..', 'common') proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS] proto_files = sum( [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH) out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] if out_files and not proto_files and not force: # We have out_files but no protos; assume they're up to date. # This is actually the common case (e.g. installation from an sdist). log.info('No proto files; using existing generated files.') return elif not out_files and not proto_files: if not os.path.exists(common): raise RuntimeError( 'Not in apache git tree; unable to find proto definitions.') else: raise RuntimeError('No proto files found in %s.' % proto_dirs) if force: regenerate = 'forced' elif not out_files: regenerate = 'no output files' elif len(out_files) < len(proto_files): regenerate = 'not enough output files' elif (min(os.path.getmtime(path) for path in out_files) <= max( os.path.getmtime(path) for path in proto_files + [os.path.realpath(__file__)])): regenerate = 'output files are out-of-date' elif len(out_files) > len(proto_files): regenerate = 'output files without corresponding .proto files' # too many output files: probably due to switching between git branches. # remove them so they don't trigger constant regeneration. for out_file in out_files: os.remove(out_file) else: regenerate = None if regenerate: try: from grpc_tools import protoc except ImportError: if platform.system() == 'Windows': # For Windows, grpcio-tools has to be installed manually. raise RuntimeError( 'Cannot generate protos for Windows since grpcio-tools package is ' 'not installed. Please install this package manually ' 'using \'pip install grpcio-tools\'.') # Use a subprocess to avoid messing with this process' path and imports. # Note that this requires a separate module from setup.py for Windows: # https://docs.python.org/2/library/multiprocessing.html#windows p = multiprocessing.Process( target=_install_grpcio_tools_and_generate_proto_files, kwargs={'force': force}) p.start() p.join() if p.exitcode: raise ValueError( "Proto generation failed (see log for details).") else: log.info('Regenerating Python proto definitions (%s).' % regenerate) builtin_protos = pkg_resources.resource_filename( 'grpc_tools', '_proto') protoc_gen_mypy = _find_protoc_gen_mypy() log.info('Found protoc_gen_mypy at %s' % protoc_gen_mypy) args = ( [sys.executable] + # expecting to be called from command line ['--proto_path=%s' % builtin_protos] + ['--proto_path=%s' % d for d in proto_dirs] + ['--python_out=%s' % out_dir] + ['--plugin=protoc-gen-mypy=%s' % protoc_gen_mypy] + ['--mypy_out=%s' % out_dir] + # TODO(robertwb): Remove the prefix once it's the default. ['--grpc_python_out=grpc_2_0:%s' % out_dir] + proto_files) ret_code = protoc.main(args) if ret_code: raise RuntimeError( 'Protoc returned non-zero status (see logs for details): ' '%s' % ret_code) # copy resource files for path in MODEL_RESOURCES: shutil.copy2(os.path.join(py_sdk_root, path), out_dir) ret_code = subprocess.call( ["futurize", "--both-stages", "--write", "--no-diff", out_dir]) if ret_code: raise RuntimeError( 'Error applying futurize to generated protobuf python files.' ) generate_urn_files(log, out_dir) else: log.info('Skipping proto regeneration: all files up to date')
"""Runs protoc with the gRPC plugin to generate messages and gRPC stubs.""" from grpc_tools import protoc protoc.main(( '', '-I./protos', '--python_out=python_grpc_mutual_tls_auth', '--grpc_python_out=python_grpc_mutual_tls_auth', './protos/mutual_tls_auth.proto', ))