def __params2args(self, argu_dict, params): arguments = "" for p in params: if argu_dict[p] is not None: if p not in self.none_params: arguments += docker_config.arg(p, argu_dict[p]) return arguments
def __params2args(self, argu_dict, params): """Convert function parameters into string of arguments for input JSON Args: argu_dict (dict): the dictionary of key and value for function parameters params (set): the paramater names of certain function Returns: str: converted string for function parameters """ arguments = "" for p in params: if argu_dict[p] is not None: if p not in self.none_params: # no need to convert arguments += docker_config.arg(p, argu_dict[p]) return arguments
def convert_model(self, model_type=None, output_onnx_path=None, model="", model_params=None, model_input_shapes=None, target_opset=None, caffe_model_prototxt=None, initial_types=None, model_inputs_names=None, model_outputs_names=None, input_json=None, convert_json=False, windows=False): # is Windows if os.name == 'nt': windows = True def mount_parameters(output_onnx_path, model, caffe_model_prototxt, input_json): # --output_onnx_path if output_onnx_path is None or output_onnx_path == '': output_onnx_path = self.convert_path output_onnx_path = self.__join_with_mount(output_onnx_path) # --model model = self.__join_with_mount(model) # --caffe_model_prototxt if caffe_model_prototxt is not None: caffe_model_prototxt = self.__join_with_mount( caffe_model_prototxt) if input_json is not None: input_json = self.__join_with_mount(input_json) return output_onnx_path, model, caffe_model_prototxt, input_json if model_type is None and input_json is None: raise RuntimeError('The conveted model type needs to be provided.') img_name = (docker_config.CONTAINER_NAME + docker_config.FUNC_NAME['onnx_converter'] + ':latest') # --input_json if input_json is not None: local_input_json = input_json output_onnx_path, model, caffe_model_prototxt, input_json = mount_parameters( output_onnx_path, model, caffe_model_prototxt, input_json) # --initial_types if initial_types is not None: if convert_json: initial_types = '[(\'' + initial_types[ 0] + '\',' + initial_types[1] + ')]' else: initial_types = '"[(\'' + initial_types[ 0] + '\',' + initial_types[1] + ')]"' # create test directory for output if self.convert_directory is not None: test_path = posixpath.join(self.path, self.convert_directory) if not os.path.exists(test_path): os.makedirs(test_path) json_filename = input_json parameters = self.convert_model.__code__.co_varnames[ 1:self.convert_model.__code__.co_argcount] arguments = self.__params2args(locals(), parameters) # convert the input parameters into json file if convert_json: self.__convert_input_json(arguments, json_filename) arguments = docker_config.arg('input_json', input_json) # load by JSON file elif input_json is not None: with open(posixpath.join(self.path, local_input_json), 'r') as f: json_data = json.load(f) if 'output_onnx_path' in json_data and ( json_data['output_onnx_path'] is not None and json_data['output_onnx_path'] != ''): output_onnx_path = json_data['output_onnx_path'] self.convert_path = output_onnx_path params = mount_parameters(output_onnx_path, model, caffe_model_prototxt, input_json) output_onnx_path = params[0] if 'model' in json_data: model = json_data['model'] if 'caffe_model_prototxt' in json_data: caffe_model_prototxt = json_data['caffe_model_prototxt'] # convert to mount path _, model, caffe_model_prototxt, _ = mount_parameters( output_onnx_path, model, caffe_model_prototxt, input_json) # write back to JSON json_data['output_onnx_path'] = output_onnx_path if 'model' in json_data: json_data['model'] = model if 'caffe_model_prototxt' in json_data: json_data['caffe_model_prototxt'] = caffe_model_prototxt with open(posixpath.join(self.path, local_input_json), 'w') as f: json.dump(json_data, f) stream = self.client.containers.run( image=img_name, command=arguments, volumes={self.path: { 'bind': self.mount_path, 'mode': 'rw' }}, detach=True) if self.print_logs: self.__print_docker_logs(stream, windows) return output_onnx_path
def perf_tuning(self, model=None, result=None, config=None, mode=None, execution_provider=None, repeated_times=None, duration_times=None, threadpool_size=None, num_threads=None, top_n=None, parallel=None, runtime=True, input_json=None, convert_json=False, windows=False): # is Windows, there is no runtime if os.name == 'nt': runtime = False windows = True def mount_parameters(model, result, input_json): # --model if model is None: model = self.convert_path model = self.__join_with_mount(model) result = self.__join_with_mount(self.result) # --input_json if input_json is not None: input_json = self.__join_with_mount(input_json) return model, result, input_json json_filename = input_json # --input_json if input_json is not None: local_input_json = input_json if result is not None: self.result = result model, result, input_json = mount_parameters(model, result, input_json) img_name = (docker_config.CONTAINER_NAME + docker_config.FUNC_NAME['perf_tuning'] + ':latest') parameters = self.perf_tuning.__code__.co_varnames[ 1:self.perf_tuning.__code__.co_argcount] arguments = self.__params2args(locals(), parameters) # convert the input parameters into json file if convert_json: self.__convert_input_json(arguments, json_filename) arguments = docker_config.arg('input_json', input_json) # load by JSON file elif input_json is not None: with open(posixpath.join(self.path, local_input_json)) as f: json_data = json.load(f) if 'result' in json_data: result = json_data['result'] if result[:len(self.mount_path)] == self.mount_path: self.result = str(result[len(self.mount_path) + 1:]) else: self.result = result params = mount_parameters(model, None, input_json) result = params[1] if 'runtime' in json_data: runtime = json_data['runtime'] if 'model' in json_data: model = json_data['model'] # convert to mount path model, _, _ = mount_parameters(model, result, input_json) # write back to JSON json_data['result'] = result if 'model' in json_data: json_data['model'] = model with open(posixpath.join(self.path, local_input_json), 'w') as f: json.dump(json_data, f) runtime = 'nvidia' if runtime else '' stream = self.client.containers.run( image=img_name, command=arguments, volumes={self.path: { 'bind': self.mount_path, 'mode': 'rw' }}, runtime=runtime, detach=True) if self.print_logs: self.__print_docker_logs(stream, windows) return posixpath.join(self.path, self.result)
def perf_tuning(self, model=None, result=None, config=None, mode=None, execution_provider=None, repeated_times=None, duration_times=None, inter_op_num_threads=None, intra_op_num_threads=None, top_n=None, parallel=None, runtime=True, input_json=None, convert_json=False, windows=False): """Parameters usage could reference: https://github.com/microsoft/OLive/blob/master/notebook/onnx-pipeline.ipynb""" # is Windows, there is no runtime if os.name == 'nt': runtime = False windows = True # add mounted path into local path and handle missing parameters with default path def mount_parameters(model, result, input_json): # --model if model is None: model = self.convert_path model = self.__join_with_mount(model) result = self.__join_with_mount(self.result) # --input_json if input_json is not None: input_json = self.__join_with_mount(input_json) return model, result, input_json json_filename = input_json # --input_json if input_json is not None: local_input_json = input_json if result is not None: self.result = result model, result, input_json = mount_parameters(model, result, input_json) img_name = (docker_config.CONTAINER_NAME + docker_config.FUNC_NAME['perf_tuning'] + ':latest') parameters = self.perf_tuning.__code__.co_varnames[ 1:self.perf_tuning.__code__.co_argcount] arguments = self.__params2args(locals(), parameters) # convert the input parameters into json file if convert_json: self.__convert_input_json(arguments, json_filename) arguments = docker_config.arg('input_json', input_json) # load by JSON file and handle missing parameters with default path, add mounted path into original path elif input_json is not None: with open(posixpath.join(self.path, local_input_json)) as f: json_data = json.load(f) if 'result' in json_data: result = json_data['result'] if result[:len(self.mount_path)] == self.mount_path: self.result = str(result[len(self.mount_path) + 1:]) else: self.result = result params = mount_parameters(model, None, input_json) result = params[1] if 'runtime' in json_data: runtime = json_data['runtime'] if 'model' in json_data: model = json_data['model'] # convert to mount path model, _, _ = mount_parameters(model, result, input_json) # write back to JSON json_data['result'] = result if 'model' in json_data: json_data['model'] = model # after parameter modification, overlap the original JSON file with open(posixpath.join(self.path, local_input_json), 'w') as f: json.dump(json_data, f) # run nvidia if enabled runtime = 'nvidia' if runtime else '' # run docker commands to execute stream = self.client.containers.run( image=img_name, command=arguments, volumes={self.path: { 'bind': self.mount_path, 'mode': 'rw' }}, runtime=runtime, detach=True) if self.print_logs: self.__print_docker_logs(stream, windows) return posixpath.join(self.path, self.result)
def main(): main_parser = argparse.ArgumentParser(prog="cosmicpi", description="CosmicPi acquisition process", add_help=False) main_parser.add_argument("--config", help="Path to configuration file", default="/etc/cosmicpi.yaml") args, remaining_argv = main_parser.parse_known_args() # Merge the default config with the configuration file config = load_config(args.config) # Parse the command line for overrides parser = argparse.ArgumentParser(parents=[main_parser]) parser.set_defaults(**config) parser.add_argument("-i", "--host", **arg("broker.host", "Message broker host")) parser.add_argument("-p", "--port", **arg("broker.port", "Message broker port", type=int)) parser.add_argument("-a", "--username", **arg("broker.username", "Message broker username")) parser.add_argument("-b", "--password", **arg("broker.password", "Message broker password")) parser.add_argument("-n", "--no-publish", **arg("broker.enabled", "Disable event publication")) parser.add_argument("-u", "--usb", **arg("usb.device", "USB device name")) parser.add_argument("-d", "--debug", **arg("debug", "Enable debug mode")) parser.add_argument("-o", "--log-config", **arg("logging.config", "Path to logging configuration")) parser.add_argument("-l", "--no-log", **arg("logging.enabled", "Disable file logging")) parser.add_argument("-v", "--no-vib", **arg("monitoring.vibration", "Disable vibration monitoring")) parser.add_argument("-w", "--no-weather", **arg("monitoring.weather", "Disable weather monitoring")) parser.add_argument("-c", "--no-cosmics", **arg("monitoring.cosmics", "Disable cosmic ray monitoring")) parser.add_argument("-k", "--patk", **arg("patok", "Server push notification token")) options = parser.parse_args() log_config = options.logging["config"] print ("INFO: using logging configuration from %s" % log_config) logging.config.fileConfig(log_config, disable_existing_loggers=False) console = logging.getLogger(__name__) if options.debug: print_config(options) try: publisher = EventPublisher(options) except: console.error("Exception: Can't connect to broker") sys.exit(1) try: usb = UsbHandler(options.usb['device'], 9600, 60) usb.open() except Exception as e: console.error("Exception: Can't open USB device: %s" % e) sys.exit(1) detector = Detector(usb, publisher, options) try: detector.start() command_handler = CommandHandler(detector, usb, options) command_handler.start() while True: time.sleep(1) except Exception as e: console.info("Exception: main: %s" % e) traceback.print_exc() finally: detector.stop() console.info("Quitting ...") time.sleep(1) usb.close() publisher.close() sys.exit(0)
import os import sys import tensorflow as tf from tensorflow import keras import argparse import models import dataloader from config import Config, Argument as arg IMG_WIDTH = 320 IMG_HEIGHT = 180 D_STATE = 1 config = Config( dataset=arg(str, 'dataset', 'TFRecords file'), validation=arg(str, 'validation', 'TFRecords file'), model_name=arg(str, 'bc', 'Output directory'), batch_size=arg(int, 64, "Batch size"), epochs=arg(int, 50, 'How many epochs to train'), lr=arg(float, 1e-4, 'Learning rate'), shuffle_buffer=arg(int, 8000, 'Shuffle buffer size'), workers=arg(int, 4, 'How many threads to use for preprocessing'), augment=Config(flip_prob=arg( float, 0.2, 'Probability of flipping image in preprocessing'), translate=arg(int, 25, 'translation magnitude in preprocessing'), min_contrast=arg(float, 0.9, 'Min contrast augmentation'), max_contrast=arg(float, 1.1, 'Max contrast augmentation'), min_saturation=arg(float, 0.9, 'Min saturation'), max_satutation=arg(float, 1.1, 'Max saturation'), hue=arg(float, 0.1, 'Hue max delta')))