def main(): parser = argparse.ArgumentParser(description='catapult tests runner') parser.add_argument('--image', help='docker tests image', required=True) parser.add_argument('--compiler-configuration', help='path to compiler configuration yaml', required=True) parser.add_argument('--user', help='docker user', required=True) parser.add_argument('--mode', help='test mode', choices=('bench', 'test', 'lint'), required=True) parser.add_argument('--verbosity', help='verbosity level', default='max') parser.add_argument('--dry-run', help='outputs desired commands without running them', action='store_true') args = parser.parse_args() process_manager = ProcessManager(args.dry_run) compose_template_directory = Path(__file__).parent / 'templates' compose_template_filepath = compose_template_directory / 'Run{}.yaml'.format( args.mode.capitalize()) print('processing template from {}'.format(compose_template_filepath)) prepare_replacements = { 'image_name': args.image, 'compiler_configuration': args.compiler_configuration, 'user': args.user, 'verbosity': args.verbosity } prepare_docker_compose_file(compose_template_filepath, prepare_replacements, sys.stdout) if not args.dry_run: with open('docker-compose.yaml', 'wt') as outfile: prepare_docker_compose_file(compose_template_filepath, prepare_replacements, outfile) environment_manager = EnvironmentManager(args.dry_run) environment_manager.set_env_var('COMPOSE_HTTP_TIMEOUT', '200') environment_manager.mkdirs(OUTPUT_DIR / 'logs', exist_ok=True) environment_manager.mkdirs(OUTPUT_DIR / 'workdir', exist_ok=True) if 'test' == args.mode: environment_manager.mkdirs(MONGO_DIR / get_image_label(args.image)) docker_compose_args = create_docker_compose_command(args.mode) if process_manager.dispatch_subprocess(docker_compose_args, handle_error=False): print('tests failed') sys.exit(1) print('tests succeeded')
def main(): parser = argparse.ArgumentParser(description='catapult bench runner') parser.add_argument('--exe-path', help='path to executables', required=True) parser.add_argument('--out-dir', help='directory in which to store bench output files', required=True) parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true') args = parser.parse_args() process_manager = ProcessManager(args.dry_run) environment_manager = EnvironmentManager(args.dry_run) for filepath in environment_manager.find_glob(args.exe_path, 'bench*'): output_filepath = Path(args.out_dir) / (filepath.name + '.json') process_manager.dispatch_subprocess([filepath, '--benchmark_out=' + str(output_filepath)])
def __init__(self): environment = EnvironmentManager() # Categories contiene la scelta delle feature (teen/notteen) (employed/notemployed) self.categories = environment.GetIndexedCategories() self.features = environment.features self.featureSpace = environment.featureSpace self.probabilities = np.array(environment.probabilities) self.possiblePrices = np.array(environment.prices) self.nArms = len(self.possiblePrices) self.expLogs = [] self.betaParams = [] self.week = -1
def __init__(self, portname): # TODO make function to transform fullpath to portname self.portname = portname self.platform = get_platform() self.version = '' self.name = '' self.default_version = {} self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src' self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads' self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build' self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz' self.environment = {} self.env = EnvironmentManager()
def __init__(self, budget=5.0, nArms=6): self.budget = budget self.nArms = nArms self.budgets = np.linspace(0.0, self.budget, self.nArms) advertisingEnvironment = EnvironmentManager() self.realValues = None self.phaseLabels = advertisingEnvironment.phaseLabels self.phaseWeights = advertisingEnvironment.GetPhaseWeights() self.featureLabels = advertisingEnvironment.featureLabels self.clickFunctions = advertisingEnvironment.clickFunctions self.sigma = advertisingEnvironment.sigma self.OptSuperArmReward = None self.optRewardsPerExperiment = [] self.gptsRewardsPerExperiment = []
def main(): parser = argparse.ArgumentParser( description='catapult project build generator') parser.add_argument('--compiler-configuration', help='path to compiler configuration yaml', required=True) parser.add_argument('--build-configuration', help='path to build configuration yaml', required=True) parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true') args = parser.parse_args() process_manager = ProcessManager(args.dry_run) environment_manager = EnvironmentManager(args.dry_run) builder = BuildManager(args, process_manager, environment_manager) env = LinuxEnvironment(builder.use_conan, process_manager, environment_manager) env.prepare() if builder.use_conan: env.prepare_conan({ 'version': builder.compiler.version, 'libcxx': builder.stl.lib }) env.run_conan_install() builder.run_cmake() builder.build() builder.copy_files()
class Port(object): def __init__(self, portname): # TODO make function to transform fullpath to portname self.portname = portname self.platform = get_platform() self.version = '' self.name = '' self.default_version = {} self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src' self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads' self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build' self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz' self.environment = {} self.env = EnvironmentManager() def get_default_version(self): return self.default_version.get(get_platform()) def sources_root(self): return os.path.expanduser( self.sources_root_scheme.format(PORTNAME=self.portname)) def download_dir(self): return os.path.expanduser( self.download_dir_scheme.format(PORTNAME=self.portname)) def download_filename(self): return os.path.expanduser( self.download_dir_scheme.format(PORTNAME=self.portname) + '/' + self.filename_scheme.format(PORTNAME=self.portname, VERSION=self.version)) def build_root(self): return os.path.expanduser( self.build_dir_scheme.format(PORTNAME=self.portname)) def getEnvironment(self): return self.env.getEnvironment() def getEnvironmentVariable(self, key): return self.env.getVariable(key) def getEnvironmentOptions(self): return self.env.getOptions() def updateEnvironmentVariable(self, key, value): self.env.updateVariable(key, value)
def __init__(self, budget=10.0, n_arms=10, sampleFactor=10): self.budget = budget self.nArms = n_arms self.budgets = np.linspace(0.0, self.budget, self.nArms) env = EnvironmentManager() self.phaseLabels = env.phaseLabels self.phaseWeights = env.GetPhaseWeights() self.phaseList = env.GetPhaseList(sampleFactor) self.phaseLen = len(self.phaseList) self.featureLabels = env.featureLabels self.clickFunctions = env.clickFunctions self.sigma = env.sigma self.optimalSuperArmRewards = self.RunClairvoyant() self.optRewardsPerExperiment = [] self.gptsRewardsPerExperiment = [] self.swgptsReardsPerExperiment = [] self.windowSize = None
class Port(object): def __init__(self, portname): # TODO make function to transform fullpath to portname self.portname = portname self.platform = get_platform() self.version = '' self.name = '' self.default_version = {} self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src' self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads' self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build' self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz' self.environment = {} self.env = EnvironmentManager() def get_default_version(self): return self.default_version.get(get_platform()) def sources_root(self): return os.path.expanduser(self.sources_root_scheme.format(PORTNAME=self.portname)) def download_dir(self): return os.path.expanduser(self.download_dir_scheme.format(PORTNAME=self.portname)) def download_filename(self): return os.path.expanduser( self.download_dir_scheme.format(PORTNAME=self.portname) + '/' + self.filename_scheme.format( PORTNAME=self.portname, VERSION=self.version) ) def build_root(self): return os.path.expanduser(self.build_dir_scheme.format(PORTNAME=self.portname)) def getEnvironment(self): return self.env.getEnvironment() def getEnvironmentVariable(self, key): return self.env.getVariable(key) def getEnvironmentOptions(self): return self.env.getOptions() def updateEnvironmentVariable(self, key, value): self.env.updateVariable(key, value)
def __init__(self, budget=10, nArms=10, singlePrice=False): self.budget = budget self.nArms = nArms self.budgets = np.linspace(0.0, self.budget, self.nArms) environment = EnvironmentManager() self.phaseLabels = environment.phaseLabels self.phaseWeights = environment.GetPhaseWeights() self.featureLabels = environment.featureLabels self.clickFunctions = environment.clickFunctions self.sigma = environment.sigma self.categories = environment.GetIndexedCategories() self.features = environment.features self.featureSpace = environment.featureSpace self.personType = np.array(environment.probabilities) self.possiblePrices = np.array(environment.prices) self.singlePrice = singlePrice self.optSuperArmReward = self.RunClairvoyant() self.optRewardsPerExperiment = [] self.gptsRewardsPerExperiment = []
def main(): parser = argparse.ArgumentParser( description='catapult project build generator') parser.add_argument('--disposition', help='type of image to create', choices=('tests', 'private', 'public'), required=True) parser.add_argument('--dry-run', help='outputs desired commands without running them', action='store_true') args = parser.parse_args() print('preparing {} image'.format(args.disposition)) process_manager = ProcessManager(args.dry_run) environment_manager = EnvironmentManager(args.dry_run) is_dev_build = 'tests' == args.disposition if is_dev_build: for name in ['seed', 'scripts', 'resources']: environment_manager.copy_tree_with_symlinks( DATA_VOLUME / name, USER_HOME / name) bin_folder_names = ['bin', 'deps', 'lib'] if is_dev_build: bin_folder_names.append('tests') for name in bin_folder_names: environment_manager.copy_tree_with_symlinks( DATA_VOLUME / 'binaries' / name, USER_HOME / name) # LD_LIBRARY_PATH is not passed when llvm-symbolizer is started via asan, so move libs to system location if is_dev_build: system_bin_path = environment_manager.system_bin_path environment_manager.move_glob_with_symlinks(USER_HOME / 'deps', 'libLLVM*', system_bin_path) process_manager.dispatch_subprocess(['ls', '-laF', system_bin_path]) process_manager.dispatch_subprocess(['ls', '-laF', USER_HOME]) ls_folder_names = ['bin', 'deps', 'lib'] if is_dev_build: ls_folder_names.extend(['seed', 'scripts', 'resources', 'tests']) for name in ls_folder_names: process_manager.dispatch_subprocess(['ls', '-laF', USER_HOME / name])
def main(): parser = argparse.ArgumentParser(description='catapult lint runner') parser.add_argument('--out-dir', help='directory in which to store lint output files', required=True) parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true') args = parser.parse_args() process_manager = ProcessManager(args.dry_run) environment_manager = EnvironmentManager(args.dry_run) environment_manager.set_env_var('HOME', '/tmp') run_cpp_linters(process_manager, args.out_dir) environment_manager.chdir('scripts') linter_runner = LinterRunner(process_manager, args.out_dir, args.dry_run) run_shell_linters(linter_runner, find_files_with_extension(environment_manager, '.sh')) run_python_linters(linter_runner, find_files_with_extension(environment_manager, '.py'))
def main(): parser = argparse.ArgumentParser(description='catapult project build generator') parser.add_argument('--compiler-configuration', help='path to compiler configuration yaml', required=True) parser.add_argument('--build-configuration', help='path to build configuration yaml', required=True) parser.add_argument('--operating-system', help='operating system', required=True) parser.add_argument('--user', help='docker user', required=True) parser.add_argument('--destination-image-label', help='docker destination image label', required=True) parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true') parser.add_argument('--base-image-names-only', help='only output the base image names', action='store_true') args = parser.parse_args() options = OptionsManager(args) if args.base_image_names_only: print(options.build_base_image_name) print(options.prepare_base_image_name) return docker_run = create_docker_run_command(options, args.compiler_configuration, args.build_configuration, args.user) environment_manager = EnvironmentManager(args.dry_run) environment_manager.rmtree(OUTPUT_DIR) environment_manager.mkdirs(BINARIES_DIR) environment_manager.mkdirs(options.ccache_path / 'tmp', exist_ok=True) environment_manager.mkdirs(options.conan_path, exist_ok=True) print('building project') process_manager = ProcessManager(args.dry_run) return_code = process_manager.dispatch_subprocess(docker_run) if return_code: sys.exit(return_code) print('copying files') environment_manager.chdir(OUTPUT_DIR) for folder_name in ['scripts', 'seed', 'resources']: environment_manager.copy_tree_with_symlinks(SRC_DIR / folder_name, folder_name) environment_manager.chdir(SRC_DIR) print('building docker image') container_id = '<dry_run_container_id>' if args.dry_run else None prepare_docker_image(process_manager, container_id, { 'base_image_name': options.prepare_base_image_name, 'destination_image_label': args.destination_image_label, 'build_disposition': options.build_disposition })
from dqn import DQN from qlearning import train if __name__ == "__main__": batch_size = 256 gamma = 0.999 eps_start = 1 eps_end = 0.01 eps_decay = 0.001 target_update = 10 memory_size = 100000 lr = 0.001 num_episodes = 1000 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") env_manager = EnvironmentManager(device) strategy = EpsilonGreedyStrategy(eps_start, eps_end, eps_decay) agent = Agent(strategy, env_manager.num_actions_available(), device) memory = ReplayMemory(memory_size) input_shape = (3, 60, 40) n_actions = 4 policy_net = DQN(input_shape, n_actions).to(device) target_net = DQN(input_shape, n_actions).to(device) target_net.load_state_dict(policy_net.state_dict()) target_net.eval() # Switch target to inference mode optimizer = optim.Adam(params=policy_net.parameters(), lr=lr)
def main(): parser = argparse.ArgumentParser( description='download and install catapult dependencies locally') parser.add_argument('--target', help='target dependencies directory', required=True) parser.add_argument('--versions', help='locked versions file', required=True) parser.add_argument('--download', help='download all dependencies', action='store_true') parser.add_argument('--build', help='build all dependencies', action='store_true') parser.add_argument('--use-clang', help='uses clang compiler instead of gcc', action='store_true') parser.add_argument('--dry-run', help='outputs desired commands without running them', action='store_true') parser.add_argument('--force', help='purges any existing files', action='store_true') args = parser.parse_args() versions = load_versions_map(args.versions) environment_manager = EnvironmentManager(args.dry_run) if args.force: environment_manager.rmtree(args.target) target_directory = Path(args.target).absolute() source_directory = Path(args.target) / SOURCE_DIR_NAME environment_manager.mkdirs(source_directory, exist_ok=True) environment_manager.chdir(source_directory) process_manager = ProcessManager(args.dry_run) dependency_repositories = [('google', 'googletest'), ('google', 'benchmark'), ('mongodb', 'mongo-c-driver'), ('mongodb', 'mongo-cxx-driver'), ('zeromq', 'libzmq'), ('zeromq', 'cppzmq'), ('facebook', 'rocksdb')] if args.download: print('[x] downloading all dependencies') downloader = Downloader(versions, process_manager) downloader.download_boost() for repository in dependency_repositories: downloader.download_git_dependency(repository[0], repository[1]) if args.build: print('[x] building all dependencies') builder = Builder(target_directory, versions, process_manager, environment_manager) if args.use_clang: builder.use_clang() builder.build_boost() for repository in dependency_repositories: builder.build_git_dependency(repository[0], repository[1])
def main(): parser = argparse.ArgumentParser(description='catapult test runner') parser.add_argument('--compiler-configuration', help='path to compiler configuration yaml', required=True) parser.add_argument('--exe-path', help='path to executables', required=True) parser.add_argument('--out-dir', help='directory in which to store test output files', required=True) parser.add_argument('--verbosity', help='output verbosity', choices=('suite', 'test', 'max'), default='max') parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true') args = parser.parse_args() process_manager = ProcessManager(args.dry_run) environment_manager = EnvironmentManager(args.dry_run) compiler_configuration = load_compiler_configuration( args.compiler_configuration) sanitizer_environment = SanitizerEnvironment( environment_manager, compiler_configuration.sanitizers) sanitizer_environment.prepare() prepare_tests(environment_manager) process_manager.dispatch_subprocess(['ls', '-laF', '.']) process_manager.dispatch_subprocess(['ls', '-laF', '/catapult-data']) process_manager.dispatch_subprocess(['ls', '-laF', '/catapult-src']) failed_test_suites = [] for test_exe_filepath in environment_manager.find_glob( args.exe_path, 'tests*'): base_output_filepath = Path(args.out_dir) / test_exe_filepath.name output_filepath = '{}.xml'.format(base_output_filepath) test_args = [ test_exe_filepath, '--gtest_output=xml:{}'.format(output_filepath), Path(args.exe_path) / '..' / 'lib' ] if process_manager.dispatch_test_subprocess(test_args, args.verbosity): for core_path in Path('.').glob('core*'): handle_core_file(process_manager, core_path, test_exe_filepath, base_output_filepath) failed_test_suites.append(test_exe_filepath) process_sanitizer_logs_all(environment_manager, Path(args.out_dir), test_exe_filepath.name) if failed_test_suites: print('test failures detected') for test_suite in sorted(failed_test_suites): print('[*] {}'.format(test_suite)) sys.exit(len(failed_test_suites)) else: print('all tests succeeded')