def main():
parser = argparse.ArgumentParser(description='catapult tests runner')
parser.add_argument('--image', help='docker tests image', required=True)
parser.add_argument('--compiler-configuration',
help='path to compiler configuration yaml',
required=True)
parser.add_argument('--user', help='docker user', required=True)
parser.add_argument('--mode',
help='test mode',
choices=('bench', 'test', 'lint'),
required=True)
parser.add_argument('--verbosity', help='verbosity level', default='max')
parser.add_argument('--dry-run',
help='outputs desired commands without running them',
action='store_true')
args = parser.parse_args()
process_manager = ProcessManager(args.dry_run)
compose_template_directory = Path(__file__).parent / 'templates'
compose_template_filepath = compose_template_directory / 'Run{}.yaml'.format(
args.mode.capitalize())
print('processing template from {}'.format(compose_template_filepath))
prepare_replacements = {
'image_name': args.image,
'compiler_configuration': args.compiler_configuration,
'user': args.user,
'verbosity': args.verbosity
}
prepare_docker_compose_file(compose_template_filepath,
prepare_replacements, sys.stdout)
if not args.dry_run:
with open('docker-compose.yaml', 'wt') as outfile:
prepare_docker_compose_file(compose_template_filepath,
prepare_replacements, outfile)
environment_manager = EnvironmentManager(args.dry_run)
environment_manager.set_env_var('COMPOSE_HTTP_TIMEOUT', '200')
environment_manager.mkdirs(OUTPUT_DIR / 'logs', exist_ok=True)
environment_manager.mkdirs(OUTPUT_DIR / 'workdir', exist_ok=True)
if 'test' == args.mode:
environment_manager.mkdirs(MONGO_DIR / get_image_label(args.image))
docker_compose_args = create_docker_compose_command(args.mode)
if process_manager.dispatch_subprocess(docker_compose_args,
handle_error=False):
print('tests failed')
sys.exit(1)
print('tests succeeded')
def main():
parser = argparse.ArgumentParser(description='catapult bench runner')
parser.add_argument('--exe-path', help='path to executables', required=True)
parser.add_argument('--out-dir', help='directory in which to store bench output files', required=True)
parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true')
args = parser.parse_args()
process_manager = ProcessManager(args.dry_run)
environment_manager = EnvironmentManager(args.dry_run)
for filepath in environment_manager.find_glob(args.exe_path, 'bench*'):
output_filepath = Path(args.out_dir) / (filepath.name + '.json')
process_manager.dispatch_subprocess([filepath, '--benchmark_out=' + str(output_filepath)])
def __init__(self):
environment = EnvironmentManager()
# Categories contiene la scelta delle feature (teen/notteen) (employed/notemployed)
self.categories = environment.GetIndexedCategories()
self.features = environment.features
self.featureSpace = environment.featureSpace
self.probabilities = np.array(environment.probabilities)
self.possiblePrices = np.array(environment.prices)
self.nArms = len(self.possiblePrices)
self.expLogs = []
self.betaParams = []
self.week = -1
def __init__(self, portname):
# TODO make function to transform fullpath to portname
self.portname = portname
self.platform = get_platform()
self.version = ''
self.name = ''
self.default_version = {}
self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src'
self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads'
self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build'
self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz'
self.environment = {}
self.env = EnvironmentManager()
def __init__(self, budget=5.0, nArms=6):
self.budget = budget
self.nArms = nArms
self.budgets = np.linspace(0.0, self.budget, self.nArms)
advertisingEnvironment = EnvironmentManager()
self.realValues = None
self.phaseLabels = advertisingEnvironment.phaseLabels
self.phaseWeights = advertisingEnvironment.GetPhaseWeights()
self.featureLabels = advertisingEnvironment.featureLabels
self.clickFunctions = advertisingEnvironment.clickFunctions
self.sigma = advertisingEnvironment.sigma
self.OptSuperArmReward = None
self.optRewardsPerExperiment = []
self.gptsRewardsPerExperiment = []
def main():
parser = argparse.ArgumentParser(
description='catapult project build generator')
parser.add_argument('--compiler-configuration',
help='path to compiler configuration yaml',
required=True)
parser.add_argument('--build-configuration',
help='path to build configuration yaml',
required=True)
parser.add_argument('--dry-run',
help='outputs desired commands without runing them',
action='store_true')
args = parser.parse_args()
process_manager = ProcessManager(args.dry_run)
environment_manager = EnvironmentManager(args.dry_run)
builder = BuildManager(args, process_manager, environment_manager)
env = LinuxEnvironment(builder.use_conan, process_manager,
environment_manager)
env.prepare()
if builder.use_conan:
env.prepare_conan({
'version': builder.compiler.version,
'libcxx': builder.stl.lib
})
env.run_conan_install()
builder.run_cmake()
builder.build()
builder.copy_files()
class Port(object):
def __init__(self, portname):
# TODO make function to transform fullpath to portname
self.portname = portname
self.platform = get_platform()
self.version = ''
self.name = ''
self.default_version = {}
self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src'
self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads'
self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build'
self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz'
self.environment = {}
self.env = EnvironmentManager()
def get_default_version(self):
return self.default_version.get(get_platform())
def sources_root(self):
return os.path.expanduser(
self.sources_root_scheme.format(PORTNAME=self.portname))
def download_dir(self):
return os.path.expanduser(
self.download_dir_scheme.format(PORTNAME=self.portname))
def download_filename(self):
return os.path.expanduser(
self.download_dir_scheme.format(PORTNAME=self.portname) + '/' +
self.filename_scheme.format(PORTNAME=self.portname,
VERSION=self.version))
def build_root(self):
return os.path.expanduser(
self.build_dir_scheme.format(PORTNAME=self.portname))
def getEnvironment(self):
return self.env.getEnvironment()
def getEnvironmentVariable(self, key):
return self.env.getVariable(key)
def getEnvironmentOptions(self):
return self.env.getOptions()
def updateEnvironmentVariable(self, key, value):
self.env.updateVariable(key, value)
def __init__(self, budget=10.0, n_arms=10, sampleFactor=10):
self.budget = budget
self.nArms = n_arms
self.budgets = np.linspace(0.0, self.budget, self.nArms)
env = EnvironmentManager()
self.phaseLabels = env.phaseLabels
self.phaseWeights = env.GetPhaseWeights()
self.phaseList = env.GetPhaseList(sampleFactor)
self.phaseLen = len(self.phaseList)
self.featureLabels = env.featureLabels
self.clickFunctions = env.clickFunctions
self.sigma = env.sigma
self.optimalSuperArmRewards = self.RunClairvoyant()
self.optRewardsPerExperiment = []
self.gptsRewardsPerExperiment = []
self.swgptsReardsPerExperiment = []
self.windowSize = None
class Port(object):
def __init__(self, portname):
# TODO make function to transform fullpath to portname
self.portname = portname
self.platform = get_platform()
self.version = ''
self.name = ''
self.default_version = {}
self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src'
self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads'
self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build'
self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz'
self.environment = {}
self.env = EnvironmentManager()
def get_default_version(self):
return self.default_version.get(get_platform())
def sources_root(self):
return os.path.expanduser(self.sources_root_scheme.format(PORTNAME=self.portname))
def download_dir(self):
return os.path.expanduser(self.download_dir_scheme.format(PORTNAME=self.portname))
def download_filename(self):
return os.path.expanduser(
self.download_dir_scheme.format(PORTNAME=self.portname) + '/' + self.filename_scheme.format(
PORTNAME=self.portname, VERSION=self.version)
)
def build_root(self):
return os.path.expanduser(self.build_dir_scheme.format(PORTNAME=self.portname))
def getEnvironment(self):
return self.env.getEnvironment()
def getEnvironmentVariable(self, key):
return self.env.getVariable(key)
def getEnvironmentOptions(self):
return self.env.getOptions()
def updateEnvironmentVariable(self, key, value):
self.env.updateVariable(key, value)
def __init__(self, budget=10, nArms=10, singlePrice=False):
self.budget = budget
self.nArms = nArms
self.budgets = np.linspace(0.0, self.budget, self.nArms)
environment = EnvironmentManager()
self.phaseLabels = environment.phaseLabels
self.phaseWeights = environment.GetPhaseWeights()
self.featureLabels = environment.featureLabels
self.clickFunctions = environment.clickFunctions
self.sigma = environment.sigma
self.categories = environment.GetIndexedCategories()
self.features = environment.features
self.featureSpace = environment.featureSpace
self.personType = np.array(environment.probabilities)
self.possiblePrices = np.array(environment.prices)
self.singlePrice = singlePrice
self.optSuperArmReward = self.RunClairvoyant()
self.optRewardsPerExperiment = []
self.gptsRewardsPerExperiment = []
def __init__(self, portname):
# TODO make function to transform fullpath to portname
self.portname = portname
self.platform = get_platform()
self.version = ''
self.name = ''
self.default_version = {}
self.sources_root_scheme = '~/.ports/cache/{PORTNAME}/src'
self.download_dir_scheme = '~/.ports/cache/{PORTNAME}/downloads'
self.build_dir_scheme = '~/.ports/cache/{PORTNAME}/build'
self.filename_scheme = '{PORTNAME}-{VERSION}.tar.gz'
self.environment = {}
self.env = EnvironmentManager()
def main():
parser = argparse.ArgumentParser(
description='catapult project build generator')
parser.add_argument('--disposition',
help='type of image to create',
choices=('tests', 'private', 'public'),
required=True)
parser.add_argument('--dry-run',
help='outputs desired commands without running them',
action='store_true')
args = parser.parse_args()
print('preparing {} image'.format(args.disposition))
process_manager = ProcessManager(args.dry_run)
environment_manager = EnvironmentManager(args.dry_run)
is_dev_build = 'tests' == args.disposition
if is_dev_build:
for name in ['seed', 'scripts', 'resources']:
environment_manager.copy_tree_with_symlinks(
DATA_VOLUME / name, USER_HOME / name)
bin_folder_names = ['bin', 'deps', 'lib']
if is_dev_build:
bin_folder_names.append('tests')
for name in bin_folder_names:
environment_manager.copy_tree_with_symlinks(
DATA_VOLUME / 'binaries' / name, USER_HOME / name)
# LD_LIBRARY_PATH is not passed when llvm-symbolizer is started via asan, so move libs to system location
if is_dev_build:
system_bin_path = environment_manager.system_bin_path
environment_manager.move_glob_with_symlinks(USER_HOME / 'deps',
'libLLVM*',
system_bin_path)
process_manager.dispatch_subprocess(['ls', '-laF', system_bin_path])
process_manager.dispatch_subprocess(['ls', '-laF', USER_HOME])
ls_folder_names = ['bin', 'deps', 'lib']
if is_dev_build:
ls_folder_names.extend(['seed', 'scripts', 'resources', 'tests'])
for name in ls_folder_names:
process_manager.dispatch_subprocess(['ls', '-laF', USER_HOME / name])
def main():
parser = argparse.ArgumentParser(description='catapult lint runner')
parser.add_argument('--out-dir', help='directory in which to store lint output files', required=True)
parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true')
args = parser.parse_args()
process_manager = ProcessManager(args.dry_run)
environment_manager = EnvironmentManager(args.dry_run)
environment_manager.set_env_var('HOME', '/tmp')
run_cpp_linters(process_manager, args.out_dir)
environment_manager.chdir('scripts')
linter_runner = LinterRunner(process_manager, args.out_dir, args.dry_run)
run_shell_linters(linter_runner, find_files_with_extension(environment_manager, '.sh'))
run_python_linters(linter_runner, find_files_with_extension(environment_manager, '.py'))
def main():
parser = argparse.ArgumentParser(description='catapult project build generator')
parser.add_argument('--compiler-configuration', help='path to compiler configuration yaml', required=True)
parser.add_argument('--build-configuration', help='path to build configuration yaml', required=True)
parser.add_argument('--operating-system', help='operating system', required=True)
parser.add_argument('--user', help='docker user', required=True)
parser.add_argument('--destination-image-label', help='docker destination image label', required=True)
parser.add_argument('--dry-run', help='outputs desired commands without runing them', action='store_true')
parser.add_argument('--base-image-names-only', help='only output the base image names', action='store_true')
args = parser.parse_args()
options = OptionsManager(args)
if args.base_image_names_only:
print(options.build_base_image_name)
print(options.prepare_base_image_name)
return
docker_run = create_docker_run_command(options, args.compiler_configuration, args.build_configuration, args.user)
environment_manager = EnvironmentManager(args.dry_run)
environment_manager.rmtree(OUTPUT_DIR)
environment_manager.mkdirs(BINARIES_DIR)
environment_manager.mkdirs(options.ccache_path / 'tmp', exist_ok=True)
environment_manager.mkdirs(options.conan_path, exist_ok=True)
print('building project')
process_manager = ProcessManager(args.dry_run)
return_code = process_manager.dispatch_subprocess(docker_run)
if return_code:
sys.exit(return_code)
print('copying files')
environment_manager.chdir(OUTPUT_DIR)
for folder_name in ['scripts', 'seed', 'resources']:
environment_manager.copy_tree_with_symlinks(SRC_DIR / folder_name, folder_name)
environment_manager.chdir(SRC_DIR)
print('building docker image')
container_id = '<dry_run_container_id>' if args.dry_run else None
prepare_docker_image(process_manager, container_id, {
'base_image_name': options.prepare_base_image_name,
'destination_image_label': args.destination_image_label,
'build_disposition': options.build_disposition
})
from dqn import DQN
from qlearning import train
if __name__ == "__main__":
batch_size = 256
gamma = 0.999
eps_start = 1
eps_end = 0.01
eps_decay = 0.001
target_update = 10
memory_size = 100000
lr = 0.001
num_episodes = 1000
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
env_manager = EnvironmentManager(device)
strategy = EpsilonGreedyStrategy(eps_start, eps_end, eps_decay)
agent = Agent(strategy, env_manager.num_actions_available(), device)
memory = ReplayMemory(memory_size)
input_shape = (3, 60, 40)
n_actions = 4
policy_net = DQN(input_shape, n_actions).to(device)
target_net = DQN(input_shape, n_actions).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval() # Switch target to inference mode
optimizer = optim.Adam(params=policy_net.parameters(), lr=lr)
def main():
parser = argparse.ArgumentParser(
description='download and install catapult dependencies locally')
parser.add_argument('--target',
help='target dependencies directory',
required=True)
parser.add_argument('--versions',
help='locked versions file',
required=True)
parser.add_argument('--download',
help='download all dependencies',
action='store_true')
parser.add_argument('--build',
help='build all dependencies',
action='store_true')
parser.add_argument('--use-clang',
help='uses clang compiler instead of gcc',
action='store_true')
parser.add_argument('--dry-run',
help='outputs desired commands without running them',
action='store_true')
parser.add_argument('--force',
help='purges any existing files',
action='store_true')
args = parser.parse_args()
versions = load_versions_map(args.versions)
environment_manager = EnvironmentManager(args.dry_run)
if args.force:
environment_manager.rmtree(args.target)
target_directory = Path(args.target).absolute()
source_directory = Path(args.target) / SOURCE_DIR_NAME
environment_manager.mkdirs(source_directory, exist_ok=True)
environment_manager.chdir(source_directory)
process_manager = ProcessManager(args.dry_run)
dependency_repositories = [('google', 'googletest'),
('google', 'benchmark'),
('mongodb', 'mongo-c-driver'),
('mongodb', 'mongo-cxx-driver'),
('zeromq', 'libzmq'), ('zeromq', 'cppzmq'),
('facebook', 'rocksdb')]
if args.download:
print('[x] downloading all dependencies')
downloader = Downloader(versions, process_manager)
downloader.download_boost()
for repository in dependency_repositories:
downloader.download_git_dependency(repository[0], repository[1])
if args.build:
print('[x] building all dependencies')
builder = Builder(target_directory, versions, process_manager,
environment_manager)
if args.use_clang:
builder.use_clang()
builder.build_boost()
for repository in dependency_repositories:
builder.build_git_dependency(repository[0], repository[1])
def main():
parser = argparse.ArgumentParser(description='catapult test runner')
parser.add_argument('--compiler-configuration',
help='path to compiler configuration yaml',
required=True)
parser.add_argument('--exe-path',
help='path to executables',
required=True)
parser.add_argument('--out-dir',
help='directory in which to store test output files',
required=True)
parser.add_argument('--verbosity',
help='output verbosity',
choices=('suite', 'test', 'max'),
default='max')
parser.add_argument('--dry-run',
help='outputs desired commands without runing them',
action='store_true')
args = parser.parse_args()
process_manager = ProcessManager(args.dry_run)
environment_manager = EnvironmentManager(args.dry_run)
compiler_configuration = load_compiler_configuration(
args.compiler_configuration)
sanitizer_environment = SanitizerEnvironment(
environment_manager, compiler_configuration.sanitizers)
sanitizer_environment.prepare()
prepare_tests(environment_manager)
process_manager.dispatch_subprocess(['ls', '-laF', '.'])
process_manager.dispatch_subprocess(['ls', '-laF', '/catapult-data'])
process_manager.dispatch_subprocess(['ls', '-laF', '/catapult-src'])
failed_test_suites = []
for test_exe_filepath in environment_manager.find_glob(
args.exe_path, 'tests*'):
base_output_filepath = Path(args.out_dir) / test_exe_filepath.name
output_filepath = '{}.xml'.format(base_output_filepath)
test_args = [
test_exe_filepath, '--gtest_output=xml:{}'.format(output_filepath),
Path(args.exe_path) / '..' / 'lib'
]
if process_manager.dispatch_test_subprocess(test_args, args.verbosity):
for core_path in Path('.').glob('core*'):
handle_core_file(process_manager, core_path, test_exe_filepath,
base_output_filepath)
failed_test_suites.append(test_exe_filepath)
process_sanitizer_logs_all(environment_manager, Path(args.out_dir),
test_exe_filepath.name)
if failed_test_suites:
print('test failures detected')
for test_suite in sorted(failed_test_suites):
print('[*] {}'.format(test_suite))
sys.exit(len(failed_test_suites))
else:
print('all tests succeeded')
