def test_simulation_exit(self): simulator = Simulator() with NamedTemporaryFile( suffix=".csv") as machine_config, NamedTemporaryFile( suffix=".csv") as job_config: with open(machine_config.name, "w") as write_stream: write_stream.write( "TotalSlotCPUs TotalSlotDisk TotalSlotMemory Count\n" "1 44624348.0 8000 1") with open(job_config.name, "w") as write_stream: write_stream.write( "QDate RequestCpus RequestWalltime RequestMemory RequestDisk " "RemoteWallClockTime MemoryUsage DiskUsage_RAW RemoteSysCpu " "RemoteUserCpu\n" "1567155456 1 60 2000 6000000 100.0 2867 41898 10.0 40.0") job_input = open(job_config.name, "r+") machine_input = open(machine_config.name, "r+") simulator.create_job_generator(job_input=job_input, job_reader=htcondor_job_reader) simulator.create_scheduler(scheduler_type=CondorJobScheduler) simulator.create_pools( pool_input=machine_input, pool_reader=htcondor_pool_reader, pool_type=StaticPool, ) simulator.run() assert 180 == simulator.duration
def static(ctx, job_file, pool_file): click.echo("starting static environment") simulator = Simulator(seed=ctx.obj["seed"]) file, file_type = job_file simulator.create_job_generator( job_input=file, job_reader=job_import_mapper[file_type] ) simulator.create_scheduler(scheduler_type=CondorJobScheduler) for current_pool in pool_file: pool_file, pool_file_type = current_pool simulator.create_pools( pool_input=pool_file, pool_reader=pool_import_mapper[pool_file_type], pool_type=StaticPool, ) simulator.run(until=ctx.obj["until"])
def hybrid(ctx, job_file, static_pool_file, dynamic_pool_file): click.echo("starting hybrid environment") simulator = Simulator(seed=ctx.obj["seed"]) file, file_type = job_file simulator.create_job_generator( job_input=file, job_reader=job_import_mapper[file_type] ) simulator.create_scheduler(scheduler_type=CondorJobScheduler) for current_pool in static_pool_file: file, file_type = current_pool simulator.create_pools( pool_input=file, pool_reader=pool_import_mapper[file_type], pool_type=StaticPool, ) for current_pool in dynamic_pool_file: file, file_type = current_pool simulator.create_pools( pool_input=file, pool_reader=pool_import_mapper[file_type], pool_type=Pool, controller=SimulatedLinearController, ) simulator.run(until=ctx.obj["until"])
def test_full_simulation_with_hitratebased_caching(self): with NamedTemporaryFile( suffix=".csv") as machine_config, NamedTemporaryFile( suffix=".csv") as storage_config, NamedTemporaryFile( suffix=".json") as job_config: with open(machine_config.name, "w") as write_stream: write_stream.write( "TotalSlotCPUs TotalSlotDisk TotalSlotMemory Count sitename \n" "1 44624348.0 4000 1 mysite") with open(job_config.name, "w") as write_stream: job_description = [{ "RequestCpus": 1, "RequestWalltime": 60, "RequestMemory": 2000, "RequestDisk": 6000000, "QDate": 0, "RemoteWallClockTime": 42, "Number of Allocated Processors": 1, "MemoryUsage": 1500, "DiskUsage_RAW": 41898, "RemoteSysCpu": 40, "RemoteUserCpu": 2, "Inputfiles": { "a.root": { "filesize": 5, "usedsize": 5, "hitrates": { "mysite": 1.0 }, }, "b.root": { "filesize": 5, "usedsize": 5, "hitrates": { "mysite": 0.0 }, }, }, }] json.dump(job_description, write_stream) with open(storage_config.name, "w") as write_stream: write_stream.write( "name sitename cachesizeGB throughput_limit \n" "mycache mysite 1000 1.0") job_input = open(job_config.name, "r+") machine_input = open(machine_config.name, "r+") storage_input = open(storage_config.name, "r+") storage_content_input = None simulator = Simulator() simulator.create_job_generator(job_input=job_input, job_reader=htcondor_job_reader) simulator.create_scheduler( scheduler_type=CondorClassadJobScheduler) simulator.create_connection_module(remote_throughput=0.1, filebased_caching=False) simulator.create_storage( storage_input=storage_input, storage_content_input=storage_content_input, storage_reader=storage_reader, storage_type=FileBasedHitrateStorage, ) simulator.create_pools( pool_input=machine_input, pool_reader=htcondor_pool_reader, pool_type=StaticPool, ) simulator.enable_monitoring() simulator.run() assert 180 == simulator.duration job_input.close() storage_input.close() machine_input.close()
def ini_and_run( job_file, pool_files, storage_file, storage_type, log_file="test_{}.log".format(time()), remote_throughput=1.0, seed=1234, until=None, calculation_efficiency=1.0, log_telegraf=False, pre_job_rank=pre_job_rank_default, machine_ads=machine_ad_defaults, job_ads=job_ad_defaults, additional_identifier=None, ): # ini logging to file monitoring_logger = logging.getLogger() monitoring_logger.setLevel(logging.DEBUG) time_filter = SimulationTimeFilter() monitoring_logger.addFilter(time_filter) streamHandler = logging.StreamHandler(stream=open(log_file, "w")) streamHandler.setFormatter(JsonFormatter()) monitoring_logger.addHandler(streamHandler) if log_telegraf: telegrafHandler = LoggingUDPSocketHandler( "localhost", logging.handlers.DEFAULT_UDP_LOGGING_PORT) telegrafHandler.setFormatter(LineProtocolFormatter(resolution=1)) monitoring_logger.addHandler(telegrafHandler) # ini simulation print("starting static environment") simulator = Simulator(seed=seed) file_type = "htcondor" file = job_file simulator.create_job_generator( job_input=open(file, "r"), job_reader=partial(job_import_mapper[file_type], calculation_efficiency=calculation_efficiency), ) print("scheduler configuration: \n " "\tpre job rank: {} \n\n" "\tmachine classad:\n \t{}\n\n" "\tjob classad: {}".format(pre_job_rank, machine_ads, job_ads)) simulator.job_scheduler = CondorClassadJobScheduler( job_queue=simulator.job_queue, pre_job_rank=pre_job_rank, machine_ad=machine_ads, job_ad=job_ads, ) simulator.connection = Connection(remote_throughput * 1000 * 1000 * 1000, filebased_caching=False) dummy_pool_connection = Connection(float("Inf")) print("dummy:", dummy_pool_connection.remote_connection.connection) with open(storage_file, "r") as storage_file: simulator.create_storage( storage_input=storage_file, storage_content_input=None, storage_reader=storage_import_mapper[storage_type], storage_type=FileBasedHitrateStorage, ) for pool_file in pool_files: with open(pool_file, "r") as pool_file: # Attention: dummy_pool_connection is currently not part of # monitoring as it is not known within the simulator itself # TODO: do you need this in monitoring? create_pool_in_simulator( simulator=simulator, pool_input=pool_file, pool_reader=pool_import_mapper["htcondor"], pool_type=StaticPool, connection=dummy_pool_connection if "dummycluster" in pool_file.name else simulator.connection, ) simulator.enable_monitoring() # run simulation simulator.run(until=until)
def static(ctx, job_file, pool_file, storage_files, remote_throughput, cache_hitrate): click.echo("starting static environment") simulator = Simulator(seed=ctx.obj["seed"]) file, file_type = job_file simulator.create_job_generator( job_input=file, job_reader=partial( job_import_mapper[file_type], calculation_efficiency=ctx.obj["calculation_efficiency"], ), ) simulator.create_scheduler(scheduler_type=CondorJobScheduler) if all(storage_files): simulator.create_connection_module(remote_throughput, False) storage_file, storage_content_file, storage_type = storage_files simulator.create_storage( storage_input=storage_file, storage_content_input=storage_content_file, storage_reader=storage_import_mapper[storage_type], storage_type=FileBasedHitrateStorage, ) for current_pool in pool_file: pool_file, pool_file_type = current_pool simulator.create_pools( pool_input=pool_file, pool_reader=pool_import_mapper[pool_file_type], pool_type=StaticPool, ) simulator.enable_monitoring() simulator.run(until=ctx.obj["until"])
def dynamic(ctx, job_file, pool_file): click.echo("starting dynamic environment") simulator = Simulator(seed=ctx.obj["seed"]) file, file_type = job_file simulator.create_job_generator( job_input=file, job_reader=partial( job_import_mapper[file_type], calculation_efficiency=ctx.obj["calculation_efficiency"], ), ) simulator.create_scheduler(scheduler_type=CondorJobScheduler) for current_pool in pool_file: file, file_type = current_pool simulator.create_pools( pool_input=file, pool_reader=pool_import_mapper[file_type], pool_type=Pool, controller=SimulatedLinearController, ) simulator.enable_monitoring() simulator.run(until=ctx.obj["until"])
def cli( seed, until, log_tcp, log_file, log_telegraf, calculation_efficiency, job_file, pre_job_rank, machine_ads, job_ads, scheduler_type, static_pool_files, dynamic_pool_files, storage_files, remote_throughput, filebased_caching, cache_hitrate, ): monitoring_logger = logging.getLogger() monitoring_logger.setLevel(logging.DEBUG) time_filter = SimulationTimeFilter() monitoring_logger.addFilter(time_filter) if log_tcp: socketHandler = LoggingSocketHandler( "localhost", logging.handlers.DEFAULT_TCP_LOGGING_PORT) socketHandler.setFormatter(JsonFormatter()) monitoring_logger.addHandler(socketHandler) if log_file: streamHandler = logging.StreamHandler(stream=log_file) streamHandler.setFormatter(JsonFormatter()) monitoring_logger.addHandler(streamHandler) if log_telegraf: telegrafHandler = LoggingUDPSocketHandler( "localhost", logging.handlers.DEFAULT_UDP_LOGGING_PORT) telegrafHandler.setFormatter(LineProtocolFormatter(resolution=1)) monitoring_logger.addHandler(telegrafHandler) click.echo("starting hybrid environment") simulator = Simulator(seed=seed) infile, file_type = job_file simulator.create_job_generator( job_input=infile, job_reader=partial( job_import_mapper[file_type], calculation_efficiency=calculation_efficiency, ), ) if scheduler_import_mapper[ scheduler_type] == CondorClassadJobScheduler and any( (pre_job_rank, machine_ads, job_ads)): simulator.job_scheduler = CondorClassadJobScheduler( job_queue=simulator.job_queue, pre_job_rank=pre_job_rank, machine_ad=machine_ads, job_ad=job_ads, ) else: simulator.create_scheduler( scheduler_type=scheduler_import_mapper[scheduler_type]) for current_storage_files in storage_files: assert all( current_storage_files), "All storage inputs have to be available" simulator.create_connection_module(remote_throughput, filebased_caching) storage_file, storage_content_file, storage_type = current_storage_files simulator.create_storage( storage_input=storage_file, storage_content_input=storage_content_file, storage_reader=storage_import_mapper[storage_type], storage_type=FileBasedHitrateStorage, # TODO: Generalize this ) for current_pool in static_pool_files: pool_file, pool_file_type = current_pool if "dummycluster" in pool_file.name: simulator.create_connection_module(float("Inf")) simulator.create_pools( pool_input=pool_file, pool_reader=pool_import_mapper[pool_file_type], pool_type=StaticPool, ) for current_pool in dynamic_pool_files: pool_file, pool_file_type = current_pool if "dummycluster" in pool_file.name: simulator.create_connection_module(float("Inf")) simulator.create_pools( pool_input=pool_file, pool_reader=pool_import_mapper[pool_file_type], pool_type=Pool, controller=SimulatedLinearController, ) click.echo("scheduler configuration: \n " f"\tscheduler type: {scheduler_type}\n\n" f"\tpre job rank: {pre_job_rank} \n\n" f"\tmachine classads:\n \t{machine_ads}\n\n" f"\tjob classads: {job_ads}") simulator.enable_monitoring() simulator.run(until=until)