def static(ctx, job_file, pool_file, storage_files, remote_throughput,
cache_hitrate):
click.echo("starting static environment")
simulator = Simulator(seed=ctx.obj["seed"])
file, file_type = job_file
simulator.create_job_generator(
job_input=file,
job_reader=partial(
job_import_mapper[file_type],
calculation_efficiency=ctx.obj["calculation_efficiency"],
),
)
simulator.create_scheduler(scheduler_type=CondorJobScheduler)
if all(storage_files):
simulator.create_connection_module(remote_throughput, False)
storage_file, storage_content_file, storage_type = storage_files
simulator.create_storage(
storage_input=storage_file,
storage_content_input=storage_content_file,
storage_reader=storage_import_mapper[storage_type],
storage_type=FileBasedHitrateStorage,
)
for current_pool in pool_file:
pool_file, pool_file_type = current_pool
simulator.create_pools(
pool_input=pool_file,
pool_reader=pool_import_mapper[pool_file_type],
pool_type=StaticPool,
)
simulator.enable_monitoring()
simulator.run(until=ctx.obj["until"])
def test_full_simulation_with_hitratebased_caching(self):
with NamedTemporaryFile(
suffix=".csv") as machine_config, NamedTemporaryFile(
suffix=".csv") as storage_config, NamedTemporaryFile(
suffix=".json") as job_config:
with open(machine_config.name, "w") as write_stream:
write_stream.write(
"TotalSlotCPUs TotalSlotDisk TotalSlotMemory Count sitename \n"
"1 44624348.0 4000 1 mysite")
with open(job_config.name, "w") as write_stream:
job_description = [{
"RequestCpus": 1,
"RequestWalltime": 60,
"RequestMemory": 2000,
"RequestDisk": 6000000,
"QDate": 0,
"RemoteWallClockTime": 42,
"Number of Allocated Processors": 1,
"MemoryUsage": 1500,
"DiskUsage_RAW": 41898,
"RemoteSysCpu": 40,
"RemoteUserCpu": 2,
"Inputfiles": {
"a.root": {
"filesize": 5,
"usedsize": 5,
"hitrates": {
"mysite": 1.0
},
},
"b.root": {
"filesize": 5,
"usedsize": 5,
"hitrates": {
"mysite": 0.0
},
},
},
}]
json.dump(job_description, write_stream)
with open(storage_config.name, "w") as write_stream:
write_stream.write(
"name sitename cachesizeGB throughput_limit \n"
"mycache mysite 1000 1.0")
job_input = open(job_config.name, "r+")
machine_input = open(machine_config.name, "r+")
storage_input = open(storage_config.name, "r+")
storage_content_input = None
simulator = Simulator()
simulator.create_job_generator(job_input=job_input,
job_reader=htcondor_job_reader)
simulator.create_scheduler(
scheduler_type=CondorClassadJobScheduler)
simulator.create_connection_module(remote_throughput=0.1,
filebased_caching=False)
simulator.create_storage(
storage_input=storage_input,
storage_content_input=storage_content_input,
storage_reader=storage_reader,
storage_type=FileBasedHitrateStorage,
)
simulator.create_pools(
pool_input=machine_input,
pool_reader=htcondor_pool_reader,
pool_type=StaticPool,
)
simulator.enable_monitoring()
simulator.run()
assert 180 == simulator.duration
job_input.close()
storage_input.close()
machine_input.close()
def ini_and_run(
job_file,
pool_files,
storage_file,
storage_type,
log_file="test_{}.log".format(time()),
remote_throughput=1.0,
seed=1234,
until=None,
calculation_efficiency=1.0,
log_telegraf=False,
pre_job_rank=pre_job_rank_default,
machine_ads=machine_ad_defaults,
job_ads=job_ad_defaults,
additional_identifier=None,
):
# ini logging to file
monitoring_logger = logging.getLogger()
monitoring_logger.setLevel(logging.DEBUG)
time_filter = SimulationTimeFilter()
monitoring_logger.addFilter(time_filter)
streamHandler = logging.StreamHandler(stream=open(log_file, "w"))
streamHandler.setFormatter(JsonFormatter())
monitoring_logger.addHandler(streamHandler)
if log_telegraf:
telegrafHandler = LoggingUDPSocketHandler(
"localhost", logging.handlers.DEFAULT_UDP_LOGGING_PORT)
telegrafHandler.setFormatter(LineProtocolFormatter(resolution=1))
monitoring_logger.addHandler(telegrafHandler)
# ini simulation
print("starting static environment")
simulator = Simulator(seed=seed)
file_type = "htcondor"
file = job_file
simulator.create_job_generator(
job_input=open(file, "r"),
job_reader=partial(job_import_mapper[file_type],
calculation_efficiency=calculation_efficiency),
)
print("scheduler configuration: \n "
"\tpre job rank: {} \n\n"
"\tmachine classad:\n \t{}\n\n"
"\tjob classad: {}".format(pre_job_rank, machine_ads, job_ads))
simulator.job_scheduler = CondorClassadJobScheduler(
job_queue=simulator.job_queue,
pre_job_rank=pre_job_rank,
machine_ad=machine_ads,
job_ad=job_ads,
)
simulator.connection = Connection(remote_throughput * 1000 * 1000 * 1000,
filebased_caching=False)
dummy_pool_connection = Connection(float("Inf"))
print("dummy:", dummy_pool_connection.remote_connection.connection)
with open(storage_file, "r") as storage_file:
simulator.create_storage(
storage_input=storage_file,
storage_content_input=None,
storage_reader=storage_import_mapper[storage_type],
storage_type=FileBasedHitrateStorage,
)
for pool_file in pool_files:
with open(pool_file, "r") as pool_file:
# Attention: dummy_pool_connection is currently not part of
# monitoring as it is not known within the simulator itself
# TODO: do you need this in monitoring?
create_pool_in_simulator(
simulator=simulator,
pool_input=pool_file,
pool_reader=pool_import_mapper["htcondor"],
pool_type=StaticPool,
connection=dummy_pool_connection
if "dummycluster" in pool_file.name else simulator.connection,
)
simulator.enable_monitoring()
# run simulation
simulator.run(until=until)
def cli(
seed,
until,
log_tcp,
log_file,
log_telegraf,
calculation_efficiency,
job_file,
pre_job_rank,
machine_ads,
job_ads,
scheduler_type,
static_pool_files,
dynamic_pool_files,
storage_files,
remote_throughput,
filebased_caching,
cache_hitrate,
):
monitoring_logger = logging.getLogger()
monitoring_logger.setLevel(logging.DEBUG)
time_filter = SimulationTimeFilter()
monitoring_logger.addFilter(time_filter)
if log_tcp:
socketHandler = LoggingSocketHandler(
"localhost", logging.handlers.DEFAULT_TCP_LOGGING_PORT)
socketHandler.setFormatter(JsonFormatter())
monitoring_logger.addHandler(socketHandler)
if log_file:
streamHandler = logging.StreamHandler(stream=log_file)
streamHandler.setFormatter(JsonFormatter())
monitoring_logger.addHandler(streamHandler)
if log_telegraf:
telegrafHandler = LoggingUDPSocketHandler(
"localhost", logging.handlers.DEFAULT_UDP_LOGGING_PORT)
telegrafHandler.setFormatter(LineProtocolFormatter(resolution=1))
monitoring_logger.addHandler(telegrafHandler)
click.echo("starting hybrid environment")
simulator = Simulator(seed=seed)
infile, file_type = job_file
simulator.create_job_generator(
job_input=infile,
job_reader=partial(
job_import_mapper[file_type],
calculation_efficiency=calculation_efficiency,
),
)
if scheduler_import_mapper[
scheduler_type] == CondorClassadJobScheduler and any(
(pre_job_rank, machine_ads, job_ads)):
simulator.job_scheduler = CondorClassadJobScheduler(
job_queue=simulator.job_queue,
pre_job_rank=pre_job_rank,
machine_ad=machine_ads,
job_ad=job_ads,
)
else:
simulator.create_scheduler(
scheduler_type=scheduler_import_mapper[scheduler_type])
for current_storage_files in storage_files:
assert all(
current_storage_files), "All storage inputs have to be available"
simulator.create_connection_module(remote_throughput,
filebased_caching)
storage_file, storage_content_file, storage_type = current_storage_files
simulator.create_storage(
storage_input=storage_file,
storage_content_input=storage_content_file,
storage_reader=storage_import_mapper[storage_type],
storage_type=FileBasedHitrateStorage, # TODO: Generalize this
)
for current_pool in static_pool_files:
pool_file, pool_file_type = current_pool
if "dummycluster" in pool_file.name:
simulator.create_connection_module(float("Inf"))
simulator.create_pools(
pool_input=pool_file,
pool_reader=pool_import_mapper[pool_file_type],
pool_type=StaticPool,
)
for current_pool in dynamic_pool_files:
pool_file, pool_file_type = current_pool
if "dummycluster" in pool_file.name:
simulator.create_connection_module(float("Inf"))
simulator.create_pools(
pool_input=pool_file,
pool_reader=pool_import_mapper[pool_file_type],
pool_type=Pool,
controller=SimulatedLinearController,
)
click.echo("scheduler configuration: \n "
f"\tscheduler type: {scheduler_type}\n\n"
f"\tpre job rank: {pre_job_rank} \n\n"
f"\tmachine classads:\n \t{machine_ads}\n\n"
f"\tjob classads: {job_ads}")
simulator.enable_monitoring()
simulator.run(until=until)