def main():
env = Environment.get()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Test the TOAST runtime environment.",
fromfile_prefix_chars="@")
parser.add_argument(
"--groupsize",
required=False,
type=int,
default=0,
help="size of processor groups used to distribute observations",
)
try:
args = parser.parse_args()
except SystemExit:
return
mpiworld, procs, rank = get_world()
if rank == 0:
print(env)
log.info(
"Numba threading layer set to '{}'".format(numba_threading_layer))
if mpiworld is None:
log.info("Running serially with one process")
else:
if rank == 0:
log.info("Running with {} processes".format(procs))
groupsize = args.groupsize
if groupsize <= 0:
groupsize = procs
if rank == 0:
log.info("Using group size of {} processes".format(groupsize))
comm = Comm(world=mpiworld, groupsize=groupsize)
log.info(
"Process {}: world rank {}, group {} of {}, group rank {}".format(
rank, comm.world_rank, comm.group + 1, comm.ngroups,
comm.group_rank))
return
def parse_arguments(comm, procs):
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate satellite boresight pointing and make a map.",
fromfile_prefix_chars="@",
)
add_dist_args(parser)
add_pointing_args(parser)
add_tidas_args(parser)
add_spt3g_args(parser)
add_dipole_args(parser)
add_pysm_args(parser)
add_mc_args(parser)
add_noise_args(parser)
add_todsatellite_args(parser)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument(
"--debug",
required=False,
default=False,
action="store_true",
help="Write diagnostics",
)
add_madam_args(parser)
add_binner_args(parser)
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam for map-making",
dest="use_madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libmadam for map-making [default]",
dest="use_madam",
)
parser.set_defaults(use_madam=False)
parser.add_argument(
"--focalplane",
required=False,
default=None,
help="Pickle file containing a dictionary of detector properties. "
"The keys of this dict are the detector names, and each value is also "
'a dictionary with keys "quat" (4 element ndarray), "fwhm" '
'(float, arcmin), "fknee" (float, Hz), "alpha" (float), and "NET" '
'(float). For optional plotting, the key "color" can specify a '
"valid matplotlib color string.",
)
parser.add_argument(
"--gain",
required=False,
default=None,
help="Calibrate the input timelines with a set of gains from a"
"FITS file containing 3 extensions:"
"HDU named DETECTORS : table with list of detector names in a column named DETECTORS"
"HDU named TIME: table with common timestamps column named TIME"
"HDU named GAINS: 2D image of floats with one row per detector and one column per value.",
)
try:
args = parser.parse_args()
except SystemExit:
sys.exit()
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
groupsize = args.group_size
if groupsize is None or groupsize <= 0:
groupsize = procs
# This is the 2-level toast communicator.
comm = Comm(groupsize=groupsize)
return args, comm, groupsize
def job_config(mpicomm, cases):
env = Environment.get()
log = Logger.get()
class args:
debug = False
# TOD Ground options
el_mod_step_deg = 0.0
el_mod_rate_hz = 0.0
el_mod_amplitude_deg = 1.0
el_mod_sine = False
el_nod_deg = False
el_nod_every_scan = False
start_with_el_nod = False
end_with_el_nod = False
scan_rate = 1.0
scan_rate_el = 0.0
scan_accel = 1.0
scan_accel_el = 0.0
scan_cosecant_modulate = False
sun_angle_min = 30.0
schedule = None # required
weather = "SIM"
timezone = 0
sample_rate = 100.0
coord = "C"
split_schedule = None
sort_schedule = False
hwp_rpm = 10.0
hwp_step_deg = None
hwp_step_time_s = None
elevation_noise_a = 0.0
elevation_noise_b = 0.0
freq = "150"
do_daymaps = False
do_seasonmaps = False
# Pointing options
nside = 1024
nside_submap = 16
single_precision_pointing = False
common_flag_mask = 1
# Polyfilter options
apply_polyfilter = False
poly_order = 0
# Ground filter options
apply_groundfilter = False
ground_order = 0
# Atmosphere options
simulate_atmosphere = False
simulate_coarse_atmosphere = False
focalplane_radius_deg = None
atm_verbosity = 0
atm_lmin_center = 0.01
atm_lmin_sigma = 0.001
atm_lmax_center = 10.0
atm_lmax_sigma = 10.0
atm_gain = 2.0e-5
atm_gain_coarse = 8.0e-5
atm_zatm = 40000.0
atm_zmax = 200.0
atm_xstep = 10.0
atm_ystep = 10.0
atm_zstep = 10.0
atm_nelem_sim_max = 10000
atm_wind_dist = 3000.0
atm_z0_center = 2000.0
atm_z0_sigma = 0.0
atm_T0_center = 280.0
atm_T0_sigma = 10.0
atm_cache = None
atm_apply_flags = False
# Noise simulation options
simulate_noise = False
# Gain scrambler
apply_gainscrambler = False
gain_sigma = 0.01
# Map maker
mapmaker_prefix = "toast"
mapmaker_mask = None
mapmaker_weightmap = None
mapmaker_iter_max = 20
mapmaker_precond_width = 100
mapmaker_prefilter_order = None
mapmaker_baseline_length = 200.0
mapmaker_noisefilter = False
mapmaker_fourier2D_order = None
mapmaker_fourier2D_subharmonics = None
write_hits = True
write_binmap = True
write_wcov = False
write_wcov_inv = False
zip_maps = False
# Monte Carlo
MC_start = 0
MC_count = 1
# Sky signal
input_map = None
simulate_sky = True
# Input dir
auxdir = "toast_inputs"
# Output
outdir = "toast"
tidas = None
spt3g = None
parser = argparse.ArgumentParser(
description="Run a TOAST workflow scaled appropriately to the MPI communicator size and available memory.",
fromfile_prefix_chars="@",
)
parser.add_argument(
"--node_mem_gb",
required=False,
default=None,
type=float,
help="Use this much memory per node in GB",
)
parser.add_argument(
"--dry_run",
required=False,
default=None,
type=str,
help="Comma-separated total_procs,node_procs to simulate.",
)
parser.parse_args(namespace=args)
procs = 1
rank = 0
if mpicomm is not None:
procs = mpicomm.size
rank = mpicomm.rank
avail_node_bytes = None
procs_per_node = None
if args.dry_run is not None:
dryrun_total, dryrun_node = args.dry_run.split(",")
dryrun_total = int(dryrun_total)
dryrun_node = int(dryrun_node)
if rank == 0:
log.info(
"DRY RUN simulating {} total processes with {} per node".format(
dryrun_total, dryrun_node
)
)
procs_per_node = dryrun_node
procs = dryrun_total
# We are simulating the distribution
avail_node_bytes = get_node_mem(mpicomm, 0)
else:
# Get information about the actual job size
procs_per_node, avail_node_bytes = job_size(mpicomm)
if rank == 0:
log.info(
"Minimum detected per-node memory available is {:0.2f} GB".format(
avail_node_bytes / (1024 ** 3)
)
)
if args.node_mem_gb is not None:
avail_node_bytes = int((1024 ** 3) * args.node_mem_gb)
if rank == 0:
log.info(
"Setting per-node available memory to {:0.2f} GB as requested".format(
avail_node_bytes / (1024 ** 3)
)
)
# Based on the total number of processes and count per node, choose the number of
# nodes in each observation and a focalplane such that every process has >= 4
# detectors.
n_nodes = procs // procs_per_node
if rank == 0:
log.info("Job has {} total nodes".format(n_nodes))
if rank == 0:
log.info("Examining {} possible cases to run:".format(len(cases)))
selected_case = None
selected_nodes = None
n_detector = None
time_samples = None
group_procs = None
group_nodes = None
n_group = None
group_time_samples = None
for case_name, case_samples in cases.items():
(
case_n_detector,
case_time_samples,
case_group_procs,
case_group_nodes,
case_n_group,
case_group_time_samples,
) = sample_distribution(
rank, procs_per_node, avail_node_bytes, case_samples, args.sample_rate
)
case_min_nodes = case_n_group * case_group_nodes
if rank == 0:
log.info(
" {:8s}: requires {:d} nodes for {} MPI ranks and {:0.1f}GB per node".format(
case_name,
case_min_nodes,
procs_per_node,
avail_node_bytes / (1024 ** 3),
)
)
if selected_nodes is None:
if case_min_nodes <= n_nodes:
# First case that fits in our job
selected_case = case_name
selected_nodes = case_min_nodes
n_detector = case_n_detector
time_samples = case_time_samples
group_procs = case_group_procs
group_nodes = case_group_nodes
n_group = case_n_group
group_time_samples = case_group_time_samples
else:
if (case_min_nodes <= n_nodes) and (case_min_nodes >= selected_nodes):
# This case fits in our job and is larger than the current one
selected_case = case_name
selected_nodes = case_min_nodes
n_detector = case_n_detector
time_samples = case_time_samples
group_procs = case_group_procs
group_nodes = case_group_nodes
n_group = case_n_group
group_time_samples = case_group_time_samples
if selected_case is None:
msg = (
"None of the available cases fit into aggregate memory. Use a larger job."
)
if rank == 0:
log.error(msg)
raise RuntimeError(msg)
else:
if rank == 0:
log.info("Selected case '{}'".format(selected_case))
if rank == 0:
log.info("Using groups of {} nodes".format(group_nodes))
# Adjust number of groups
if n_nodes % group_nodes != 0:
msg = "Current number of nodes ({}) is not divisible by the required group size ({})".format(
n_nodes, group_nodes
)
if rank == 0:
log.error(msg)
raise RuntimeError(msg)
n_group = n_nodes // group_nodes
group_time_samples = 1 + time_samples // n_group
group_seconds = group_time_samples / args.sample_rate
if args.simulate_atmosphere and args.weather is None:
raise RuntimeError("Cannot simulate atmosphere without a TOAST weather file")
comm = None
if mpicomm is None or args.dry_run is not None:
comm = Comm(world=None)
else:
comm = Comm(world=mpicomm, groupsize=group_procs)
jobdate = datetime.now().strftime("%Y%m%d-%H:%M:%S")
args.outdir += "_{:06d}_grp-{:04d}p-{:02d}n_{}".format(
procs, group_procs, group_nodes, jobdate
)
args.auxdir = os.path.join(args.outdir, "inputs")
if rank == 0:
os.makedirs(args.outdir)
os.makedirs(args.auxdir, exist_ok=True)
if rank == 0:
with open(os.path.join(args.outdir, "log"), "w") as f:
f.write("Running at {}\n".format(jobdate))
f.write("TOAST version = {}\n".format(env.version()))
f.write("TOAST max threads = {}\n".format(env.max_threads()))
f.write("MPI Processes = {}\n".format(procs))
f.write("MPI Processes per node = {}\n".format(procs_per_node))
f.write(
"Memory per node = {:0.2f} GB\n".format(avail_node_bytes / (1024 ** 3))
)
f.write("Number of groups = {}\n".format(n_group))
f.write("Group nodes = {}\n".format(group_nodes))
f.write("Group MPI Processes = {}\n".format(group_procs))
f.write("Case selected = {}\n".format(selected_case))
f.write("Case number of detectors = {}\n".format(n_detector))
f.write(
"Case total samples = {}\n".format(
n_group * group_time_samples * n_detector
)
)
f.write(
"Case samples per group = {}\n".format(group_time_samples * n_detector)
)
f.write("Case data seconds per group = {}\n".format(group_seconds))
f.write("Parameters:\n")
for k, v in vars(args).items():
if re.match(r"_.*", k) is None:
f.write(" {} = {}\n".format(k, v))
args.schedule = os.path.join(args.auxdir, "schedule.txt")
args.input_map = os.path.join(args.auxdir, "cmb.fits")
return args, comm, n_nodes, n_detector, selected_case, group_seconds, n_group
def parse_arguments(comm):
timer = Timer()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"and map astrophysical signal.",
fromfile_prefix_chars="@",
)
add_dist_args(parser)
add_debug_args(parser)
add_todground_args(parser)
add_pointing_args(parser)
add_polyfilter_args(parser)
add_groundfilter_args(parser)
add_gainscrambler_args(parser)
add_noise_args(parser)
add_sky_map_args(parser)
add_tidas_args(parser)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
add_madam_args(parser)
add_binner_args(parser)
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam for map-making",
dest="use_madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libmadam for map-making [default]",
dest="use_madam",
)
parser.set_defaults(use_madam=False)
parser.add_argument(
"--focalplane",
required=False,
default=None,
help="Pickle file containing a dictionary of detector "
"properties. The keys of this dict are the detector "
"names, and each value is also a dictionary with keys "
'"quat" (4 element ndarray), "fwhm" (float, arcmin), '
'"fknee" (float, Hz), "alpha" (float), and '
'"NET" (float). For optional plotting, the key "color"'
" can specify a valid matplotlib color string.",
)
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if args.tidas is not None:
if not tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if comm.comm_world is None or comm.world_rank == 0:
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.comm_world is None or comm.comm_world.rank == 0:
os.makedirs(args.outdir, exist_ok=True)
if comm.comm_world is None or comm.world_rank == 0:
timer.report_clear("Parsed parameters")
return args, comm
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
toast_tools.add_dist_args(parser)
toast_tools.add_todground_args(parser)
toast_tools.add_pointing_args(parser)
toast_tools.add_polyfilter_args(parser)
toast_tools.add_groundfilter_args(parser)
toast_tools.add_atmosphere_args(parser)
toast_tools.add_noise_args(parser)
toast_tools.add_gainscrambler_args(parser)
toast_tools.add_madam_args(parser)
toast_tools.add_mapmaker_args(parser)
toast_tools.add_filterbin_args(parser)
toast_tools.add_sky_map_args(parser)
toast_tools.add_sss_args(parser)
toast_tools.add_tidas_args(parser)
toast_tools.add_mc_args(parser)
so_tools.add_corotator_args(parser)
so_tools.add_time_constant_args(parser)
so_tools.add_demodulation_args(parser)
so_tools.add_h_n_args(parser)
so_tools.add_crosslinking_args(parser)
so_tools.add_cadence_map_args(parser)
so_tools.add_hw_args(parser)
so_tools.add_so_noise_args(parser)
so_tools.add_pysm_args(parser)
so_tools.add_export_args(parser)
toast_tools.add_debug_args(parser)
so_tools.add_import_args(parser)
so_tools.add_sim_sso_args(parser)
so_tools.add_flag_sso_args(parser)
so_tools.add_sim_hwpss_args(parser)
parser.add_argument(
"--no-maps",
required=False,
default=False,
action="store_true",
help="Disable all mapmaking.",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam for map-making",
dest="use_madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libmadam for map-making [default]",
dest="use_madam",
)
parser.set_defaults(use_madam=True)
try:
args = parser.parse_args()
except SystemExit as e:
sys.exit()
if len(args.bands.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if args.weather is None:
raise RuntimeError("You must provide a TOAST weather file")
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
if not os.path.isdir(args.outdir):
try:
os.makedirs(args.outdir)
except FileExistsError:
pass
timer.report_clear("Parse arguments")
return args, comm
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
add_dist_args(parser)
add_debug_args(parser)
add_todground_args(parser)
add_pointing_args(parser)
add_polyfilter_args(parser)
add_groundfilter_args(parser)
add_atmosphere_args(parser)
add_noise_args(parser)
add_gainscrambler_args(parser)
add_madam_args(parser)
add_sky_map_args(parser)
add_pysm_args(parser)
add_sss_args(parser)
add_tidas_args(parser)
add_spt3g_args(parser)
add_mc_args(parser)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument(
"--focalplane",
required=False,
default=None,
help="Pickle file containing a dictionary of detector "
"properties. The keys of this dict are the detector "
"names, and each value is also a dictionary with keys "
'"quat" (4 element ndarray), "fwhm" (float, arcmin), '
'"fknee" (float, Hz), "alpha" (float), and '
'"NET" (float).',
)
parser.add_argument(
"--freq",
required=True,
help="Comma-separated list of frequencies with identical focal planes."
" They override the bandpasses in the focalplane for the purpose of"
" scaling the atmospheric signal but not for simulating the sky signal.",
)
try:
args = parser.parse_args()
except SystemExit:
sys.exit(0)
if args.tidas is not None:
if not tidas_available:
raise RuntimeError("TIDAS not found- cannot export")
if args.spt3g is not None:
if not spt3g_available:
raise RuntimeError("SPT3G not found- cannot export")
if len(args.freq.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if args.simulate_atmosphere and args.weather is None:
raise RuntimeError(
"Cannot simulate atmosphere without a TOAST weather file")
if comm.world_rank == 0:
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
os.makedirs(args.outdir, exist_ok=True)
timer.stop()
if comm.world_rank == 0:
timer.report("Parsed parameters")
return args, comm
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
toast_tools.add_dist_args(parser)
toast_tools.add_todground_args(parser)
toast_tools.add_pointing_args(parser)
toast_tools.add_polyfilter_args(parser)
toast_tools.add_polyfilter2D_args(parser)
toast_tools.add_common_mode_filter_args(parser)
toast_tools.add_groundfilter_args(parser)
toast_tools.add_atmosphere_args(parser)
toast_tools.add_noise_args(parser)
toast_tools.add_gainscrambler_args(parser)
toast_tools.add_madam_args(parser)
toast_tools.add_filterbin_args(parser)
toast_tools.add_sky_map_args(parser)
toast_tools.add_sss_args(parser)
toast_tools.add_tidas_args(parser)
toast_tools.add_mc_args(parser)
s4_tools.add_hw_args(parser)
s4_tools.add_s4_noise_args(parser)
s4_tools.add_pysm_args(parser)
toast_tools.add_debug_args(parser)
parser.add_argument(
"--no-maps",
required=False,
default=False,
action="store_true",
help="Disable all mapmaking.",
)
parser.add_argument(
"--skip-madam",
required=False,
default=False,
action="store_true",
help="Skip the first Madam call.",
)
parser.add_argument(
"--pairdiff",
required=False,
default=False,
action="store_true",
help="Pair-difference TOD and pointing.",
)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
try:
args = parser.parse_args()
except SystemExit as e:
sys.exit()
if len(args.bands.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if args.simulate_atmosphere and args.weather is None:
raise RuntimeError(
"Cannot simulate atmosphere without a TOAST weather file")
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
os.makedirs(args.outdir, exist_ok=True)
timer.report_clear("Parse arguments")
return args, comm
def parse_arguments(comm):
timer = Timer()
timer.start()
log = Logger.get()
parser = argparse.ArgumentParser(
description="Simulate ground-based boresight pointing. Simulate "
"atmosphere and make maps for some number of noise Monte Carlos.",
fromfile_prefix_chars="@",
)
toast_tools.add_dist_args(parser)
toast_tools.add_todground_args(parser)
toast_tools.add_pointing_args(parser)
toast_tools.add_polyfilter_args(parser)
toast_tools.add_groundfilter_args(parser)
toast_tools.add_noise_args(parser)
toast_tools.add_sky_map_args(parser)
toast_tools.add_mc_args(parser)
so_tools.add_hw_args(parser)
so_tools.add_so_noise_args(parser)
so_tools.add_pysm_args(parser)
so_tools.add_export_args(parser)
toast_tools.add_debug_args(parser)
parser.add_argument("--outdir",
required=False,
default="out",
help="Output directory")
parser.add_argument("--map-prefix",
required=False,
default="toast",
help="Output map prefix")
parser.add_argument(
"--madam",
required=False,
action="store_true",
help="Use libmadam to bin the signal",
dest="madam",
)
parser.add_argument(
"--no-madam",
required=False,
action="store_false",
help="Do not use libMadam to bin the signal",
dest="madam",
)
parser.set_defaults(madam=False)
parser.add_argument(
"--madam-conserve-memory",
required=False,
action="store_true",
help="Stage the Madam buffer packing",
dest="madam_conserve_memory",
)
parser.add_argument(
"--no-madam-conserve-memory",
required=False,
action="store_false",
help="Do not stage the Madam buffer packing",
dest="madam_conserve_memory",
)
parser.set_defaults(madam_conserve_memory=True)
parser.add_argument(
"--madam-allreduce",
required=False,
action="store_true",
help="Use the allreduce communication pattern in Madam",
dest="madam_allreduce",
)
parser.add_argument(
"--no-madam-allreduce",
required=False,
action="store_false",
help="Do not use the allreduce communication pattern in Madam",
dest="madam_allreduce",
)
parser.set_defaults(madam_allreduce=False)
parser.add_argument(
"--madam-concatenate-messages",
required=False,
action="store_true",
help="Use the alltoallv commucation pattern in Madam",
dest="madam_concatenate_messages",
)
parser.add_argument(
"--no-madam-concatenate-messages",
required=False,
action="store_false",
help="Use the point-to-point communication pattern in Madam",
dest="madam_concatenate_messages",
)
parser.set_defaults(madam_concatenate_messages=True)
try:
args = parser.parse_args()
except SystemExit as e:
sys.exit()
if len(args.bands.split(",")) != 1:
# Multi frequency run. We don't support multiple copies of
# scanned signal.
if args.input_map:
raise RuntimeError(
"Multiple frequencies are not supported when scanning from a map"
)
if comm.world_rank == 0:
log.info("\n")
log.info("All parameters:")
for ag in vars(args):
log.info("{} = {}".format(ag, getattr(args, ag)))
log.info("\n")
if args.group_size:
comm = Comm(groupsize=args.group_size)
if comm.world_rank == 0:
if not os.path.isdir(args.outdir):
try:
os.makedirs(args.outdir)
except FileExistsError:
pass
timer.report_clear("Parse arguments")
return args, comm