def backengine(self):
""" This method drives the backengine CrystFEL.pattern_sim."""
# pattern_sim backengine does not support MPI.
return self._run()
# collect MPI arguments
if self.parameters.forced_mpi_command == "":
np = self.computeNTasks()
mpicommand = ParallelUtilities.prepareMPICommandArguments(np)
else:
mpicommand = self.parameters.forced_mpi_command
# Dump to a temporary file.
fname = IOUtilities.getTmpFileName()
self.dumpToFile(fname)
mpicommand += " ".join([" ", sys.executable, __file__, fname])
args = shlex.split(mpicommand)
if 'SIMEX_VERBOSE' in os.environ and os.environ[
'SIMEX_VERBOSE'] == 'MPI':
print("MPI command: " + mpicommand)
try:
proc = subprocess.Popen(args, universal_newlines=True)
proc.wait()
except:
raise
finally:
os.remove(fname)
return proc.returncode
def testPrepareMPICommandArguments_Adds_ExtraMPIParameters(self):
""" Test we correctly use SIMEX_EXTRA_MPI_PARAMETERS environment variable"""
os.environ["SIMEX_EXTRA_MPI_PARAMETERS"]='blabla'
self.assertIn("blabla",ParallelUtilities.prepareMPICommandArguments(10,0))
del os.environ["SIMEX_EXTRA_MPI_PARAMETERS"]
def testVendorSpecificMPIArguments_UseAllThreads(self):
""" Test we don't set OMP_NUM_THREADS by default"""
version = dict([("Vendor", "OpenMPI"), ("Version", '1.6.0')])
str = ParallelUtilities._getVendorSpecificMPIArguments(version, 0)
self.assertNotIn("OMP_NUM_THREADS", str)
def backengine(self):
""" Starts WPG simulations in parallel in a subprocess """
fname = IOUtilities.getTmpFileName()
self.dumpToFile(fname)
forcedMPIcommand = self.parameters.forced_mpi_command
if forcedMPIcommand == "":
(np, ncores) = self.computeNTasks()
mpicommand = ParallelUtilities.prepareMPICommandArguments(
np, ncores)
else:
mpicommand = forcedMPIcommand
if 'SIMEX_VERBOSE' in os.environ:
if 'MPI' in os.environ['SIMEX_VERBOSE']:
print(("WavePropagator backengine mpicommand: " + mpicommand))
if 'PYTHON' in os.environ['SIMEX_VERBOSE']:
import platform
print("Running python %s." % (platform.python_version()))
mpicommand += " python " + __file__ + " " + fname
args = shlex.split(mpicommand)
proc = subprocess.Popen(args, universal_newlines=True)
proc.wait()
os.remove(fname)
return proc.returncode
def testVendorSpecificMPIArguments_OpenMPINewVersion(self):
""" Test mpirun arguments for OpenMPI are set correctly for new version."""
version = dict([("Vendor", "OpenMPI"), ("Version", '1.9.0')])
str = ParallelUtilities._getVendorSpecificMPIArguments(version, 1)
self.assertEqual(
str,
" --map-by node --bind-to none -x OMP_NUM_THREADS=1 -x OMPI_MCA_mpi_warn_on_fork=0 -x OMPI_MCA_btl_base_warn_component_unused=0"
)
def testResourceInfoFromSlurm_WorksForSingleNode(self):
""" Test we can get resource info from SLURM for a single node."""
os.environ['SLURM_JOB_NUM_NODES'] = '1'
os.environ['SLURM_JOB_CPUS_PER_NODE'] = '40'
resource = ParallelUtilities.getParallelResourceInfo()
self.assertEqual(resource['NCores'], 40)
self.assertEqual(resource['NNodes'], 1)
def testResourceInfoFromSlurm_WorksForMultipleNodeWithDifferentAmountOfCores(self):
""" Test we can get resource info from SLURM for multiple heterogeneous nodes."""
os.environ['SLURM_JOB_NUM_NODES']='3'
os.environ['SLURM_JOB_CPUS_PER_NODE']='40x(2),20x(1),10x(10)'
resource = ParallelUtilities.getParallelResourceInfo()
self.assertEqual(resource['NCores'],200)
self.assertEqual(resource['NNodes'],3)
def testVendorSpecificMPIArguments_OpenMPIOldVersion(self):
""" Test mpirun arguments for OpenMPi are set correctly for old version."""
version = dict([("Vendor", "OpenMPI"), ("Version", '1.6.0')])
str = ParallelUtilities._getVendorSpecificMPIArguments(version, 1)
self.assertEqual(
str,
" --bynode -x OMP_NUM_THREADS=1 -x OMPI_MCA_mpi_warn_on_fork=0 -x OMPI_MCA_btl_base_warn_component_unused=0 "
"--mca mpi_cuda_support 0 --mca btl_openib_warn_no_device_params_found 0"
)
def testResourceInfoFromSlurm_WorksForMultipleNodeWithSameAmountOfCores(
self):
""" Test we can get resource info from SLURM for multiple homogeneous nodes."""
os.environ['SLURM_JOB_NUM_NODES'] = '3'
os.environ['SLURM_JOB_CPUS_PER_NODE'] = '40(x3)'
resource = ParallelUtilities.getParallelResourceInfo()
self.assertEqual(resource['NCores'], 120 / self.threads_per_core)
self.assertEqual(resource['NNodes'], 3)
def testGetVersionInfo(self):
""" Test we can extract MPI version infromation."""
version = ParallelUtilities._getMPIVersionInfo()
self.assertIn('Vendor', version)
self.assertIn('Version', version)
self.assertIn(version['Vendor'], ["OpenMPI", "MPICH"])
self.assertIsInstance(version['Version'], str)
self.assertGreater(StrictVersion(version['Version']),
StrictVersion("0.0.0"))
def computeNTasks(self):
resources = ParallelUtilities.getParallelResourceInfo()
ncores = resources['NCores']
nnodes = resources['NNodes']
if self.parameters.cpus_per_task == "MAX":
np = nnodes
else:
np = max(int(ncores / int(self.parameters.cpus_per_task)), 1)
return np
def testResourceInfoNothingWorked(self):
""" Test that we if we cannot get info, the default values are used."""
# we set SIMEX_MPICOMMAND so that it call return error
os.environ["SIMEX_MPICOMMAND"] = 'blabla'
resource = ParallelUtilities.getParallelResourceInfo()
self.assertEqual(resource['NCores'], 0)
self.assertEqual(resource['NNodes'], 1)
del os.environ["SIMEX_MPICOMMAND"]
def testResourceInfoFromSimexWorks(self):
""" Test we can set resource info via environment variables."""
os.environ["SIMEX_NNODES"] = '1'
os.environ["SIMEX_NCORES"] = '1'
resource = ParallelUtilities.getParallelResourceInfo()
self.assertEqual(resource['NCores'], 1)
self.assertEqual(resource['NNodes'], 1)
os.environ["SIMEX_NNODES"] = '-1'
os.environ["SIMEX_NCORES"] = '-1'
self.assertRaises(IOError, ParallelUtilities.getParallelResourceInfo)
def computeNTasks(self):
resources = ParallelUtilities.getParallelResourceInfo()
nnodes = resources['NNodes']
ncores = resources['NCores']
cpusPerTask = self.parameters.cpus_per_task
if cpusPerTask == "MAX":
np = nnodes
ncores = 0
else:
np = max(1, int(ncores / int(cpusPerTask)))
ncores = int(cpusPerTask)
return (np, ncores)
def computeNTasks(self):
""" Calculate the number of MPI tasks as function of available resources and
assigned cpus per task."""
resources = ParallelUtilities.getParallelResourceInfo()
ncores = resources['NCores']
nnodes = resources['NNodes']
if nnodes > 1:
raise RuntimeError("Backengine does not support MPI parallelism. ")
if self.parameters.cpus_per_task == "MAX":
np = nnodes
else:
np = max(int(ncores / int(self.parameters.cpus_per_task)), 1)
return np
def backengine(self):
""" Starts EMC simulations in parallel in a subprocess """
# Set paths.
self._setupPaths()
fname = IOUtilities.getTmpFileName()
self.dumpToFile(fname)
# collect MPI arguments
if self.parameters.forced_mpi_command == "":
np = self.computeNTasks()
mpicommand = ParallelUtilities.prepareMPICommandArguments(np)
else:
mpicommand = self.parameters.forced_mpi_command
# collect program arguments
command_sequence = [
'python',
__file__,
fname,
]
# put MPI and program arguments together
args = shlex.split(mpicommand) + command_sequence
if 'SIMEX_VERBOSE' in os.environ:
if 'MPI' in os.environ['SIMEX_VERBOSE']:
print(("EMCOrientation backengine mpicommand: " + mpicommand))
if 'PYTHON' in os.environ['SIMEX_VERBOSE']:
import platform
print("Running python version %s." %
(platform.python_version()))
# Run the backengine command.
proc = subprocess.Popen(args)
proc.wait()
os.remove(fname)
# Return the return code from the backengine.
return proc.returncode
def _backengineWithPdb(self):
""" """
"""
Run the diffraction simulation if the sample is a pdb.
Codes is based on pysingfel/tests/test_particle.test_calFromPDB
"""
# Dump self to file.
fname = IOUtilities.getTmpFileName()
self.dumpToFile(fname)
# Setup the mpi call.
forcedMPIcommand = self.parameters.forced_mpi_command
if forcedMPIcommand == "" or forcedMPIcommand is None:
(np, ncores) = self.computeNTasks()
mpicommand = ParallelUtilities.prepareMPICommandArguments(
np, ncores)
else:
mpicommand = forcedMPIcommand
mpicommand += " ".join(("", sys.executable, __file__, fname))
if 'SIMEX_VERBOSE' in os.environ:
if 'MPI' in os.environ['SIMEX_VERBOSE']:
print(("SingFELPhotonDiffractor backengine mpicommand: " +
mpicommand))
# Launch the system command.
args = shlex.split(mpicommand)
with subprocess.Popen(args, universal_newlines=True) as proc:
out, err = proc.communicate()
# Remove the dumped class.
os.remove(fname)
return proc.returncode
def testResourceInfoFromMPI(self):
""" Test we can get resource info from MPI command."""
resource = ParallelUtilities.getParallelResourceInfo()
self.assertGreater(resource['NCores'], 0)
self.assertEqual(resource['NNodes'], 1)
def testPrepareMPICommandArguments_Adds_NumberOfTasks(self):
""" Test we correctly set number of tasks"""
self.assertIn("-np 10",
ParallelUtilities.prepareMPICommandArguments(10, 0))
def setUpClass(cls):
""" Setting up the test class. """
cls.threads_per_core = ParallelUtilities.getThreadsPerCoreFromSlurm()
def testVendorSpecificMPIArguments_MPICH(self):
""" Test mpirun arguments for MPICH are set correctly."""
version = dict([("Vendor", "MPICH"), ("Version", '1.9.0')])
str = ParallelUtilities._getVendorSpecificMPIArguments(version, 1)
self.assertEqual(str, " -map-by node -env OMP_NUM_THREADS 1")
def backengine(self):
""" This method drives the backengine CrystFEL.pattern_sim."""
# Setup directory structure as needed.
if not os.path.isdir( self.output_path ):
os.makedirs( self.output_path )
output_file_base = os.path.join( self.output_path, "diffr_out")
if self.parameters.number_of_diffraction_patterns == 1:
output_file_base += "_0000001.h5"
# collect MPI arguments
if self.parameters.forced_mpi_command=="":
np=self.computeNTasks()
mpicommand=ParallelUtilities.prepareMPICommandArguments(np)
else:
mpicommand=self.parameters.forced_mpi_command
# Setup command, minimum set first.
command_sequence = ['pattern_sim',
'-p %s' % self.parameters.sample,
'--geometry=%s' % self.parameters.detector_geometry,
'--output=%s' % output_file_base,
'--number=%d' % self.parameters.number_of_diffraction_patterns,
]
# Handle random rotation is requested.
if self.parameters.uniform_rotation is True:
command_sequence.append('--random-orientation')
command_sequence.append('--really-random')
if self.parameters.beam_parameters is not None:
command_sequence.append('--photon-energy=%f' % (self.parameters.beam_parameters.photon_energy.m_as(electronvolt)))
command_sequence.append('--beam-bandwidth=%f' % (self.parameters.beam_parameters.photon_energy_relative_bandwidth))
nphotons = self.parameters.beam_parameters.pulse_energy / self.parameters.beam_parameters.photon_energy
command_sequence.append('--nphotons=%e' % (nphotons))
command_sequence.append('--beam-radius=%e' % (self.parameters.beam_parameters.beam_diameter_fwhm.m_as(meter)/2.))
command_sequence.append('--spectrum=%s' % (self.parameters.beam_parameters.photon_energy_spectrum_type.lower()))
if self.parameters.beam_parameters.photon_energy_spectrum_type.lower() == "sase":
command_sequence.append('--sample-spectrum=512')
# Handle intensities list if present.
if self.parameters.intensities_file is not None:
command_sequence.append('--intensities=%s' % (self.parameters.intensities_file))
# Handle powder if present.
if self.parameters.powder is True:
command_sequence.append('--powder=%s' % (os.path.join(self.output_path, "powder.h5")))
# Handle size range if present.
if self.parameters.crystal_size_min is not None:
command_sequence.append('--min-size=%f' % (self.parameters.crystal_size_min.m_as(1e-9*meter) ))
if self.parameters.crystal_size_max is not None:
command_sequence.append('--max-size=%f' % (self.parameters.crystal_size_max.m_as(1e-9*meter) ))
# put MPI and program arguments together
args = shlex.split(mpicommand) + command_sequence
#args = command_sequence
command = " ".join(args)
if 'SIMEX_VERBOSE' in os.environ:
print("CrystFELPhotonDiffractor backengine command: "+command)
# Run the backengine command.
proc = subprocess.Popen(command, shell=True)
proc.wait()
# Return the return code from the backengine.
return proc.returncode
def backengine(self):
""" This method drives the backengine singFEL."""
uniform_rotation = self.parameters.uniform_rotation
calculate_Compton = int(self.parameters.calculate_Compton)
slice_interval = self.parameters.slice_interval
number_of_slices = self.parameters.number_of_slices
pmi_start_ID = self.parameters.pmi_start_ID
pmi_stop_ID = self.parameters.pmi_stop_ID
number_of_diffraction_patterns = self.parameters.number_of_diffraction_patterns
if not os.path.isdir(self.output_path):
os.mkdir(self.output_path)
self.__output_dir = self.output_path
# If the sample is passed as a pdb, branch out to separate backengine implementation.
if self.input_path.split(".")[-1].lower() == 'pdb':
if not os.path.isfile(self.input_path):
# Attempt to query from pdb.
self.input_path = IOUtilities.checkAndGetPDB(self.input_path)
return self._backengineWithPdb()
# Ok, not a pdb, proceed.
# Serialize the geometry file.
beam_geometry_file = "tmp.geom"
self.parameters.detector_geometry.serialize(beam_geometry_file)
# Setup directory to pmi output.
# Backengine expects a directory name, so have to check if
# input_path is dir or file and handle accordingly.
if os.path.isdir(self.input_path):
input_dir = self.input_path
elif os.path.isfile(self.input_path):
input_dir = os.path.dirname(self.input_path)
config_file = '/dev/null'
# collect MPI arguments
if self.parameters.forced_mpi_command == "":
np, ncores = self.computeNTasks()
mpicommand = ParallelUtilities.prepareMPICommandArguments(np,1)
else:
mpicommand = self.parameters.forced_mpi_command
# collect program arguments
command_sequence = ['radiationDamageMPI',
'--inputDir', str(input_dir),
'--outputDir', str(self.__output_dir),
'--geomFile', str(beam_geometry_file),
'--configFile', str(config_file),
'--uniformRotation', str(uniform_rotation),
'--calculateCompton', str(calculate_Compton),
'--sliceInterval', str(slice_interval),
'--numSlices', str(number_of_slices),
'--pmiStartID', str(pmi_start_ID),
'--pmiEndID', str(pmi_stop_ID),
'--numDP', str(number_of_diffraction_patterns),
]
if self.parameters.beam_parameters is not None:
beam_parameter_file = "tmp.beam"
self.parameters.beam_parameters.serialize(beam_parameter_file)
command_sequence.append('--beamFile')
command_sequence.append(str(beam_parameter_file))
# put MPI and program arguments together
args = shlex.split(mpicommand) + command_sequence
if 'SIMEX_VERBOSE' in os.environ:
print(("SingFELPhotonDiffractor backengine command: "
+ " ".join(args)),flush=True)
# Run the backengine command.
proc = subprocess.Popen(args)
proc.wait()
# Return the return code from the backengine.
return proc.returncode
