#SBATCH --clusters=serial #SBATCH --time=96:00:00 # # if a job might take over 2 days to be executed import os import sys import stat import math #Classes containing sweet compile/run basic option from mule_local.JobGeneration import * from mule.SWEETRuntimeParametersScenarios import * #Create main compile/run options jg = JobGeneration() # Request dedicated compile script jg.compilecommand_in_jobscript = False # Wallclock time max_wallclock_seconds = 2 * 24 * 60 * 60 ref_max_wallclock_seconds = 48 * 60 * 60 #Get Earth parameters (if necessary) earth = EarthMKSDimensions() # # Run simulation on plane or sphere # #Basic plane options
def __load_job_raw_data(self, jobdir=None): """ Parse all output.out files and extract all kind of job output information Return a dictionary with content from the job directories { # # Dictionary with data from job generation # (read from [jobdir]/jobgeneration.pickle) # 'jobgeneration': # From jobgeneration.pickle { 'compile': [...], 'runtime': [...], 'parallelization': [...], 'platforms_platform': [...], 'platform_resources': [...], }, 'output': # From output.out with prefix [MULE] { 'simulation_benchmark_timings.main': [value], 'simulation_benchmark_timings.main_simulationLoop': [value], [...] }, '[filename(.pickle)]': { [...] } }, '[filename(.pickle)]': { [...] } """ # self.__job_raw_data = {} if self.verbosity > 5: self.info("") self.info("Processing '" + jobdir + "'") """ * Process 'output.out' """ if self.verbosity > 5: self.info("Loading job output file 'output.out'") outfile = jobdir + '/output.out' try: #if True: with open(outfile, 'r') as f: content = f.readlines() self.__job_raw_data['output'] = self.__parse_job_output( content) except Exception as err: print("*" * 80) print("* WARNINNG: opening '" + outfile + "' (ignoring)") print("* " + str(err)) print("*" * 80) """ Process 'jobgeneration.pickle' """ # Ensure that 'jobgeneration.pickle' exists jobgenerationfile = jobdir + '/jobgeneration.pickle' if self.verbosity > 5: self.info("Loading 'jobgeneration.pickle'") j = JobGeneration(dummy_init=True) self.__job_raw_data['jobgeneration'] = j.load_attributes_dict( jobgenerationfile) """ Process other '*.pickle' """ pickle_files = glob.glob(jobdir + '/*.pickle') # Iterate over all found pickle files for picklefile in pickle_files: filename = os.path.basename(picklefile) tag = filename.replace('.pickle', '') if tag == 'jobgeneration': continue if self.verbosity > 5: self.info("Loading pickle file '" + filename + "'") import pickle with open(picklefile, 'rb') as f: self.__job_raw_data[tag] = pickle.load(f)