def create_script(self): ''' create the script ''' self.script_name = 'Run.' + self.runid + '.' + str(self.current_job) fidt = open(self.template, 'r') lines = fidt.readlines() fidt.close() cdir = self.userspace fid = open(self.script_name, 'w') for line in lines: line = line.replace('<NCORES>', str(self.ncores)) line = line.replace('<CASE>', str(self.confcase)) line = line.replace('<JOB>', str(self.current_job)) line = line.replace('<NAMELIST>', str(self.namelist_current)) line = line.replace('<CDIR>', str(cdir)) line = line.replace('<WALLTIME>', str(self.walltime)) line = line.replace('<PROJECTCODE>', str(self.projectcode)) line = line.replace('<QUEUE>', str(self.queue)) fid.write(line) fid.close() os_utils.execute('chmod +x ' + self.script_name) return None
def create_script(self): ''' create the script ''' self.script_name = 'Run.' + self.runid + '.' + str(self.current_job) fidt = open(self.template,'r') lines = fidt.readlines() fidt.close() cdir = self.userspace fid = open(self.script_name,'w') for line in lines: line = line.replace('<NCORES>',str(self.ncores)) line = line.replace('<CASE>',str(self.confcase)) line = line.replace('<JOB>',str(self.current_job)) line = line.replace('<NAMELIST>',str(self.namelist_current)) line = line.replace('<CDIR>',str(cdir)) line = line.replace('<WALLTIME>',str(self.walltime)) line = line.replace('<PROJECTCODE>',str(self.projectcode)) line = line.replace('<QUEUE>',str(self.queue)) fid.write(line) fid.close() os_utils.execute('chmod +x ' + self.script_name) return None
def run(self,scrpt): # go to run directory command cdcmd = ' cd ' + self.run_opts['output_dir'] # run the script command runcmd = self.sub_opts['subcmd'] + scrpt.script_name # call system : this is where all the magic happens os_utils.execute( cdcmd + ' ; ' + runcmd ) return None
def run(self, scrpt): # go to run directory command cdcmd = ' cd ' + self.run_opts['output_dir'] # run the script command runcmd = self.sub_opts['subcmd'] + scrpt.script_name # call system : this is where all the magic happens os_utils.execute(cdcmd + ' ; ' + runcmd) return None
def _create_script(self): '''create control script for one run ''' fidr = open( self.rmanager_root + 'user/template/template_script_ctl.py', 'r' ) fidw = open( self.myrmanager + '/' + self.runname + '_ctl.py', 'w' ) for line in fidr.readlines(): fidw.write( line.replace( '<MY_RMANAGER>', self.myrmanager ).replace( '<MY_RUN>', self.runname ) ) fidr.close() fidw.close() os_utils.execute('chmod +x ' + self.myrmanager + '/' + self.runname + '_ctl.py' ) return None
def set_value(self, textToSearch, textToReplace): '''replace <VALUE> by its value ''' fidr = open('namelist.tmp1', 'r') fidw = open('namelist.tmp2', 'w') for line in fidr.readlines(): fidw.write(line.replace(textToSearch, textToReplace)) fidr.close() fidw.close() os_utils.execute('mv namelist.tmp2 namelist.tmp1') return None
def set_value(self,textToSearch, textToReplace): '''replace <VALUE> by its value ''' fidr = open( 'namelist.tmp1', 'r' ) fidw = open( 'namelist.tmp2', 'w' ) for line in fidr.readlines(): fidw.write( line.replace( textToSearch, textToReplace ) ) fidr.close() fidw.close() os_utils.execute('mv namelist.tmp2 namelist.tmp1') return None
def create_hostfile(self): self.fileout = 'hostfile.' + self.run_opts['runid'] offset = int( self.sub_opts['host_offset'] ) fid = open(self.fileout,'w') for kp in np.arange(self.nodes_needed): fid.write('node' + str(int(kp+offset+1)) + ' slots=16 max_slots=16 \n') # 16 harcoded fid.close() # move this directly to run directory rundir = self.run_opts['output_dir'] os_utils.execute('mv ' + self.fileout + ' ' + rundir + '/.') return None
def archive_rst(self): '''copy restart in a safe place''' rstin = self.restart_file rstout = self.restart_file.replace('.nc','.nc' + '.' + str(self.previous_job)) status = os_utils.execute('rsync -av ' + self.output_dir + '/' + rstin + ' ' + self.rstdir + '/' + rstout) # tidal filter files list_filtered = os_utils.get_output('ls ' + self.output_dir + ' | grep ocean_fil*nc') if len(list_filtered) > 0: status = os_utils.execute('rsync -av ' + self.output_dir + '/ocean_fil*nc' + ' ' + self.rstdir + '/.') else: pass return None
def create_hostfile(self): self.fileout = 'hostfile.' + self.run_opts['runid'] offset = int(self.sub_opts['host_offset']) fid = open(self.fileout, 'w') for kp in np.arange(self.nodes_needed): fid.write('node' + str(int(kp + offset + 1)) + ' slots=16 max_slots=16 \n') # 16 harcoded fid.close() # move this directly to run directory rundir = self.run_opts['output_dir'] os_utils.execute('mv ' + self.fileout + ' ' + rundir + '/.') return None
def rename_file(self): ''' call unix command mv ''' # remove filein from full path wrk = self.filein.replace('/',' ').split()[0:-1] # re-create path path = '' for part in wrk: path = path + '/' + part # rename file only if different if self.filein == path + '/' + self.fileout: pass else: os_utils.execute('mv ' + self.filein + ' ' + path + '/' + self.fileout) return None
def _create_build_script(self): '''create custom build.bash script ''' self.model_dir = '' while self.model_dir == '': self.model_dir = raw_input('Where is the ROMS code installed ? \n >>> ') fidr = open( self.rmanager_root + 'user/template/build.bash.template', 'r' ) fidw = open( self.myrmanager + '/' + self.runname + '/build.bash', 'w' ) for line in fidr.readlines(): fidw.write( line.replace( '<ROMS_APP>', self.roms_app ).replace( '<ROMS_DIR>', self.model_dir ).replace( '<MY_TMPDIR>', self.tmpdir ) ) fidr.close() fidw.close() os_utils.execute('chmod +x ' + self.myrmanager + '/' + self.runname + '/build.bash' ) return None
def create_archive_dir(self): ''' check existence of directories for restart and outputs and create them if needed''' #--- restarts directory inq_restart = os_utils.execute('test -d ' + self.rootrst) if inq_restart == 0: pass else: print '<RMANAGER> creating ' + self.rootrst status = os_utils.execute('mkdir ' + self.rootrst) #--- this restart inq_restart = os_utils.execute('test -d ' + self.rstdir) if inq_restart == 0: pass else: print '<RMANAGER> creating ' + self.rstdir status = os_utils.execute('mkdir ' + self.rstdir) #--- output directory inq_arch = os_utils.execute('test -d ' + self.sarchive_dir) if inq_arch == 0: pass else: print '<RMANAGER> creating ' + self.sarchive_dir status = os_utils.execute('mkdir ' + self.sarchive_dir) #---- by year for year in self.list_years: inq_arch = os_utils.execute('test -d ' + self.sarchive_dir + '/' + str(year)) if inq_arch == 0: pass else: print '<RMANAGER> creating ' + self.sarchive_dir + '/' + str(year) status = os_utils.execute('mkdir ' + self.sarchive_dir + '/' + str(year)) return None
def rename_file(self): ''' call unix command mv ''' # remove filein from full path wrk = self.filein.replace('/', ' ').split()[0:-1] # re-create path path = '' for part in wrk: path = path + '/' + part # rename file only if different if self.filein == path + '/' + self.fileout: pass else: os_utils.execute('mv ' + self.filein + ' ' + path + '/' + self.fileout) return None
def _create_script(self): '''create control script for one run ''' fidr = open( self.rmanager_root + 'user/template/template_script_ctl.py', 'r') fidw = open(self.myrmanager + '/' + self.runname + '_ctl.py', 'w') for line in fidr.readlines(): fidw.write( line.replace('<MY_RMANAGER>', self.myrmanager).replace('<MY_RUN>', self.runname)) fidr.close() fidw.close() os_utils.execute('chmod +x ' + self.myrmanager + '/' + self.runname + '_ctl.py') return None
def write_log_timeline(self): ''' write the current status of run time line in a log file ''' ftimeline = self.sarchive_dir + '/timeline_run' # if file already exists, load it check = os_utils.execute('test -f ' + ftimeline) if check == 0: f = open(ftimeline, 'r') lines = f.readlines() f.close() list_timeline = [] for line in lines: list_timeline.append(line.split()[0]) print 'already in timeline_run :', list_timeline # update timeline list_timeline.extend(self.list_years) list_timeline = list(set(list_timeline)) print 'timeline updated :', list_timeline else: list_timeline = self.list_years list_timeline = list(set(list_timeline)) print 'new timeline_run', list_timeline # overwrite with the new list f = open(ftimeline, 'w') for year in list_timeline: f.write(year + '\n') f.close() return None
def archive_output(self): '''move outputs files''' list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep :') for myfile in list_files: year = myfile.replace(self.confcase,'').replace('_',' ').replace('-',' ').split()[1] status = os_utils.execute('mv ' + self.output_dir + '/' + myfile + ' ' + self.sarchive_dir + '/' + str(year) + '/.') return None
def write_log_timeline(self): ''' write the current status of run time line in a log file ''' ftimeline = self.sarchive_dir + '/timeline_run' # if file already exists, load it check = os_utils.execute('test -f ' + ftimeline ) if check ==0: f = open(ftimeline,'r') ; lines = f.readlines() ; f.close() list_timeline = [] for line in lines: list_timeline.append(line.split()[0]) print 'already in timeline_run :', list_timeline # update timeline list_timeline.extend(self.list_years) list_timeline = list(set(list_timeline)) print 'timeline updated :', list_timeline else: list_timeline = self.list_years list_timeline = list(set(list_timeline)) print 'new timeline_run', list_timeline # overwrite with the new list f = open(ftimeline,'w') for year in list_timeline: f.write( year + '\n') f.close() return None
def archive_rst(self): '''copy restart in a safe place''' rstin = self.restart_file rstout = self.restart_file.replace( '.nc', '.nc' + '.' + str(self.previous_job)) status = os_utils.execute('rsync -av ' + self.output_dir + '/' + rstin + ' ' + self.rstdir + '/' + rstout) # tidal filter files list_filtered = os_utils.get_output('ls ' + self.output_dir + ' | grep ocean_fil*nc') if len(list_filtered) > 0: status = os_utils.execute('rsync -av ' + self.output_dir + '/ocean_fil*nc' + ' ' + self.rstdir + '/.') else: pass return None
def _create_build_script(self): '''create custom build.bash script ''' self.model_dir = '' while self.model_dir == '': self.model_dir = raw_input( 'Where is the ROMS code installed ? \n >>> ') fidr = open(self.rmanager_root + 'user/template/build.bash.template', 'r') fidw = open(self.myrmanager + '/' + self.runname + '/build.bash', 'w') for line in fidr.readlines(): fidw.write( line.replace('<ROMS_APP>', self.roms_app).replace( '<ROMS_DIR>', self.model_dir).replace('<MY_TMPDIR>', self.tmpdir)) fidr.close() fidw.close() os_utils.execute('chmod +x ' + self.myrmanager + '/' + self.runname + '/build.bash') return None
def archive_output(self): '''move outputs files''' list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep :') for myfile in list_files: year = myfile.replace(self.confcase, '').replace('_', ' ').replace('-', ' ').split()[1] status = os_utils.execute('mv ' + self.output_dir + '/' + myfile + ' ' + self.sarchive_dir + '/' + str(year) + '/.') return None
def _create_run_directory(self): self.tmpdir = self.scratch + '/tmpdir_'+ self.runname # create the appropriate directories os_utils.execute('mkdir ' + self.myrmanager + '/' + self.runname) os_utils.execute('mkdir ' + self.myrmanager + '/' + self.runname + '/src') os_utils.execute('mkdir ' + self.tmpdir ) return None
def transfer_one_file(self,file,fromdir,todir): exit_code = 0 # 0 normal, 1 problem checksum = 0 check = 0 # first check if file already on storage exist = self.check_existence_file(todir + file) # verify it is the same file if exist: sum_original = self.compute_md5sum(fromdir + file) sum_copy = self.compute_md5sum(todir + file) # or copy to a different file if sum_original != sum_copy: datenow = dt.datetime.now() os_utils.execute('mv ' + todir + file + ' ' + todir + file + '.' + datenow.isoformat() ) checksum = 1 self.run_copy(file,fromdir,todir) if (check != 0) or (checksum != 0): exit_code = 1 return exit_code
def _create_run_directory(self): self.tmpdir = self.scratch + '/tmpdir_' + self.runname # create the appropriate directories os_utils.execute('mkdir ' + self.myrmanager + '/' + self.runname) os_utils.execute('mkdir ' + self.myrmanager + '/' + self.runname + '/src') os_utils.execute('mkdir ' + self.tmpdir) return None
def transfer_one_file(self, file, fromdir, todir): exit_code = 0 # 0 normal, 1 problem checksum = 0 check = 0 # first check if file already on storage exist = self.check_existence_file(todir + file) # verify it is the same file if exist: sum_original = self.compute_md5sum(fromdir + file) sum_copy = self.compute_md5sum(todir + file) # or copy to a different file if sum_original != sum_copy: datenow = dt.datetime.now() os_utils.execute('mv ' + todir + file + ' ' + todir + file + '.' + datenow.isoformat()) checksum = 1 self.run_copy(file, fromdir, todir) if (check != 0) or (checksum != 0): exit_code = 1 return exit_code
def copy2rundir(self,namlist,scrpt,infiles): rundir = self.run_opts['output_dir'] # namelist os_utils.execute('mv ' + namlist.namelist_current + ' ' + rundir +'/.') # script os_utils.execute('mv ' + scrpt.script_name + ' ' + rundir +'/.') # varinfo.dat and all other files ice.in, bio.in,... for infile in infiles: os_utils.execute('cp ' + infile + ' ' + rundir ) return None
def copy2rundir(self, namlist, scrpt, infiles): rundir = self.run_opts['output_dir'] # namelist os_utils.execute('mv ' + namlist.namelist_current + ' ' + rundir + '/.') # script os_utils.execute('mv ' + scrpt.script_name + ' ' + rundir + '/.') # varinfo.dat and all other files ice.in, bio.in,... for infile in infiles: os_utils.execute('cp ' + infile + ' ' + rundir) return None
def create_archive_dir(self): ''' check existence of directories for restart and outputs and create them if needed''' #--- restarts directory inq_restart = os_utils.execute('test -d ' + self.rootrst) if inq_restart == 0: pass else: print '<RMANAGER> creating ' + self.rootrst status = os_utils.execute('mkdir ' + self.rootrst) #--- this restart inq_restart = os_utils.execute('test -d ' + self.rstdir) if inq_restart == 0: pass else: print '<RMANAGER> creating ' + self.rstdir status = os_utils.execute('mkdir ' + self.rstdir) #--- output directory inq_arch = os_utils.execute('test -d ' + self.sarchive_dir) if inq_arch == 0: pass else: print '<RMANAGER> creating ' + self.sarchive_dir status = os_utils.execute('mkdir ' + self.sarchive_dir) #---- by year for year in self.list_years: inq_arch = os_utils.execute('test -d ' + self.sarchive_dir + '/' + str(year)) if inq_arch == 0: pass else: print '<RMANAGER> creating ' + self.sarchive_dir + '/' + str( year) status = os_utils.execute('mkdir ' + self.sarchive_dir + '/' + str(year)) return None
def copy_skel(self): '''copy into a tmp file''' os_utils.execute('cp ' + self.userspace + '/' + self.namelist_skel + ' namelist.tmp1') return None
def finalize_namelist(self): '''copy into the final file''' os_utils.execute('mv namelist.tmp1 ' + self.namelist_current) return None
def run_copy(self,file,fromdir,todir): check = os_utils.execute('nice -0 rsync -av ' + fromdir + file + ' ' + todir + file ) return check
def make_dir(self,dir): os_utils.execute('mkdir ' + dir) return None
def check_existence_dir(self,mydir): exist=True check = os_utils.execute('test -d ' + mydir) if check != 0: exist=False return exist
def check_existence_file(self,myfile): exist=True check = os_utils.execute('test -f ' + myfile) if check != 0: exist=False return exist
def finalize_namelist(self): '''copy into the final file''' os_utils.execute('mv namelist.tmp1 ' + self.namelist_current) return None
def _prepare_transfer(self): ''' check presence of directories on workdir ''' check = os_utils.execute('test -d ' + self.run_opts['output_dir'] ) if check != 0: self._stop('Error : the tmpdir for this run does not exist on the scrath') check = os_utils.execute('test -d ' + self.myworkdir ) if check != 0: self._stop('Error : the workdir does not exist or path is wrong') check = os_utils.execute('test -d ' + self.storagedir ) if check != 0: os_utils.execute('mkdir '+ self.storagedir ) check = os_utils.execute('test -d ' + self.storagedir + 'outputs' ) if check != 0: os_utils.execute('mkdir '+ self.storagedir + 'outputs' ) check = os_utils.execute('test -d ' + self.storagedir + 'restarts' ) if check != 0: os_utils.execute('mkdir '+ self.storagedir + 'restarts' ) return None
def run_copy(self, file, fromdir, todir): check = os_utils.execute('nice -0 rsync -av ' + fromdir + file + ' ' + todir + file) return check
def copy_skel(self): '''copy into a tmp file''' os_utils.execute('cp ' + self.userspace + '/' + self.namelist_skel + ' namelist.tmp1') return None
def _prepare_transfer(self): ''' check presence of directories on workdir ''' check = os_utils.execute('test -d ' + self.run_opts['output_dir']) if check != 0: self._stop( 'Error : the tmpdir for this run does not exist on the scrath') check = os_utils.execute('test -d ' + self.myworkdir) if check != 0: self._stop('Error : the workdir does not exist or path is wrong') check = os_utils.execute('test -d ' + self.storagedir) if check != 0: os_utils.execute('mkdir ' + self.storagedir) check = os_utils.execute('test -d ' + self.storagedir + 'outputs') if check != 0: os_utils.execute('mkdir ' + self.storagedir + 'outputs') check = os_utils.execute('test -d ' + self.storagedir + 'restarts') if check != 0: os_utils.execute('mkdir ' + self.storagedir + 'restarts') return None
def check_existence_file(self, myfile): exist = True check = os_utils.execute('test -f ' + myfile) if check != 0: exist = False return exist
def check_existence_dir(self, mydir): exist = True check = os_utils.execute('test -d ' + mydir) if check != 0: exist = False return exist
def make_dir(self, dir): os_utils.execute('mkdir ' + dir) return None