def transfer_restarts(self):
# get list of restarts
list_restarts = os_utils.get_output('ls ' + self.run_opts['output_dir'] + '/restarts' )
for restart in list_restarts:
my_todir = self.storagedir + 'restarts/' + restart + '/'
if not self.check_existence_dir(my_todir):
self.make_dir(my_todir)
my_fromdir = self.run_opts['output_dir'] + '/restarts/' + restart + '/'
list_files = os_utils.get_output('ls ' + my_fromdir )
for file in list_files:
error = self.transfer_one_file(file,my_fromdir,my_todir)
if error != 0:
print 'An error has occured while transfering file ', file
return None
def archive_output(self):
'''move outputs files'''
list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep :')
for myfile in list_files:
year = myfile.replace(self.confcase,'').replace('_',' ').replace('-',' ').split()[1]
status = os_utils.execute('mv ' + self.output_dir + '/' + myfile + ' ' + self.sarchive_dir + '/' + str(year) + '/.')
return None
def transfer_outputs(self):
# get list of years
list_years = os_utils.get_output('ls ' + self.run_opts['output_dir'] +
'/outputs')
for year in list_years:
my_todir = self.storagedir + 'outputs/' + year + '/'
if not self.check_existence_dir(my_todir):
self.make_dir(my_todir)
my_fromdir = self.run_opts['output_dir'] + '/outputs/' + year + '/'
list_files = os_utils.get_output('ls ' + my_fromdir)
for file in list_files:
error = self.transfer_one_file(file, my_fromdir, my_todir)
if error != 0:
print 'An error has occured while transfering file ', file
return None
def create_list_years(self):
'''look at output files and find out what years are currently present'''
list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep :')
list_years = []
for myfile in list_files:
tmp2 = myfile.replace(self.confcase,'').replace('_',' ').replace('-',' ').split()[1]
list_years.append(tmp2)
self.list_years = list(set(list_years))
return None
def archive_output(self):
'''move outputs files'''
list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep :')
for myfile in list_files:
year = myfile.replace(self.confcase,
'').replace('_', ' ').replace('-',
' ').split()[1]
status = os_utils.execute('mv ' + self.output_dir + '/' + myfile +
' ' + self.sarchive_dir + '/' +
str(year) + '/.')
return None
def create_list_years(self):
'''look at output files and find out what years are currently present'''
list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep :')
list_years = []
for myfile in list_files:
tmp2 = myfile.replace(self.confcase,
'').replace('_', ' ').replace('-',
' ').split()[1]
list_years.append(tmp2)
self.list_years = list(set(list_years))
return None
def archive_rst(self):
'''copy restart in a safe place'''
rstin = self.restart_file
rstout = self.restart_file.replace('.nc','.nc' + '.' + str(self.previous_job))
status = os_utils.execute('rsync -av ' + self.output_dir + '/' + rstin + ' ' + self.rstdir + '/' + rstout)
# tidal filter files
list_filtered = os_utils.get_output('ls ' + self.output_dir + ' | grep ocean_fil*nc')
if len(list_filtered) > 0:
status = os_utils.execute('rsync -av ' + self.output_dir + '/ocean_fil*nc' + ' ' + self.rstdir + '/.')
else:
pass
return None
def tags_files(self,ftype='avg'):
'''rename roms files with a proper datetime tag'''
list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep ' + ftype + ' | grep nc')
for myfile in list_files:
tag = dtroms.tagfile(self.output_dir + '/' + myfile)
if self.use_leap_years == 'True':
tag()
elif self.use_leap_years == 'False':
tag(leap=False)
else:
# do not rename files
pass
return None
def check_blowup(self):
logfile = self.run_opts['output_dir'] + '/' + 'log.' + str( self.run_opts['current_job'] )
out = os_utils.get_output('grep Blowing-up ' + logfile)
print '<RMANAGER> checking for blowup in ' + logfile + '...'
if ( len(out) == 0):
print '<RMANAGER> log seems normal'
pass
else:
print ''
print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
print '!!! RUN BLEW UP , EMERGENCY STOP !!!'
print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
exit()
def __init__(self, user=None):
# define user
self.login = os_utils.get_output('whoami')[0]
if user is None:
self.user = self.login
else:
self.user = user # to override if RMANAGER user is different from login (e.g. on yellowstone)
# define machine on which we run
self.list_machine = [
'workstation', 'yellowstone', 'triton16', 'triton24'
]
self.machine = ''
print 'Answer with one of these :', self.list_machine
while self.machine not in self.list_machine:
self.machine = raw_input(
'On which supercomputer are we running ? \n >>> ')
if self.machine in ['workstation', 'triton16', 'triton24']:
try:
self.scratch = os_utils.get_envvar('SCRATCH')
except:
exit(
'SCRATCH environment variable misssing, please add to .bashrc'
)
elif self.machine in ['yellowstone']:
self.scratch = '/glade/scratch/' + self.login
# find where RMANAGER is installed
self.rmanager_root = None
pythonpath = os_utils.get_envvar('PYTHONPATH')
list_pythonpath = pythonpath.replace(':', ' ').split()
for dir in list_pythonpath:
if (dir.find('RMANAGER') != -1 and dir.find(self.login) != -1):
self.rmanager_root = dir.replace('src/python', '')
break # we want the first occurence of RMANAGER in PYTHONPATH
if self.rmanager_root is None:
print 'Unable to find out where RMANAGER is installed or incorrect install'
exit()
else:
print 'RMANAGER is installed in : ', self.rmanager_root
print 'If this seems incorrect, please check your install, paths,...'
print 'You can now run setup = libmanager.setup_simulation()'
# define where user will create the run folder
self.myrmanager = self.rmanager_root + 'user/' + self.user
print 'Your user space is ', self.myrmanager
print 'You can now run setup()'
return None
def tags_files(self, ftype='avg'):
'''rename roms files with a proper datetime tag'''
list_files = os_utils.get_output('ls ' + self.output_dir + ' | grep ' +
ftype + ' | grep nc')
for myfile in list_files:
tag = dtroms.tagfile(self.output_dir + '/' + myfile)
if self.use_leap_years == 'True':
tag()
elif self.use_leap_years == 'False':
tag(leap=False)
else:
# do not rename files
pass
return None
def check_blowup(self):
logfile = self.run_opts['output_dir'] + '/' + 'log.' + str(
self.run_opts['current_job'])
out = os_utils.get_output('grep Blowing-up ' + logfile)
print '<RMANAGER> checking for blowup in ' + logfile + '...'
if (len(out) == 0):
print '<RMANAGER> log seems normal'
pass
else:
print ''
print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
print '!!! RUN BLEW UP , EMERGENCY STOP !!!'
print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
exit()
def archive_rst(self):
'''copy restart in a safe place'''
rstin = self.restart_file
rstout = self.restart_file.replace(
'.nc', '.nc' + '.' + str(self.previous_job))
status = os_utils.execute('rsync -av ' + self.output_dir + '/' +
rstin + ' ' + self.rstdir + '/' + rstout)
# tidal filter files
list_filtered = os_utils.get_output('ls ' + self.output_dir +
' | grep ocean_fil*nc')
if len(list_filtered) > 0:
status = os_utils.execute('rsync -av ' + self.output_dir +
'/ocean_fil*nc' + ' ' + self.rstdir +
'/.')
else:
pass
return None
def __init__(self,user=None):
# define user
self.login = os_utils.get_output('whoami')[0]
if user is None:
self.user = self.login
else:
self.user = user # to override if RMANAGER user is different from login (e.g. on yellowstone)
# define machine on which we run
self.list_machine = ['workstation','yellowstone','triton16','triton24']
self.machine = ''
print 'Answer with one of these :', self.list_machine
while self.machine not in self.list_machine:
self.machine = raw_input('On which supercomputer are we running ? \n >>> ')
if self.machine in ['workstation','triton16','triton24']:
try:
self.scratch = os_utils.get_envvar('SCRATCH')
except:
exit('SCRATCH environment variable misssing, please add to .bashrc')
elif self.machine in ['yellowstone']:
self.scratch = '/glade/scratch/' + self.login
# find where RMANAGER is installed
self.rmanager_root = None
pythonpath = os_utils.get_envvar('PYTHONPATH')
list_pythonpath = pythonpath.replace(':',' ').split()
for dir in list_pythonpath:
if (dir.find('RMANAGER') != -1 and dir.find(self.login) != -1):
self.rmanager_root = dir.replace('src/python','')
break # we want the first occurence of RMANAGER in PYTHONPATH
if self.rmanager_root is None:
print 'Unable to find out where RMANAGER is installed or incorrect install'
exit()
else:
print 'RMANAGER is installed in : ', self.rmanager_root
print 'If this seems incorrect, please check your install, paths,...'
print 'You can now run setup = libmanager.setup_simulation()'
# define where user will create the run folder
self.myrmanager = self.rmanager_root + 'user/' + self.user
print 'Your user space is ', self.myrmanager
print 'You can now run setup()'
return None
def compute_md5sum(self,myfile):
md5sum = os_utils.get_output('md5sum ' + myfile )[0]
return md5sum
def compute_md5sum(self, myfile):
md5sum = os_utils.get_output('md5sum ' + myfile)[0]
return md5sum