def check_presn(path, base):
# Hmm, we do not want to check all the time, so we want some
# record that the run is OK. We may need some run info data base
# in the run directory, or somewhere in general.
dump_filename = os.path.join(path, base + '#presn')
if not os.path.exists(dump_filename):
return False
invalid_tokens = ('broken', )
for token in invalid_tokens:
broken_filename = os.path.join(path, token)
if os.path.exists(broken_filename):
return False
presnok_filename = os.path.join(path, 'presn')
if not os.path.exists(presnok_filename):
# sometimes file is not quite written to disk, then wait.
for i in range(10):
try:
d = kepdump._load(dump_filename)
except RecLenError:
time.sleep(i + 1)
continue
else:
break
ok = d.is_presn
if not ok:
touch(broken_filename)
print('broken: ', dump_filename)
else:
touch(presnok_filename)
print('ok: ', dump_filename)
return ok
return True
def lastdump(self):
for i in range(10):
try:
d = kepdump._load(self.lastfile())
return d
except:
time.sleep(1)
def presn(self):
f = os.path.join(self.cwd, self.run + '#presn')
if os.path.isfile(f):
d = kepdump._load(f)
return d
else:
return None
def check_presn(path, base):
# Hmm, we do not want to check all the time, so we want some
# record that the run is OK. We may need some run info data base
# in the run directory, or somewhere in general.
dump_filename = os.path.join(path, base + '#presn')
if not os.path.exists(dump_filename):
return False
invalid_tokens = ('broken', )
for token in invalid_tokens:
broken_filename = os.path.join(path, token)
if os.path.exists(broken_filename):
return False
presnok_filename = os.path.join(path, 'presn')
if not os.path.exists(presnok_filename):
# sometimes file is not quite written to disk, then wait.
for i in range(10):
try:
d = kepdump._load(dump_filename)
except RecLenError:
time.sleep(i + 1)
continue
else:
break
ok = d.is_presn
if not ok:
touch(broken_filename)
print('broken: ', dump_filename)
else:
touch(presnok_filename)
print('ok: ', dump_filename)
return ok
return True
def lastdump(self):
for i in range(10):
try:
d = kepdump._load(self.lastfile())
return d
except:
time.sleep(1)
def presn(self):
f = os.path.join(self.cwd, self.run + '#presn')
if os.path.isfile(f):
d = kepdump._load(f)
return d
else:
return None
def check_all_presn(
path=default_path,
fix=False,
):
paths = glob.glob(os.path.join(path, '*'))
for x in paths:
print('.', end='', flush=True)
run = os.path.basename(x)
dump_filename = os.path.join(x, run + '#presn')
presn_filename = os.path.join(x, 'presn')
broken_filename = os.path.join(x, 'broken')
if os.path.exists(dump_filename):
b = os.path.exists(broken_filename)
p = os.path.exists(presn_filename)
d = kepdump._load(dump_filename)
if d.is_presn:
if b:
print('presn seems OK, but marked broken: {}'.format(run))
if fix:
os.remove(broken_filename)
if not p:
print(
'presn seems OK, but not marked presn: {}'.format(run))
if fix:
touch(presn_filename)
else:
if p:
print('presn not OK, but marked presn: {}'.format(run))
if fix:
os.remove(presn_filename)
os.remove(dump_filename)
touch(broken_filename)
else:
b = os.path.exists(broken_filename)
p = os.path.exists(presn_filename)
if p:
print('presn not present, but marked presn: {}'.format(run))
if fix:
os.remove(presn_filename)
touch(broken_filename)
def check_all_presn(path = default_path,
fix = False,
):
paths = glob.glob(os.path.join(path, '*'))
for x in paths:
print('.', end = '', flush = True)
run = os.path.basename(x)
dump_filename = os.path.join(x, run + '#presn')
presn_filename = os.path.join(x, 'presn')
broken_filename = os.path.join(x, 'broken')
if os.path.exists(dump_filename):
b = os.path.exists(broken_filename)
p = os.path.exists(presn_filename)
d = kepdump._load(dump_filename)
if d.is_presn:
if b:
print('presn seems OK, but marked broken: {}'.format(run))
if fix:
os.remove(broken_filename)
if not p:
print('presn seems OK, but not marked presn: {}'.format(run))
if fix:
touch(presn_filename)
else:
if p:
print('presn not OK, but marked presn: {}'.format(run))
if fix:
os.remove(presn_filename)
os.remove(dump_filename)
touch(broken_filename)
else:
b = os.path.exists(broken_filename)
p = os.path.exists(presn_filename)
if p:
print('presn not present, but marked presn: {}'.format(run))
if fix:
os.remove(presn_filename)
touch(broken_filename)
def clean_broken_presn(path = default_path,
fix = False,
paths = None,
):
if paths is None:
paths = glob.glob(os.path.join(path, '*'))
for x in paths:
run = os.path.basename(x)
dump_filename = os.path.join(x, run + '#presn')
broken_filename = os.path.join(x, 'broken')
if (os.path.exists(broken_filename) and
os.path.exists(dump_filename)):
d = kepdump._load(dump_filename)
if d.is_presn:
print('presn seems OK, but marked proken')
if fix:
os.remove(broken_filename)
continue
if fix:
print('removing {}'.format(dump_filename))
os.remove(dump_filename)
else:
print('found broken {}'.format(dump_filename))
def clean_broken_presn(
path=default_path,
fix=False,
paths=None,
):
if paths is None:
paths = glob.glob(os.path.join(path, '*'))
for x in paths:
run = os.path.basename(x)
dump_filename = os.path.join(x, run + '#presn')
broken_filename = os.path.join(x, 'broken')
if (os.path.exists(broken_filename) and os.path.exists(dump_filename)):
d = kepdump._load(dump_filename)
if d.is_presn:
print('presn seems OK, but marked proken')
if fix:
os.remove(broken_filename)
continue
if fix:
print('removing {}'.format(dump_filename))
os.remove(dump_filename)
else:
print('found broken {}'.format(dump_filename))
def get_dump(self, dump = 'z'):
if not (dump in self.dumps):
filename = self.get_dump_filename(dump)
self.dumps[dump] = kepdump._load(filename)
return self.dumps[dump]
def z(self):
d = kepdump._load(os.path.join(self.cwd, self.run + 'z'))
return d
def check_aboarded(path = default_path,
filename = None,
fix = False,
paths = None,
):
if filename is None:
timestamp = 0
else:
timestamp = os.path.getmtime(filename)
dirs = dict()
if paths is None:
paths = glob.glob(os.path.join(path, '*'))
for x in paths:
run = os.path.basename(x)
# filter out runs just set up
if not (os.path.exists(os.path.join(x, run + '.cnv')) or
os.path.exists(os.path.join(x, run + '.wnd')) or
os.path.exists(os.path.join(x, run + '.log')) or
os.path.exists(os.path.join(x, run + 'z')) or
os.path.exists(os.path.join(x, run + 'z1'))):
continue
# filter out runs with online status
if (os.path.exists(os.path.join(x, 'presn')) or
os.path.exists(os.path.join(x, 'continue')) or
os.path.exists(os.path.join(x, 'broken')) or
os.path.exists(os.path.join(x, 'nofix'))):
continue
if os.path.getmtime(x) < timestamp:
continue
dirs[x] = dict()
#if fix is not True:
# return dirs
# find last OK dumps
logs = {
'cnv': convdata._load,
'wnd': winddata._load,
'log': logdata._load,
}
for x, v in dirs.items():
run = os.path.basename(x)
backup = os.path.join(os.path.dirname(x), 'backup', run)
if not os.path.exists(backup):
print('copying directory {} --> {}'.format(x, backup))
if fix:
shutil.copytree(x, backup)
print('checking {} ... '.format(run), end = '', flush = True)
dumps = [os.path.join(x, run + 'z'),
os.path.join(x, run + 'z1')]
for f in glob.glob(os.path.join(x, run + '#*')):
dumps += [f]
dumps_broken = []
dumps_ok = []
for f in dumps:
if not os.path.exists(f):
continue
try:
d = kepdump._load(f)
except:
dumps_broken += [f]
else:
dumps_ok += [(d.ncyc, f)]
print(len(dumps_broken), len(dumps_ok))
dumps_ok = sorted(dumps_ok, key = lambda x: x[0])
v['dumps_ok'] = dumps_ok
v['dumps_broken'] = dumps_broken
for d in dumps_broken:
print('Removing: ', d, os.path.getsize(d))
if fix:
os.remove(d)
print('last OK dump ', dumps_ok[-1])
# find last OK history file
for ext, loader in logs.items():
data = loader(os.path.join(x, run + '.' + ext),
silent = 40,
raise_exceptions = False)
u = data.ncyc
jj, = np.where(np.not_equal(u[1:], u[:-1]+1))
if len(jj) == 0:
ncyc = u[-1]
print(ext, ncyc)
else:
ncyc = u[jj[0]]
print(ext, 'CORRUPT', ncyc, u[-1], '{:5.2f}%'.format(100* ncyc/ u[-1]))
v[ext] = ncyc
max_seq = min([v[ext] for ext in logs.keys()])
restart_file = None
for ncyc, f in dumps_ok[::-1]:
if ncyc <= max_seq:
restart_file = f
break
else:
print('Removing: ', ncyc, f)
if fix:
os.remove(f)
print('*'*72)
print('*** Last complete model: {:d} {}'.format(ncyc, restart_file))
print('*'*72)
z_file = os.path.join(x, run + 'z')
if restart_file != z_file:
try:
print('Removing {}'.format(z_file))
if fix:
os.remove(z_file)
except FileNotFoundError:
pass
print('Copying {} --> {}'.format(restart_file, z_file))
if fix:
shutil.copy2(restart_file, z_file)
if fix:
touch(os.path.join(x, 'continue'), verbose = True)
def get_dump(self, dump='z'):
if not (dump in self.dumps):
filename = self.get_dump_filename(dump)
self.dumps[dump] = kepdump._load(filename)
return self.dumps[dump]
def z(self):
d = kepdump._load(os.path.join(self.cwd, self.run + 'z'))
return d
def check_aboarded(
path=default_path,
filename=None,
fix=False,
paths=None,
):
if filename is None:
timestamp = 0
else:
timestamp = os.path.getmtime(filename)
dirs = dict()
if paths is None:
paths = glob.glob(os.path.join(path, '*'))
for x in paths:
run = os.path.basename(x)
# filter out runs just set up
if not (os.path.exists(os.path.join(x, run + '.cnv'))
or os.path.exists(os.path.join(x, run + '.wnd'))
or os.path.exists(os.path.join(x, run + '.log'))
or os.path.exists(os.path.join(x, run + 'z'))
or os.path.exists(os.path.join(x, run + 'z1'))):
continue
# filter out runs with online status
if (os.path.exists(os.path.join(x, 'presn'))
or os.path.exists(os.path.join(x, 'continue'))
or os.path.exists(os.path.join(x, 'broken'))
or os.path.exists(os.path.join(x, 'nofix'))):
continue
if os.path.getmtime(x) < timestamp:
continue
dirs[x] = dict()
#if fix is not True:
# return dirs
# find last OK dumps
logs = {
'cnv': convdata._load,
'wnd': winddata._load,
'log': logdata._load,
}
for x, v in dirs.items():
run = os.path.basename(x)
backup = os.path.join(os.path.dirname(x), 'backup', run)
if not os.path.exists(backup):
print('copying directory {} --> {}'.format(x, backup))
if fix:
shutil.copytree(x, backup)
print('checking {} ... '.format(run), end='', flush=True)
dumps = [os.path.join(x, run + 'z'), os.path.join(x, run + 'z1')]
for f in glob.glob(os.path.join(x, run + '#*')):
dumps += [f]
dumps_broken = []
dumps_ok = []
for f in dumps:
if not os.path.exists(f):
continue
try:
d = kepdump._load(f)
except:
dumps_broken += [f]
else:
dumps_ok += [(d.ncyc, f)]
print(len(dumps_broken), len(dumps_ok))
dumps_ok = sorted(dumps_ok, key=lambda x: x[0])
v['dumps_ok'] = dumps_ok
v['dumps_broken'] = dumps_broken
for d in dumps_broken:
print('Removing: ', d, os.path.getsize(d))
if fix:
os.remove(d)
print('last OK dump ', dumps_ok[-1])
# find last OK history file
for ext, loader in logs.items():
data = loader(os.path.join(x, run + '.' + ext),
silent=40,
raise_exceptions=False)
u = data.ncyc
jj, = np.where(np.not_equal(u[1:], u[:-1] + 1))
if len(jj) == 0:
ncyc = u[-1]
print(ext, ncyc)
else:
ncyc = u[jj[0]]
print(ext, 'CORRUPT', ncyc, u[-1],
'{:5.2f}%'.format(100 * ncyc / u[-1]))
v[ext] = ncyc
max_seq = min([v[ext] for ext in logs.keys()])
restart_file = None
for ncyc, f in dumps_ok[::-1]:
if ncyc <= max_seq:
restart_file = f
break
else:
print('Removing: ', ncyc, f)
if fix:
os.remove(f)
print('*' * 72)
print('*** Last complete model: {:d} {}'.format(ncyc, restart_file))
print('*' * 72)
z_file = os.path.join(x, run + 'z')
if restart_file != z_file:
try:
print('Removing {}'.format(z_file))
if fix:
os.remove(z_file)
except FileNotFoundError:
pass
print('Copying {} --> {}'.format(restart_file, z_file))
if fix:
shutil.copy2(restart_file, z_file)
if fix:
touch(os.path.join(x, 'continue'), verbose=True)