def check_presn(path, base): # Hmm, we do not want to check all the time, so we want some # record that the run is OK. We may need some run info data base # in the run directory, or somewhere in general. dump_filename = os.path.join(path, base + '#presn') if not os.path.exists(dump_filename): return False invalid_tokens = ('broken', ) for token in invalid_tokens: broken_filename = os.path.join(path, token) if os.path.exists(broken_filename): return False presnok_filename = os.path.join(path, 'presn') if not os.path.exists(presnok_filename): # sometimes file is not quite written to disk, then wait. for i in range(10): try: d = kepdump._load(dump_filename) except RecLenError: time.sleep(i + 1) continue else: break ok = d.is_presn if not ok: touch(broken_filename) print('broken: ', dump_filename) else: touch(presnok_filename) print('ok: ', dump_filename) return ok return True
def lastdump(self): for i in range(10): try: d = kepdump._load(self.lastfile()) return d except: time.sleep(1)
def presn(self): f = os.path.join(self.cwd, self.run + '#presn') if os.path.isfile(f): d = kepdump._load(f) return d else: return None
def check_presn(path, base): # Hmm, we do not want to check all the time, so we want some # record that the run is OK. We may need some run info data base # in the run directory, or somewhere in general. dump_filename = os.path.join(path, base + '#presn') if not os.path.exists(dump_filename): return False invalid_tokens = ('broken', ) for token in invalid_tokens: broken_filename = os.path.join(path, token) if os.path.exists(broken_filename): return False presnok_filename = os.path.join(path, 'presn') if not os.path.exists(presnok_filename): # sometimes file is not quite written to disk, then wait. for i in range(10): try: d = kepdump._load(dump_filename) except RecLenError: time.sleep(i + 1) continue else: break ok = d.is_presn if not ok: touch(broken_filename) print('broken: ', dump_filename) else: touch(presnok_filename) print('ok: ', dump_filename) return ok return True
def lastdump(self): for i in range(10): try: d = kepdump._load(self.lastfile()) return d except: time.sleep(1)
def presn(self): f = os.path.join(self.cwd, self.run + '#presn') if os.path.isfile(f): d = kepdump._load(f) return d else: return None
def check_all_presn( path=default_path, fix=False, ): paths = glob.glob(os.path.join(path, '*')) for x in paths: print('.', end='', flush=True) run = os.path.basename(x) dump_filename = os.path.join(x, run + '#presn') presn_filename = os.path.join(x, 'presn') broken_filename = os.path.join(x, 'broken') if os.path.exists(dump_filename): b = os.path.exists(broken_filename) p = os.path.exists(presn_filename) d = kepdump._load(dump_filename) if d.is_presn: if b: print('presn seems OK, but marked broken: {}'.format(run)) if fix: os.remove(broken_filename) if not p: print( 'presn seems OK, but not marked presn: {}'.format(run)) if fix: touch(presn_filename) else: if p: print('presn not OK, but marked presn: {}'.format(run)) if fix: os.remove(presn_filename) os.remove(dump_filename) touch(broken_filename) else: b = os.path.exists(broken_filename) p = os.path.exists(presn_filename) if p: print('presn not present, but marked presn: {}'.format(run)) if fix: os.remove(presn_filename) touch(broken_filename)
def check_all_presn(path = default_path, fix = False, ): paths = glob.glob(os.path.join(path, '*')) for x in paths: print('.', end = '', flush = True) run = os.path.basename(x) dump_filename = os.path.join(x, run + '#presn') presn_filename = os.path.join(x, 'presn') broken_filename = os.path.join(x, 'broken') if os.path.exists(dump_filename): b = os.path.exists(broken_filename) p = os.path.exists(presn_filename) d = kepdump._load(dump_filename) if d.is_presn: if b: print('presn seems OK, but marked broken: {}'.format(run)) if fix: os.remove(broken_filename) if not p: print('presn seems OK, but not marked presn: {}'.format(run)) if fix: touch(presn_filename) else: if p: print('presn not OK, but marked presn: {}'.format(run)) if fix: os.remove(presn_filename) os.remove(dump_filename) touch(broken_filename) else: b = os.path.exists(broken_filename) p = os.path.exists(presn_filename) if p: print('presn not present, but marked presn: {}'.format(run)) if fix: os.remove(presn_filename) touch(broken_filename)
def clean_broken_presn(path = default_path, fix = False, paths = None, ): if paths is None: paths = glob.glob(os.path.join(path, '*')) for x in paths: run = os.path.basename(x) dump_filename = os.path.join(x, run + '#presn') broken_filename = os.path.join(x, 'broken') if (os.path.exists(broken_filename) and os.path.exists(dump_filename)): d = kepdump._load(dump_filename) if d.is_presn: print('presn seems OK, but marked proken') if fix: os.remove(broken_filename) continue if fix: print('removing {}'.format(dump_filename)) os.remove(dump_filename) else: print('found broken {}'.format(dump_filename))
def clean_broken_presn( path=default_path, fix=False, paths=None, ): if paths is None: paths = glob.glob(os.path.join(path, '*')) for x in paths: run = os.path.basename(x) dump_filename = os.path.join(x, run + '#presn') broken_filename = os.path.join(x, 'broken') if (os.path.exists(broken_filename) and os.path.exists(dump_filename)): d = kepdump._load(dump_filename) if d.is_presn: print('presn seems OK, but marked proken') if fix: os.remove(broken_filename) continue if fix: print('removing {}'.format(dump_filename)) os.remove(dump_filename) else: print('found broken {}'.format(dump_filename))
def get_dump(self, dump = 'z'): if not (dump in self.dumps): filename = self.get_dump_filename(dump) self.dumps[dump] = kepdump._load(filename) return self.dumps[dump]
def z(self): d = kepdump._load(os.path.join(self.cwd, self.run + 'z')) return d
def check_aboarded(path = default_path, filename = None, fix = False, paths = None, ): if filename is None: timestamp = 0 else: timestamp = os.path.getmtime(filename) dirs = dict() if paths is None: paths = glob.glob(os.path.join(path, '*')) for x in paths: run = os.path.basename(x) # filter out runs just set up if not (os.path.exists(os.path.join(x, run + '.cnv')) or os.path.exists(os.path.join(x, run + '.wnd')) or os.path.exists(os.path.join(x, run + '.log')) or os.path.exists(os.path.join(x, run + 'z')) or os.path.exists(os.path.join(x, run + 'z1'))): continue # filter out runs with online status if (os.path.exists(os.path.join(x, 'presn')) or os.path.exists(os.path.join(x, 'continue')) or os.path.exists(os.path.join(x, 'broken')) or os.path.exists(os.path.join(x, 'nofix'))): continue if os.path.getmtime(x) < timestamp: continue dirs[x] = dict() #if fix is not True: # return dirs # find last OK dumps logs = { 'cnv': convdata._load, 'wnd': winddata._load, 'log': logdata._load, } for x, v in dirs.items(): run = os.path.basename(x) backup = os.path.join(os.path.dirname(x), 'backup', run) if not os.path.exists(backup): print('copying directory {} --> {}'.format(x, backup)) if fix: shutil.copytree(x, backup) print('checking {} ... '.format(run), end = '', flush = True) dumps = [os.path.join(x, run + 'z'), os.path.join(x, run + 'z1')] for f in glob.glob(os.path.join(x, run + '#*')): dumps += [f] dumps_broken = [] dumps_ok = [] for f in dumps: if not os.path.exists(f): continue try: d = kepdump._load(f) except: dumps_broken += [f] else: dumps_ok += [(d.ncyc, f)] print(len(dumps_broken), len(dumps_ok)) dumps_ok = sorted(dumps_ok, key = lambda x: x[0]) v['dumps_ok'] = dumps_ok v['dumps_broken'] = dumps_broken for d in dumps_broken: print('Removing: ', d, os.path.getsize(d)) if fix: os.remove(d) print('last OK dump ', dumps_ok[-1]) # find last OK history file for ext, loader in logs.items(): data = loader(os.path.join(x, run + '.' + ext), silent = 40, raise_exceptions = False) u = data.ncyc jj, = np.where(np.not_equal(u[1:], u[:-1]+1)) if len(jj) == 0: ncyc = u[-1] print(ext, ncyc) else: ncyc = u[jj[0]] print(ext, 'CORRUPT', ncyc, u[-1], '{:5.2f}%'.format(100* ncyc/ u[-1])) v[ext] = ncyc max_seq = min([v[ext] for ext in logs.keys()]) restart_file = None for ncyc, f in dumps_ok[::-1]: if ncyc <= max_seq: restart_file = f break else: print('Removing: ', ncyc, f) if fix: os.remove(f) print('*'*72) print('*** Last complete model: {:d} {}'.format(ncyc, restart_file)) print('*'*72) z_file = os.path.join(x, run + 'z') if restart_file != z_file: try: print('Removing {}'.format(z_file)) if fix: os.remove(z_file) except FileNotFoundError: pass print('Copying {} --> {}'.format(restart_file, z_file)) if fix: shutil.copy2(restart_file, z_file) if fix: touch(os.path.join(x, 'continue'), verbose = True)
def get_dump(self, dump='z'): if not (dump in self.dumps): filename = self.get_dump_filename(dump) self.dumps[dump] = kepdump._load(filename) return self.dumps[dump]
def z(self): d = kepdump._load(os.path.join(self.cwd, self.run + 'z')) return d
def check_aboarded( path=default_path, filename=None, fix=False, paths=None, ): if filename is None: timestamp = 0 else: timestamp = os.path.getmtime(filename) dirs = dict() if paths is None: paths = glob.glob(os.path.join(path, '*')) for x in paths: run = os.path.basename(x) # filter out runs just set up if not (os.path.exists(os.path.join(x, run + '.cnv')) or os.path.exists(os.path.join(x, run + '.wnd')) or os.path.exists(os.path.join(x, run + '.log')) or os.path.exists(os.path.join(x, run + 'z')) or os.path.exists(os.path.join(x, run + 'z1'))): continue # filter out runs with online status if (os.path.exists(os.path.join(x, 'presn')) or os.path.exists(os.path.join(x, 'continue')) or os.path.exists(os.path.join(x, 'broken')) or os.path.exists(os.path.join(x, 'nofix'))): continue if os.path.getmtime(x) < timestamp: continue dirs[x] = dict() #if fix is not True: # return dirs # find last OK dumps logs = { 'cnv': convdata._load, 'wnd': winddata._load, 'log': logdata._load, } for x, v in dirs.items(): run = os.path.basename(x) backup = os.path.join(os.path.dirname(x), 'backup', run) if not os.path.exists(backup): print('copying directory {} --> {}'.format(x, backup)) if fix: shutil.copytree(x, backup) print('checking {} ... '.format(run), end='', flush=True) dumps = [os.path.join(x, run + 'z'), os.path.join(x, run + 'z1')] for f in glob.glob(os.path.join(x, run + '#*')): dumps += [f] dumps_broken = [] dumps_ok = [] for f in dumps: if not os.path.exists(f): continue try: d = kepdump._load(f) except: dumps_broken += [f] else: dumps_ok += [(d.ncyc, f)] print(len(dumps_broken), len(dumps_ok)) dumps_ok = sorted(dumps_ok, key=lambda x: x[0]) v['dumps_ok'] = dumps_ok v['dumps_broken'] = dumps_broken for d in dumps_broken: print('Removing: ', d, os.path.getsize(d)) if fix: os.remove(d) print('last OK dump ', dumps_ok[-1]) # find last OK history file for ext, loader in logs.items(): data = loader(os.path.join(x, run + '.' + ext), silent=40, raise_exceptions=False) u = data.ncyc jj, = np.where(np.not_equal(u[1:], u[:-1] + 1)) if len(jj) == 0: ncyc = u[-1] print(ext, ncyc) else: ncyc = u[jj[0]] print(ext, 'CORRUPT', ncyc, u[-1], '{:5.2f}%'.format(100 * ncyc / u[-1])) v[ext] = ncyc max_seq = min([v[ext] for ext in logs.keys()]) restart_file = None for ncyc, f in dumps_ok[::-1]: if ncyc <= max_seq: restart_file = f break else: print('Removing: ', ncyc, f) if fix: os.remove(f) print('*' * 72) print('*** Last complete model: {:d} {}'.format(ncyc, restart_file)) print('*' * 72) z_file = os.path.join(x, run + 'z') if restart_file != z_file: try: print('Removing {}'.format(z_file)) if fix: os.remove(z_file) except FileNotFoundError: pass print('Copying {} --> {}'.format(restart_file, z_file)) if fix: shutil.copy2(restart_file, z_file) if fix: touch(os.path.join(x, 'continue'), verbose=True)