def get_value_from_file(self, filename, quantity):
""" Use to read in a quantity from *.in or *.local files.
Args:
file: can be "run.in", "start.in", "cparam.local"
quantity: variable to read in from file
"""
from os.path import join
from pencilnew.math import is_number
from pencilnew.math import is_float
from pencilnew.math import is_int
if filename in ['run.in', 'start.in']:
filepath = join(self.dir, filename)
elif filename in ['cparam.local']:
filepath = join(self.dir, 'src', filename)
else:
print('!! Quantity ' + quantity + ' not found in ' + filepath)
return False
with open(filepath, 'r') as f:
data = f.readlines() # open file and read content
for id, line in enumerate(data): # check lines for quantity
if line.find(quantity) >= 0:
# special cases:
if quantity == 'Lxyz':
line = line.replace(' ', '')
line = line.split(quantity + '=')[-1]
return [float(i) for i in line.split(',')]
# default case:
for i in line.split(quantity + '=')[-1]:
if is_number(i) == False and not i in ['.', 'e', '-', '+']:
break
oldvalue = line.split(quantity + '=')[-1].split(i)[
0] # extract quantity value (not needed here)
if is_int(oldvalue): return int(float(oldvalue))
elif is_float(oldvalue): return float(oldvalue)
else:
print(
'?? WARNING: value from file is neither float nor int! value is: '
+ oldvalue)
return oldvalue
else:
print('!! quantity ' + quantity + ' not found in ' + filepath)
return False
def get_value_from_file(filename, quantity, change_quantity_to=None, sim=False, filepath=False, DEBUG=False, silent=False):
""" Use to read in a quantity from
- *.in
- *.local
- submit*, i.e. submit.sh, submit.csh, files, only works if computer is readily specified in pencilnew.io.get_systemid
Please add further functionallity by yourself!
Args:
filename: can be "run.in", "start.in", "cparam.local", path to that file is extraced from filepath or sim object
quantity: variable to read in from file
sim: put simulation object here, file will be found by filename automatically
filepath: normally not needed, specify here where to find the file with filename, can be a list of paths if unshure
DEBUG: make dry run, tell me what you would do but dont change anything!
silent: suppress certain output by setting True
Return:
Returns None if not successful
"""
import os, pencilnew
import numpy as np
from os.path import join, abspath, exists, split, isfile
from pencilnew.math import is_number, is_float, is_int
from pencilnew.io import timestamp, debug_breakpoint, mkdir
def string_to_tuple(s):
q = s.split(',')
if is_number(q[0]):
q = np.array([float(t) for t in q])
q_type = 'TUPLE_FLOAT'
return q, q_type
if q[0] == 'T' or q[0] == 'F':
q = np.array([bool(t=='T') for t in q])
q_type = 'TUPLE_BOOL'
return q, q_type
if type(q[0]) == type('string'):
q = [t.replace('"','').replace("'", '') for t in q]
q_type = 'TUPLE_STRING'
return q, q_type
print('! ERROR: Could not parse string '+s+' into a tuple!')
print('! DEBUG_BREAKPOINT AKTIVATED - check out the following variables: string s, tuple q, first entry in tuple q[0]')
debug_breakpoint(); return None, None
def tuple_to_string(t, q_type):
return ','.join([str(a) for a in t])
######## prepare filename and quantity
filename = filename.strip() # get rid of whitespaces
quantity = quantity.strip()
q_type = False # q_type will store the type of the quantity value once found and identified
split_filename = split(filename)
if sim == False and split_filename[0] != '' and filepath == False:
filepath = split_filename[0]
filename = split_filename[1]
######## find correct file
# prepare search_path list to search filename in
if filepath == False:
if sim == False:
sim = pencilnew.get_sim()
else:
filepath = sim.path
search_paths = [sim.path, join(sim.path, 'src')] # add other search paths here!!
elif type(filepath) == type('string'):
if filepath.endswith(filename): filepath = filepath[:-len(filename)] # clean filepath if filename occures to be in there at the end
search_paths = [abspath(filepath.strip())] # correct path format
elif type(filepath) == type(['list']):
search_paths = filepath
else:
print('! ERROR: Filename '+str(filename)+' could not be interprated or found!'); return None
absolute_filepath = None
for search_path in search_paths:
tmp_path = join(search_path, filename)
if os.path.isfile(tmp_path):
absolute_filepath = tmp_path
break
# Traps the case of not being able to find the file
if absolute_filepath is None:
if DEBUG:
print('~ DEBUG: File {0} not found in {1}!'.format(filename, search_paths))
return None
######## open file
# now having absolute filepath to file, lets check that file and find quantity inside!
if DEBUG: print('~ DEBUG: Found file {0} in {1}'.format(filename,filepath))
with open(absolute_filepath, 'r') as f:
data_raw = f.readlines()
######## find line in file which quantity in
line_matches = []
# scan through file for differently for different files
if filename.endswith('.in') or 'cparam.local' in filename :
FILE_IS = 'IN_LOCAL'
SYM_COMMENT = '!'
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.strip().startswith('&'): continue # filter out lines with &something, e.g. &density_run_pars
if quantity in line.split(SYM_COMMENT)[0]: line_matches.append(ii)
elif filename.startswith('submit') and filename.split('.')[-1] in ['csh', 'sh']:
FILE_IS = 'SUBMIT'
SYM_COMMENT = False
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.replace(' ', '').startswith('#@') and quantity in line: line_matches.append(ii)
else:
print('! ERROR: Filename unknown! No parsing possible! Please enhance this function to work with '+filename)
if len(line_matches) > 1: print('! ERROR: Found more than one line with keyword "'+quantity+'" inside!'); return None
if len(line_matches) == 0:
if silent == False: print('! ERROR: Found no line with keyword "'+quantity+'" inside '+join(filepath, filename)+'!')
return None
filename = os.path.basename(filename)
######## get line with quantity inside
line = data_raw[line_matches[0]]
######## do separation of quantity from rest of line, i.e. get rid of comments and other quantities defined in this line
comment = ''
if SYM_COMMENT:
tmp = line.partition(SYM_COMMENT) # strip away comment
line = tmp[0]
if tmp[-1] != '': comment = SYM_COMMENT+tmp[-1] # and store for later
line = line.replace(' ','').replace('\n', '') # do cleanup in this line
qs = line.partition(quantity+SYM_ASSIGN)
if SYM_ASSIGN in qs[-1]:
qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
#qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
qs = qs[:2]+qs[2].rpartition(',')+qs[3:]
qs = list(qs)
q = qs[2]
while q.endswith('\t'): q = q[:-1]; comment = '\t'+comment # take care of trailing tabulator
while q.endswith(','): q = q[:-1] # remove trailing ,
######## do a cleanup of quantity value q and convert into string, float, int or array, also remember data type of q
if q.startswith("'") and q.endswith("'"): # quantity q is string in form of 'STRING'
q = q[1:-1]
q_type = 'STRING'
elif q.startswith('"') and q.endswith('"'): # quantity q is string in form of "STRING"
q = q[1:-1]
q_type = 'STRING'
elif not is_number(q[0]): # quantity q is string in form of not beeing a number
q = q.strip().replace('"','').replace("'", '')
q_type = 'STRING'
try:
float(q)
q_type = 'FLOAT'
if is_int(q):
q = int(q)
q_type = 'INT'
except:
if type(q) == type('string') and ',' in q:
q, q_type = string_to_tuple(q) # q is a TULPE_something
if type(q) == type('string') and q in ['F', 'f']: # q is BOOL
q = False
q_type = 'BOOL'
if type(q)== type('string') and q in ['T', 't']:
q = True
q_type = 'BOOL'
if type(q) == type('string'):
if is_number(q[0]):
q_type = 'STRING'
if q_type == False: # catch if type of q was not recognized
print('! ERROR: Couldnt identify the data type of the quantity value: '+str(q))
DEBUG = True
debug_breakpoint()
elif DEBUG:
print('~ DEBUG: q_type = '+q_type)
if q_type == 'FLOAT':
q = float(q)
elif q_type == 'INT':
q = int(q)
######## if value of quantity has to be changed do:
if change_quantity_to != None:
####### prepare change_quantity_to for string injection
if q_type == 'STRING':
if not FILE_IS=='SUBMIT':
change_quantity_to = "'"+change_quantity_to+"'"
elif q_type == 'BOOL':
change_quantity_to = bool(change_quantity_to in ['T', 't', True])
if change_quantity_to == True:
change_quantity_to = 'T'
elif change_quantity_to == False:
change_quantity_to = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to should be bool...')
debug_breakpoint(); return None
elif q_type == 'FLOAT':
change_quantity_to = '%e' % change_quantity_to
elif q_type.startswith('TUPLE'):
if q_type.endswith('BOOL'):
if type(change_quantity_to) == type(['list', 'of', 'bool', 'or', 'strings']):
for ii, val in enumerate(change_quantity_to):
if val in ['T', 't', True]:
change_quantity_to[ii] = 'T'
elif val in ['F', 'f', False]:
change_quantity_to[ii] = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to['+str(ii)+'] should be bool or string representation, but it is '+str(change_quantity_to[ii]))
debug_breakpoint(); return None
change_quantity_to = ','.join([str(t) for t in change_quantity_to])
if DEBUG: print('~ DEBUG: Would change quantity '+quantity+' from '+str(q)+' to '+str(change_quantity_to))
qs[2] = str(change_quantity_to)
######## further formatting
new_line = ''.join(qs).replace(SYM_SEPARATOR, SYM_SEPARATOR+' ')+'\t'+comment # create new line and add comment stripped away before
if not (FILE_IS == 'SUBMIT' or filename == 'cparam.local'): new_line = ' '+new_line
new_line = new_line.rstrip() # clean empty spaces on the right, no one needs that...
if new_line[-1] != '\n': new_line = new_line+'\n'
if FILE_IS=='SUBMIT': new_line = new_line.replace('#@', '#@ ').replace('=', ' = ') # optimizing format of submit script
if DEBUG:
print('~ DEBUG: old line: '+str(data_raw[line_matches[0]])[:-1])
print('~ DEBUG: new line: '+str(new_line)[:-1])
if not DEBUG:
####### do backup of file before changing it
from shutil import copyfile
target = join(sim.path, 'pc/backups/'+timestamp())
mkdir(target); target = join(target, filename)
copyfile(absolute_filepath, target)
# replace line in raw data
data_raw[line_matches[0]] = new_line
# save on drive
f.close()
with open(absolute_filepath, 'w') as f:
for l in data_raw: f.write(l)
######## DONE!
return q
def get_value_from_file(filename, quantity, change_quantity_to=False, sim=False, filepath=False, DEBUG=False, silent=False):
""" Use to read in a quantity from
- *.in
- *.local
- submit*, i.e. submit.sh, submit.csh, files, only works if computer is readily specified in pencilnew.io.get_systemid
Please add further functionallity by yourself!
Args:
filename: can be "run.in", "start.in", "cparam.local", path to that file is extraced from filepath or sim object
quantity: variable to read in from file
sim: put simulation object here, file will be found by filename automatically
filepath: normally not needed, specify here where to find the file with filename, can be a list of paths if unshure
DEBUG: make dry run, tell me what you would do but dont change anything!
silent: suppress certain output by setting True
"""
import os, pencilnew
import numpy as np
from os.path import join, abspath, exists, split, isfile
from pencilnew.math import is_number, is_float, is_int
from pencilnew.io import timestamp, debug_breakpoint, mkdir
def string_to_tuple(s):
q = s.split(',')
if is_number(q[0]):
q = np.array([float(t) for t in q])
q_type = 'TUPLE_FLOAT'
return q, q_type
if q[0] == 'T' or q[0] == 'F':
q = np.array([bool(t=='T') for t in q])
q_type = 'TUPLE_BOOL'
return q, q_type
if type(q[0]) == type('string'):
q = [t.replace('"','').replace("'", '') for t in q]
q_type = 'TUPLE_STRING'
return q, q_type
print('! ERROR: Could not parse string '+s+' into a tuple!')
print('! DEBUG_BREAKPOINT AKTIVATED - check out the following variables: string s, tuple q, first entry in tuple q[0]')
debug_breakpoint(); return None, None
def tuple_to_string(t, q_type):
return ','.join([str(a) for a in t])
######## prepare filename and quantity
filename = filename.strip() # get rid of whitespaces
quantity = quantity.strip()
q_type = False # q_type will store the type of the quantity value once found and identified
split_filename = split(filename)
if sim == False and split_filename[0] != '' and filepath == False:
filepath = split_filename[0]
filename = split_filename[1]
######## find correct file
# prepare search_path list to search filename in
if filepath == False:
if sim == False:
sim = pencilnew.get_sim()
else:
filepath = sim.path
search_paths = [sim.path, join(sim.path, 'src')] # add other search paths here!!
elif type(filepath) == type('string'):
if filepath.endswith(filename): filepath = filepath[:-len(filename)] # clean filepath if filename occures to be in there at the end
search_paths = [abspath(filepath.strip())] # correct path format
elif type(filepath) == type(['list']):
search_paths = filepath
else:
print('! ERROR: Filename '+str(filename)+' could not be interprated or found!'); return False
absolute_filepath = None
for search_path in search_paths:
tmp_path = join(search_path, filename)
if os.path.isfile(tmp_path):
absolute_filepath = tmp_path
break
# Traps the case of not being able to find the file
if absolute_filepath is None:
if DEBUG:
print('~ DEBUG: File {0} not found in {1}!'.format(filename, search_paths))
return None
######## open file
# now having absolute filepath to file, lets check that file and find quantity inside!
if DEBUG: print('~ DEBUG: Found suiting file {0} in {1}'.format(filename,filepath))
with open(absolute_filepath, 'r') as f:
data_raw = f.readlines()
######## find line in file which quantity in
line_matches = []
# scan through file for differently for different files
if filename.endswith('.in') or 'cparam.local' in filename :
FILE_IS = 'IN_LOCAL'
SYM_COMMENT = '!'
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.strip().startswith('&'): continue # filter out lines with &something, e.g. &density_run_pars
if quantity in line.split(SYM_COMMENT)[0]: line_matches.append(ii)
elif filename.startswith('submit') and filename.split('.')[-1] in ['csh', 'sh']:
FILE_IS = 'SUBMIT'
SYM_COMMENT = False
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.replace(' ', '').startswith('#@') and quantity in line: line_matches.append(ii)
else:
print('! ERROR: Filename unknown! No parsing possible! Please enhance this function to work with '+filename)
if len(line_matches) > 1: print('! ERROR: Found more than one line with keyword "'+quantity+'" inside!'); return None
if len(line_matches) == 0:
if silent == False: print('! ERROR: Found no line with keyword "'+quantity+'" inside '+join(filepath, filename)+'!')
return None
######## get line with quantity inside
line = data_raw[line_matches[0]]
######## do separation of quantity from rest of line, i.e. get rid of comments and other quantities defined in this line
comment = ''
if SYM_COMMENT:
tmp = line.partition(SYM_COMMENT) # strip away comment
line = tmp[0]
if tmp[-1] != '': comment = SYM_COMMENT+tmp[-1] # and store for later
line = line.replace(' ','').replace('\n', '') # do cleanup in this line
qs = line.partition(quantity+SYM_ASSIGN)
if SYM_ASSIGN in qs[-1]:
qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
#qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
qs = qs[:2]+qs[2].rpartition(',')+qs[3:]
qs = list(qs)
q = qs[2]
while q.endswith('\t'): q = q[:-1]; comment = '\t'+comment # take care of trailing tabulator
while q.endswith(','): q = q[:-1] # remove trailing ,
######## do a cleanup of quantity value q and convert into string, float, int or array, also remember data type of q
if q.startswith("'") and q.endswith("'"): # quantity q is string in form of 'STRING'
q = q[1:-1]
q_type = 'STRING'
elif q.startswith('"') and q.endswith('"'): # quantity q is string in form of "STRING"
q = q[1:-1]
q_type = 'STRING'
elif not is_number(q[0]): # quantity q is string in form of not beeing a number
q = q.strip().replace('"','').replace("'", '')
q_type = 'STRING'
elif is_float(q):
q = float(q)
q_type = 'FLOAT'
if is_int(q):
q = int(q)
q_type = 'INT'
if type(q) == type('string') and ',' in q:
q, q_type = string_to_tuple(q) # q is a TULPE_something
if type(q) == type('string') and q in ['F', 'f']: # q is BOOL
q = False
q_type = 'BOOL'
if type(q)== type('string') and q in ['T', 't']:
q = True
q_type = 'BOOL'
if type(q) == type('string'):
if is_number(q[0]):
q_type = 'STRING'
if q_type == False: # catch if type of q was not recognized
print('! ERROR: Couldnt identify the data type of the quantity value: '+str(q))
DEBUG = True
debug_breakpoint()
elif DEBUG:
print('~ DEBUG: q_type = '+q_type)
######## if value of quantity has to be changed do:
if change_quantity_to != False:
####### prepare change_quantity_to for string injection
if q_type == 'STRING':
if not FILE_IS=='SUBMIT':
change_quantity_to = "'"+change_quantity_to+"'"
elif q_type == 'BOOL':
change_quantity_to = bool(change_quantity_to in ['T', 't', True])
if change_quantity_to == True:
change_quantity_to = 'T'
elif change_quantity_to == False:
change_quantity_to = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to should be bool...')
debug_breakpoint(); return None
elif q_type == 'FLOAT':
change_quantity_to = '%e' % change_quantity_to
elif q_type.startswith('TUPLE'):
if q_type.endswith('BOOL'):
if type(change_quantity_to) == type(['list', 'of', 'bool', 'or', 'strings']):
for ii, val in enumerate(change_quantity_to):
if val in ['T', 't', True]:
change_quantity_to[ii] = 'T'
elif val in ['F', 'f', False]:
change_quantity_to[ii] = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to['+str(ii)+'] should be bool or string representation, but it is '+str(change_quantity_to[ii]))
debug_breakpoint(); return None
change_quantity_to = ','.join([str(t) for t in change_quantity_to])
if DEBUG: print('~ DEBUG: Would change quantity '+quantity+' from '+str(q)+' to '+str(change_quantity_to))
qs[2] = str(change_quantity_to)
######## further formatting
new_line = ''.join(qs).replace(SYM_SEPARATOR, SYM_SEPARATOR+' ')+'\t\t'+comment # create new line and add comment stripped away before
if not FILE_IS == 'SUBMIT': new_line = ' '+new_line
if new_line[-1] != '\n': new_line = new_line+'\n'
if FILE_IS=='SUBMIT': new_line = new_line.replace('#@', '#@ ').replace('=', ' = ') # optimizing format of submit script
if DEBUG:
print('~ DEBUG: old line: '+str(data_raw[line_matches[0]])[:-1])
print('~ DEBUG: new line: '+str(new_line)[:-1])
if not DEBUG:
####### do backup of file before changing it
from shutil import copyfile
target = join(sim.path, '.pc/backups/'+timestamp())
mkdir(target); target = join(target, filename)
copyfile(filepath, target)
# replace line in raw data
data_raw[line_matches[0]] = new_line
# save on drive
f.close()
with open(filename, 'w') as f:
for l in data_raw: f.write(l)
######## DONE!
return q
def resume_from_var(self, sim_source, varno, DEBUG=False):
"""
Copies everything to resume a run from an older state.
It uses VAR-file number >varno< as new VAR0 and var.dat.
Does copy PVAR as well if available.
Args:
sim_source: simulation from where to copy all the files
varno: provide number # of which var-file VAR# should be copied from
"""
from os import listdir
from os.path import exists, join, isdir
import glob
from pencilnew.math import is_int
from pencilnew.io import mkdir
def copyfile(src, dst, DEBUG=False):
from shutil import copy2
from os.path import exists
if not exists(src): return False
if DEBUG: print('< '+src)
if DEBUG: print('> '+dst)
copy2(src, dst)
src = sim_source.datadir; dst = self.datadir
if is_int(varno): varno ='VAR'+str(int(varno))
if not exists(src): print('! ERROR: Source data directory does not exits: '+str(src)); return False
if not exists(dst): print('! ERROR: Destination data directory does not exits: '+str(dst)); return False
if not varno in sim_source.get_varlist(): print('! ERROR: Could not find '+varno+' in procX folder of sim_source: '+sim_source.name); return False
data_folders = [p for p in listdir(src) if isdir(join(src, p))]
procX_folder = [p for p in data_folders if p.startswith('proc')]
for p in data_folders: mkdir(join(dst,p))
# data/
files = ['def_var.pro', 'dim.dat', 'index.pro', 'move-me.list', 'particles_stalker_header.dat',
'params.log', 'pc_constants.pro', 'pdim.dat', 'pencils.list', 'pvarname.dat', 'svnid.dat', 'var.general', 'variables.pro', 'varname.dat']
for f in files: copyfile(join(src, f), dst, DEBUG=DEBUG)
# data/allprocs/
files = ['grid.dat']
for f in files: copyfile(join(src, 'allprocs', f), join(dst, 'allprocs/'), DEBUG=DEBUG)
# data/procX
files = ['dim.dat', 'grid.dat', 'proc_bounds.dat']
for X in procX_folder:
for f in files:
copyfile(join(src, X, f), join(dst, X), DEBUG=DEBUG)
copyfile(join(src, X, varno), join(dst, X, 'VAR0'), DEBUG=DEBUG)
copyfile(join(src, X, 'P'+varno), join(dst, X, 'PVAR0'), DEBUG=DEBUG)
copyfile(join(src, X, varno), join(dst, X, 'var.dat'), DEBUG=DEBUG)
copyfile(join(src, X, 'P'+varno), join(dst, X, 'pvar.dat'), DEBUG=DEBUG)
print('? WARNING: KNOWN ERRORS:')
print('? RUN MIGHT NOT START BECAUSE data/param.nml can get damaged in a run that crashes. This is not fixed by this routine.')
print('? TRY AND START A SINGLE CORE RUN WITH THIS SETUP AND USE THE CREATED param.nml FOR YOUR PURPOSE INSTEAD.')
print('? SAME FOR: - tstalk.dat')
return True