def get_value_from_file(filename, quantity, change_quantity_to=None, sim=False, filepath=False, DEBUG=False, silent=False):
""" Use to read in a quantity from
- *.in
- *.local
- submit*, i.e. submit.sh, submit.csh, files, only works if computer is readily specified in pencilnew.io.get_systemid
Please add further functionallity by yourself!
Args:
filename: can be "run.in", "start.in", "cparam.local", path to that file is extraced from filepath or sim object
quantity: variable to read in from file
sim: put simulation object here, file will be found by filename automatically
filepath: normally not needed, specify here where to find the file with filename, can be a list of paths if unshure
DEBUG: make dry run, tell me what you would do but dont change anything!
silent: suppress certain output by setting True
Return:
Returns None if not successful
"""
import os, pencilnew
import numpy as np
from os.path import join, abspath, exists, split, isfile
from pencilnew.math import is_number, is_float, is_int
from pencilnew.io import timestamp, debug_breakpoint, mkdir
def string_to_tuple(s):
q = s.split(',')
if is_number(q[0]):
q = np.array([float(t) for t in q])
q_type = 'TUPLE_FLOAT'
return q, q_type
if q[0] == 'T' or q[0] == 'F':
q = np.array([bool(t=='T') for t in q])
q_type = 'TUPLE_BOOL'
return q, q_type
if type(q[0]) == type('string'):
q = [t.replace('"','').replace("'", '') for t in q]
q_type = 'TUPLE_STRING'
return q, q_type
print('! ERROR: Could not parse string '+s+' into a tuple!')
print('! DEBUG_BREAKPOINT AKTIVATED - check out the following variables: string s, tuple q, first entry in tuple q[0]')
debug_breakpoint(); return None, None
def tuple_to_string(t, q_type):
return ','.join([str(a) for a in t])
######## prepare filename and quantity
filename = filename.strip() # get rid of whitespaces
quantity = quantity.strip()
q_type = False # q_type will store the type of the quantity value once found and identified
split_filename = split(filename)
if sim == False and split_filename[0] != '' and filepath == False:
filepath = split_filename[0]
filename = split_filename[1]
######## find correct file
# prepare search_path list to search filename in
if filepath == False:
if sim == False:
sim = pencilnew.get_sim()
else:
filepath = sim.path
search_paths = [sim.path, join(sim.path, 'src')] # add other search paths here!!
elif type(filepath) == type('string'):
if filepath.endswith(filename): filepath = filepath[:-len(filename)] # clean filepath if filename occures to be in there at the end
search_paths = [abspath(filepath.strip())] # correct path format
elif type(filepath) == type(['list']):
search_paths = filepath
else:
print('! ERROR: Filename '+str(filename)+' could not be interprated or found!'); return None
absolute_filepath = None
for search_path in search_paths:
tmp_path = join(search_path, filename)
if os.path.isfile(tmp_path):
absolute_filepath = tmp_path
break
# Traps the case of not being able to find the file
if absolute_filepath is None:
if DEBUG:
print('~ DEBUG: File {0} not found in {1}!'.format(filename, search_paths))
return None
######## open file
# now having absolute filepath to file, lets check that file and find quantity inside!
if DEBUG: print('~ DEBUG: Found file {0} in {1}'.format(filename,filepath))
with open(absolute_filepath, 'r') as f:
data_raw = f.readlines()
######## find line in file which quantity in
line_matches = []
# scan through file for differently for different files
if filename.endswith('.in') or 'cparam.local' in filename :
FILE_IS = 'IN_LOCAL'
SYM_COMMENT = '!'
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.strip().startswith('&'): continue # filter out lines with &something, e.g. &density_run_pars
if quantity in line.split(SYM_COMMENT)[0]: line_matches.append(ii)
elif filename.startswith('submit') and filename.split('.')[-1] in ['csh', 'sh']:
FILE_IS = 'SUBMIT'
SYM_COMMENT = False
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.replace(' ', '').startswith('#@') and quantity in line: line_matches.append(ii)
else:
print('! ERROR: Filename unknown! No parsing possible! Please enhance this function to work with '+filename)
if len(line_matches) > 1: print('! ERROR: Found more than one line with keyword "'+quantity+'" inside!'); return None
if len(line_matches) == 0:
if silent == False: print('! ERROR: Found no line with keyword "'+quantity+'" inside '+join(filepath, filename)+'!')
return None
filename = os.path.basename(filename)
######## get line with quantity inside
line = data_raw[line_matches[0]]
######## do separation of quantity from rest of line, i.e. get rid of comments and other quantities defined in this line
comment = ''
if SYM_COMMENT:
tmp = line.partition(SYM_COMMENT) # strip away comment
line = tmp[0]
if tmp[-1] != '': comment = SYM_COMMENT+tmp[-1] # and store for later
line = line.replace(' ','').replace('\n', '') # do cleanup in this line
qs = line.partition(quantity+SYM_ASSIGN)
if SYM_ASSIGN in qs[-1]:
qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
#qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
qs = qs[:2]+qs[2].rpartition(',')+qs[3:]
qs = list(qs)
q = qs[2]
while q.endswith('\t'): q = q[:-1]; comment = '\t'+comment # take care of trailing tabulator
while q.endswith(','): q = q[:-1] # remove trailing ,
######## do a cleanup of quantity value q and convert into string, float, int or array, also remember data type of q
if q.startswith("'") and q.endswith("'"): # quantity q is string in form of 'STRING'
q = q[1:-1]
q_type = 'STRING'
elif q.startswith('"') and q.endswith('"'): # quantity q is string in form of "STRING"
q = q[1:-1]
q_type = 'STRING'
elif not is_number(q[0]): # quantity q is string in form of not beeing a number
q = q.strip().replace('"','').replace("'", '')
q_type = 'STRING'
try:
float(q)
q_type = 'FLOAT'
if is_int(q):
q = int(q)
q_type = 'INT'
except:
if type(q) == type('string') and ',' in q:
q, q_type = string_to_tuple(q) # q is a TULPE_something
if type(q) == type('string') and q in ['F', 'f']: # q is BOOL
q = False
q_type = 'BOOL'
if type(q)== type('string') and q in ['T', 't']:
q = True
q_type = 'BOOL'
if type(q) == type('string'):
if is_number(q[0]):
q_type = 'STRING'
if q_type == False: # catch if type of q was not recognized
print('! ERROR: Couldnt identify the data type of the quantity value: '+str(q))
DEBUG = True
debug_breakpoint()
elif DEBUG:
print('~ DEBUG: q_type = '+q_type)
if q_type == 'FLOAT':
q = float(q)
elif q_type == 'INT':
q = int(q)
######## if value of quantity has to be changed do:
if change_quantity_to != None:
####### prepare change_quantity_to for string injection
if q_type == 'STRING':
if not FILE_IS=='SUBMIT':
change_quantity_to = "'"+change_quantity_to+"'"
elif q_type == 'BOOL':
change_quantity_to = bool(change_quantity_to in ['T', 't', True])
if change_quantity_to == True:
change_quantity_to = 'T'
elif change_quantity_to == False:
change_quantity_to = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to should be bool...')
debug_breakpoint(); return None
elif q_type == 'FLOAT':
change_quantity_to = '%e' % change_quantity_to
elif q_type.startswith('TUPLE'):
if q_type.endswith('BOOL'):
if type(change_quantity_to) == type(['list', 'of', 'bool', 'or', 'strings']):
for ii, val in enumerate(change_quantity_to):
if val in ['T', 't', True]:
change_quantity_to[ii] = 'T'
elif val in ['F', 'f', False]:
change_quantity_to[ii] = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to['+str(ii)+'] should be bool or string representation, but it is '+str(change_quantity_to[ii]))
debug_breakpoint(); return None
change_quantity_to = ','.join([str(t) for t in change_quantity_to])
if DEBUG: print('~ DEBUG: Would change quantity '+quantity+' from '+str(q)+' to '+str(change_quantity_to))
qs[2] = str(change_quantity_to)
######## further formatting
new_line = ''.join(qs).replace(SYM_SEPARATOR, SYM_SEPARATOR+' ')+'\t'+comment # create new line and add comment stripped away before
if not (FILE_IS == 'SUBMIT' or filename == 'cparam.local'): new_line = ' '+new_line
new_line = new_line.rstrip() # clean empty spaces on the right, no one needs that...
if new_line[-1] != '\n': new_line = new_line+'\n'
if FILE_IS=='SUBMIT': new_line = new_line.replace('#@', '#@ ').replace('=', ' = ') # optimizing format of submit script
if DEBUG:
print('~ DEBUG: old line: '+str(data_raw[line_matches[0]])[:-1])
print('~ DEBUG: new line: '+str(new_line)[:-1])
if not DEBUG:
####### do backup of file before changing it
from shutil import copyfile
target = join(sim.path, 'pc/backups/'+timestamp())
mkdir(target); target = join(target, filename)
copyfile(absolute_filepath, target)
# replace line in raw data
data_raw[line_matches[0]] = new_line
# save on drive
f.close()
with open(absolute_filepath, 'w') as f:
for l in data_raw: f.write(l)
######## DONE!
return q
def rename_in_submit_script(new_name,
submit_script_path=False,
sim=False,
old_name=False):
import os
from os.path import exists, join, abspath, dirname
from pencilnew.io import timestamp, get_systemid, mkdir
if submit_script_path != False:
path = dirname(abspath(submit_script_path))
filename = basename(abspath(submit_script_path))
# locate submit script file if not given
if submit_script_path == False:
# get path of simulation folder where submit script should be around
if sim == False:
path = '.'
else:
path = sim.path
list_of_possible_submit_scripts = [
f for f in os.listdir(path)
if f.startswith('submit') and f.split('.')[-1] in ['sh', 'csh']
]
# only works if a single submit script could be identified, else prompt error and let the user do it manually
if len(list_of_possible_submit_scripts) == 0:
print('!! ERROR: Could not find a submit script in ' + str(path))
return False
elif len(list_of_possible_submit_scripts) > 1:
print(
'!! ERROR: Could not identify submit script, please specify manually:'
+ str(list_of_possible_submit_scripts))
return False
else:
filename = list_of_possible_submit_scripts[0]
# path to submit script should now be clear, but better check its a string
if type(path) != type('STRING'):
print(
'!! ERROR: Could not identify submit script path, please check manually: '
+ str(path))
path_filename = join(path, filename)
# open submit script as f and read content into s
with open(path_filename) as f:
s = f.read()
# if the old name is known, we can simply replace that string with the new name
if old_name != False and type(old_name) == type('STRING'):
with open(path_filename) as f:
s = f.read()
if old_name in s:
s = s.replace(old_name, new_name)
else:
print('?? ERROR: Could not find old_name ' + str(old_name) +
' in submit script ' + str(path_filename))
return False
# else we need to look for specific identifiers, that differ from queue system and cluster
else:
# get submit name line identifier
identify = get_systemid()[2]
if identify == False:
print(
'!! ERROR: Could not identify an submit script name identifier, please update pencilnew.io.get_systemid.py by adding your machine.'
)
if identify in s:
if s.count(identify) > 1:
print(
'ERROR: Job name identifier has multiple appearences in submit script!'
)
return False
s = s.split('\n')
for ii, line in enumerate(s):
if identify in line: break
s[ii] = identify + ' ' + new_name
s = '\n'.join(s)
else:
print(
'!! ERROR: Could not find name identifier in submit script, identifier is '
+ str(identify))
# s should now be updated, we can save it in now in the file submit_script_path, but let us backup the old one
# backup
from shutil import copyfile
target_dir = join(path, 'pc/backups/')
mkdir(target_dir)
copyfile(path_filename,
join(target_dir, filename + '.BAK' + str(timestamp())))
# save new submit script
with open(path_filename, 'w') as f:
f.write(s)
# done
return True
def get_value_from_file(filename, quantity, change_quantity_to=False, sim=False, filepath=False, DEBUG=False, silent=False):
""" Use to read in a quantity from
- *.in
- *.local
- submit*, i.e. submit.sh, submit.csh, files, only works if computer is readily specified in pencilnew.io.get_systemid
Please add further functionallity by yourself!
Args:
filename: can be "run.in", "start.in", "cparam.local", path to that file is extraced from filepath or sim object
quantity: variable to read in from file
sim: put simulation object here, file will be found by filename automatically
filepath: normally not needed, specify here where to find the file with filename, can be a list of paths if unshure
DEBUG: make dry run, tell me what you would do but dont change anything!
silent: suppress certain output by setting True
"""
import os, pencilnew
import numpy as np
from os.path import join, abspath, exists, split, isfile
from pencilnew.math import is_number, is_float, is_int
from pencilnew.io import timestamp, debug_breakpoint, mkdir
def string_to_tuple(s):
q = s.split(',')
if is_number(q[0]):
q = np.array([float(t) for t in q])
q_type = 'TUPLE_FLOAT'
return q, q_type
if q[0] == 'T' or q[0] == 'F':
q = np.array([bool(t=='T') for t in q])
q_type = 'TUPLE_BOOL'
return q, q_type
if type(q[0]) == type('string'):
q = [t.replace('"','').replace("'", '') for t in q]
q_type = 'TUPLE_STRING'
return q, q_type
print('! ERROR: Could not parse string '+s+' into a tuple!')
print('! DEBUG_BREAKPOINT AKTIVATED - check out the following variables: string s, tuple q, first entry in tuple q[0]')
debug_breakpoint(); return None, None
def tuple_to_string(t, q_type):
return ','.join([str(a) for a in t])
######## prepare filename and quantity
filename = filename.strip() # get rid of whitespaces
quantity = quantity.strip()
q_type = False # q_type will store the type of the quantity value once found and identified
split_filename = split(filename)
if sim == False and split_filename[0] != '' and filepath == False:
filepath = split_filename[0]
filename = split_filename[1]
######## find correct file
# prepare search_path list to search filename in
if filepath == False:
if sim == False:
sim = pencilnew.get_sim()
else:
filepath = sim.path
search_paths = [sim.path, join(sim.path, 'src')] # add other search paths here!!
elif type(filepath) == type('string'):
if filepath.endswith(filename): filepath = filepath[:-len(filename)] # clean filepath if filename occures to be in there at the end
search_paths = [abspath(filepath.strip())] # correct path format
elif type(filepath) == type(['list']):
search_paths = filepath
else:
print('! ERROR: Filename '+str(filename)+' could not be interprated or found!'); return False
absolute_filepath = None
for search_path in search_paths:
tmp_path = join(search_path, filename)
if os.path.isfile(tmp_path):
absolute_filepath = tmp_path
break
# Traps the case of not being able to find the file
if absolute_filepath is None:
if DEBUG:
print('~ DEBUG: File {0} not found in {1}!'.format(filename, search_paths))
return None
######## open file
# now having absolute filepath to file, lets check that file and find quantity inside!
if DEBUG: print('~ DEBUG: Found suiting file {0} in {1}'.format(filename,filepath))
with open(absolute_filepath, 'r') as f:
data_raw = f.readlines()
######## find line in file which quantity in
line_matches = []
# scan through file for differently for different files
if filename.endswith('.in') or 'cparam.local' in filename :
FILE_IS = 'IN_LOCAL'
SYM_COMMENT = '!'
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.strip().startswith('&'): continue # filter out lines with &something, e.g. &density_run_pars
if quantity in line.split(SYM_COMMENT)[0]: line_matches.append(ii)
elif filename.startswith('submit') and filename.split('.')[-1] in ['csh', 'sh']:
FILE_IS = 'SUBMIT'
SYM_COMMENT = False
SYM_ASSIGN = '='
SYM_SEPARATOR = ','
for ii, line in enumerate(data_raw):
if line.replace(' ', '').startswith('#@') and quantity in line: line_matches.append(ii)
else:
print('! ERROR: Filename unknown! No parsing possible! Please enhance this function to work with '+filename)
if len(line_matches) > 1: print('! ERROR: Found more than one line with keyword "'+quantity+'" inside!'); return None
if len(line_matches) == 0:
if silent == False: print('! ERROR: Found no line with keyword "'+quantity+'" inside '+join(filepath, filename)+'!')
return None
######## get line with quantity inside
line = data_raw[line_matches[0]]
######## do separation of quantity from rest of line, i.e. get rid of comments and other quantities defined in this line
comment = ''
if SYM_COMMENT:
tmp = line.partition(SYM_COMMENT) # strip away comment
line = tmp[0]
if tmp[-1] != '': comment = SYM_COMMENT+tmp[-1] # and store for later
line = line.replace(' ','').replace('\n', '') # do cleanup in this line
qs = line.partition(quantity+SYM_ASSIGN)
if SYM_ASSIGN in qs[-1]:
qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
#qs = qs[:2]+qs[-1].partition(SYM_ASSIGN)
qs = qs[:2]+qs[2].rpartition(',')+qs[3:]
qs = list(qs)
q = qs[2]
while q.endswith('\t'): q = q[:-1]; comment = '\t'+comment # take care of trailing tabulator
while q.endswith(','): q = q[:-1] # remove trailing ,
######## do a cleanup of quantity value q and convert into string, float, int or array, also remember data type of q
if q.startswith("'") and q.endswith("'"): # quantity q is string in form of 'STRING'
q = q[1:-1]
q_type = 'STRING'
elif q.startswith('"') and q.endswith('"'): # quantity q is string in form of "STRING"
q = q[1:-1]
q_type = 'STRING'
elif not is_number(q[0]): # quantity q is string in form of not beeing a number
q = q.strip().replace('"','').replace("'", '')
q_type = 'STRING'
elif is_float(q):
q = float(q)
q_type = 'FLOAT'
if is_int(q):
q = int(q)
q_type = 'INT'
if type(q) == type('string') and ',' in q:
q, q_type = string_to_tuple(q) # q is a TULPE_something
if type(q) == type('string') and q in ['F', 'f']: # q is BOOL
q = False
q_type = 'BOOL'
if type(q)== type('string') and q in ['T', 't']:
q = True
q_type = 'BOOL'
if type(q) == type('string'):
if is_number(q[0]):
q_type = 'STRING'
if q_type == False: # catch if type of q was not recognized
print('! ERROR: Couldnt identify the data type of the quantity value: '+str(q))
DEBUG = True
debug_breakpoint()
elif DEBUG:
print('~ DEBUG: q_type = '+q_type)
######## if value of quantity has to be changed do:
if change_quantity_to != False:
####### prepare change_quantity_to for string injection
if q_type == 'STRING':
if not FILE_IS=='SUBMIT':
change_quantity_to = "'"+change_quantity_to+"'"
elif q_type == 'BOOL':
change_quantity_to = bool(change_quantity_to in ['T', 't', True])
if change_quantity_to == True:
change_quantity_to = 'T'
elif change_quantity_to == False:
change_quantity_to = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to should be bool...')
debug_breakpoint(); return None
elif q_type == 'FLOAT':
change_quantity_to = '%e' % change_quantity_to
elif q_type.startswith('TUPLE'):
if q_type.endswith('BOOL'):
if type(change_quantity_to) == type(['list', 'of', 'bool', 'or', 'strings']):
for ii, val in enumerate(change_quantity_to):
if val in ['T', 't', True]:
change_quantity_to[ii] = 'T'
elif val in ['F', 'f', False]:
change_quantity_to[ii] = 'F'
else:
print('! ERROR: There is something deeply wrong here! change_quantity_to['+str(ii)+'] should be bool or string representation, but it is '+str(change_quantity_to[ii]))
debug_breakpoint(); return None
change_quantity_to = ','.join([str(t) for t in change_quantity_to])
if DEBUG: print('~ DEBUG: Would change quantity '+quantity+' from '+str(q)+' to '+str(change_quantity_to))
qs[2] = str(change_quantity_to)
######## further formatting
new_line = ''.join(qs).replace(SYM_SEPARATOR, SYM_SEPARATOR+' ')+'\t\t'+comment # create new line and add comment stripped away before
if not FILE_IS == 'SUBMIT': new_line = ' '+new_line
if new_line[-1] != '\n': new_line = new_line+'\n'
if FILE_IS=='SUBMIT': new_line = new_line.replace('#@', '#@ ').replace('=', ' = ') # optimizing format of submit script
if DEBUG:
print('~ DEBUG: old line: '+str(data_raw[line_matches[0]])[:-1])
print('~ DEBUG: new line: '+str(new_line)[:-1])
if not DEBUG:
####### do backup of file before changing it
from shutil import copyfile
target = join(sim.path, '.pc/backups/'+timestamp())
mkdir(target); target = join(target, filename)
copyfile(filepath, target)
# replace line in raw data
data_raw[line_matches[0]] = new_line
# save on drive
f.close()
with open(filename, 'w') as f:
for l in data_raw: f.write(l)
######## DONE!
return q
def rename_in_submit_script(new_name, submit_script_path=False, sim=False, old_name=False):
import os
from os.path import exists, join, abspath, dirname
from pencilnew.io import timestamp, get_systemid, mkdir
if submit_script_path != False:
path = dirname(abspath(submit_script_path))
filename = basename(abspath(submit_script_path))
# locate submit script file if not given
if submit_script_path == False:
# get path of simulation folder where submit script should be around
if sim == False:
path = '.'
else:
path = sim.path
list_of_possible_submit_scripts = [f for f in os.listdir(path) if f.startswith('submit') and f.split('.')[-1] in ['sh', 'csh']]
# only works if a single submit script could be identified, else prompt error and let the user do it manually
if len(list_of_possible_submit_scripts) == 0:
print('!! ERROR: Could not find a submit script in '+str(path)); return False
elif len(list_of_possible_submit_scripts) > 1:
print('!! ERROR: Could not identify submit script, please specify manually:'+str(list_of_possible_submit_scripts)); return False
else:
filename = list_of_possible_submit_scripts[0]
# path to submit script should now be clear, but better check its a string
if type(path) != type('STRING'):
print('!! ERROR: Could not identify submit script path, please check manually: '+str(path))
path_filename = join(path, filename)
# open submit script as f and read content into s
with open(path_filename) as f: s = f.read()
# if the old name is known, we can simply replace that string with the new name
if old_name != False and type(old_name) == type('STRING'):
with open(path_filename) as f:
s = f.read()
if old_name in s:
s = s.replace(old_name, new_name)
else:
print('?? ERROR: Could not find old_name '+str(old_name)+' in submit script '+str(path_filename))
return False
# else we need to look for specific identifiers, that differ from queue system and cluster
else:
# get submit name line identifier
identify = get_systemid()[2]
if identify == False: print('!! ERROR: Could not identify an submit script name identifier, please update pencilnew.io.get_systemid.py by adding your machine.')
if identify in s:
if s.count(identify) > 1: print('ERROR: Job name identifier has multiple appearences in submit script!'); return False
s = s.split('\n')
for ii, line in enumerate(s):
if identify in line: break
s[ii] = identify+' '+new_name
s = '\n'.join(s)
else:
print('!! ERROR: Could not find name identifier in submit script, identifier is '+str(identify))
# s should now be updated, we can save it in now in the file submit_script_path, but let us backup the old one
# backup
from shutil import copyfile
target_dir = join(path, 'pc/backups/'); mkdir(target_dir);
copyfile(path_filename, join(target_dir, filename+'.BAK'+str(timestamp())))
# save new submit script
with open(path_filename, 'w') as f:
f.write(s)
# done
return True