def parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
if self.token == '(':
v_idx_bounds = self.parse_indices()
v_idx = FIndex(v_idx_bounds, self.global_start_index)
# Update starting index against namelist record
if v_name in parent.start_index:
p_idx = parent.start_index[v_name]
for idx, pv in enumerate(zip(p_idx, v_idx.first)):
if all(i is None for i in pv):
i_first = None
else:
i_first = min(i for i in pv if i is not None)
v_idx.first[idx] = i_first
# Resize vector based on starting index
for i_p, i_v in zip(p_idx, v_idx.first):
if i_p is not None and i_v is not None and i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
else:
# If variable already existed without an index, then assume a
# 1-based index
# FIXME: Need to respect undefined `None` starting indexes?
if v_name in parent:
v_idx.first = [
self.default_start_index for _ in v_idx.first
]
parent.start_index[v_name] = v_idx.first
self.update_tokens()
else:
v_idx = None
# If indexed variable already exists, then re-index this new
# non-indexed variable using the global start index
# FIXME: The `hasattr` check is a hack to deal with the "list of
# parents" bug metioned in the `%` parsing block below. The real
# solution here is to prevent the list of parents being set as
# parent.
if hasattr(parent, 'start_index') and v_name in parent.start_index:
p_start = parent.start_index[v_name]
v_start = [self.default_start_index for _ in p_start]
# Resize vector based on new starting index
for i_p, i_v in zip(p_start, v_start):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.start_index[v_name] = v_start
if self.token == '%':
# Resolve the derived type
# FIXME: If we were in an index (v_idx != None) then v_parent is
# incorrectly set as the list of dicts rather than the respective
# dict (which must somehow be deduced from v_idx at some later
# stage)
# This is not causing any errors, but the parent value is nonsense
# and could break something in the future.
# Check for value in patch
v_patch_nml = None
if v_name in patch_nml:
v_patch_nml = patch_nml.pop(v_name.lower())
if parent and v_name in parent:
v_parent = parent[v_name]
else:
v_parent = Namelist()
parent[v_name] = v_parent
self.update_tokens()
self.update_tokens()
v_att, v_att_vals = self.parse_variable(v_parent,
patch_nml=v_patch_nml)
next_value = Namelist()
next_value[v_att] = v_att_vals
self.append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
prior_ws_sep = ws_sep = False
self.update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%')
or (self.prior_token, self.token) == ('=', '(')):
# Check for repeated values
if self.token == '*':
n_vals = self.parse_value()
assert isinstance(n_vals, int)
self.update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$')
and not (self.prior_token == ','
and self.token in ('/', '&', '$'))):
self.append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self.update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$')
and self.prior_token == '*')):
next_value = None
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self.parse_value()
# Check for escaped strings
if (v_values and isinstance(v_values[-1], str)
and isinstance(next_value, str)
and not prior_ws_sep):
quote_char = self.prior_token[0]
v_values[-1] = quote_char.join(
[v_values[-1], next_value])
else:
self.append_value(v_values, next_value, v_idx, n_vals)
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
prior_ws_sep = ws_sep
if patch_values:
if (p_idx < len(patch_values) and len(patch_values) > 0
and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml.f90repr(patch_values[p_idx])
p_idx += 1
ws_sep = self.update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
else:
# Skip any values beyond the patch size
skip = (p_idx >= len(patch_values))
self.update_tokens(patch_skip=skip)
else:
ws_sep = self.update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def test_findex_iteration(self):
rng = [(None, 5, None)]
fidx = iter(FIndex(rng))
for i, j in enumerate(fidx, start=1):
self.assertEqual(i, j[0])
def parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
if self.token == '(':
v_idx_bounds = self.parse_indices()
v_idx = FIndex(v_idx_bounds)
# Update starting index against namelist record
if v_name in parent.first_index:
p_idx = parent.first_index[v_name]
v_idx.first = [min(p_i, v_i)
for p_i, v_i in zip(p_idx, v_idx.first)]
# Resize vector based on starting index
for i_p, i_v in zip(p_idx, v_idx.first):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.first_index[v_name] = v_idx.first
self.update_tokens()
else:
v_idx = None
if self.token == '%':
# Resolve the derived type
if parent and v_name in parent:
v_parent = parent[v_name]
else:
v_parent = Namelist()
self.update_tokens()
self.update_tokens()
v_att, v_att_vals = self.parse_variable(v_parent)
next_value = Namelist()
next_value[v_att] = v_att_vals
self.append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
prior_ws_sep = ws_sep = False
self.update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%') or
(self.prior_token, self.token) == ('=', '(')):
# Check for repeated values
if self.token == '*':
n_vals = self.parse_value()
assert isinstance(n_vals, int)
self.update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$') and
not (self.prior_token == ',' and
self.token in ('/', '&', '$'))):
self.append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self.update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$') and
self.prior_token == '*')):
next_value = None
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self.parse_value()
# Check for escaped strings
if (v_values and isinstance(v_values[-1], str) and
isinstance(next_value, str) and not prior_ws_sep):
quote_char = self.prior_token[0]
v_values[-1] = quote_char.join([v_values[-1],
next_value])
else:
self.append_value(v_values, next_value, v_idx, n_vals)
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
prior_ws_sep = ws_sep
if (patch_values and p_idx < len(patch_values) and
len(patch_values) > 0 and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml.f90repr(patch_values[p_idx])
p_idx += 1
ws_sep = self.update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
else:
ws_sep = self.update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def _parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
# Derived type parent index (see notes below)
dt_idx = None
if self.token == '(':
v_idx_bounds = self._parse_indices()
v_idx = FIndex(v_idx_bounds, self.global_start_index)
# Update starting index against namelist record
if v_name.lower() in parent.start_index:
p_idx = parent.start_index[v_name.lower()]
for idx, pv in enumerate(zip(p_idx, v_idx.first)):
if all(i is None for i in pv):
i_first = None
else:
i_first = min(i for i in pv if i is not None)
v_idx.first[idx] = i_first
# Resize vector based on starting index
parent[v_name] = prepad_array(parent[v_name], p_idx,
v_idx.first)
else:
# If variable already existed without an index, then assume a
# 1-based index
# FIXME: Need to respect undefined `None` starting indexes?
if v_name in parent:
v_idx.first = [
self.default_start_index for _ in v_idx.first
]
parent.start_index[v_name.lower()] = v_idx.first
self._update_tokens()
# Derived type parent check
# NOTE: This assumes single-dimension derived type vectors
# (which I think is the only case supported in Fortran)
if self.token == '%':
assert v_idx_bounds[0][1] - v_idx_bounds[0][0] == 1
dt_idx = v_idx_bounds[0][0] - v_idx.first[0]
# NOTE: This is the sensible play to call `parse_variable`
# but not yet sure how to implement it, so we currently pass
# along `dt_idx` to the `%` handler.
else:
v_idx = None
# If indexed variable already exists, then re-index this new
# non-indexed variable using the global start index
if v_name in parent.start_index:
p_start = parent.start_index[v_name.lower()]
v_start = [self.default_start_index for _ in p_start]
# Resize vector based on new starting index
for i_p, i_v in zip(p_start, v_start):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.start_index[v_name.lower()] = v_start
if self.token == '%':
# Resolve the derived type
# Check for value in patch
v_patch_nml = None
if v_name in patch_nml:
v_patch_nml = patch_nml.pop(v_name.lower())
if parent:
vpar = parent.get(v_name.lower())
if vpar and isinstance(vpar, list):
# If new element is not a list, then assume it's the first
# element of the list.
if dt_idx is None:
dt_idx = self.default_start_index
try:
v_parent = vpar[dt_idx]
except IndexError:
v_parent = Namelist()
elif vpar:
v_parent = vpar
else:
v_parent = Namelist()
else:
v_parent = Namelist()
parent[v_name] = v_parent
self._update_tokens()
self._update_tokens()
v_att, v_att_vals = self._parse_variable(v_parent,
patch_nml=v_patch_nml)
next_value = Namelist()
next_value[v_att] = v_att_vals
self._append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
self._update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%')
or (self.prior_token, self.token) in (('=', '('),
(',', '('))):
# Check for repeated values
if self.token == '*':
n_vals = self._parse_value()
assert isinstance(n_vals, int)
self._update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$')
and not (self.prior_token == ','
and self.token in ('/', '&', '$'))):
self._append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self._update_tokens()
if (self.prior_token == ',' or self.token == '='
or (self.token in ('/', '&', '$')
and self.prior_token == '*')):
next_value = None
# XXX: Repeated ,, after N*, will be off by one...
if self.prior_token == ',' and self.token == ',':
n_vals += 1
else:
next_value = self._parse_value()
self._append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self._parse_value()
self._append_value(v_values, next_value, v_idx, n_vals)
# Reset default repeat factor for subsequent values
n_vals = 1
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
# NOTE: it is probably very inefficient to keep re-creating
# iterators upon every element; this solution reflects the
# absence of mature lookahead in the script.
#
# This is a temporary fix to address errors caused by
# patches of different length from the original value, and
# represents a direction to fully rewrite the parser using
# `tee`.
# NOTE: We may be able to assume that self.token is a value
# rather than prepending it to the iterator.
self.tokens, pre_lookahead = itertools.tee(self.tokens)
lookahead = itertools.chain([self.token], pre_lookahead)
n_vals_remain = count_values(lookahead)
if patch_values:
# TODO: Patch indices that are not set in the namelist
if (p_idx < len(patch_values) and n_vals_remain > 0
and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml._f90repr(patch_values[p_idx])
p_idx += 1
self._update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self._update_tokens(write_token=False)
self._update_tokens(write_token=False)
self._update_tokens(write_token=False)
self._update_tokens(write_token=False)
else:
# Skip any values beyond the patch size
skip = (p_idx >= len(patch_values))
self._update_tokens(patch_skip=skip)
else:
self._update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
# Derived type parent index (see notes below)
dt_idx = None
if self.token == '(':
v_idx_bounds = self.parse_indices()
v_idx = FIndex(v_idx_bounds, self.global_start_index)
# Update starting index against namelist record
if v_name.lower() in parent.start_index:
p_idx = parent.start_index[v_name.lower()]
for idx, pv in enumerate(zip(p_idx, v_idx.first)):
if all(i is None for i in pv):
i_first = None
else:
i_first = min(i for i in pv if i is not None)
v_idx.first[idx] = i_first
# Resize vector based on starting index
for i_p, i_v in zip(p_idx, v_idx.first):
if i_p is not None and i_v is not None and i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
else:
# If variable already existed without an index, then assume a
# 1-based index
# FIXME: Need to respect undefined `None` starting indexes?
if v_name in parent:
v_idx.first = [
self.default_start_index for _ in v_idx.first
]
parent.start_index[v_name.lower()] = v_idx.first
self.update_tokens()
# Derived type parent check
# NOTE: This assumes single-dimension derived type vectors
# (which I think is the only case supported in Fortran)
if self.token == '%':
assert (v_idx_bounds[0][1] - v_idx_bounds[0][0] == 1)
dt_idx = v_idx_bounds[0][0] - v_idx.first[0]
# NOTE: This is the sensible play to call `parse_variable`
# but not yet sure how to implement it, so we currently pass
# along `dt_idx` to the `%` handler.
else:
v_idx = None
# If indexed variable already exists, then re-index this new
# non-indexed variable using the global start index
if v_name in parent.start_index:
p_start = parent.start_index[v_name.lower()]
v_start = [self.default_start_index for _ in p_start]
# Resize vector based on new starting index
for i_p, i_v in zip(p_start, v_start):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.start_index[v_name.lower()] = v_start
if self.token == '%':
# Resolve the derived type
# Check for value in patch
v_patch_nml = None
if v_name in patch_nml:
v_patch_nml = patch_nml.pop(v_name.lower())
if parent:
vpar = parent.get(v_name.lower())
if vpar and isinstance(vpar, list):
assert dt_idx is not None
try:
v_parent = vpar[dt_idx]
except IndexError:
v_parent = Namelist()
elif vpar:
v_parent = vpar
else:
v_parent = Namelist()
else:
v_parent = Namelist()
parent[v_name] = v_parent
self.update_tokens()
self.update_tokens()
v_att, v_att_vals = self.parse_variable(v_parent,
patch_nml=v_patch_nml)
next_value = Namelist()
next_value[v_att] = v_att_vals
self.append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
prior_ws_sep = ws_sep = False
self.update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%')
or (self.prior_token, self.token) == ('=', '(')):
# Check for repeated values
if self.token == '*':
n_vals = self.parse_value()
assert isinstance(n_vals, int)
self.update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$')
and not (self.prior_token == ','
and self.token in ('/', '&', '$'))):
self.append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self.update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$')
and self.prior_token == '*')):
next_value = None
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self.parse_value()
# Check for escaped strings
if (v_values and isinstance(v_values[-1], str)
and isinstance(next_value, str)
and not prior_ws_sep):
quote_char = self.prior_token[0]
v_values[-1] = quote_char.join(
[v_values[-1], next_value])
else:
self.append_value(v_values, next_value, v_idx, n_vals)
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
prior_ws_sep = ws_sep
if patch_values:
if (p_idx < len(patch_values) and len(patch_values) > 0
and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml.f90repr(patch_values[p_idx])
p_idx += 1
ws_sep = self.update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
else:
# Skip any values beyond the patch size
skip = (p_idx >= len(patch_values))
self.update_tokens(patch_skip=skip)
else:
ws_sep = self.update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def _parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
# Derived type parent index (see notes below)
dt_idx = None
if self.token == '(':
v_idx_bounds = self._parse_indices()
v_idx = FIndex(v_idx_bounds, self.global_start_index)
# Update starting index against namelist record
if v_name.lower() in parent.start_index:
p_idx = parent.start_index[v_name.lower()]
for idx, pv in enumerate(zip(p_idx, v_idx.first)):
if all(i is None for i in pv):
i_first = None
else:
i_first = min(i for i in pv if i is not None)
v_idx.first[idx] = i_first
# Resize vector based on starting index
parent[v_name] = prepad_array(parent[v_name], p_idx,
v_idx.first)
else:
# If variable already existed without an index, then assume a
# 1-based index
# FIXME: Need to respect undefined `None` starting indexes?
if v_name in parent:
v_idx.first = [self.default_start_index
for _ in v_idx.first]
parent.start_index[v_name.lower()] = v_idx.first
self._update_tokens()
# Derived type parent check
# NOTE: This assumes single-dimension derived type vectors
# (which I think is the only case supported in Fortran)
if self.token == '%':
assert v_idx_bounds[0][1] - v_idx_bounds[0][0] == 1
dt_idx = v_idx_bounds[0][0] - v_idx.first[0]
# NOTE: This is the sensible play to call `parse_variable`
# but not yet sure how to implement it, so we currently pass
# along `dt_idx` to the `%` handler.
else:
v_idx = None
# If indexed variable already exists, then re-index this new
# non-indexed variable using the global start index
if v_name in parent.start_index:
p_start = parent.start_index[v_name.lower()]
v_start = [self.default_start_index for _ in p_start]
# Resize vector based on new starting index
for i_p, i_v in zip(p_start, v_start):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.start_index[v_name.lower()] = v_start
if self.token == '%':
# Resolve the derived type
# Check for value in patch
v_patch_nml = None
if v_name in patch_nml:
v_patch_nml = patch_nml.pop(v_name.lower())
if parent:
vpar = parent.get(v_name.lower())
if vpar and isinstance(vpar, list):
# If new element is not a list, then assume it's the first
# element of the list.
if dt_idx is None:
dt_idx = self.default_start_index
try:
v_parent = vpar[dt_idx]
except IndexError:
v_parent = Namelist()
elif vpar:
v_parent = vpar
else:
v_parent = Namelist()
else:
v_parent = Namelist()
parent[v_name] = v_parent
self._update_tokens()
self._update_tokens()
v_att, v_att_vals = self._parse_variable(
v_parent,
patch_nml=v_patch_nml
)
next_value = Namelist()
next_value[v_att] = v_att_vals
self._append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
self._update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%') or
(self.prior_token, self.token) in (('=', '('), (',', '('))):
# Check for repeated values
if self.token == '*':
n_vals = self._parse_value()
assert isinstance(n_vals, int)
self._update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$') and
not (self.prior_token == ',' and
self.token in ('/', '&', '$'))):
self._append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self._update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$') and
self.prior_token == '*')):
next_value = None
else:
next_value = self._parse_value()
self._append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self._parse_value()
self._append_value(v_values, next_value, v_idx, n_vals)
# Reset default repeat factor for subsequent values
n_vals = 1
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
# Get the remaining length of the unpatched vector?
# NOTE: it is probably very inefficient to keep re-creating
# iterators upon every element; this solution reflects the
# absence of mature lookahead in the script.
#
# This is a temporary fix to address errors caused by
# patches of different length from the original value, and
# represents a direction to fully rewrite the parser using
# `tee`.
self.tokens, lookahead = itertools.tee(self.tokens)
n_vals_remain = count_values(lookahead)
if patch_values:
# XXX: The (p_idx - 1) <= n_vals_remain test is dodgy
# and does not really make sense to me, but it appears
# to work.
# TODO: Patch indices that are not set in the namelist
if (p_idx < len(patch_values) and
(p_idx - 1) <= n_vals_remain and
len(patch_values) > 0 and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml._f90repr(patch_values[p_idx])
p_idx += 1
self._update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self._update_tokens(write_token=False)
self._update_tokens(write_token=False)
self._update_tokens(write_token=False)
self._update_tokens(write_token=False)
else:
# Skip any values beyond the patch size
skip = (p_idx >= len(patch_values))
self._update_tokens(patch_skip=skip)
else:
self._update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
if self.token == '(':
v_idx_bounds = self.parse_indices()
v_idx = FIndex(v_idx_bounds, self.global_start_index)
# Update starting index against namelist record
if v_name in parent.start_index:
p_idx = parent.start_index[v_name]
for idx, pv in enumerate(zip(p_idx, v_idx.first)):
if all(i is None for i in pv):
i_first = None
else:
i_first = min(i for i in pv if i is not None)
v_idx.first[idx] = i_first
# Resize vector based on starting index
for i_p, i_v in zip(p_idx, v_idx.first):
if i_p is not None and i_v is not None and i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
else:
# If variable already existed without an index, then assume a
# 1-based index
# FIXME: Need to respect undefined `None` starting indexes?
if v_name in parent:
v_idx.first = [self.default_start_index
for _ in v_idx.first]
parent.start_index[v_name] = v_idx.first
self.update_tokens()
else:
v_idx = None
# If indexed variable already exists, then re-index this new
# non-indexed variable using the global start index
# FIXME: The `hasattr` check is a hack to deal with the "list of
# parents" bug metioned in the `%` parsing block below. The real
# solution here is to prevent the list of parents being set as
# parent.
if hasattr(parent, 'start_index') and v_name in parent.start_index:
p_start = parent.start_index[v_name]
v_start = [self.default_start_index for _ in p_start]
# Resize vector based on new starting index
for i_p, i_v in zip(p_start, v_start):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.start_index[v_name] = v_start
if self.token == '%':
# Resolve the derived type
# FIXME: If we were in an index (v_idx != None) then v_parent is
# incorrectly set as the list of dicts rather than the respective
# dict (which must somehow be deduced from v_idx at some later
# stage)
# This is not causing any errors, but the parent value is nonsense
# and could break something in the future.
# Check for value in patch
v_patch_nml = None
if v_name in patch_nml:
v_patch_nml = patch_nml.pop(v_name.lower())
if parent and v_name in parent:
v_parent = parent[v_name]
else:
v_parent = Namelist()
parent[v_name] = v_parent
self.update_tokens()
self.update_tokens()
v_att, v_att_vals = self.parse_variable(v_parent,
patch_nml=v_patch_nml)
next_value = Namelist()
next_value[v_att] = v_att_vals
self.append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
prior_ws_sep = ws_sep = False
self.update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%') or
(self.prior_token, self.token) == ('=', '(')):
# Check for repeated values
if self.token == '*':
n_vals = self.parse_value()
assert isinstance(n_vals, int)
self.update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$') and
not (self.prior_token == ',' and
self.token in ('/', '&', '$'))):
self.append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self.update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$') and
self.prior_token == '*')):
next_value = None
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self.parse_value()
# Check for escaped strings
if (v_values and isinstance(v_values[-1], str) and
isinstance(next_value, str) and not prior_ws_sep):
quote_char = self.prior_token[0]
v_values[-1] = quote_char.join([v_values[-1],
next_value])
else:
self.append_value(v_values, next_value, v_idx, n_vals)
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
prior_ws_sep = ws_sep
if patch_values:
if (p_idx < len(patch_values) and
len(patch_values) > 0 and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml.f90repr(patch_values[p_idx])
p_idx += 1
ws_sep = self.update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
else:
# Skip any values beyond the patch size
skip = (p_idx >= len(patch_values))
self.update_tokens(patch_skip=skip)
else:
ws_sep = self.update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
# Derived type parent index (see notes below)
dt_idx = None
if self.token == '(':
v_idx_bounds = self.parse_indices()
v_idx = FIndex(v_idx_bounds, self.global_start_index)
# Update starting index against namelist record
if v_name.lower() in parent.start_index:
p_idx = parent.start_index[v_name.lower()]
for idx, pv in enumerate(zip(p_idx, v_idx.first)):
if all(i is None for i in pv):
i_first = None
else:
i_first = min(i for i in pv if i is not None)
v_idx.first[idx] = i_first
# Resize vector based on starting index
for i_p, i_v in zip(p_idx, v_idx.first):
if i_p is not None and i_v is not None and i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
else:
# If variable already existed without an index, then assume a
# 1-based index
# FIXME: Need to respect undefined `None` starting indexes?
if v_name in parent:
v_idx.first = [self.default_start_index
for _ in v_idx.first]
parent.start_index[v_name.lower()] = v_idx.first
self.update_tokens()
# Derived type parent check
# NOTE: This assumes single-dimension derived type vectors
# (which I think is the only case supported in Fortran)
if self.token == '%':
assert(v_idx_bounds[0][1] - v_idx_bounds[0][0] == 1)
dt_idx = v_idx_bounds[0][0] - v_idx.first[0]
# NOTE: This is the sensible play to call `parse_variable`
# but not yet sure how to implement it, so we currently pass
# along `dt_idx` to the `%` handler.
else:
v_idx = None
# If indexed variable already exists, then re-index this new
# non-indexed variable using the global start index
if v_name in parent.start_index:
p_start = parent.start_index[v_name.lower()]
v_start = [self.default_start_index for _ in p_start]
# Resize vector based on new starting index
for i_p, i_v in zip(p_start, v_start):
if i_v < i_p:
pad = [None for _ in range(i_p - i_v)]
parent[v_name] = pad + parent[v_name]
parent.start_index[v_name.lower()] = v_start
if self.token == '%':
# Resolve the derived type
# Check for value in patch
v_patch_nml = None
if v_name in patch_nml:
v_patch_nml = patch_nml.pop(v_name.lower())
if parent:
vpar = parent.get(v_name.lower())
if vpar and isinstance(vpar, list):
assert dt_idx is not None
try:
v_parent = vpar[dt_idx]
except IndexError:
v_parent = Namelist()
elif vpar:
v_parent = vpar
else:
v_parent = Namelist()
else:
v_parent = Namelist()
parent[v_name] = v_parent
self.update_tokens()
self.update_tokens()
v_att, v_att_vals = self.parse_variable(v_parent,
patch_nml=v_patch_nml)
next_value = Namelist()
next_value[v_att] = v_att_vals
self.append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
self.update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%') or
(self.prior_token, self.token) == ('=', '(')):
# Check for repeated values
if self.token == '*':
n_vals = self.parse_value()
assert isinstance(n_vals, int)
self.update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$') and
not (self.prior_token == ',' and
self.token in ('/', '&', '$'))):
self.append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self.update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$') and
self.prior_token == '*')):
next_value = None
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
# Reset default repeat factor for subsequent values
n_vals = 1
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
if patch_values:
if (p_idx < len(patch_values) and
len(patch_values) > 0 and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml.f90repr(patch_values[p_idx])
p_idx += 1
self.update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
else:
# Skip any values beyond the patch size
skip = (p_idx >= len(patch_values))
self.update_tokens(patch_skip=skip)
else:
self.update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values
def parse_variable(self, parent, patch_nml=None):
"""Parse a variable and return its name and values."""
if not patch_nml:
patch_nml = Namelist()
v_name = self.prior_token
v_values = []
# Patch state
patch_values = None
if self.token == '(':
v_idx_bounds = self.parse_indices()
v_idx = FIndex(v_idx_bounds)
self.update_tokens()
else:
v_idx = None
if self.token == '%':
# Resolve the derived type
if parent and v_name in parent:
v_parent = parent[v_name]
else:
v_parent = []
self.update_tokens()
self.update_tokens()
v_att, v_att_vals = self.parse_variable(v_parent)
next_value = Namelist()
next_value[v_att] = v_att_vals
self.append_value(v_values, next_value, v_idx)
else:
# Construct the variable array
assert self.token == '='
n_vals = None
prior_ws_sep = ws_sep = False
self.update_tokens()
# Check if value is in the namelist patch
# TODO: Edit `Namelist` to support case-insensitive `pop` calls
# (Currently only a problem in PyPy2)
if v_name in patch_nml:
patch_values = patch_nml.pop(v_name.lower())
if not isinstance(patch_values, list):
patch_values = [patch_values]
p_idx = 0
# Add variables until next variable trigger
while (self.token not in ('=', '(', '%')
or (self.prior_token, self.token) == ('=', '(')):
# Check for repeated values
if self.token == '*':
n_vals = self.parse_value()
assert isinstance(n_vals, int)
self.update_tokens()
elif not n_vals:
n_vals = 1
# First check for implicit null values
if self.prior_token in ('=', '%', ','):
if (self.token in (',', '/', '&', '$')
and not (self.prior_token == ','
and self.token in ('/', '&', '$'))):
self.append_value(v_values, None, v_idx, n_vals)
elif self.prior_token == '*':
if self.token not in ('/', '&', '$'):
self.update_tokens()
if (self.token == '=' or (self.token in ('/', '&', '$')
and self.prior_token == '*')):
next_value = None
else:
next_value = self.parse_value()
self.append_value(v_values, next_value, v_idx, n_vals)
else:
next_value = self.parse_value()
# Check for escaped strings
if (v_values and isinstance(v_values[-1], str)
and isinstance(next_value, str)
and not prior_ws_sep):
quote_char = self.prior_token[0]
v_values[-1] = quote_char.join(
[v_values[-1], next_value])
else:
self.append_value(v_values, next_value, v_idx, n_vals)
# Exit for end of nml group (/, &, $) or null broadcast (=)
if self.token in ('/', '&', '$', '='):
break
else:
prior_ws_sep = ws_sep
if (patch_values and p_idx < len(patch_values)
and len(patch_values) > 0 and self.token != ','):
p_val = patch_values[p_idx]
p_repr = patch_nml.f90repr(patch_values[p_idx])
p_idx += 1
ws_sep = self.update_tokens(override=p_repr)
if isinstance(p_val, complex):
# Skip over the complex content
# NOTE: Assumes input and patch are complex
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
self.update_tokens(write_token=False)
else:
ws_sep = self.update_tokens()
if patch_values:
v_values = patch_values
if not v_idx:
v_values = delist(v_values)
return v_name, v_values