def update(self, *attrs):
attributes = self.attributes
if len(attrs)==1 and isinstance(attrs[0],(tuple,list)):
attrs = attrs[0]
for attr in attrs:
lattr = attr.lower()
uattr = attr.upper()
if lattr.startswith('dimension'):
assert self.dimension is None, `self.dimension,attr`
l = attr[9:].lstrip()
assert l[0]+l[-1]=='()',`l`
self.set_dimension(split_comma(l[1:-1].strip(), self.parent.item))
continue
if lattr.startswith('intent'):
l = attr[6:].lstrip()
assert l[0]+l[-1]=='()',`l`
self.set_intent(specs_split_comma(l[1:-1].strip(),
self.parent.item, upper=True))
continue
if lattr.startswith('bind'):
l = attr[4:].lstrip()
assert l[0]+l[-1]=='()',`l`
self.bind = specs_split_comma(l[1:-1].strip(), self.parent.item,
upper = True)
continue
if lattr.startswith('check'):
l = attr[5:].lstrip()
assert l[0]+l[-1]=='()',`l`
self.check.extend(split_comma(l[1:-1].strip()), self.parent.item)
continue
if uattr not in attributes:
if uattr not in self.known_attributes:
self.parent.warning('unknown attribute %r' % (attr))
attributes.append(uattr)
return
def process_item(self):
line = self.item.get_line()[8:].lstrip()
if line.lower()=='none':
self.items = []
return
items = []
for item in split_comma(line, self.item):
i = item.find('(')
assert i!=-1 and item.endswith(')'),`item`
specs = []
for spec in split_comma(item[i+1:-1].strip(), self.item):
if '-' in spec:
s,e = spec.lower().split('-')
s = s.strip()
e = e.strip()
assert s in self.letters and e in self.letters,`s,e`
else:
e = s = spec.lower().strip()
assert s in self.letters,`s,e`
specs.append((s,e))
tspec = item[:i].rstrip()
stmt = None
for cls in declaration_type_spec:
if cls.match(tspec):
stmt = cls(self, self.item.copy(tspec))
if stmt.isvalid:
break
assert stmt is not None,`item,line`
items.append((stmt,specs))
self.items = items
return
def process_item(self):
line = self.item.get_line()[8:].lstrip()
if line.lower() == 'none':
self.items = []
return
items = []
for item in split_comma(line, self.item):
i = item.find('(')
assert i != -1 and item.endswith(')'), ` item `
specs = []
for spec in split_comma(item[i + 1:-1].strip(), self.item):
if '-' in spec:
s, e = spec.lower().split('-')
s = s.strip()
e = e.strip()
assert s in self.letters and e in self.letters, ` s, e `
else:
e = s = spec.lower().strip()
assert s in self.letters, ` s, e `
specs.append((s, e))
tspec = item[:i].rstrip()
stmt = None
for cls in declaration_type_spec:
if cls.match(tspec):
stmt = cls(self, self.item.copy(tspec))
if stmt.isvalid:
break
assert stmt is not None, ` item, line `
items.append((stmt, specs))
self.items = items
return
def process_item(self):
line = self.item.get_line()[4:].lstrip()
if line.startswith('('):
self.isvalid = False
return
specs = []
i = line.find('::')
if i!=-1:
for s in line[:i].split(','):
s = s.strip()
if s: specs.append(s)
line = line[i+2:].lstrip()
self.specs = specs
i = line.find('(')
if i!=-1:
self.name = line[:i].rstrip()
assert line[-1]==')',`line`
self.params = split_comma(line[i+1:-1].lstrip())
else:
self.name = line
self.params = []
if not is_name(self.name):
self.isvalid = False
return
return BeginStatement.process_item(self)
def process_item(self):
line = self.item.get_line()[4:].lstrip()
if line.startswith('('):
self.isvalid = False
return
specs = []
i = line.find('::')
if i != -1:
for s in line[:i].split(','):
s = s.strip()
if s: specs.append(s)
line = line[i + 2:].lstrip()
self.specs = specs
i = line.find('(')
if i != -1:
self.name = line[:i].rstrip()
assert line[-1] == ')', ` line `
self.params = split_comma(line[i + 1:-1].lstrip())
else:
self.name = line
self.params = []
if not is_name(self.name):
self.isvalid = False
return
return BeginStatement.process_item(self)
def update(self, *attrs):
attributes = self.attributes
if len(attrs) == 1 and isinstance(attrs[0], (tuple, list)):
attrs = attrs[0]
for attr in attrs:
lattr = attr.lower()
uattr = attr.upper()
if lattr.startswith('dimension'):
assert self.dimension is None, ` self.dimension, attr `
l = attr[9:].lstrip()
assert l[0] + l[-1] == '()', ` l `
self.set_dimension(
split_comma(l[1:-1].strip(), self.parent.item))
continue
if lattr.startswith('intent'):
l = attr[6:].lstrip()
assert l[0] + l[-1] == '()', ` l `
self.set_intent(
specs_split_comma(l[1:-1].strip(),
self.parent.item,
upper=True))
continue
if lattr.startswith('bind'):
l = attr[4:].lstrip()
assert l[0] + l[-1] == '()', ` l `
self.bind = specs_split_comma(l[1:-1].strip(),
self.parent.item,
upper=True)
continue
if lattr.startswith('check'):
l = attr[5:].lstrip()
assert l[0] + l[-1] == '()', ` l `
self.check.extend(split_comma(l[1:-1].strip()),
self.parent.item)
continue
if uattr not in attributes:
if uattr not in self.known_attributes:
self.parent.warning('unknown attribute %r' % (attr))
attributes.append(uattr)
return
def update(self, *attrs):
attributes = self.attributes
if len(attrs) == 1 and isinstance(attrs[0], (tuple, list)):
attrs = attrs[0]
for attr in attrs:
lattr = attr.lower()
uattr = attr.upper()
if lattr.startswith("dimension"):
l = attr[9:].lstrip()
assert l[0] + l[-1] == "()", ` l `
old_dim = self.dimension
self.set_dimension(split_comma(l[1:-1].strip(), self.parent.item))
assert old_dim is None or old_dim == self.dimension, (` self.dimension, attr, old_dim `)
continue
if lattr.startswith("intent"):
l = attr[6:].lstrip()
assert l[0] + l[-1] == "()", ` l `
self.set_intent(specs_split_comma(l[1:-1].strip(), self.parent.item, upper=True))
continue
if lattr.startswith("bind"):
l = attr[4:].lstrip()
assert l[0] + l[-1] == "()", ` l `
self.bind = specs_split_comma(l[1:-1].strip(), self.parent.item, upper=True)
continue
if lattr.startswith("check"):
l = attr[5:].lstrip()
assert l[0] + l[-1] == "()", ` l `
self.check.extend(split_comma(l[1:-1].strip(), self.parent.item))
continue
if lattr.startswith("depend"):
l = attr[6:].lstrip()
assert l[0] + l[-1] == "()", ` l `
self.depend.extend(split_comma(l[1:-1].strip(), self.parent.item))
continue
if uattr not in attributes:
if uattr not in self.known_attributes:
self.parent.warning("unknown attribute %r" % (attr))
attributes.append(uattr)
return
def add_predictor(decoder):
"""Adds all enabled predictors to the ``decoder``. This function
makes heavy use of the global ``args`` which contains the
SGNMT configuration. Particularly, it reads out ``args.predictors``
and adds appropriate instances to ``decoder``.
TODO: Refactor this method as it is waaaay tooooo looong
Args:
decoder (Decoder): Decoding strategy, see ``create_decoder()``.
This method will add predictors to this instance with
``add_predictor()``
"""
preds = utils.split_comma(args.predictors)
if not preds:
logging.fatal("Require at least one predictor! See the --predictors "
"argument for more information.")
if len(preds) > 1:
logging.fatal("Only 1 predictor supported at the moment")
pred = preds[0]
try:
predictor = predictors.PREDICTOR_REGISTRY[pred](args)
decoder.add_predictor(pred, predictor)
logging.info("Initialized predictor {}".format(pred))
except IOError as e:
logging.fatal("One of the files required for setting up the "
"predictors could not be read: %s" % e)
decoder.remove_predictor()
except AttributeError as e:
logging.fatal("Invalid argument for one of the predictors: %s."
"Stack trace: %s" % (e, traceback.format_exc()))
decoder.remove_predictors()
except NameError as e:
logging.fatal("Could not find external library: %s. Please make sure "
"that your PYTHONPATH and LD_LIBRARY_PATH contains all "
"paths required for the predictors. Stack trace: %s" %
(e, traceback.format_exc()))
decoder.remove_predictor()
except ValueError as e:
logging.fatal("A number format error occurred while configuring the "
"predictors: %s. Please double-check all integer- or "
"float-valued parameters such as --predictor_weights and"
" try again. Stack trace: %s" %
(e, traceback.format_exc()))
decoder.remove_predictor()
except Exception as e:
logging.fatal("An unexpected %s has occurred while setting up the pre"
"dictors: %s Stack trace: %s" %
(sys.exc_info()[0], e, traceback.format_exc()))
decoder.remove_predictor()
def parse_args(parser):
"""http://codereview.stackexchange.com/questions/79008/parse-a-config-file-
and-add-to-command-line-arguments-using-argparse-in-python """
args = parser.parse_args()
if args.config_file:
if not YAML_AVAILABLE:
logging.fatal("Install PyYAML in order to use config files.")
return args
paths = args.config_file
delattr(args, 'config_file')
arg_dict = args.__dict__
for path in utils.split_comma(paths):
_load_config_file(arg_dict, path)
return args
def _parse_char_selector(self, selector):
if not selector:
return '',''
if selector.startswith('*'):
l = selector[1:].lstrip()
if l.startswith('('):
if l.endswith(','): l = l[:-1].rstrip()
assert l.endswith(')'),`l`
l = l[1:-1].strip()
if l.lower().startswith('len'):
l = l[3:].lstrip()[1:].lstrip()
kind=''
else:
assert selector[0]+selector[-1]=='()',`selector`
l = split_comma(selector[1:-1].strip(), self.item)
if len(l)==1:
l = l[0]
if l.lower().startswith('len'):
l=l[3:].lstrip()
assert l.startswith('='),`l`
l=l[1:].lstrip()
kind = ''
elif l.lower().startswith('kind'):
kind = l[4:].lstrip()[1:].lstrip()
l = ''
else:
kind = ''
else:
assert len(l)==2
if l[0].lower().startswith('len'):
assert l[1].lower().startswith('kind'),`l`
kind = l[1][4:].lstrip()[1:].lstrip()
l = l[0][3:].lstrip()[1:].lstrip()
elif l[0].lower().startswith('kind'):
assert l[1].lower().startswith('len'),`l`
kind = l[0][4:].lstrip()[1:].lstrip()
l = l[1][3:].lstrip()[1:].lstrip()
else:
if l[1].lower().startswith('kind'):
kind = l[1][4:].lstrip()[1:].lstrip()
l = l[0]
else:
kind = l[1]
l = l[0]
return l,kind
def _parse_char_selector(self, selector):
if not selector:
return '', ''
if selector.startswith('*'):
l = selector[1:].lstrip()
if l.startswith('('):
if l.endswith(','): l = l[:-1].rstrip()
assert l.endswith(')'), ` l `
l = l[1:-1].strip()
if l.lower().startswith('len'):
l = l[3:].lstrip()[1:].lstrip()
kind = ''
else:
assert selector[0] + selector[-1] == '()', ` selector `
l = split_comma(selector[1:-1].strip(), self.item)
if len(l) == 1:
l = l[0]
if l.lower().startswith('len'):
l = l[3:].lstrip()
assert l.startswith('='), ` l `
l = l[1:].lstrip()
kind = ''
elif l.lower().startswith('kind'):
kind = l[4:].lstrip()[1:].lstrip()
l = ''
else:
kind = ''
else:
assert len(l) == 2
if l[0].lower().startswith('len'):
assert l[1].lower().startswith('kind'), ` l `
kind = l[1][4:].lstrip()[1:].lstrip()
l = l[0][3:].lstrip()[1:].lstrip()
elif l[0].lower().startswith('kind'):
assert l[1].lower().startswith('len'), ` l `
kind = l[0][4:].lstrip()[1:].lstrip()
l = l[1][3:].lstrip()[1:].lstrip()
else:
if l[1].lower().startswith('kind'):
kind = l[1][4:].lstrip()[1:].lstrip()
l = l[0]
else:
kind = l[1]
l = l[0]
return l, kind
def __init__(self, path, args):
"""Creates a Moses n-best list output handler.
Args:
path (string): Path to the n-best file to write
predictor_names: Names of the predictors whose scores
should be included in the score breakdown
in the n-best list
"""
super(NBestOutputHandler, self).__init__()
self.path = path
self.predictor_names = []
name_count = {}
for name in utils.split_comma(args.predictors):
if not name in name_count:
name_count[name] = 1
final_name = name
else:
name_count[name] += 1
final_name = "%s%d" % (name, name_count[name])
self.predictor_names.append(final_name.replace("_", "0"))
def _parse_char_selector(self, selector):
if not selector:
return "", ""
if selector.startswith("*"):
l = selector[1:].lstrip()
if l.startswith("("):
if l.endswith(","):
l = l[:-1].rstrip()
assert l.endswith(")"), ` l `
l = l[1:-1].strip()
if l.lower().startswith("len"):
l = l[3:].lstrip()[1:].lstrip()
kind = ""
else:
assert selector[0] + selector[-1] == "()", ` selector `
l = split_comma(selector[1:-1].strip(), self.item)
if len(l) == 1:
l = l[0]
key, value = self._split_char_selector(l)
if key == "len":
kind, l = "", value
elif key == "kind":
kind, l = value, ""
else:
kind = ""
else:
assert len(l) == 2, ` l `
key0, value0 = self._split_char_selector(l[0])
key1, value1 = self._split_char_selector(l[1])
if key0 == "len":
assert key1 in [None, "kind"], ` key1 `
l, kind = value0, value1
elif key0 == "kind":
assert key1 == "len", ` key1 `
l, kind = value1, value0
else:
assert key0 is None, ` key0 `
assert key1 in [None, "kind"], ` key1 `
l, kind = value0, value1
return l, kind
def _parse_char_selector(self, selector):
if not selector:
return '', ''
if selector.startswith('*'):
l = selector[1:].lstrip()
if l.startswith('('):
if l.endswith(','): l = l[:-1].rstrip()
assert l.endswith(')'), ` l `
l = l[1:-1].strip()
if l.lower().startswith('len'):
l = l[3:].lstrip()[1:].lstrip()
kind = ''
else:
assert selector[0] + selector[-1] == '()', ` selector `
l = split_comma(selector[1:-1].strip(), self.item)
if len(l) == 1:
l = l[0]
key, value = self._split_char_selector(l)
if key == 'len':
kind, l = '', value
elif key == 'kind':
kind, l = value, ''
else:
kind = ''
else:
assert len(l) == 2, ` l `
key0, value0 = self._split_char_selector(l[0])
key1, value1 = self._split_char_selector(l[1])
if key0 == 'len':
assert key1 in [None, 'kind'], ` key1 `
l, kind = value0, value1
elif key0 == 'kind':
assert key1 == 'len', ` key1 `
l, kind = value1, value0
else:
assert key0 is None, ` key0 `
assert key1 in [None, 'kind'], ` key1 `
l, kind = value0, value1
return l, kind
def create_output_handlers():
"""Creates the output handlers defined in the ``io`` module.
These handlers create output files in different formats from the
decoding results.
Args:
args: Global command line arguments.
Returns:
list. List of output handlers according --outputs
"""
if not args.outputs:
return []
outputs = []
for name in utils.split_comma(args.outputs):
path = args.output_path % name if '%s' in args.output_path else args.output_path
try:
outputs.append(output.OUTPUT_REGISTRY[name](path, args))
except KeyError:
logging.fatal("Output format %s not available. Please double-check"
" the --outputs parameter." % name)
return outputs
def _parse_char_selector(self, selector):
if not selector:
return '',''
if selector.startswith('*'):
l = selector[1:].lstrip()
if l.startswith('('):
if l.endswith(','): l = l[:-1].rstrip()
assert l.endswith(')'),`l`
l = l[1:-1].strip()
if l.lower().startswith('len'):
l = l[3:].lstrip()[1:].lstrip()
kind=''
else:
assert selector[0]+selector[-1]=='()',`selector`
l = split_comma(selector[1:-1].strip(), self.item)
if len(l)==1:
l = l[0]
key, value = self._split_char_selector(l)
if key=='len':
kind, l = '', value
elif key=='kind':
kind, l = value, ''
else:
kind = ''
else:
assert len(l)==2,`l`
key0, value0 = self._split_char_selector(l[0])
key1, value1 = self._split_char_selector(l[1])
if key0=='len':
assert key1 in [None, 'kind'],`key1`
l,kind = value0, value1
elif key0=='kind':
assert key1=='len',`key1`
l,kind = value1, value0
else:
assert key0 is None,`key0`
assert key1 in [None,'kind'],`key1`
l, kind = value0, value1
return l,kind
def process_item(self):
item = self.item
apply_map = item.apply_map
clsname = self.__class__.__name__.lower()
line = item.get_line()
from block_statements import Function
#if line.find('[')>0: import pdb; pdb.set_trace()
if not line.lower().startswith(clsname):
i = 0
j = 0
for c in line:
i += 1
if c == ' ': continue
j += 1
if j == len(clsname):
break
line = line[:i].replace(' ', '') + line[i:]
assert line.lower().startswith(clsname), ` line, clsname `
line = line[len(clsname):].lstrip()
if line.startswith('('):
i = line.find(')')
selector = apply_map(line[:i + 1].strip())
line = line[i + 1:].lstrip()
elif line.startswith('*'):
selector = '*'
line = line[1:].lstrip()
if line.startswith('('):
i = line.find(')')
selector += apply_map(line[:i + 1].rstrip())
line = line[i + 1:].lstrip()
else:
m = re.match(r'\d+(_\w+|)|[*]', line)
if not m:
self.isvalid = False
return
i = m.end()
selector += line[:i].rstrip()
line = line[i:].lstrip()
else:
selector = ''
fm = Function.match(line)
if fm:
l2 = line[:fm.end()]
m2 = re.match(r'.*?\b(?P<name>\w+)\Z', l2)
if not m2:
self.isvalid = False
return
fname = m2.group('name')
fitem = item.copy(clsname + selector + ' :: ' + fname,
apply_map=True)
self.parent.put_item(fitem)
item.clone(line)
self.isvalid = False
return
if line.startswith(','):
line = line[1:].lstrip()
self.raw_selector = selector
if isinstance(self, Character):
self.selector = self._parse_char_selector(selector)
else:
self.selector = self._parse_kind_selector(selector)
i = line.find('::')
if i == -1:
self.attrspec = []
self.entity_decls = split_comma(line, self.item)
else:
self.attrspec = split_comma(line[:i].rstrip(), self.item)
self.entity_decls = split_comma(line[i + 2:].lstrip(), self.item)
for entity in self.entity_decls:
if not is_entity_decl(entity):
self.isvalid = False
return
if isinstance(self.parent, Function) \
and self.parent.name in self.entity_decls:
assert self.parent.typedecl is None, ` self.parent.typedecl `
self.parent.typedecl = self
self.parent.result_in_typedecl = True # OC addition
#self.ignore = True # OC deletion
if isinstance(self, Type):
self.name = self.selector[1].lower()
assert is_name(self.name), ` self.name `
else:
self.name = clsname
return
def get_pump(self):
out = self.get_values()
out = [split_comma(item) for item in out]
return out
def process_item(self):
item = self.item
apply_map = item.apply_map
clsname = self.__class__.__name__.lower()
line = item.get_line()
from block_statements import Function
if not line.lower().startswith(clsname):
i = 0
j = 0
for c in line:
i += 1
if c==' ': continue
j += 1
if j==len(clsname):
break
line = line[:i].replace(' ','') + line[i:]
assert line.lower().startswith(clsname),`line,clsname`
line = line[len(clsname):].lstrip()
if line.startswith('('):
i = line.find(')')
selector = apply_map(line[:i+1].strip())
line = line[i+1:].lstrip()
elif line.startswith('*'):
selector = '*'
line = line[1:].lstrip()
if line.startswith('('):
i = line.find(')')
selector += apply_map(line[:i+1].rstrip())
line = line[i+1:].lstrip()
else:
m = re.match(r'\d+(_\w+|)|[*]',line)
if not m:
self.isvalid = False
return
i = m.end()
selector += line[:i].rstrip()
line = line[i:].lstrip()
else:
selector = ''
fm = Function.match(line)
if fm:
l2 = line[:fm.end()]
m2 = re.match(r'.*?\b(?P<name>\w+)\Z',l2)
if not m2:
self.isvalid = False
return
fname = m2.group('name')
fitem = item.copy(clsname+selector+' :: '+fname,
apply_map=True)
self.parent.put_item(fitem)
item.clone(line)
self.isvalid = False
return
if line.startswith(','):
line = line[1:].lstrip()
self.raw_selector = selector
if isinstance(self, Character):
self.selector = self._parse_char_selector(selector)
else:
self.selector = self._parse_kind_selector(selector)
i = line.find('::')
if i==-1:
self.attrspec = []
self.entity_decls = split_comma(line, self.item)
else:
self.attrspec = split_comma(line[:i].rstrip(), self.item)
self.entity_decls = split_comma(line[i+2:].lstrip(), self.item)
for entity in self.entity_decls:
if not is_entity_decl(entity):
self.isvalid = False
return
if isinstance(self.parent, Function) \
and self.parent.name in self.entity_decls:
assert self.parent.typedecl is None,`self.parent.typedecl`
self.parent.typedecl = self
self.ignore = True
if isinstance(self, Type):
self.name = self.selector[1].lower()
assert is_name(self.name),`self.name`
else:
self.name = clsname
return
def process_item(self):
item = self.item
apply_map = item.apply_map
clsname = self.__class__.__name__.lower()
# line = item.get_line() # KGEN deletion
itemline = item.get_line() # KGEN addition
line = itemline[:] # KGEN addition
from block_statements import Function
# if line.find('[')>0: import pdb; pdb.set_trace()
if not line.lower().startswith(clsname):
i = 0
j = 0
for c in line:
i += 1
if c == " ":
continue
j += 1
if j == len(clsname):
break
line = line[:i].replace(" ", "") + line[i:]
assert line.lower().startswith(clsname), ` line, clsname `
line = line[len(clsname) :].lstrip()
if line.startswith("("):
i = line.find(")")
selector = apply_map(line[: i + 1].strip())
line = line[i + 1 :].lstrip()
elif line.startswith("*"):
selector = "*"
line = line[1:].lstrip()
if line.startswith("("):
i = line.find(")")
selector += apply_map(line[: i + 1].rstrip())
line = line[i + 1 :].lstrip()
else:
m = re.match(r"\d+(_\w+|)|[*]", line)
if not m:
self.isvalid = False
return
i = m.end()
selector += line[:i].rstrip()
line = line[i:].lstrip()
else:
selector = ""
fm = Function.match(line)
if fm:
l2 = line[: fm.end()]
m2 = re.match(r".*?\b(?P<name>\w+)\Z", l2)
if not m2:
self.isvalid = False
return
fname = m2.group("name")
# start of KGEN deletion
# prevending adding typedecl stmt for Function result
# fitem = item.copy(clsname+selector+' :: '+fname,
# apply_map=True)
# self.parent.put_item(fitem)
# end of KGEN deletion
# start of KGEN addition
pos = itemline.find(line)
if pos > 0:
if not hasattr(self.parent, "funcresult_in_stmt"):
self.parent.funcresult_in_stmt = {}
self.parent.funcresult_in_stmt[fname] = self.item.apply_map(itemline[:pos].rstrip())
self.parent.result_in_typedecl = False
# end of KGEN addition
item.clone(line)
self.isvalid = False
return
if line.startswith(","):
line = line[1:].lstrip()
self.raw_selector = selector
if isinstance(self, Character):
self.selector = self._parse_char_selector(selector)
else:
self.selector = self._parse_kind_selector(selector)
i = line.find("::")
if i == -1:
self.attrspec = []
self.entity_decls = split_comma(line, self.item)
else:
self.attrspec = split_comma(line[:i].rstrip(), self.item)
self.entity_decls = split_comma(line[i + 2 :].lstrip(), self.item)
for entity in self.entity_decls:
if not is_entity_decl(entity):
self.isvalid = False
return
if isinstance(self.parent, Function) and self.parent.name in self.entity_decls:
assert self.parent.typedecl is None, ` self.parent.typedecl `
self.parent.typedecl = self
self.parent.result_in_typedecl = True # KGEN addition
# self.ignore = True # KGEN deletion
if isinstance(self, Type):
self.name = self.selector[1].lower()
assert is_name(self.name), ` self.name `
else:
self.name = clsname
return