def __init__(self, input_size, output_size, initialization='normal', uniform=False):
# print("Inside initialization method of FCLayer")
# print("Input size:{}".format(input_size))
# print("Output size:{}".format(output_size))
"""
:param input_size: (int) the number of nodes in the previous layer
:param output_size: (int) the number of nodes in 'this' layer
:param initialization: (str) whether to use Xavier initialization
:param uniform: (bool) in Xavier initialization, whether to use uniform or truncated normal distribution
"""
if initialization == 'xavier':
# print("Chosen initialization is Xavier")
# Xavier initialization followed the implementation in Tensorflow.
fan_in = input_size
fan_out = output_size
n = (fan_in + fan_out) / 2.0
if uniform:
limit = math.sqrt(3.0 / n)
weights = np.random.uniform(-limit, limit, size=(fan_in, fan_out))
bias = np.zeros((1, output_size))
else:
trunc_std = math.sqrt(1.3 / n)
a, b = -2, 2 # truncated between [- 2 * std, 2 * std]
weights = stats.truncnorm.rvs(a, b, loc=0.0, scale=trunc_std, size=(fan_in, fan_out))
bias = np.zeros((1, output_size))
else:
weights = np.random.randn(input_size, output_size)
bias = np.random.randn(1, output_size)
self.weights = Parameter(weights) # instantiate Parameter object by passing the initialized values
self.bias = Parameter(bias) # instantiate Parameter object by passing the initialized values
self.param = [self.weights, self.bias]
self.forward_outputs = []
self.backprop_deltas = []
self.all_weights = []
self.all_weights_grads = []
self.all_bias_grads = []
self.backward_counter = 0
# print("Weights are: {}".format(self.weights.value))
# print("Shape of weights matrix is: {}".format(self.weights.value.shape))
# print("Bias matrix is: {}".format(self.bias.value))
# print("Shape of Bias matrix is: {}".format(self.bias.value.shape))
# print("self.param: {}".format(self.param))
# print("Len of self.param list is: {}".format(len(self.param)))
#
# print("FC Layer created")
# print("..................................................................\n")
self.backward_counter = 0
def __init__(self, linkage, name, parent, L, theta, phi=None, ux=None, uz=None, locked=False):
super(FromPointLine, self).__init__(linkage, name)
self.parent = parent
self.locked = locked
self.L = L
phi = np.pi/2 if phi is None else phi*np.pi/180
self.ux = ux
self.uz = uz
self.p1 = OnPointPoint(self.linkage, '{}.{}'.format(self.name, '1'), parent=parent)
self.p2 = CalculatedAlphaPoint(self.linkage, '{}.{}'.format(self.name, '2'), parent=self)
self.params.theta = Parameter([theta*np.pi/180/ANGLE_FACTOR],
self, 'theta', range=[0,2*np.pi], units='rad', locked=self.locked)
self.params.phi = Parameter([phi/ANGLE_FACTOR],
self, 'phi', range=[0,2*np.pi], units='rad', locked=True)
def __init__(self, linkage, name, parent, L, theta, phi=None, ux=None, uz=None, beta=None):
super(OnPointLine, self).__init__(linkage, name)
self.parent = parent
self.L = L
phi = np.pi/2 if phi is None else phi*np.pi/180
self.ux = ux
self.uz = uz
beta = 0.5 if beta is None else beta
self.p1 = CalculatedAnteriorPoint(self.linkage, '{}.{}'.format(self.name, '1'), parent=self)
self.p2 = CalculatedPosteriorPoint(self.linkage, '{}.{}'.format(self.name, '2'), parent=self)
self.params.theta = Parameter([theta*np.pi/180/ANGLE_FACTOR],
self, 'theta', range=[0,2*np.pi], units='rad', locked=False)
self.params.phi = Parameter([phi/ANGLE_FACTOR],
self, 'phi', range=[0,2*np.pi], units='rad', locked=True)
self.params.beta = Parameter([beta],
self, 'beta', range=[0,1], units=None, locked=False)
def process_vehicle(vehicle):
debug("===\n\n\nProcessing %s" % vehicle.name)
current_file = vehicle.path + '/Parameters.cpp'
f = open(current_file)
p_text = f.read()
f.close()
group_matches = prog_groups.findall(p_text)
debug(group_matches)
for group_match in group_matches:
lib = Library(group_match[0])
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
else:
error("group: unknown parameter metadata field '%s'" %
field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
libraries.append(lib)
param_matches = []
if not args.emit_sitl:
param_matches = prog_param.findall(p_text)
for param_match in param_matches:
(only_vehicles, param_name,
field_text) = (param_match[0], param_match[1], param_match[2])
if len(only_vehicles):
only_vehicles_list = [x.strip() for x in only_vehicles.split(",")]
for only_vehicle in only_vehicles_list:
if only_vehicle not in valid_truenames:
raise ValueError("Invalid only_vehicle %s" % only_vehicle)
if vehicle.truename not in only_vehicles_list:
continue
p = Parameter(vehicle.reference + ":" + param_name, current_file)
debug(p.name + ' ')
global current_param
current_param = p.name
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', "", field[1]).rstrip()
setattr(p, field[0], value)
elif field[0] == "CopyValuesFrom":
setattr(p, field[0], field[1])
else:
error("param: unknown parameter metadata field '%s'" %
field[0])
for req_field in required_param_fields:
if req_field not in field_list:
error("missing parameter metadata field '%s' in %s" %
(req_field, field_text))
vehicle.params.append(p)
current_file = None
debug("Processed %u params" % len(vehicle.params))
def __init__(self, linkage, name, parent1, parent2, L, gamma=None):
super(OnPointsLine, self).__init__(linkage, name)
self.parent1 = parent1
self.parent2 = parent2
self.L = L
gamma = 0.5 if gamma is None else gamma
self.p1 = CalculatedAnteriorGammaPoint(self.linkage, '{}.{}'.format(self.name, '1'), parent=self)
self.p2 = CalculatedPosteriorGammaPoint(self.linkage, '{}.{}'.format(self.name, '2'), parent=self)
self.params.gamma = Parameter([gamma], self, 'gamma', range=[0,1], units=None, locked=False)
def process_library(library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
if len(vehicles) != 1:
print("Unable to handle multiple vehicles with .pde library")
continue
libraryfname = os.path.join(vehicles[0].path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s" % (path, library.name))
continue
param_matches = prog_group_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
p = Parameter(library.name + param_match[0])
debug(p.name + ' ')
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
for field in fields:
if field[0] in known_param_fields:
setattr(p, field[0], field[1])
else:
error("unknown parameter metadata field %s" % field[0])
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
l = Library(group)
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(l, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(l.name == parsed_l.name for parsed_l in libraries):
l.name = library.name + l.name
debug("Group name: %s" % l.name)
process_library(l, os.path.dirname(libraryfname))
alllibs.append(l)
def __init__(self, linkage, name, parent, alpha):
super(OnLinePoint, self).__init__(linkage, name)
self.parent = parent
alpha = 0.5 if alpha is None else alpha
self.params.alpha = Parameter([alpha],
self,
'alpha',
range=[0, 1],
units=None,
locked=False)
def __init__(self,
input_size,
output_size,
initialization='normal',
uniform=False):
"""
:param input_size: (int) the number of nodes in the previous layer
:param output_size: (int) the number of nodes in 'this' layer
:param initialization: (str) whether to use Xavier initialization
:param uniform: (bool) in Xavier initialization, whether to use uniform or truncated normal distribution
"""
if initialization == 'xavier':
# Xavier initialization followed the implementation in Tensorflow.
fan_in = input_size
fan_out = output_size
n = (fan_in + fan_out) / 2.0
if uniform:
limit = math.sqrt(3.0 / n)
weights = np.random.uniform(-limit,
limit,
size=(fan_in, fan_out))
bias = np.zeros((1, output_size))
else:
trunc_std = math.sqrt(1.3 / n)
a, b = -2, 2 # truncated between [- 2 * std, 2 * std]
weights = stats.truncnorm.rvs(a,
b,
loc=0.0,
scale=trunc_std,
size=(fan_in, fan_out))
bias = np.zeros((1, output_size))
else:
weights = np.random.randn(input_size, output_size)
bias = np.random.randn(1, output_size)
self.weights = Parameter(
weights
) # instantiate Parameter object by passing the initialized values
self.bias = Parameter(
bias
) # instantiate Parameter object by passing the initialized values
self.param = [self.weights,
self.bias] # store the weight and bias to a list
def __init__(self, linkage, name, at):
super(AnchorPoint, self).__init__(linkage, name)
self.params.x = Parameter(at[0],
self,
'x',
range=[-10, 10],
units='m',
locked=True)
self.params.y = Parameter(at[1],
self,
'y',
range=[-10, 10],
units='m',
locked=True)
self.params.z = Parameter(at[2],
self,
'z',
range=[-10, 10],
units='m',
locked=True)
def __init__(self, linkage, name, at, parent):
super(ToPointPoint, self).__init__(linkage, name)
self.parent = parent
self.params.x = Parameter(at[0],
self,
'x',
range=[-10, 10],
units='m',
locked=False)
self.params.y = Parameter(at[1],
self,
'y',
range=[-10, 10],
units='m',
locked=False)
self.params.z = Parameter(at[2],
self,
'z',
range=[-10, 10],
units='m',
locked=False)
def run_turkish():
params = Parameter()
params.UseTransRules = True
params.DoPruning = False
params.DoCompound = False
params.ExcludeUnreliable = False
params.BestNCandSuffix = 150
infile_train = r'data/wordlist.2010.tur.utf8.txt'
infile_test_gold = r'data/mit/gold.tur.txt'
run_experiment(infile_train, infile_test_gold, params)
def run_finnish():
"""Runs an experiment on Finnish data against gold standard results."""
params = Parameter()
params.UseTransRules = False
params.DoPruning = True
params.DoCompound = True
params.ExcludeUnreliable = True
params.BestNCandSuffix = 150
infile_train = r'data/wordlist.2010.fin.utf8.txt'
infile_test_gold = r'data/mit/gold.fin.txt'
run_experiment(infile_train, infile_test_gold, params)
def _get_basic_type(_p: param.Parameter) -> Union[type, Callable]:
"""
Given a parameter, get its basic Python type, e.g.: param.Boolean -> bool.
Throw exception if it is not supported.
:param _p: parameter to get type and nargs for.
:return: Type
"""
if isinstance(_p, param.Boolean):
p_type: Union[type,
Callable] = lambda x: (str(x).lower() == 'true')
elif isinstance(_p, param.Integer):
p_type = lambda x: _p.default if x == "" else int(x)
elif isinstance(_p, param.Number):
p_type = lambda x: _p.default if x == "" else float(x)
elif isinstance(_p, param.String):
p_type = str
elif isinstance(_p, param.List):
p_type = lambda x: [_p.class_(item) for item in x.split(',')]
elif isinstance(_p, param.NumericTuple):
float_or_int = lambda y: int(y) if isinstance(_p, IntTuple
) else float(y)
p_type = lambda x: tuple(
[float_or_int(item) for item in x.split(',')])
elif isinstance(_p, param.ClassSelector):
p_type = _p.class_
elif isinstance(_p, ListOrDictParam):
def list_or_dict(x: str) -> Union[Dict, List]:
import json
if x.startswith("{") or x.startswith('['):
res = json.loads(x)
else:
res = [str(item) for item in x.split(',')]
if isinstance(res, Dict):
return res
elif isinstance(res, List):
return res
else:
raise ValueError(
f"Parameter of type {_p} should resolve to List or Dict"
)
p_type = list_or_dict
else:
raise TypeError(
"Parameter of type: {} is not supported".format(_p))
return p_type
def build_param(key, value):
return str(Parameter(key, value))
def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
global current_file
current_file = path
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
if len(vehicles) != 1:
print("Unable to handle multiple vehicles with .pde library")
continue
libraryfname = os.path.join(vehicles[0].path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s (fname=%s)" %
(path, library.name, libraryfname))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
p = Parameter(library.name + param_match[0], current_file)
debug(p.name + ' ')
global current_param
current_param = p.name
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
non_vehicle_specific_values_seen = False
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
if field[0] == "Values":
non_vehicle_specific_values_seen = True
else:
error("param: unknown parameter metadata field %s" %
field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
this_vehicle_values_seen = False
this_vehicle_value = None
other_vehicle_values_seen = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [
x.rstrip().lstrip() for x in only_for_vehicles
]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug(
"field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n"
% (field[0], vehicle.name, vehicle.truename, field[1],
str(only_for_vehicles)))
value = re.sub('@PREFIX@', library.name, field[2])
if field[0] == "Values":
if vehicle.truename in only_for_vehicles:
this_vehicle_values_seen = True
this_vehicle_value = value
if len(only_for_vehicles) > 1:
other_vehicle_values_seen = True
elif len(only_for_vehicles):
other_vehicle_values_seen = True
if field[0] in known_param_fields:
setattr(p, field[0], value)
else:
error(
"tagged param<: unknown parameter metadata field '%s'"
% field[0])
if ((non_vehicle_specific_values_seen
or not other_vehicle_values_seen)
or this_vehicle_values_seen):
if this_vehicle_values_seen and field[0] == 'Values':
setattr(p, field[0], this_vehicle_value)
# debug("Appending (non_vehicle_specific_values_seen=%u "
# "other_vehicle_values_seen=%u this_vehicle_values_seen=%u)" %
# (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen))
p.path = path # Add path. Later deleted - only used for duplicates
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
done_groups = dict()
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
do_append = True
if group in done_groups:
# this is to handle cases like the RangeFinder
# parameters, where the wasp stuff gets tack into the
# same RNGFND1_ group
lib = done_groups[group]
do_append = False
else:
lib = Library(group)
done_groups[group] = lib
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
if do_append:
lib.name = library.name + lib.name
debug("Group name: %s" % lib.name)
process_library(vehicle, lib, os.path.dirname(libraryfname))
if do_append:
alllibs.append(lib)
current_file = None
debug(group_matches)
for group_match in group_matches:
lib = Library(group_match[0])
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
else:
error("group: unknown parameter metadata field '%s'" %
field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
libraries.append(lib)
for param_match in param_matches:
p = Parameter(vehicle.name + ":" + param_match[0], current_file)
debug(p.name + ' ')
current_param = p.name
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', "", field[1]).rstrip()
setattr(p, field[0], value)
else:
error("param: unknown parameter metadata field '%s'" %
field[0])
for req_field in required_param_fields:
if req_field not in field_list:
def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
global current_file
current_file = path
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
if len(vehicles) != 1:
print("Unable to handle multiple vehicles with .pde library")
continue
libraryfname = os.path.join(vehicles[0].path, path)
else:
libraryfname = os.path.normpath(os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s" % (path, library.name))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
p = Parameter(library.name+param_match[0], current_file)
debug(p.name + ' ')
global current_param
current_param = p.name
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
non_vehicle_specific_values_seen = False
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
if field[0] == "Values":
non_vehicle_specific_values_seen = True
else:
error("param: unknown parameter metadata field %s" % field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
this_vehicle_values_seen = False
this_vehicle_value = None
other_vehicle_values_seen = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [x.rstrip().lstrip() for x in only_for_vehicles]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug("field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n" %
(field[0], vehicle.name, vehicle.truename, field[1], str(only_for_vehicles)))
value = re.sub('@PREFIX@', library.name, field[2])
if field[0] == "Values":
if vehicle.truename in only_for_vehicles:
this_vehicle_values_seen = True
this_vehicle_value = value
if len(only_for_vehicles) > 1:
other_vehicle_values_seen = True
elif len(only_for_vehicles):
other_vehicle_values_seen = True
if field[0] in known_param_fields:
setattr(p, field[0], value)
else:
error("tagged param<: unknown parameter metadata field '%s'" % field[0])
if ((non_vehicle_specific_values_seen or not other_vehicle_values_seen) or this_vehicle_values_seen):
if this_vehicle_values_seen:
debug("Setting vehicle-specific value (%s)" % str(this_vehicle_value))
setattr(p, field[0], this_vehicle_value)
# debug("Appending (non_vehicle_specific_values_seen=%u "
# "other_vehicle_values_seen=%u this_vehicle_values_seen=%u)" %
# (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen))
p.path = path # Add path. Later deleted - only used for duplicates
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
lib = Library(group)
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
lib.name = library.name + lib.name
debug("Group name: %s" % lib.name)
process_library(vehicle, lib, os.path.dirname(libraryfname))
alllibs.append(lib)
current_file = None
error("group: unknown parameter metadata field '%s'" % field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
libraries.append(lib)
for param_match in param_matches:
(only_vehicles, param_name, field_text) = (param_match[0],
param_match[1],
param_match[2])
if len(only_vehicles):
only_vehicles_list = [x.strip() for x in only_vehicles.split(",")]
for only_vehicle in only_vehicles_list:
if only_vehicle not in valid_truenames:
raise ValueError("Invalid only_vehicle %s" % only_vehicle)
if vehicle.truename not in only_vehicles_list:
continue
p = Parameter(vehicle.name+":"+param_name, current_file)
debug(p.name + ' ')
current_param = p.name
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', "", field[1]).rstrip()
setattr(p, field[0], value)
else:
error("param: unknown parameter metadata field '%s'" % field[0])
for req_field in required_param_fields:
if req_field not in field_list:
error("missing parameter metadata field '%s' in %s" % (req_field, field_text))
group_matches = prog_groups.findall(p_text)
debug(group_matches)
for group_match in group_matches:
l = Library(group_match[0])
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(l, field[0], field[1])
else:
error("group: unknown parameter metadata field '%s'" % field[0])
if not any(l.name == parsed_l.name for parsed_l in libraries):
libraries.append(l)
for param_match in param_matches:
p = Parameter(vehicle.name+":"+param_match[0])
debug(p.name + ' ')
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
setattr(p, field[0], field[1])
else:
error("param: unknown parameter metadata field '%s'" % field[0])
for req_field in required_param_fields:
if req_field not in field_list:
error("missing parameter metadata field '%s' in %s" % (req_field, field_text))
vehicle.params.append(p)
def run(infile, outfile, params):
print('| Reading data...')
word_freq_list = read_word_freq_list(infile)
print('| Analyzing...')
morph_analyzer = MorphAnalyzer(params)
morph_analyzer.train(word_freq_list)
print('| Segmenting...')
word_list = [word for word, _freq in word_freq_list]
word_segs = morph_analyzer.segment_token_list(word_list)
print('| Saving result...')
save_segmentations(zip(word_list, word_segs), outfile)
print('| Done!')
if __name__ == '__main__':
params = Parameter()
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('infile', help='The input file containing a word list with line format: <word> <freq>')
arg_parser.add_argument('outfile', help='The output file to save the segmentation result')
arg_parser.add_argument('-p', '--prune', help='Whether use pruning (1|0, default:%s)' % params.DoPruning, type=bool, default=params.DoPruning)
arg_parser.add_argument('-t', '--trans', help='Whether use transformation rules (1|0, default:%s)' % params.UseTransRules, type=bool, default=params.UseTransRules)
arg_parser.add_argument('-c', '--comp', help='Whether process compounding (1|0, default:%s)' % params.DoCompound, type=bool, default=params.DoCompound)
arg_parser.add_argument('-e', '--excl', help='Whether exclude unreliable roots (1|0, default:%s)' % params.ExcludeUnreliable, type=bool, default=params.ExcludeUnreliable)
arg_parser.add_argument('-n', '--hyphen', help='Whether explicitly deal with hyphen words (1|0, default:%s)' % params.DoHyphen, type=bool, default=params.DoHyphen)
arg_parser.add_argument('-a', '--apos', help='Whether explicitly deal with apostrophes (1|0, default:%s)' % params.DoApostrophe, type=bool, default=params.DoApostrophe)
arg_parser.add_argument('-r', '--root', help='Minimal length of roots that will be possibly segmented (default:%s)' % params.MinStemLen, type=int, default=params.MinStemLen)
arg_parser.add_argument('-s', '--suff', help='Maximal length of suffixes (default:%s)' % params.MaxSuffixLen, type=int, default=params.MaxSuffixLen)
args = arg_parser.parse_args()
params.DoPruning = args.prune
params.UseTransRules = args.trans
params.DoCompound = args.comp
def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
global current_file
current_file = path
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
libraryfname = os.path.join(vehicle.path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s (fname=%s)" %
(path, library.name, libraryfname))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
(only_vehicles, param_name,
field_text) = (param_match[0], param_match[1], param_match[2])
if len(only_vehicles):
only_vehicles_list = [
x.strip() for x in only_vehicles.split(",")
]
for only_vehicle in only_vehicles_list:
if only_vehicle not in valid_truenames:
raise ValueError("Invalid only_vehicle %s" %
only_vehicle)
if vehicle.name not in only_vehicles_list:
continue
p = Parameter(library.name + param_name, current_file)
debug(p.name + ' ')
global current_param
current_param = p.name
fields = prog_param_fields.findall(field_text)
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
elif field[0] == "CopyValuesFrom":
setattr(p, field[0], field[1])
else:
error("param: unknown parameter metadata field %s" %
field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
# a parameter is considered to be vehicle-specific if
# there does not exist a Values: or Values{VehicleName}
# for that vehicle but @Values{OtherVehicle} exists.
seen_values_or_bitmask_for_other_vehicle = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [
x.rstrip().lstrip() for x in only_for_vehicles
]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug(
"field[0]=%s vehicle=%s field[1]=%s only_for_vehicles=%s\n"
%
(field[0], vehicle.name, field[1], str(only_for_vehicles)))
if field[0] not in known_param_fields:
error(
"tagged param: unknown parameter metadata field '%s'" %
field[0])
continue
if vehicle.name not in only_for_vehicles:
if len(only_for_vehicles) and field[0] in [
'Values', 'Bitmask'
]:
seen_values_or_bitmask_for_other_vehicle = True
continue
value = re.sub('@PREFIX@', library.name, field[2])
setattr(p, field[0], value)
if (getattr(p, 'Values', None) is not None
and getattr(p, 'Bitmask', None) is not None):
error("Both @Values and @Bitmask present")
if (getattr(p, 'Values', None) is None
and getattr(p, 'Bitmask', None) is None):
# values and Bitmask available for this vehicle
if seen_values_or_bitmask_for_other_vehicle:
# we've (e.g.) seen @Values{Copter} when we're
# processing for Rover, and haven't seen either
# @Values: or @Vales{Rover} - so we omit this
# parameter on the assumption that it is not
# applicable for this vehicle.
continue
p.path = path # Add path. Later deleted - only used for duplicates
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
done_groups = dict()
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
do_append = True
if group in done_groups:
# this is to handle cases like the RangeFinder
# parameters, where the wasp stuff gets tack into the
# same RNGFND1_ group
lib = done_groups[group]
do_append = False
else:
lib = Library(group)
done_groups[group] = lib
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
if do_append:
lib.set_name(library.name + lib.name)
debug("Group name: %s" % lib.name)
process_library(vehicle, lib, os.path.dirname(libraryfname))
if do_append:
alllibs.append(lib)
current_file = None
#!/usr/bin/env python3
from param import Register, MemAddr, Parameter
from testcase import Testcase
# Choose out of various operands
reg8 = Register('al')
reg16 = Register('ax')
reg32 = Register('eax')
reg64 = Register('rax')
xmm = Register('xmm0')
ymm = Register('ymm0')
zmm = Register('zmm0')
mem0 = MemAddr('(%rax, %esi, 4)')
imd1 = Parameter('IMD')
# -----------------------------------------------
# -USER INPUT------------------------------------
# -----------------------------------------------
# Enter your mnemonic
mnemonic = 'add'
# Define your operands. If you don't need it, just type in None
dst = mem0
op1 = imd1
op2 = None
# Define the number of instructions per loop (default: 12)
per_loop = '32'
# -----------------------------------------------
def add_param(self, param_id, shape, post=None, value=None):
if value is None:
value = Parameter(shape, self.init, name=param_id, post=post)
setattr(self, param_id, value)
self.params[param_id] = value
group_matches = prog_groups.findall(p_text)
debug(group_matches)
for group_match in group_matches:
l = Library(group_match[0])
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(l, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(l.name == parsed_l.name for parsed_l in libraries):
libraries.append(l)
for param_match in param_matches:
p = Parameter(vehicle.name + ":" + param_match[0])
debug(p.name + ' ')
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
setattr(p, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
for req_field in required_param_fields:
if req_field not in field_list:
error("missing parameter metadata field '%s' in %s" %
(req_field, field_text))
def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
if len(vehicles) != 1:
print("Unable to handle multiple vehicles with .pde library")
continue
libraryfname = os.path.join(vehicles[0].path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s" % (path, library.name))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
p = Parameter(library.name + param_match[0])
debug(p.name + ' ')
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
non_vehicle_specific_values_seen = False
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
if field[0] == "Values":
non_vehicle_specific_values_seen = True
else:
error("param: unknown parameter metadata field %s" %
field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
this_vehicle_values_seen = False
this_vehicle_value = None
other_vehicle_values_seen = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [
x.rstrip().lstrip() for x in only_for_vehicles
]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug(
"field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n"
% (field[0], vehicle.name, vehicle.truename, field[1],
str(only_for_vehicles)))
value = re.sub('@PREFIX@', library.name, field[2])
if field[0] == "Values":
if vehicle.truename in only_for_vehicles:
this_vehicle_values_seen = True
this_vehicle_value = value
if len(only_for_vehicles) > 1:
other_vehicle_values_seen = True
elif len(only_for_vehicles):
other_vehicle_values_seen = True
if field[0] in known_param_fields:
setattr(p, field[0], value)
else:
error(
"tagged param<: unknown parameter metadata field '%s'"
% field[0])
if ((non_vehicle_specific_values_seen
or not other_vehicle_values_seen)
or this_vehicle_values_seen):
if this_vehicle_values_seen:
debug("Setting vehicle-specific value (%s)" %
str(this_vehicle_value))
setattr(p, field[0], this_vehicle_value)
debug(
"Appending (non_vehicle_specific_values_seen=%u other_vehicle_values_seen=%u this_vehicle_values_seen=%u!)"
% (non_vehicle_specific_values_seen,
other_vehicle_values_seen, this_vehicle_values_seen))
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
l = Library(group)
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(l, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(l.name == parsed_l.name for parsed_l in libraries):
l.name = library.name + l.name
debug("Group name: %s" % l.name)
process_library(vehicle, l, os.path.dirname(libraryfname))
alllibs.append(l)
