def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
global current_file
current_file = path
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
if len(vehicles) != 1:
print("Unable to handle multiple vehicles with .pde library")
continue
libraryfname = os.path.join(vehicles[0].path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s (fname=%s)" %
(path, library.name, libraryfname))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
(only_vehicles, param_name,
field_text) = (param_match[0], param_match[1], param_match[2])
if len(only_vehicles):
only_vehicles_list = [
x.strip() for x in only_vehicles.split(",")
]
for only_vehicle in only_vehicles_list:
if only_vehicle not in valid_truenames:
raise ValueError("Invalid only_vehicle %s" %
only_vehicle)
if vehicle.truename not in only_vehicles_list:
continue
p = Parameter(library.name + param_name, current_file)
debug(p.name + ' ')
global current_param
current_param = p.name
fields = prog_param_fields.findall(field_text)
non_vehicle_specific_values_seen = False
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
if field[0] == "Values":
non_vehicle_specific_values_seen = True
else:
error("param: unknown parameter metadata field %s" %
field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
this_vehicle_values_seen = False
this_vehicle_value = None
other_vehicle_values_seen = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [
x.rstrip().lstrip() for x in only_for_vehicles
]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug(
"field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n"
% (field[0], vehicle.name, vehicle.truename, field[1],
str(only_for_vehicles)))
value = re.sub('@PREFIX@', library.name, field[2])
if field[0] == "Values":
if vehicle.truename in only_for_vehicles:
this_vehicle_values_seen = True
this_vehicle_value = value
if len(only_for_vehicles) > 1:
other_vehicle_values_seen = True
elif len(only_for_vehicles):
other_vehicle_values_seen = True
if field[0] in known_param_fields:
setattr(p, field[0], value)
else:
error(
"tagged param<: unknown parameter metadata field '%s'"
% field[0])
if ((non_vehicle_specific_values_seen
or not other_vehicle_values_seen)
or this_vehicle_values_seen):
if this_vehicle_values_seen and field[0] == 'Values':
setattr(p, field[0], this_vehicle_value)
# debug("Appending (non_vehicle_specific_values_seen=%u "
# "other_vehicle_values_seen=%u this_vehicle_values_seen=%u)" %
# (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen))
p.path = path # Add path. Later deleted - only used for duplicates
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
done_groups = dict()
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
do_append = True
if group in done_groups:
# this is to handle cases like the RangeFinder
# parameters, where the wasp stuff gets tack into the
# same RNGFND1_ group
lib = done_groups[group]
do_append = False
else:
lib = Library(group)
done_groups[group] = lib
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
if do_append:
lib.name = library.name + lib.name
debug("Group name: %s" % lib.name)
process_library(vehicle, lib, os.path.dirname(libraryfname))
if do_append:
alllibs.append(lib)
current_file = None
def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
global current_file
current_file = path
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
libraryfname = os.path.join(vehicle.path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s (fname=%s)" %
(path, library.name, libraryfname))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
(only_vehicles, param_name,
field_text) = (param_match[0], param_match[1], param_match[2])
if len(only_vehicles):
only_vehicles_list = [
x.strip() for x in only_vehicles.split(",")
]
for only_vehicle in only_vehicles_list:
if only_vehicle not in valid_truenames:
raise ValueError("Invalid only_vehicle %s" %
only_vehicle)
if vehicle.name not in only_vehicles_list:
continue
p = Parameter(library.name + param_name, current_file)
debug(p.name + ' ')
global current_param
current_param = p.name
fields = prog_param_fields.findall(field_text)
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
elif field[0] == "CopyValuesFrom":
setattr(p, field[0], field[1])
else:
error("param: unknown parameter metadata field %s" %
field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
# a parameter is considered to be vehicle-specific if
# there does not exist a Values: or Values{VehicleName}
# for that vehicle but @Values{OtherVehicle} exists.
seen_values_or_bitmask_for_other_vehicle = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [
x.rstrip().lstrip() for x in only_for_vehicles
]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug(
"field[0]=%s vehicle=%s field[1]=%s only_for_vehicles=%s\n"
%
(field[0], vehicle.name, field[1], str(only_for_vehicles)))
if field[0] not in known_param_fields:
error(
"tagged param: unknown parameter metadata field '%s'" %
field[0])
continue
if vehicle.name not in only_for_vehicles:
if len(only_for_vehicles) and field[0] in [
'Values', 'Bitmask'
]:
seen_values_or_bitmask_for_other_vehicle = True
continue
value = re.sub('@PREFIX@', library.name, field[2])
setattr(p, field[0], value)
if (getattr(p, 'Values', None) is not None
and getattr(p, 'Bitmask', None) is not None):
error("Both @Values and @Bitmask present")
if (getattr(p, 'Values', None) is None
and getattr(p, 'Bitmask', None) is None):
# values and Bitmask available for this vehicle
if seen_values_or_bitmask_for_other_vehicle:
# we've (e.g.) seen @Values{Copter} when we're
# processing for Rover, and haven't seen either
# @Values: or @Vales{Rover} - so we omit this
# parameter on the assumption that it is not
# applicable for this vehicle.
continue
p.path = path # Add path. Later deleted - only used for duplicates
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
done_groups = dict()
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
do_append = True
if group in done_groups:
# this is to handle cases like the RangeFinder
# parameters, where the wasp stuff gets tack into the
# same RNGFND1_ group
lib = done_groups[group]
do_append = False
else:
lib = Library(group)
done_groups[group] = lib
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(lib, field[0], field[1])
elif field[0] == "CopyValuesFrom":
setattr(p, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(lib.name == parsed_l.name for parsed_l in libraries):
if do_append:
lib.set_name(library.name + lib.name)
debug("Group name: %s" % lib.name)
process_library(vehicle, lib, os.path.dirname(libraryfname))
if do_append:
alllibs.append(lib)
current_file = None
param_matches = []
if not args.emit_sitl:
param_matches = prog_param.findall(p_text)
for param_match in param_matches:
(only_vehicles, param_name,
field_text) = (param_match[0], param_match[1], param_match[2])
if len(only_vehicles):
only_vehicles_list = [x.strip() for x in only_vehicles.split(",")]
for only_vehicle in only_vehicles_list:
if only_vehicle not in valid_truenames:
raise ValueError("Invalid only_vehicle %s" % only_vehicle)
if vehicle.truename not in only_vehicles_list:
continue
p = Parameter(vehicle.name + ":" + param_name, current_file)
debug(p.name + ' ')
current_param = p.name
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', "", field[1]).rstrip()
setattr(p, field[0], value)
else:
error("param: unknown parameter metadata field '%s'" %
field[0])
for req_field in required_param_fields:
if req_field not in field_list:
error("missing parameter metadata field '%s' in %s" %
def process_library(vehicle, library, pathprefix=None):
'''process one library'''
paths = library.Path.split(',')
for path in paths:
path = path.strip()
debug("\n Processing file '%s'" % path)
if pathprefix is not None:
libraryfname = os.path.join(pathprefix, path)
elif path.find('/') == -1:
if len(vehicles) != 1:
print("Unable to handle multiple vehicles with .pde library")
continue
libraryfname = os.path.join(vehicles[0].path, path)
else:
libraryfname = os.path.normpath(
os.path.join(apm_path + '/libraries/' + path))
if path and os.path.exists(libraryfname):
f = open(libraryfname)
p_text = f.read()
f.close()
else:
error("Path %s not found for library %s" % (path, library.name))
continue
param_matches = prog_param.findall(p_text)
debug("Found %u documented parameters" % len(param_matches))
for param_match in param_matches:
p = Parameter(library.name + param_match[0])
debug(p.name + ' ')
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
non_vehicle_specific_values_seen = False
for field in fields:
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[1])
setattr(p, field[0], value)
if field[0] == "Values":
non_vehicle_specific_values_seen = True
else:
error("param: unknown parameter metadata field %s" %
field[0])
debug("matching %s" % field_text)
fields = prog_param_tagged_fields.findall(field_text)
this_vehicle_values_seen = False
other_vehicle_values_seen = False
for field in fields:
only_for_vehicles = field[1].split(",")
only_for_vehicles = [
x.rstrip().lstrip() for x in only_for_vehicles
]
delta = set(only_for_vehicles) - set(truename_map.values())
if len(delta):
error("Unknown vehicles (%s)" % delta)
debug(
"field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n"
% (field[0], vehicle.name, vehicle.truename, field[1],
str(only_for_vehicles)))
if field[0] == "Values":
if vehicle.truename in only_for_vehicles:
this_vehicle_values_seen = True
if len(only_for_vehicles) > 1:
other_vehicle_values_seen = True
elif len(only_for_vehicles):
other_vehicle_values_seen = True
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', library.name, field[2])
setattr(p, field[0], value)
else:
error(
"tagged param<: unknown parameter metadata field '%s'"
% field[0])
if ((non_vehicle_specific_values_seen
or not other_vehicle_values_seen)
or this_vehicle_values_seen):
library.params.append(p)
group_matches = prog_groups.findall(p_text)
debug("Found %u groups" % len(group_matches))
debug(group_matches)
for group_match in group_matches:
group = group_match[0]
debug("Group: %s" % group)
l = Library(group)
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(l, field[0], field[1])
else:
error("unknown parameter metadata field '%s'" % field[0])
if not any(l.name == parsed_l.name for parsed_l in libraries):
l.name = library.name + l.name
debug("Group name: %s" % l.name)
process_library(vehicle, l, os.path.dirname(libraryfname))
alllibs.append(l)
debug(group_matches)
for group_match in group_matches:
l = Library(group_match[0])
fields = prog_param_fields.findall(group_match[1])
for field in fields:
if field[0] in known_group_fields:
setattr(l, field[0], field[1])
else:
error("group: unknown parameter metadata field '%s'" %
field[0])
if not any(l.name == parsed_l.name for parsed_l in libraries):
libraries.append(l)
for param_match in param_matches:
p = Parameter(vehicle.name + ":" + param_match[0])
debug(p.name + ' ')
field_text = param_match[1]
fields = prog_param_fields.findall(field_text)
field_list = []
for field in fields:
field_list.append(field[0])
if field[0] in known_param_fields:
value = re.sub('@PREFIX@', "", field[1])
setattr(p, field[0], value)
else:
error("param: unknown parameter metadata field '%s'" %
field[0])
for req_field in required_param_fields:
if req_field not in field_list:
error("missing parameter metadata field '%s' in %s" %
def add_param(self, param_id, shape, post=None, value=None):
if value is None:
value = Parameter(shape, self.init, name=param_id, post=post)
setattr(self, param_id, value)
self.params[param_id] = value
"""Run morphological segmentation on frequency data in `infile`, and save results in `outfile`."""
print('| Reading data...')
word_freq_list = read_word_freq_list(infile)
print('| Analyzing...')
morph_analyzer = MorphAnalyzer(params)
morph_analyzer.train(word_freq_list)
print('| Segmenting...')
word_list = [word for word, _freq in word_freq_list]
word_segs = morph_analyzer.segment_token_list(word_list)
print('| Saving result...')
save_segmentations(zip(word_list, word_segs), outfile)
print('| Done!')
if __name__ == '__main__':
parameters = Parameter()
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
'infile',
help=
'The input file containing a word list with line format: <word> <freq>'
)
arg_parser.add_argument(
'outfile', help='The output file to save the segmentation result')
arg_parser.add_argument('-p',
'--prune',
help='Whether use pruning (1|0, default:%s)' %
parameters.DoPruning,
type=bool,
default=parameters.DoPruning)
arg_parser.add_argument(