def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() global current_file current_file = path debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: if len(vehicles) != 1: print("Unable to handle multiple vehicles with .pde library") continue libraryfname = os.path.join(vehicles[0].path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s (fname=%s)" % (path, library.name, libraryfname)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: (only_vehicles, param_name, field_text) = (param_match[0], param_match[1], param_match[2]) if len(only_vehicles): only_vehicles_list = [ x.strip() for x in only_vehicles.split(",") ] for only_vehicle in only_vehicles_list: if only_vehicle not in valid_truenames: raise ValueError("Invalid only_vehicle %s" % only_vehicle) if vehicle.truename not in only_vehicles_list: continue p = Parameter(library.name + param_name, current_file) debug(p.name + ' ') global current_param current_param = p.name fields = prog_param_fields.findall(field_text) non_vehicle_specific_values_seen = False for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) if field[0] == "Values": non_vehicle_specific_values_seen = True else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) this_vehicle_values_seen = False this_vehicle_value = None other_vehicle_values_seen = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [ x.rstrip().lstrip() for x in only_for_vehicles ] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug( "field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, vehicle.truename, field[1], str(only_for_vehicles))) value = re.sub('@PREFIX@', library.name, field[2]) if field[0] == "Values": if vehicle.truename in only_for_vehicles: this_vehicle_values_seen = True this_vehicle_value = value if len(only_for_vehicles) > 1: other_vehicle_values_seen = True elif len(only_for_vehicles): other_vehicle_values_seen = True if field[0] in known_param_fields: setattr(p, field[0], value) else: error( "tagged param<: unknown parameter metadata field '%s'" % field[0]) if ((non_vehicle_specific_values_seen or not other_vehicle_values_seen) or this_vehicle_values_seen): if this_vehicle_values_seen and field[0] == 'Values': setattr(p, field[0], this_vehicle_value) # debug("Appending (non_vehicle_specific_values_seen=%u " # "other_vehicle_values_seen=%u this_vehicle_values_seen=%u)" % # (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen)) p.path = path # Add path. Later deleted - only used for duplicates library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) done_groups = dict() for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) do_append = True if group in done_groups: # this is to handle cases like the RangeFinder # parameters, where the wasp stuff gets tack into the # same RNGFND1_ group lib = done_groups[group] do_append = False else: lib = Library(group) done_groups[group] = lib fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): if do_append: lib.name = library.name + lib.name debug("Group name: %s" % lib.name) process_library(vehicle, lib, os.path.dirname(libraryfname)) if do_append: alllibs.append(lib) current_file = None
def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() global current_file current_file = path debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: libraryfname = os.path.join(vehicle.path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s (fname=%s)" % (path, library.name, libraryfname)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: (only_vehicles, param_name, field_text) = (param_match[0], param_match[1], param_match[2]) if len(only_vehicles): only_vehicles_list = [ x.strip() for x in only_vehicles.split(",") ] for only_vehicle in only_vehicles_list: if only_vehicle not in valid_truenames: raise ValueError("Invalid only_vehicle %s" % only_vehicle) if vehicle.name not in only_vehicles_list: continue p = Parameter(library.name + param_name, current_file) debug(p.name + ' ') global current_param current_param = p.name fields = prog_param_fields.findall(field_text) for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) elif field[0] == "CopyValuesFrom": setattr(p, field[0], field[1]) else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) # a parameter is considered to be vehicle-specific if # there does not exist a Values: or Values{VehicleName} # for that vehicle but @Values{OtherVehicle} exists. seen_values_or_bitmask_for_other_vehicle = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [ x.rstrip().lstrip() for x in only_for_vehicles ] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug( "field[0]=%s vehicle=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, field[1], str(only_for_vehicles))) if field[0] not in known_param_fields: error( "tagged param: unknown parameter metadata field '%s'" % field[0]) continue if vehicle.name not in only_for_vehicles: if len(only_for_vehicles) and field[0] in [ 'Values', 'Bitmask' ]: seen_values_or_bitmask_for_other_vehicle = True continue value = re.sub('@PREFIX@', library.name, field[2]) setattr(p, field[0], value) if (getattr(p, 'Values', None) is not None and getattr(p, 'Bitmask', None) is not None): error("Both @Values and @Bitmask present") if (getattr(p, 'Values', None) is None and getattr(p, 'Bitmask', None) is None): # values and Bitmask available for this vehicle if seen_values_or_bitmask_for_other_vehicle: # we've (e.g.) seen @Values{Copter} when we're # processing for Rover, and haven't seen either # @Values: or @Vales{Rover} - so we omit this # parameter on the assumption that it is not # applicable for this vehicle. continue p.path = path # Add path. Later deleted - only used for duplicates library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) done_groups = dict() for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) do_append = True if group in done_groups: # this is to handle cases like the RangeFinder # parameters, where the wasp stuff gets tack into the # same RNGFND1_ group lib = done_groups[group] do_append = False else: lib = Library(group) done_groups[group] = lib fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) elif field[0] == "CopyValuesFrom": setattr(p, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): if do_append: lib.set_name(library.name + lib.name) debug("Group name: %s" % lib.name) process_library(vehicle, lib, os.path.dirname(libraryfname)) if do_append: alllibs.append(lib) current_file = None
param_matches = [] if not args.emit_sitl: param_matches = prog_param.findall(p_text) for param_match in param_matches: (only_vehicles, param_name, field_text) = (param_match[0], param_match[1], param_match[2]) if len(only_vehicles): only_vehicles_list = [x.strip() for x in only_vehicles.split(",")] for only_vehicle in only_vehicles_list: if only_vehicle not in valid_truenames: raise ValueError("Invalid only_vehicle %s" % only_vehicle) if vehicle.truename not in only_vehicles_list: continue p = Parameter(vehicle.name + ":" + param_name, current_file) debug(p.name + ' ') current_param = p.name fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: value = re.sub('@PREFIX@', "", field[1]).rstrip() setattr(p, field[0], value) else: error("param: unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list: error("missing parameter metadata field '%s' in %s" %
def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: if len(vehicles) != 1: print("Unable to handle multiple vehicles with .pde library") continue libraryfname = os.path.join(vehicles[0].path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s" % (path, library.name)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: p = Parameter(library.name + param_match[0]) debug(p.name + ' ') field_text = param_match[1] fields = prog_param_fields.findall(field_text) non_vehicle_specific_values_seen = False for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) if field[0] == "Values": non_vehicle_specific_values_seen = True else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) this_vehicle_values_seen = False other_vehicle_values_seen = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [ x.rstrip().lstrip() for x in only_for_vehicles ] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug( "field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, vehicle.truename, field[1], str(only_for_vehicles))) if field[0] == "Values": if vehicle.truename in only_for_vehicles: this_vehicle_values_seen = True if len(only_for_vehicles) > 1: other_vehicle_values_seen = True elif len(only_for_vehicles): other_vehicle_values_seen = True if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[2]) setattr(p, field[0], value) else: error( "tagged param<: unknown parameter metadata field '%s'" % field[0]) if ((non_vehicle_specific_values_seen or not other_vehicle_values_seen) or this_vehicle_values_seen): library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) l = Library(group) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(l, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(l.name == parsed_l.name for parsed_l in libraries): l.name = library.name + l.name debug("Group name: %s" % l.name) process_library(vehicle, l, os.path.dirname(libraryfname)) alllibs.append(l)
debug(group_matches) for group_match in group_matches: l = Library(group_match[0]) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(l, field[0], field[1]) else: error("group: unknown parameter metadata field '%s'" % field[0]) if not any(l.name == parsed_l.name for parsed_l in libraries): libraries.append(l) for param_match in param_matches: p = Parameter(vehicle.name + ":" + param_match[0]) debug(p.name + ' ') field_text = param_match[1] fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: value = re.sub('@PREFIX@', "", field[1]) setattr(p, field[0], value) else: error("param: unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list: error("missing parameter metadata field '%s' in %s" %
def add_param(self, param_id, shape, post=None, value=None): if value is None: value = Parameter(shape, self.init, name=param_id, post=post) setattr(self, param_id, value) self.params[param_id] = value
"""Run morphological segmentation on frequency data in `infile`, and save results in `outfile`.""" print('| Reading data...') word_freq_list = read_word_freq_list(infile) print('| Analyzing...') morph_analyzer = MorphAnalyzer(params) morph_analyzer.train(word_freq_list) print('| Segmenting...') word_list = [word for word, _freq in word_freq_list] word_segs = morph_analyzer.segment_token_list(word_list) print('| Saving result...') save_segmentations(zip(word_list, word_segs), outfile) print('| Done!') if __name__ == '__main__': parameters = Parameter() arg_parser = argparse.ArgumentParser() arg_parser.add_argument( 'infile', help= 'The input file containing a word list with line format: <word> <freq>' ) arg_parser.add_argument( 'outfile', help='The output file to save the segmentation result') arg_parser.add_argument('-p', '--prune', help='Whether use pruning (1|0, default:%s)' % parameters.DoPruning, type=bool, default=parameters.DoPruning) arg_parser.add_argument(