def __init__(self, input_size, output_size, initialization='normal', uniform=False): # print("Inside initialization method of FCLayer") # print("Input size:{}".format(input_size)) # print("Output size:{}".format(output_size)) """ :param input_size: (int) the number of nodes in the previous layer :param output_size: (int) the number of nodes in 'this' layer :param initialization: (str) whether to use Xavier initialization :param uniform: (bool) in Xavier initialization, whether to use uniform or truncated normal distribution """ if initialization == 'xavier': # print("Chosen initialization is Xavier") # Xavier initialization followed the implementation in Tensorflow. fan_in = input_size fan_out = output_size n = (fan_in + fan_out) / 2.0 if uniform: limit = math.sqrt(3.0 / n) weights = np.random.uniform(-limit, limit, size=(fan_in, fan_out)) bias = np.zeros((1, output_size)) else: trunc_std = math.sqrt(1.3 / n) a, b = -2, 2 # truncated between [- 2 * std, 2 * std] weights = stats.truncnorm.rvs(a, b, loc=0.0, scale=trunc_std, size=(fan_in, fan_out)) bias = np.zeros((1, output_size)) else: weights = np.random.randn(input_size, output_size) bias = np.random.randn(1, output_size) self.weights = Parameter(weights) # instantiate Parameter object by passing the initialized values self.bias = Parameter(bias) # instantiate Parameter object by passing the initialized values self.param = [self.weights, self.bias] self.forward_outputs = [] self.backprop_deltas = [] self.all_weights = [] self.all_weights_grads = [] self.all_bias_grads = [] self.backward_counter = 0 # print("Weights are: {}".format(self.weights.value)) # print("Shape of weights matrix is: {}".format(self.weights.value.shape)) # print("Bias matrix is: {}".format(self.bias.value)) # print("Shape of Bias matrix is: {}".format(self.bias.value.shape)) # print("self.param: {}".format(self.param)) # print("Len of self.param list is: {}".format(len(self.param))) # # print("FC Layer created") # print("..................................................................\n") self.backward_counter = 0
def __init__(self, linkage, name, parent, L, theta, phi=None, ux=None, uz=None, locked=False): super(FromPointLine, self).__init__(linkage, name) self.parent = parent self.locked = locked self.L = L phi = np.pi/2 if phi is None else phi*np.pi/180 self.ux = ux self.uz = uz self.p1 = OnPointPoint(self.linkage, '{}.{}'.format(self.name, '1'), parent=parent) self.p2 = CalculatedAlphaPoint(self.linkage, '{}.{}'.format(self.name, '2'), parent=self) self.params.theta = Parameter([theta*np.pi/180/ANGLE_FACTOR], self, 'theta', range=[0,2*np.pi], units='rad', locked=self.locked) self.params.phi = Parameter([phi/ANGLE_FACTOR], self, 'phi', range=[0,2*np.pi], units='rad', locked=True)
def __init__(self, linkage, name, parent, L, theta, phi=None, ux=None, uz=None, beta=None): super(OnPointLine, self).__init__(linkage, name) self.parent = parent self.L = L phi = np.pi/2 if phi is None else phi*np.pi/180 self.ux = ux self.uz = uz beta = 0.5 if beta is None else beta self.p1 = CalculatedAnteriorPoint(self.linkage, '{}.{}'.format(self.name, '1'), parent=self) self.p2 = CalculatedPosteriorPoint(self.linkage, '{}.{}'.format(self.name, '2'), parent=self) self.params.theta = Parameter([theta*np.pi/180/ANGLE_FACTOR], self, 'theta', range=[0,2*np.pi], units='rad', locked=False) self.params.phi = Parameter([phi/ANGLE_FACTOR], self, 'phi', range=[0,2*np.pi], units='rad', locked=True) self.params.beta = Parameter([beta], self, 'beta', range=[0,1], units=None, locked=False)
def process_vehicle(vehicle): debug("===\n\n\nProcessing %s" % vehicle.name) current_file = vehicle.path + '/Parameters.cpp' f = open(current_file) p_text = f.read() f.close() group_matches = prog_groups.findall(p_text) debug(group_matches) for group_match in group_matches: lib = Library(group_match[0]) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) else: error("group: unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): libraries.append(lib) param_matches = [] if not args.emit_sitl: param_matches = prog_param.findall(p_text) for param_match in param_matches: (only_vehicles, param_name, field_text) = (param_match[0], param_match[1], param_match[2]) if len(only_vehicles): only_vehicles_list = [x.strip() for x in only_vehicles.split(",")] for only_vehicle in only_vehicles_list: if only_vehicle not in valid_truenames: raise ValueError("Invalid only_vehicle %s" % only_vehicle) if vehicle.truename not in only_vehicles_list: continue p = Parameter(vehicle.reference + ":" + param_name, current_file) debug(p.name + ' ') global current_param current_param = p.name fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: value = re.sub('@PREFIX@', "", field[1]).rstrip() setattr(p, field[0], value) elif field[0] == "CopyValuesFrom": setattr(p, field[0], field[1]) else: error("param: unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list: error("missing parameter metadata field '%s' in %s" % (req_field, field_text)) vehicle.params.append(p) current_file = None debug("Processed %u params" % len(vehicle.params))
def __init__(self, linkage, name, parent1, parent2, L, gamma=None): super(OnPointsLine, self).__init__(linkage, name) self.parent1 = parent1 self.parent2 = parent2 self.L = L gamma = 0.5 if gamma is None else gamma self.p1 = CalculatedAnteriorGammaPoint(self.linkage, '{}.{}'.format(self.name, '1'), parent=self) self.p2 = CalculatedPosteriorGammaPoint(self.linkage, '{}.{}'.format(self.name, '2'), parent=self) self.params.gamma = Parameter([gamma], self, 'gamma', range=[0,1], units=None, locked=False)
def process_library(library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: if len(vehicles) != 1: print("Unable to handle multiple vehicles with .pde library") continue libraryfname = os.path.join(vehicles[0].path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s" % (path, library.name)) continue param_matches = prog_group_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: p = Parameter(library.name + param_match[0]) debug(p.name + ' ') field_text = param_match[1] fields = prog_param_fields.findall(field_text) for field in fields: if field[0] in known_param_fields: setattr(p, field[0], field[1]) else: error("unknown parameter metadata field %s" % field[0]) library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) l = Library(group) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(l, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(l.name == parsed_l.name for parsed_l in libraries): l.name = library.name + l.name debug("Group name: %s" % l.name) process_library(l, os.path.dirname(libraryfname)) alllibs.append(l)
def __init__(self, linkage, name, parent, alpha): super(OnLinePoint, self).__init__(linkage, name) self.parent = parent alpha = 0.5 if alpha is None else alpha self.params.alpha = Parameter([alpha], self, 'alpha', range=[0, 1], units=None, locked=False)
def __init__(self, input_size, output_size, initialization='normal', uniform=False): """ :param input_size: (int) the number of nodes in the previous layer :param output_size: (int) the number of nodes in 'this' layer :param initialization: (str) whether to use Xavier initialization :param uniform: (bool) in Xavier initialization, whether to use uniform or truncated normal distribution """ if initialization == 'xavier': # Xavier initialization followed the implementation in Tensorflow. fan_in = input_size fan_out = output_size n = (fan_in + fan_out) / 2.0 if uniform: limit = math.sqrt(3.0 / n) weights = np.random.uniform(-limit, limit, size=(fan_in, fan_out)) bias = np.zeros((1, output_size)) else: trunc_std = math.sqrt(1.3 / n) a, b = -2, 2 # truncated between [- 2 * std, 2 * std] weights = stats.truncnorm.rvs(a, b, loc=0.0, scale=trunc_std, size=(fan_in, fan_out)) bias = np.zeros((1, output_size)) else: weights = np.random.randn(input_size, output_size) bias = np.random.randn(1, output_size) self.weights = Parameter( weights ) # instantiate Parameter object by passing the initialized values self.bias = Parameter( bias ) # instantiate Parameter object by passing the initialized values self.param = [self.weights, self.bias] # store the weight and bias to a list
def __init__(self, linkage, name, at): super(AnchorPoint, self).__init__(linkage, name) self.params.x = Parameter(at[0], self, 'x', range=[-10, 10], units='m', locked=True) self.params.y = Parameter(at[1], self, 'y', range=[-10, 10], units='m', locked=True) self.params.z = Parameter(at[2], self, 'z', range=[-10, 10], units='m', locked=True)
def __init__(self, linkage, name, at, parent): super(ToPointPoint, self).__init__(linkage, name) self.parent = parent self.params.x = Parameter(at[0], self, 'x', range=[-10, 10], units='m', locked=False) self.params.y = Parameter(at[1], self, 'y', range=[-10, 10], units='m', locked=False) self.params.z = Parameter(at[2], self, 'z', range=[-10, 10], units='m', locked=False)
def run_turkish(): params = Parameter() params.UseTransRules = True params.DoPruning = False params.DoCompound = False params.ExcludeUnreliable = False params.BestNCandSuffix = 150 infile_train = r'data/wordlist.2010.tur.utf8.txt' infile_test_gold = r'data/mit/gold.tur.txt' run_experiment(infile_train, infile_test_gold, params)
def run_finnish(): """Runs an experiment on Finnish data against gold standard results.""" params = Parameter() params.UseTransRules = False params.DoPruning = True params.DoCompound = True params.ExcludeUnreliable = True params.BestNCandSuffix = 150 infile_train = r'data/wordlist.2010.fin.utf8.txt' infile_test_gold = r'data/mit/gold.fin.txt' run_experiment(infile_train, infile_test_gold, params)
def _get_basic_type(_p: param.Parameter) -> Union[type, Callable]: """ Given a parameter, get its basic Python type, e.g.: param.Boolean -> bool. Throw exception if it is not supported. :param _p: parameter to get type and nargs for. :return: Type """ if isinstance(_p, param.Boolean): p_type: Union[type, Callable] = lambda x: (str(x).lower() == 'true') elif isinstance(_p, param.Integer): p_type = lambda x: _p.default if x == "" else int(x) elif isinstance(_p, param.Number): p_type = lambda x: _p.default if x == "" else float(x) elif isinstance(_p, param.String): p_type = str elif isinstance(_p, param.List): p_type = lambda x: [_p.class_(item) for item in x.split(',')] elif isinstance(_p, param.NumericTuple): float_or_int = lambda y: int(y) if isinstance(_p, IntTuple ) else float(y) p_type = lambda x: tuple( [float_or_int(item) for item in x.split(',')]) elif isinstance(_p, param.ClassSelector): p_type = _p.class_ elif isinstance(_p, ListOrDictParam): def list_or_dict(x: str) -> Union[Dict, List]: import json if x.startswith("{") or x.startswith('['): res = json.loads(x) else: res = [str(item) for item in x.split(',')] if isinstance(res, Dict): return res elif isinstance(res, List): return res else: raise ValueError( f"Parameter of type {_p} should resolve to List or Dict" ) p_type = list_or_dict else: raise TypeError( "Parameter of type: {} is not supported".format(_p)) return p_type
def build_param(key, value): return str(Parameter(key, value))
def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() global current_file current_file = path debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: if len(vehicles) != 1: print("Unable to handle multiple vehicles with .pde library") continue libraryfname = os.path.join(vehicles[0].path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s (fname=%s)" % (path, library.name, libraryfname)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: p = Parameter(library.name + param_match[0], current_file) debug(p.name + ' ') global current_param current_param = p.name field_text = param_match[1] fields = prog_param_fields.findall(field_text) non_vehicle_specific_values_seen = False for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) if field[0] == "Values": non_vehicle_specific_values_seen = True else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) this_vehicle_values_seen = False this_vehicle_value = None other_vehicle_values_seen = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [ x.rstrip().lstrip() for x in only_for_vehicles ] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug( "field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, vehicle.truename, field[1], str(only_for_vehicles))) value = re.sub('@PREFIX@', library.name, field[2]) if field[0] == "Values": if vehicle.truename in only_for_vehicles: this_vehicle_values_seen = True this_vehicle_value = value if len(only_for_vehicles) > 1: other_vehicle_values_seen = True elif len(only_for_vehicles): other_vehicle_values_seen = True if field[0] in known_param_fields: setattr(p, field[0], value) else: error( "tagged param<: unknown parameter metadata field '%s'" % field[0]) if ((non_vehicle_specific_values_seen or not other_vehicle_values_seen) or this_vehicle_values_seen): if this_vehicle_values_seen and field[0] == 'Values': setattr(p, field[0], this_vehicle_value) # debug("Appending (non_vehicle_specific_values_seen=%u " # "other_vehicle_values_seen=%u this_vehicle_values_seen=%u)" % # (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen)) p.path = path # Add path. Later deleted - only used for duplicates library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) done_groups = dict() for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) do_append = True if group in done_groups: # this is to handle cases like the RangeFinder # parameters, where the wasp stuff gets tack into the # same RNGFND1_ group lib = done_groups[group] do_append = False else: lib = Library(group) done_groups[group] = lib fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): if do_append: lib.name = library.name + lib.name debug("Group name: %s" % lib.name) process_library(vehicle, lib, os.path.dirname(libraryfname)) if do_append: alllibs.append(lib) current_file = None
debug(group_matches) for group_match in group_matches: lib = Library(group_match[0]) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) else: error("group: unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): libraries.append(lib) for param_match in param_matches: p = Parameter(vehicle.name + ":" + param_match[0], current_file) debug(p.name + ' ') current_param = p.name field_text = param_match[1] fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: value = re.sub('@PREFIX@', "", field[1]).rstrip() setattr(p, field[0], value) else: error("param: unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list:
def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() global current_file current_file = path debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: if len(vehicles) != 1: print("Unable to handle multiple vehicles with .pde library") continue libraryfname = os.path.join(vehicles[0].path, path) else: libraryfname = os.path.normpath(os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s" % (path, library.name)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: p = Parameter(library.name+param_match[0], current_file) debug(p.name + ' ') global current_param current_param = p.name field_text = param_match[1] fields = prog_param_fields.findall(field_text) non_vehicle_specific_values_seen = False for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) if field[0] == "Values": non_vehicle_specific_values_seen = True else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) this_vehicle_values_seen = False this_vehicle_value = None other_vehicle_values_seen = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [x.rstrip().lstrip() for x in only_for_vehicles] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug("field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, vehicle.truename, field[1], str(only_for_vehicles))) value = re.sub('@PREFIX@', library.name, field[2]) if field[0] == "Values": if vehicle.truename in only_for_vehicles: this_vehicle_values_seen = True this_vehicle_value = value if len(only_for_vehicles) > 1: other_vehicle_values_seen = True elif len(only_for_vehicles): other_vehicle_values_seen = True if field[0] in known_param_fields: setattr(p, field[0], value) else: error("tagged param<: unknown parameter metadata field '%s'" % field[0]) if ((non_vehicle_specific_values_seen or not other_vehicle_values_seen) or this_vehicle_values_seen): if this_vehicle_values_seen: debug("Setting vehicle-specific value (%s)" % str(this_vehicle_value)) setattr(p, field[0], this_vehicle_value) # debug("Appending (non_vehicle_specific_values_seen=%u " # "other_vehicle_values_seen=%u this_vehicle_values_seen=%u)" % # (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen)) p.path = path # Add path. Later deleted - only used for duplicates library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) lib = Library(group) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): lib.name = library.name + lib.name debug("Group name: %s" % lib.name) process_library(vehicle, lib, os.path.dirname(libraryfname)) alllibs.append(lib) current_file = None
error("group: unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): libraries.append(lib) for param_match in param_matches: (only_vehicles, param_name, field_text) = (param_match[0], param_match[1], param_match[2]) if len(only_vehicles): only_vehicles_list = [x.strip() for x in only_vehicles.split(",")] for only_vehicle in only_vehicles_list: if only_vehicle not in valid_truenames: raise ValueError("Invalid only_vehicle %s" % only_vehicle) if vehicle.truename not in only_vehicles_list: continue p = Parameter(vehicle.name+":"+param_name, current_file) debug(p.name + ' ') current_param = p.name fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: value = re.sub('@PREFIX@', "", field[1]).rstrip() setattr(p, field[0], value) else: error("param: unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list: error("missing parameter metadata field '%s' in %s" % (req_field, field_text))
group_matches = prog_groups.findall(p_text) debug(group_matches) for group_match in group_matches: l = Library(group_match[0]) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(l, field[0], field[1]) else: error("group: unknown parameter metadata field '%s'" % field[0]) if not any(l.name == parsed_l.name for parsed_l in libraries): libraries.append(l) for param_match in param_matches: p = Parameter(vehicle.name+":"+param_match[0]) debug(p.name + ' ') field_text = param_match[1] fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: setattr(p, field[0], field[1]) else: error("param: unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list: error("missing parameter metadata field '%s' in %s" % (req_field, field_text)) vehicle.params.append(p)
def run(infile, outfile, params): print('| Reading data...') word_freq_list = read_word_freq_list(infile) print('| Analyzing...') morph_analyzer = MorphAnalyzer(params) morph_analyzer.train(word_freq_list) print('| Segmenting...') word_list = [word for word, _freq in word_freq_list] word_segs = morph_analyzer.segment_token_list(word_list) print('| Saving result...') save_segmentations(zip(word_list, word_segs), outfile) print('| Done!') if __name__ == '__main__': params = Parameter() arg_parser = argparse.ArgumentParser() arg_parser.add_argument('infile', help='The input file containing a word list with line format: <word> <freq>') arg_parser.add_argument('outfile', help='The output file to save the segmentation result') arg_parser.add_argument('-p', '--prune', help='Whether use pruning (1|0, default:%s)' % params.DoPruning, type=bool, default=params.DoPruning) arg_parser.add_argument('-t', '--trans', help='Whether use transformation rules (1|0, default:%s)' % params.UseTransRules, type=bool, default=params.UseTransRules) arg_parser.add_argument('-c', '--comp', help='Whether process compounding (1|0, default:%s)' % params.DoCompound, type=bool, default=params.DoCompound) arg_parser.add_argument('-e', '--excl', help='Whether exclude unreliable roots (1|0, default:%s)' % params.ExcludeUnreliable, type=bool, default=params.ExcludeUnreliable) arg_parser.add_argument('-n', '--hyphen', help='Whether explicitly deal with hyphen words (1|0, default:%s)' % params.DoHyphen, type=bool, default=params.DoHyphen) arg_parser.add_argument('-a', '--apos', help='Whether explicitly deal with apostrophes (1|0, default:%s)' % params.DoApostrophe, type=bool, default=params.DoApostrophe) arg_parser.add_argument('-r', '--root', help='Minimal length of roots that will be possibly segmented (default:%s)' % params.MinStemLen, type=int, default=params.MinStemLen) arg_parser.add_argument('-s', '--suff', help='Maximal length of suffixes (default:%s)' % params.MaxSuffixLen, type=int, default=params.MaxSuffixLen) args = arg_parser.parse_args() params.DoPruning = args.prune params.UseTransRules = args.trans params.DoCompound = args.comp
def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() global current_file current_file = path debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: libraryfname = os.path.join(vehicle.path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s (fname=%s)" % (path, library.name, libraryfname)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: (only_vehicles, param_name, field_text) = (param_match[0], param_match[1], param_match[2]) if len(only_vehicles): only_vehicles_list = [ x.strip() for x in only_vehicles.split(",") ] for only_vehicle in only_vehicles_list: if only_vehicle not in valid_truenames: raise ValueError("Invalid only_vehicle %s" % only_vehicle) if vehicle.name not in only_vehicles_list: continue p = Parameter(library.name + param_name, current_file) debug(p.name + ' ') global current_param current_param = p.name fields = prog_param_fields.findall(field_text) for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) elif field[0] == "CopyValuesFrom": setattr(p, field[0], field[1]) else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) # a parameter is considered to be vehicle-specific if # there does not exist a Values: or Values{VehicleName} # for that vehicle but @Values{OtherVehicle} exists. seen_values_or_bitmask_for_other_vehicle = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [ x.rstrip().lstrip() for x in only_for_vehicles ] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug( "field[0]=%s vehicle=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, field[1], str(only_for_vehicles))) if field[0] not in known_param_fields: error( "tagged param: unknown parameter metadata field '%s'" % field[0]) continue if vehicle.name not in only_for_vehicles: if len(only_for_vehicles) and field[0] in [ 'Values', 'Bitmask' ]: seen_values_or_bitmask_for_other_vehicle = True continue value = re.sub('@PREFIX@', library.name, field[2]) setattr(p, field[0], value) if (getattr(p, 'Values', None) is not None and getattr(p, 'Bitmask', None) is not None): error("Both @Values and @Bitmask present") if (getattr(p, 'Values', None) is None and getattr(p, 'Bitmask', None) is None): # values and Bitmask available for this vehicle if seen_values_or_bitmask_for_other_vehicle: # we've (e.g.) seen @Values{Copter} when we're # processing for Rover, and haven't seen either # @Values: or @Vales{Rover} - so we omit this # parameter on the assumption that it is not # applicable for this vehicle. continue p.path = path # Add path. Later deleted - only used for duplicates library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) done_groups = dict() for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) do_append = True if group in done_groups: # this is to handle cases like the RangeFinder # parameters, where the wasp stuff gets tack into the # same RNGFND1_ group lib = done_groups[group] do_append = False else: lib = Library(group) done_groups[group] = lib fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(lib, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(lib.name == parsed_l.name for parsed_l in libraries): if do_append: lib.set_name(library.name + lib.name) debug("Group name: %s" % lib.name) process_library(vehicle, lib, os.path.dirname(libraryfname)) if do_append: alllibs.append(lib) current_file = None
#!/usr/bin/env python3 from param import Register, MemAddr, Parameter from testcase import Testcase # Choose out of various operands reg8 = Register('al') reg16 = Register('ax') reg32 = Register('eax') reg64 = Register('rax') xmm = Register('xmm0') ymm = Register('ymm0') zmm = Register('zmm0') mem0 = MemAddr('(%rax, %esi, 4)') imd1 = Parameter('IMD') # ----------------------------------------------- # -USER INPUT------------------------------------ # ----------------------------------------------- # Enter your mnemonic mnemonic = 'add' # Define your operands. If you don't need it, just type in None dst = mem0 op1 = imd1 op2 = None # Define the number of instructions per loop (default: 12) per_loop = '32' # -----------------------------------------------
def add_param(self, param_id, shape, post=None, value=None): if value is None: value = Parameter(shape, self.init, name=param_id, post=post) setattr(self, param_id, value) self.params[param_id] = value
group_matches = prog_groups.findall(p_text) debug(group_matches) for group_match in group_matches: l = Library(group_match[0]) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(l, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(l.name == parsed_l.name for parsed_l in libraries): libraries.append(l) for param_match in param_matches: p = Parameter(vehicle.name + ":" + param_match[0]) debug(p.name + ' ') field_text = param_match[1] fields = prog_param_fields.findall(field_text) field_list = [] for field in fields: field_list.append(field[0]) if field[0] in known_param_fields: setattr(p, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) for req_field in required_param_fields: if req_field not in field_list: error("missing parameter metadata field '%s' in %s" % (req_field, field_text))
def process_library(vehicle, library, pathprefix=None): '''process one library''' paths = library.Path.split(',') for path in paths: path = path.strip() debug("\n Processing file '%s'" % path) if pathprefix is not None: libraryfname = os.path.join(pathprefix, path) elif path.find('/') == -1: if len(vehicles) != 1: print("Unable to handle multiple vehicles with .pde library") continue libraryfname = os.path.join(vehicles[0].path, path) else: libraryfname = os.path.normpath( os.path.join(apm_path + '/libraries/' + path)) if path and os.path.exists(libraryfname): f = open(libraryfname) p_text = f.read() f.close() else: error("Path %s not found for library %s" % (path, library.name)) continue param_matches = prog_param.findall(p_text) debug("Found %u documented parameters" % len(param_matches)) for param_match in param_matches: p = Parameter(library.name + param_match[0]) debug(p.name + ' ') field_text = param_match[1] fields = prog_param_fields.findall(field_text) non_vehicle_specific_values_seen = False for field in fields: if field[0] in known_param_fields: value = re.sub('@PREFIX@', library.name, field[1]) setattr(p, field[0], value) if field[0] == "Values": non_vehicle_specific_values_seen = True else: error("param: unknown parameter metadata field %s" % field[0]) debug("matching %s" % field_text) fields = prog_param_tagged_fields.findall(field_text) this_vehicle_values_seen = False this_vehicle_value = None other_vehicle_values_seen = False for field in fields: only_for_vehicles = field[1].split(",") only_for_vehicles = [ x.rstrip().lstrip() for x in only_for_vehicles ] delta = set(only_for_vehicles) - set(truename_map.values()) if len(delta): error("Unknown vehicles (%s)" % delta) debug( "field[0]=%s vehicle=%s truename=%s field[1]=%s only_for_vehicles=%s\n" % (field[0], vehicle.name, vehicle.truename, field[1], str(only_for_vehicles))) value = re.sub('@PREFIX@', library.name, field[2]) if field[0] == "Values": if vehicle.truename in only_for_vehicles: this_vehicle_values_seen = True this_vehicle_value = value if len(only_for_vehicles) > 1: other_vehicle_values_seen = True elif len(only_for_vehicles): other_vehicle_values_seen = True if field[0] in known_param_fields: setattr(p, field[0], value) else: error( "tagged param<: unknown parameter metadata field '%s'" % field[0]) if ((non_vehicle_specific_values_seen or not other_vehicle_values_seen) or this_vehicle_values_seen): if this_vehicle_values_seen: debug("Setting vehicle-specific value (%s)" % str(this_vehicle_value)) setattr(p, field[0], this_vehicle_value) debug( "Appending (non_vehicle_specific_values_seen=%u other_vehicle_values_seen=%u this_vehicle_values_seen=%u!)" % (non_vehicle_specific_values_seen, other_vehicle_values_seen, this_vehicle_values_seen)) library.params.append(p) group_matches = prog_groups.findall(p_text) debug("Found %u groups" % len(group_matches)) debug(group_matches) for group_match in group_matches: group = group_match[0] debug("Group: %s" % group) l = Library(group) fields = prog_param_fields.findall(group_match[1]) for field in fields: if field[0] in known_group_fields: setattr(l, field[0], field[1]) else: error("unknown parameter metadata field '%s'" % field[0]) if not any(l.name == parsed_l.name for parsed_l in libraries): l.name = library.name + l.name debug("Group name: %s" % l.name) process_library(vehicle, l, os.path.dirname(libraryfname)) alllibs.append(l)