def _adjust_ge100_branches(levellist): """This adjust branches that are greater than or equal to 100% to be 100% - sum(other branches). This helps prevent unphysical errors downstream. """ n = len(levellist) brsum = defaultdict(float) bridx = defaultdict(lambda: (-1, -1.0)) baddies = [] for i, (nuc, rx, hl, lvl, br, ms, sp) in enumerate(levellist): if rx == 0: continue if br >= bridx[nuc][1]: bridx[nuc] = (i, br) brsum[nuc] += br nucrx = (nuc, rx) if nucrx in _BAD_RX: baddies.append(i) # adjust branch ratios for nuc, (i, br) in bridx.items(): row = levellist[i] # this line ensures that all branches sum to 100.0 within floating point new_br = 100.0 - brsum[nuc] + br new_row = row[:4] + (new_br,) + row[5:] levellist[i] = new_row # remove bad reaction rows for i in baddies[::-1]: del levellist[i]
def resume(self): """ Resets states so that it can behave in a resumed mode """ self._end_buffering_state = dict() self._is_flushed = defaultdict(lambda: False) self._sequence_no = defaultdict(lambda: 0xFF) self._last_packet_received = defaultdict(lambda: None) self._last_packet_sent = defaultdict(lambda: None) self._end_buffering_sequence_no = dict()
def resume(self): """ Resets states so that it can behave in a resumed mode """ self._end_buffering_state = dict() self._is_flushed = defaultdict(lambda: False) self._sequence_no = defaultdict(lambda: 0xFF) self._last_packet_received = defaultdict(lambda: None) self._last_packet_sent = defaultdict(lambda: None) self._end_buffering_sequence_no = dict()
def reset(self): if os.path.exists(self._db_file): if self._db: self._db.close() os.remove(self._db_file) self._db = SqlLiteDatabase(self._db_file) self._is_flushed = defaultdict(lambda: False) self._sequence_no = defaultdict(lambda: 0xFF) self._last_packet_received = defaultdict(lambda: None) self._last_packet_sent = defaultdict(lambda: None) self._end_buffering_sequence_no = dict() self._end_buffering_state = dict()
def reset(self): if os.path.exists(self._db_file): if self._db: self._db.close() os.remove(self._db_file) self._db = SqlLiteDatabase(self._db_file) self._is_flushed = defaultdict(lambda: False) self._sequence_no = defaultdict(lambda: 0xFF) self._last_packet_received = defaultdict(lambda: None) self._last_packet_sent = defaultdict(lambda: None) self._end_buffering_sequence_no = dict() self._end_buffering_state = dict()
def __call__(self, live_packet_gatherer_parameters, machine, machine_graph, application_graph=None, graph_mapper=None): """ Add LPG vertices on Ethernet connected chips as required. :param live_packet_gatherer_parameters:\ the Live Packet Gatherer parameters requested by the script :param machine: the SpiNNaker machine as discovered :param application_graph: the application graph :param machine_graph: the machine graph :return: mapping between LPG parameters and LPG vertex """ # pylint: disable=too-many-arguments # create progress bar progress = ProgressBar(machine.ethernet_connected_chips, string_describing_what_being_progressed=( "Adding Live Packet Gatherers to Graph")) # Keep track of the vertices added by parameters and board address lpg_params_to_vertices = defaultdict(dict) # for every Ethernet connected chip, add the gatherers required for chip in progress.over(machine.ethernet_connected_chips): for params in live_packet_gatherer_parameters: if (params.board_address is None or params.board_address == chip.ip_address): lpg_params_to_vertices[params][chip.x, chip.y] = \ self._add_lpg_vertex(application_graph, graph_mapper, machine_graph, chip, params) return lpg_params_to_vertices
def __call__( self, live_packet_gatherer_parameters, machine, machine_graph, application_graph=None, graph_mapper=None): """ Add LPG vertices on Ethernet connected chips as required. :param live_packet_gatherer_parameters:\ the Live Packet Gatherer parameters requested by the script :param machine: the SpiNNaker machine as discovered :param application_graph: the application graph :param machine_graph: the machine graph :return: mapping between LPG parameters and LPG vertex """ # pylint: disable=too-many-arguments # create progress bar progress = ProgressBar( machine.ethernet_connected_chips, string_describing_what_being_progressed=( "Adding Live Packet Gatherers to Graph")) # Keep track of the vertices added by parameters and board address lpg_params_to_vertices = defaultdict(dict) # for every Ethernet connected chip, add the gatherers required for chip in progress.over(machine.ethernet_connected_chips): for params in live_packet_gatherer_parameters: if (params.board_address is None or params.board_address == chip.ip_address): lpg_params_to_vertices[params][chip.x, chip.y] = \ self._add_lpg_vertex(application_graph, graph_mapper, machine_graph, chip, params) return lpg_params_to_vertices
def __init__(self): # Mapping of (board address, tag) to IPTag self._ip_tags = dict() # Mapping of (board address, tag) to ReverseIPTag self._reverse_ip_tags = dict() # Mapping of vertex to list of IPTag self._ip_tags_by_vertex = defaultdict(list) # Mapping of vertex to list of ReverseIPTag self._reverse_ip_tags_by_vertex = defaultdict(list) # Set of ports already assigned on a board self._ports_assigned = set()
def __init__( self, x, y, ip_address, report_default_directory, write_data_speed_up_report, constraints=None): super(DataSpeedUpPacketGatherMachineVertex, self).__init__( label="mc_data_speed_up_packet_gatherer_on_{}_{}".format(x, y), constraints=constraints) # data holders for the output, and sequence numbers self._view = None self._max_seq_num = None self._output = None # Create a connection to be used self._x = x self._y = y self._ip_address = ip_address self._remote_tag = None # local provenance storage self._provenance_data_items = defaultdict(list) # create report if it doesn't already exist self._report_path = \ os.path.join(report_default_directory, self.REPORT_NAME) self._write_data_speed_up_report = write_data_speed_up_report # Stored reinjection status for resetting timeouts self._last_status = None
def __init__(self, n_neurons, delay_per_stage, source_vertex, machine_time_step, timescale_factor, constraints=None, label="DelayExtension"): """ :param n_neurons: the number of neurons :param delay_per_stage: the delay per stage :param source_vertex: where messages are coming from :param machine_time_step: how long is the machine time step :param timescale_factor: what slowdown factor has been applied :param constraints: the vertex constraints :param label: the vertex label """ # pylint: disable=too-many-arguments super(DelayExtensionVertex, self).__init__(label, constraints, 256) self._source_vertex = source_vertex self._n_delay_stages = 0 self._delay_per_stage = delay_per_stage self._delay_generator_data = defaultdict(list) self._n_subvertices = 0 self._n_data_specs = 0 # atom store self._n_atoms = n_neurons # Dictionary of vertex_slice -> delay block for data specification self._delay_blocks = dict() self.add_constraint( SameAtomsAsVertexConstraint(source_vertex))
def test_pg_files(self): """ That the pg files command tells us which files are associated with a particular PG """ file_count = 20 self.mount_a.run_shell(["mkdir", "mydir"]) self.mount_a.create_n_files("mydir/myfile", file_count) # Some files elsewhere in the system that we will ignore # to check that the tool is filtering properly self.mount_a.run_shell(["mkdir", "otherdir"]) self.mount_a.create_n_files("otherdir/otherfile", file_count) pgs_to_files = defaultdict(list) # Rough (slow) reimplementation of the logic for i in range(0, file_count): file_path = "mydir/myfile_{0}".format(i) ino = self.mount_a.path_to_ino(file_path) obj = "{0:x}.{1:08x}".format(ino, 0) pgid = json.loads(self.fs.mon_manager.raw_cluster_cmd( "osd", "map", self.fs.get_data_pool_name(), obj, "--format=json-pretty" ))['pgid'] pgs_to_files[pgid].append(file_path) log.info("{0}: {1}".format(file_path, pgid)) pg_count = self.fs.get_pgs_per_fs_pool() for pg_n in range(0, pg_count): pg_str = "{0}.{1}".format(self.fs.get_data_pool_id(), pg_n) out = self.fs.data_scan(["pg_files", "mydir", pg_str]) lines = [l for l in out.split("\n") if l] log.info("{0}: {1}".format(pg_str, lines)) self.assertSetEqual(set(lines), set(pgs_to_files[pg_str]))
def __init__(self, n_neurons, delay_per_stage, source_vertex, machine_time_step, timescale_factor, constraints=None, label="DelayExtension"): """ :param n_neurons: the number of neurons :param delay_per_stage: the delay per stage :param source_vertex: where messages are coming from :param machine_time_step: how long is the machine time step :param timescale_factor: what slowdown factor has been applied :param constraints: the vertex constraints :param label: the vertex label """ # pylint: disable=too-many-arguments super(DelayExtensionVertex, self).__init__(label, constraints, 256) self._source_vertex = source_vertex self._n_delay_stages = 0 self._delay_per_stage = delay_per_stage self._delay_generator_data = defaultdict(list) self._n_subvertices = 0 self._n_data_specs = 0 # atom store self._n_atoms = n_neurons # Dictionary of vertex_slice -> delay block for data specification self._delay_blocks = dict() self.add_constraint( SameAtomsAsVertexConstraint(source_vertex))
def _structure_dict_entry(): """Static method to generate entries for the structure dict.""" return { "pin": set(), "rdesc": set(), "rprop": set(), "pin_rdesc_rprop": defaultdict(lambda: {"data_tuples": []}), }
def _structure_dict_entry(): """Static method to generate entries for the structure dict.""" return { 'pin': set(), 'rdesc': set(), 'rprop': set(), 'pin_rdesc_rprop': defaultdict(lambda: {'data_tuples': []}) }
def get_all_perf_counters(self, prio_limit=PRIO_USEFUL): """ Return the perf counters currently known to this ceph-mgr instance, filtered by priority equal to or greater than `prio_limit`. The result is a map of string to dict, associating services (like "osd.123") with their counters. The counter dict for each service maps counter paths to a counter info structure, which is the information from the schema, plus an additional "value" member with the latest value. """ result = defaultdict(dict) for server in self.list_servers(): for service in server['services']: if service['type'] not in ("rgw", "mds", "osd", "mon"): continue schema = self.get_perf_schema(service['type'], service['id']) if not schema: self.log.warn("No perf counter schema for {0}.{1}".format( service['type'], service['id'])) continue # Value is returned in a potentially-multi-service format, # get just the service we're asking about svc_full_name = "{0}.{1}".format(service['type'], service['id']) schema = schema[svc_full_name] # Populate latest values for counter_path, counter_schema in schema.items(): # self.log.debug("{0}: {1}".format( # counter_path, json.dumps(counter_schema) # )) if counter_schema['priority'] < prio_limit: continue counter_info = dict(counter_schema) # Also populate count for the long running avgs if counter_schema['type'] & self.PERFCOUNTER_LONGRUNAVG: v, c = self.get_latest_avg(service['type'], service['id'], counter_path) counter_info['value'], counter_info['count'] = v, c result[svc_full_name][counter_path] = counter_info else: counter_info['value'] = self.get_latest( service['type'], service['id'], counter_path) result[svc_full_name][counter_path] = counter_info self.log.debug("returning {0} counter".format(len(result))) return result
def __init__(self, connection_selector): super(ReadIOBufProcess, self).__init__(connection_selector) # A dictionary of (x, y, p) -> iobuf address self._iobuf_address = dict() # A dictionary of (x, y, p) -> OrderedDict(n) -> bytearray self._iobuf = defaultdict(OrderedDict) # A dictionary of (x, y, p) -> OrderedDict(n) -> memoryview self._iobuf_view = defaultdict(OrderedDict) # A list of extra reads that need to be done as a result of the first # read = list of (x, y, p, n, base_address, size, offset) self._extra_reads = list() # A list of next reads that need to be done as a result of the first # read = list of (x, y, p, n, next_address, first_read_size) self._next_reads = list()
def __init__(self, fh): self.structure = defaultdict(Library._structure_dict_entry) self.intdict = { 0: self._linlin, 2: self._linlin, 3: self._loglin, 4: self._linlog, 5: self._loglog, } self.fh = fh # read headers for all tables self._read_headers()
def _get_all_values(worksheet, evaluate_formulas): data = worksheet.spreadsheet.values_get( worksheet.title, params={ "valueRenderOption": ( "UNFORMATTED_VALUE" if evaluate_formulas else "FORMULA" ), "dateTimeRenderOption": "FORMATTED_STRING", }, ) (row_offset, column_offset) = (1, 1) (last_row, last_column) = (worksheet.row_count, worksheet.col_count) values = data.get("values", []) rect_values = fill_gaps( values, rows=last_row - row_offset + 1, cols=last_column - column_offset + 1, ) cells = [ Cell(row=i + row_offset, col=j + column_offset, value=value) for i, row in enumerate(rect_values) for j, value in enumerate(row) ] # defaultdicts fill in gaps for empty rows/cells not returned by gdocs rows = defaultdict(lambda: defaultdict(str)) for cell in cells: row = rows.setdefault(int(cell.row), defaultdict(str)) row[cell.col] = cell.value if not rows: return [] all_row_keys = chain.from_iterable(row.keys() for row in rows.values()) rect_cols = range(1, max(all_row_keys) + 1) rect_rows = range(1, max(rows.keys()) + 1) return [[rows[i][j] for j in rect_cols] for i in rect_rows]
class TestRecordableSpikeInjector(BaseTestCase): _n_spikes = defaultdict(lambda: 0) _n_neurons = 100 def _inject(self, label, connection): time.sleep(0.1) for _ in range(5000): neuron_id = randint(0, self._n_neurons - 1) self._n_spikes[neuron_id] += 1 connection.send_spike(label, neuron_id) time.sleep(0.001) def recordable_spike_injector(self): p.setup(1.0) pop = p.Population(self._n_neurons, p.external_devices.SpikeInjector(), label="input") pop.record("spikes") connection = p.external_devices.SpynnakerLiveSpikesConnection( send_labels=["input"]) connection.add_start_callback("input", self._inject) p.run(10000) spikes = pop.get_data("spikes").segments[0].spiketrains p.end() spike_trains = dict() for spiketrain in spikes: i = spiketrain.annotations['source_index'] if __name__ == "__main__": if self._n_spikes[i] != len(spiketrain): print( "Incorrect number of spikes, expected {} but got {}:". format(self._n_spikes[i], len(spiketrain))) print(spiketrain) else: assert self._n_spikes[i] == len(spiketrain) spike_trains[i] = spiketrain for (index, count) in iteritems(self._n_spikes): if __name__ == "__main__": if index not in spike_trains: print("Neuron {} should have spiked {} times but didn't". format(index, count)) else: assert index in spike_trains def test_recordable_spike_injector(self): self.runsafe(self.recordable_spike_injector)
def test_too_many_ip_tags_for_1_board(self): n_extra_vertices = 3 machine = VirtualMachine(12, 12, with_wrap_arounds=True) eth_chips = machine.ethernet_connected_chips eth_chip = eth_chips[0] eth_chip_2 = machine.get_chip_at(eth_chip.x + 1, eth_chip.y + 1) eth_procs = [ proc.processor_id for proc in eth_chip.processors if not proc.is_monitor] procs = [proc for proc in eth_chip_2.processors if not proc.is_monitor] eth2_procs = [proc.processor_id for proc in procs] proc = procs[-1] eth_vertices = [ SimpleMachineVertex( ResourceContainer(iptags=[IPtagResource( "127.0.0.1", port=tag, strip_sdp=True)]), label="Ethernet Vertex {}".format(proc)) for tag in eth_chip.tag_ids] eth2_vertices = [ SimpleMachineVertex( ResourceContainer(iptags=[IPtagResource( "127.0.0.1", port=10000 + tag, strip_sdp=True)]), label="Ethernet 2 Vertex {}".format(proc)) for tag in range(n_extra_vertices)] placements = Placements( Placement(vertex, eth_chip.x, eth_chip.y, proc) for proc, vertex in zip(eth_procs, eth_vertices)) placements.add_placements( Placement(vertex, eth_chip_2.x, eth_chip_2.y, proc) for proc, vertex in zip(eth2_procs, eth2_vertices)) allocator = BasicTagAllocator() _, _, tags = allocator( machine, plan_n_timesteps=None, placements=placements) tags_by_board = defaultdict(set) for vertices in (eth_vertices, eth2_vertices): for vertex in vertices: iptags = tags.get_ip_tags_for_vertex(vertex) self.assertEqual( len(iptags), 1, "Incorrect number of tags assigned") placement = placements.get_placement_of_vertex(vertex) print(placement, "has tag", iptags[0]) self.assertFalse( iptags[0].tag in tags_by_board[iptags[0].board_address], "Tag used more than once") tags_by_board[iptags[0].board_address].add(iptags[0].tag) self.assertEqual( len(tags_by_board[eth_chip.ip_address]), len(eth_chip.tag_ids), "Wrong number of tags assigned to first Ethernet")
def __init__(self, ctx, fscid=None, name='cephfs', create=False): # Deliberately skip calling parent constructor self._ctx = ctx self.id = None self.name = None self.ec_profile = None self.metadata_pool_name = None self.metadata_overlay = False self.data_pool_name = None self.data_pools = None # Hack: cheeky inspection of ceph.conf to see what MDSs exist self.mds_ids = set() for line in open("ceph.conf").readlines(): match = re.match("^\[mds\.(.+)\]$", line) if match: self.mds_ids.add(match.group(1)) if not self.mds_ids: raise RuntimeError("No MDSs found in ceph.conf!") self.mds_ids = list(self.mds_ids) log.info("Discovered MDS IDs: {0}".format(self.mds_ids)) self.mon_manager = LocalCephManager() self.mds_daemons = dict([(id_, LocalDaemon("mds", id_)) for id_ in self.mds_ids]) self.client_remote = LocalRemote() self._conf = defaultdict(dict) if name is not None: if fscid is not None: raise RuntimeError("cannot specify fscid when creating fs") if create and not self.legacy_configured(): self.create() else: if fscid is not None: self.id = fscid self.getinfo(refresh=True) # Stash a reference to the first created filesystem on ctx, so # that if someone drops to the interactive shell they can easily # poke our methods. if not hasattr(self._ctx, "filesystem"): self._ctx.filesystem = self
def get_pool_list_with_stats(cls, application=None): # pylint: disable=too-many-locals pools = cls.get_pool_list(application) pools_w_stats = [] pg_summary = mgr.get("pg_summary") pool_stats = defaultdict( lambda: defaultdict(lambda: collections.deque(maxlen=10))) df = mgr.get("df") pool_stats_dict = dict([(p['id'], p['stats']) for p in df['pools']]) now = time.time() for pool_id, stats in pool_stats_dict.items(): for stat_name, stat_val in stats.items(): pool_stats[pool_id][stat_name].appendleft((now, stat_val)) for pool in pools: pool['pg_status'] = pg_summary['by_pool'][pool['pool'].__str__()] stats = pool_stats[pool['pool']] s = {} def get_rate(series): if len(series) >= 2: return differentiate(*series[0:1]) return 0 for stat_name, stat_series in stats.items(): s[stat_name] = { 'latest': stat_series[0][1], 'rate': get_rate(stat_series), 'series': [i for i in stat_series] } pool['stats'] = s pools_w_stats.append(pool) return pools_w_stats
def get_versioned_symbols(libs): """Get versioned symbols used in libraries :param libs: {realpath: soname} dict to search for versioned symbols e.g. {'/path/to/external_ref.so.1.2.3': 'external_ref.so.1'} :return: {soname: {depname: set([symbol_version])}} e.g. {'external_ref.so.1': {'libc.so.6', set(['GLIBC_2.5','GLIBC_2.12'])}} """ result = {} for path, elf in elf_file_filter(libs.keys()): # {depname: set(symbol_version)}, e.g. # {'libc.so.6', set(['GLIBC_2.5','GLIBC_2.12'])} elf_versioned_symbols = defaultdict(lambda: set()) for key, value in elf_find_versioned_symbols(elf): log.debug('path %s, key %s, value %s', path, key, value) elf_versioned_symbols[key].add(value) result[libs[path]] = elf_versioned_symbols return result
def __init__(self, population): """ :param population: the population to record for """ self._population = population # file flags, allows separate files for the recorded variables self._write_to_files_indicators = { 'spikes': None, 'gsyn_exc': None, 'gsyn_inh': None, 'v': None} # Create a dict of variable name -> bool array of indices in population # that are recorded (initially all False) self._indices_to_record = defaultdict( lambda: numpy.repeat(False, population.size))
def __init__(self, population): """ :param population: the population to record for """ self.__population = population # file flags, allows separate files for the recorded variables self.__write_to_files_indicators = { 'spikes': None, 'gsyn_exc': None, 'gsyn_inh': None, 'v': None} # Create a dict of variable name -> bool array of indices in population # that are recorded (initially all False) self.__indices_to_record = defaultdict( lambda: numpy.repeat(False, population.size))
def __call__(self, machine, file_path): """ :param machine: :param file_path: """ progress = ProgressBar( (machine.max_chip_x + 1) * (machine.max_chip_y + 1) + 2, "Converting to JSON machine") # write basic stuff json_obj = { "width": machine.max_chip_x + 1, "height": machine.max_chip_y + 1, "chip_resources": { "cores": CHIP_HOMOGENEOUS_CORES, "sdram": CHIP_HOMOGENEOUS_SDRAM, "sram": CHIP_HOMOGENEOUS_SRAM, "router_entries": ROUTER_HOMOGENEOUS_ENTRIES, "tags": CHIP_HOMOGENEOUS_TAGS}, "dead_chips": [], "dead_links": []} # handle exceptions (dead chips) exceptions = defaultdict(dict) for x in range(0, machine.max_chip_x + 1): for y in progress.over(range(0, machine.max_chip_y + 1), False): self._add_exceptions(json_obj, machine, x, y, exceptions) json_obj["chip_resource_exceptions"] = [ [x, y, exceptions[x, y]] for x, y in exceptions] progress.update() # dump to json file with open(file_path, "w") as f: json.dump(json_obj, f) progress.update() # validate the schema file_format_schemas.validate(json_obj, "machine.json") # update and complete progress bar progress.end() return file_path
def list_machines(t, machines, jobs): """ Display a table summarising the available machines and their load. Parameters ---------- t : :py:class:`.Terminal` An output styling object for stdout. machines : [{...}, ...] The list of machines and their properties returned from the server. jobs : [{...}, ...] The list of jobs and their properties returned from the server. Returns ------- An error code: 0 on success. """ machine_jobs = defaultdict(list) for job in jobs: machine_jobs[job["allocated_machine_name"]].append(job) table = [[ (t.underscore_bright, "Name"), (t.underscore_bright, "Num boards"), (t.underscore_bright, "In-use"), (t.underscore_bright, "Jobs"), (t.underscore_bright, "Tags"), ]] for machine in machines: table.append([ machine["name"], ((machine["width"] * machine["height"] * 3) - len(machine["dead_boards"])), sum(len(job["boards"]) for job in machine_jobs[machine["name"]]), len(machine_jobs[machine["name"]]), ", ".join(machine["tags"]), ]) print(render_table(table))
def __call__(self, machine_graph, plan_n_timesteps, file_path): """ :param machine_graph: The graph to convert :param plan_n_timesteps: number of timesteps to plan for :type plan_n_timesteps: int :param file_path: Where to write the JSON """ progress = ProgressBar( machine_graph.n_vertices + 1, "Converting to JSON graph") # write basic stuff json_graph = dict() # write vertices data vertices_resources = dict() json_graph["vertices_resources"] = vertices_resources edges_resources = defaultdict() json_graph["edges"] = edges_resources vertex_by_id = dict() partition_by_id = dict() for vertex in progress.over(machine_graph.vertices, False): self._convert_vertex( vertex, vertex_by_id, vertices_resources, edges_resources, machine_graph, plan_n_timesteps, partition_by_id) with open(file_path, "w") as f: json.dump(json_graph, f) progress.update() file_format_schemas.validate(json_graph, "machine_graph.json") progress.end() return file_path, vertex_by_id, partition_by_id
def _sort_left_over_verts_based_on_incoming_packets( self, machine_graph, placed_vertices, n_keys_map): """ sort left overs verts so that the ones with the most costly verts are at the front of the list :param MachineGraph machine_graph: machine graph :param set(MachineVertex) placed_vertices: the verts already placed :param AbstractMachinePartitionNKeysMap n_keys_map: map between partition to n keys. :return: new list of verts to process. :rtype: list(MachineVertex) """ vert_list = list() incoming_size_map = defaultdict(list) for vertex in machine_graph.vertices: if vertex not in placed_vertices: incoming_size = self._get_cost(vertex, machine_graph, n_keys_map) incoming_size_map[incoming_size].append(vertex) sorted_keys = sorted(incoming_size_map.keys(), reverse=True) for key in sorted_keys: vert_list.extend(incoming_size_map[key]) return vert_list
if os.environ.get('READTHEDOCS', None) != 'True': # scipy must be added in config.py as a mock install_requires.append('scipy') install_requires.append('csa') # Build a list of all project modules, as well as supplementary files main_package = "spynnaker" extensions = { ".aplx", ".boot", ".cfg", ".json", ".sql", ".template", ".xml", ".xsd", ".dict" } main_package_dir = os.path.join(os.path.dirname(__file__), main_package) start = len(main_package_dir) packages = [] package_data = defaultdict(list) for dirname, dirnames, filenames in os.walk(main_package_dir): if '__init__.py' in filenames: package = "{}{}".format(main_package, dirname[start:].replace(os.sep, '.')) packages.append(package) for filename in filenames: _, ext = os.path.splitext(filename) if ext in extensions: package = "{}{}".format(main_package, dirname[start:].replace(os.sep, '.')) package_data[package].append(filename) setup(name="sPyNNaker", version=__version__, description="SpiNNaker implementation of PyNN",
def get_wheel_elfdata(wheel_fn: str): full_elftree = {} nonpy_elftree = {} full_external_refs = {} versioned_symbols = defaultdict(lambda: set()) # type: Dict[str, Set[str]] uses_ucs2_symbols = False uses_PyFPE_jbuf = False with InGenericPkgCtx(wheel_fn) as ctx: shared_libraries_in_purelib = [] for fn, elf in elf_file_filter(ctx.iter_files()): # Check for invalid binary wheel format: no shared library should # be found in purelib so_path_split = fn.split(os.sep) # If this is in purelib, add it to the list of shared libraries in purelib if 'purelib' in so_path_split: shared_libraries_in_purelib.append(so_path_split[-1]) # If at least one shared library exists in purelib, this is going to # fail and there's no need to do further checks if not shared_libraries_in_purelib: log.debug('processing: %s', fn) elftree = lddtree(fn) for key, value in elf_find_versioned_symbols(elf): log.debug('key %s, value %s', key, value) versioned_symbols[key].add(value) is_py_ext, py_ver = elf_is_python_extension(fn, elf) # If the ELF is a Python extention, we definitely need to include # its external dependencies. if is_py_ext: full_elftree[fn] = elftree uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) if py_ver == 2: uses_ucs2_symbols |= any( True for _ in elf_find_ucs2_symbols(elf)) full_external_refs[fn] = lddtree_external_references(elftree, ctx.path) else: # If the ELF is not a Python extension, it might be included in # the wheel already because auditwheel repair vendored it, so # we will check whether we should include its internal # references later. nonpy_elftree[fn] = elftree # If at least one shared library exists in purelib, raise an error if shared_libraries_in_purelib: raise RuntimeError( ( 'Invalid binary wheel, found the following shared library/libraries in purelib folder:\n' '\t%s\n' 'The wheel has to be platlib compliant in order to be repaired by auditwheel.' ) % '\n\t'.join(shared_libraries_in_purelib) ) # Get a list of all external libraries needed by ELFs in the wheel. needed_libs = { lib for elf in itertools.chain(full_elftree.values(), nonpy_elftree.values()) for lib in elf['needed'] } for fn in nonpy_elftree.keys(): # If a non-pyextension ELF file is not needed by something else # inside the wheel, then it was not checked by the logic above and # we should walk its elftree. if basename(fn) not in needed_libs: full_elftree[fn] = nonpy_elftree[fn] full_external_refs[fn] = lddtree_external_references( nonpy_elftree[fn], ctx.path) log.debug(json.dumps(full_elftree, indent=4)) return (full_elftree, full_external_refs, versioned_symbols, uses_ucs2_symbols, uses_PyFPE_jbuf)
def __init__(self): self._data = defaultdict(dict)
class AbstractPyNNModel(object): """ A Model that can be passed in to a Population object in PyNN """ _max_atoms_per_core = defaultdict(lambda: sys.maxsize) @classmethod def set_model_max_atoms_per_core(cls, n_atoms=sys.maxsize): """ Set the maximum number of atoms per core for this model :param n_atoms: The new maximum, or None for the largest possible :type n_atoms: int or None """ AbstractPyNNModel._max_atoms_per_core[cls] = n_atoms @classmethod def get_max_atoms_per_core(cls): """ Get the maximum number of atoms per core for this model :rtype: int """ return AbstractPyNNModel._max_atoms_per_core[cls] @staticmethod def _get_init_params_and_svars(cls): init = getattr(cls, "__init__") params = None if hasattr(init, "_parameters"): params = getattr(init, "_parameters") svars = None if hasattr(init, "_state_variables"): svars = getattr(init, "_state_variables") return init, params, svars @classproperty def default_parameters(cls): """ Get the default values for the parameters of the model. :rtype: dict(str, object) """ init, params, svars = cls._get_init_params_and_svars(cls) return get_dict_from_init(init, skip=svars, include=params) @classproperty def default_initial_values(cls): """ Get the default initial values for the state variables of the model :rtype: dict(str, object) """ init, params, svars = cls._get_init_params_and_svars(cls) if params is None and svars is None: return {} return get_dict_from_init(init, skip=params, include=svars) @classmethod def get_parameter_names(cls): """ Get the names of the parameters of the model :rtype: list(str) """ return cls.default_parameters.keys() @classmethod def has_parameter(cls, name): """ Determine if the model has a parameter with the given name :param name: The name of the parameter to check for :type name: str :rtype: bool """ return name in cls.default_parameters @abstractproperty @staticmethod def default_population_parameters(): """ Get the default values for the parameters at the population-level;\ these are parameters that can be passed in to the Population\ constructor in addition to the standard PyNN options :rtype: dict(str, object) """ @abstractmethod def create_vertex(self, n_neurons, label, constraints): """ Create a vertex for a population of the model
def __call__(self, machine_graph, machine, n_keys_map, plan_n_timesteps): """ :param MachineGraph machine_graph: the machine graph :param ~spinn_machine.Machine machine: the SpiNNaker machine :param AbstractMachinePartitionNKeysMap n_keys_map: the n keys from partition map :param int plan_n_timesteps: number of timesteps to plan for :return: placements. :rtype: Placements """ # create progress bar progress_bar = ProgressBar( (machine_graph.n_vertices * self.ITERATIONS) + self.STEPS, "Placing graph vertices via spreading over an entire machine") # check that the algorithm can handle the constraints self._check_constraints( machine_graph.vertices, additional_placement_constraints={SameChipAsConstraint}) progress_bar.update() # get same chip groups same_chip_vertex_groups = get_same_chip_vertex_groups(machine_graph) progress_bar.update() # get chip and core placed verts hard_chip_constraints = self._locate_hard_placement_verts( machine_graph) progress_bar.update() # get one to one groups one_to_one_groups = create_vertices_groups( machine_graph.vertices, functools.partial(self._find_one_to_one_vertices, graph=machine_graph)) progress_bar.update() # sort chips so that they are radial from a given point and other # init data structs chips_in_order = self._determine_chip_list(machine) resource_tracker = ResourceTracker(machine, plan_n_timesteps, chips=chips_in_order) placements = Placements() placed_vertices = set() cost_per_chip = defaultdict(int) progress_bar.update() # allocate hard ones for hard_vertex in hard_chip_constraints: (x, y, p, _, _) = resource_tracker.allocate_constrained_resources( hard_vertex.resources_required, hard_vertex.constraints) placements.add_placement(Placement(hard_vertex, x, y, p)) placed_vertices.add(hard_vertex) cost_per_chip[x, y] += self._get_cost(hard_vertex, machine_graph, n_keys_map) # place groups of verts that need the same chip on the same chip, self._place_same_chip_verts(same_chip_vertex_groups, chips_in_order, placements, progress_bar, resource_tracker, placed_vertices, cost_per_chip, machine_graph, n_keys_map) # place 1 group per chip if possible on same chip as any already # placed verts. if not then radially from it. self._place_one_to_one_verts(one_to_one_groups, chips_in_order, placements, progress_bar, resource_tracker, placed_vertices, cost_per_chip, machine_graph, n_keys_map, machine) # place vertices which don't have annoying placement constraints. # spread them over the chips so that they have minimal impact on the # overall incoming packet cost per router. self._place_left_over_verts(machine_graph, chips_in_order, placements, progress_bar, resource_tracker, placed_vertices, cost_per_chip, n_keys_map) progress_bar.end() # return the built placements return placements
def _fit(self, X, y, groups, parameter_iterable): """ Actual fitting, performing the search over parameters. Taken from https://github.com/scikit-learn/scikit-learn/blob/0.18.X .../sklearn/model_selection/_search.py """ estimator = self.estimator cv = sklearn.model_selection._validation.check_cv( self.cv, y, classifier=is_classifier(estimator)) self.scorer_ = check_scoring(self.estimator, scoring=self.scoring) X, y, groups = indexable(X, y, groups) n_splits = cv.get_n_splits(X, y, groups) if self.verbose > 0 and isinstance(parameter_iterable, Sized): n_candidates = len(parameter_iterable) self.printfn( "Fitting {0} folds for each of {1} candidates, totalling" " {2} fits".format(n_splits, n_candidates, n_candidates * n_splits)) base_estimator = clone(self.estimator) pre_dispatch = self.pre_dispatch cv_iter = list(cv.split(X, y, groups)) out = Parallel( n_jobs=self.n_jobs, verbose=self.verbose, pre_dispatch=pre_dispatch)( delayed(sklearn.model_selection._validation._fit_and_score)( clone(base_estimator), X, y, self.scorer_, train, test, self.verbose, parameters, fit_params=self.fit_params, return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=True, error_score=self.error_score) for parameters in parameter_iterable for train, test in cv_iter) # if one choose to see train score, "out" will contain train score info if self.return_train_score: (train_scores, test_scores, test_sample_counts, fit_time, score_time, parameters) = zip(*out) else: (test_scores, test_sample_counts, fit_time, score_time, parameters) = zip(*out) candidate_params = parameters[::n_splits] n_candidates = len(candidate_params) results = dict() def _store(key_name, array, weights=None, splits=False, rank=False): """A small helper to store the scores/times to the cv_results_""" array = np.array(array, dtype=np.float64).reshape(n_candidates, n_splits) if splits: for split_i in range(n_splits): results["split%d_%s" % (split_i, key_name)] = array[:, split_i] array_means = np.average(array, axis=1, weights=weights) results['mean_%s' % key_name] = array_means # Weighted std is not directly available in numpy array_stds = np.sqrt( np.average((array - array_means[:, np.newaxis])**2, axis=1, weights=weights)) results['std_%s' % key_name] = array_stds if rank: results["rank_%s" % key_name] = np.asarray(rankdata( -array_means, method='min'), dtype=np.int32) # Computed the (weighted) mean and std for test scores alone # NOTE test_sample counts (weights) remain the same for all candidates test_sample_counts = np.array(test_sample_counts[:n_splits], dtype=np.int) _store('test_score', test_scores, splits=True, rank=True, weights=test_sample_counts if self.iid else None) if self.return_train_score: _store('train_score', train_scores, splits=True) _store('fit_time', fit_time) _store('score_time', score_time) best_index = np.flatnonzero(results["rank_test_score"] == 1)[0] best_parameters = candidate_params[best_index] # Use one MaskedArray and mask all the places where the param is not # applicable for that candidate. Use defaultdict as each candidate may # not contain all the params param_results = defaultdict( partial(MaskedArray, np.empty(n_candidates, ), mask=True, dtype=object)) for cand_i, params in enumerate(candidate_params): for name, value in params.items(): # An all masked empty array gets created for the key # `"param_%s" % name` at the first occurence of `name`. # Setting the value at an index also unmasks that index param_results["param_%s" % name][cand_i] = value results.update(param_results) # Store a list of param dicts at the key 'params' results['params'] = candidate_params self.cv_results_ = results self.best_index_ = best_index self.n_splits_ = n_splits if self.refit: # fit the best estimator using the entire dataset # clone first to work around broken estimators best_estimator = clone(base_estimator).set_params( **best_parameters) if y is not None: best_estimator.fit(X, y, **self.fit_params) else: best_estimator.fit(X, **self.fit_params) self.best_estimator_ = best_estimator return self
except ImportError: from collections import defaultdict import os __version__ = None exec(open("spinn_front_end_common/_version.py").read()) assert __version__ # Build a list of all project modules, as well as supplementary files main_package = "spinn_front_end_common" extensions = {".aplx", ".boot", ".cfg", ".json", ".sql", ".template", ".xml", ".xsd", ".dict"} main_package_dir = os.path.join(os.path.dirname(__file__), main_package) start = len(main_package_dir) packages = [] package_data = defaultdict(list) for dirname, dirnames, filenames in os.walk(main_package_dir): if '__init__.py' in filenames: package = "{}{}".format( main_package, dirname[start:].replace(os.sep, '.')) packages.append(package) for filename in filenames: _, ext = os.path.splitext(filename) if ext in extensions: package = "{}{}".format( main_package, dirname[start:].replace(os.sep, '.')) package_data[package].append(filename) setup( name="SpiNNFrontEndCommon", version=__version__,
def fs_status(self, fs_id): mds_versions = defaultdict(list) fsmap = mgr.get("fs_map") filesystem = None for fs in fsmap['filesystems']: if fs['id'] == fs_id: filesystem = fs break if filesystem is None: raise cherrypy.HTTPError(404, "CephFS id {0} not found".format(fs_id)) rank_table = [] mdsmap = filesystem['mdsmap'] client_count = 0 for rank in mdsmap["in"]: up = "mds_{0}".format(rank) in mdsmap["up"] if up: gid = mdsmap['up']["mds_{0}".format(rank)] info = mdsmap['info']['gid_{0}'.format(gid)] dns = mgr.get_latest("mds", info['name'], "mds.inodes") inos = mgr.get_latest("mds", info['name'], "mds_mem.ino") if rank == 0: client_count = mgr.get_latest("mds", info['name'], "mds_sessions.session_count") elif client_count == 0: # In case rank 0 was down, look at another rank's # sessionmap to get an indication of clients. client_count = mgr.get_latest("mds", info['name'], "mds_sessions.session_count") laggy = "laggy_since" in info state = info['state'].split(":")[1] if laggy: state += "(laggy)" # Populate based on context of state, e.g. client # ops for an active daemon, replay progress, reconnect # progress if state == "active": activity = CephService.get_rate("mds", info['name'], "mds_server.handle_client_request") else: activity = 0.0 metadata = mgr.get_metadata('mds', info['name']) mds_versions[metadata.get('ceph_version', 'unknown')].append( info['name']) rank_table.append( { "rank": rank, "state": state, "mds": info['name'], "activity": activity, "dns": dns, "inos": inos } ) else: rank_table.append( { "rank": rank, "state": "failed", "mds": "", "activity": 0.0, "dns": 0, "inos": 0 } ) # Find the standby replays # pylint: disable=unused-variable for gid_str, daemon_info in mdsmap['info'].items(): if daemon_info['state'] != "up:standby-replay": continue inos = mgr.get_latest("mds", daemon_info['name'], "mds_mem.ino") dns = mgr.get_latest("mds", daemon_info['name'], "mds.inodes") activity = CephService.get_rate( "mds", daemon_info['name'], "mds_log.replay") rank_table.append( { "rank": "{0}-s".format(daemon_info['rank']), "state": "standby-replay", "mds": daemon_info['name'], "activity": activity, "dns": dns, "inos": inos } ) df = mgr.get("df") pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) osdmap = mgr.get("osd_map") pools = dict([(p['pool'], p) for p in osdmap['pools']]) metadata_pool_id = mdsmap['metadata_pool'] data_pool_ids = mdsmap['data_pools'] pools_table = [] for pool_id in [metadata_pool_id] + data_pool_ids: pool_type = "metadata" if pool_id == metadata_pool_id else "data" stats = pool_stats[pool_id] pools_table.append({ "pool": pools[pool_id]['pool_name'], "type": pool_type, "used": stats['bytes_used'], "avail": stats['max_avail'] }) standby_table = [] for standby in fsmap['standbys']: metadata = mgr.get_metadata('mds', standby['name']) mds_versions[metadata.get('ceph_version', 'unknown')].append( standby['name']) standby_table.append({ 'name': standby['name'] }) return { "cephfs": { "id": fs_id, "name": mdsmap['fs_name'], "client_count": client_count, "ranks": rank_table, "pools": pools_table }, "standbys": standby_table, "versions": mds_versions }
def handle_fs_status(self, cmd): output = "" fs_filter = cmd.get('fs', None) mds_versions = defaultdict(list) fsmap = self.get("fs_map") for filesystem in fsmap['filesystems']: if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: continue rank_table = PrettyTable( ("Rank", "State", "MDS", "Activity", "dns", "inos"), hrules=prettytable.FRAME) mdsmap = filesystem['mdsmap'] client_count = 0 for rank in mdsmap["in"]: up = "mds_{0}".format(rank) in mdsmap["up"] if up: gid = mdsmap['up']["mds_{0}".format(rank)] info = mdsmap['info']['gid_{0}'.format(gid)] dns = self.get_latest("mds", info['name'], "mds_mem.dn") inos = self.get_latest("mds", info['name'], "mds_mem.ino") if rank == 0: client_count = self.get_latest( "mds", info['name'], "mds_sessions.session_count") elif client_count == 0: # In case rank 0 was down, look at another rank's # sessionmap to get an indication of clients. client_count = self.get_latest( "mds", info['name'], "mds_sessions.session_count") laggy = "laggy_since" in info state = info['state'].split(":")[1] if laggy: state += "(laggy)" if state == "active" and not laggy: c_state = self.colorize(state, self.GREEN) else: c_state = self.colorize(state, self.YELLOW) # Populate based on context of state, e.g. client # ops for an active daemon, replay progress, reconnect # progress activity = "" if state == "active": activity = "Reqs: " + self.format_dimless( self.get_rate("mds", info['name'], "mds_server.handle_client_request"), 5) + "/s" metadata = self.get_metadata('mds', info['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(info['name']) rank_table.add_row([ self.bold(rank.__str__()), c_state, info['name'], activity, self.format_dimless(dns, 5), self.format_dimless(inos, 5) ]) else: rank_table.add_row([rank, "failed", "", "", "", ""]) # Find the standby replays for gid_str, daemon_info in six.iteritems(mdsmap['info']): if daemon_info['state'] != "up:standby-replay": continue inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") activity = "Evts: " + self.format_dimless( self.get_rate("mds", daemon_info['name'], "mds_log.replayed"), 5) + "/s" metadata = self.get_metadata('mds', daemon_info['name']) mds_versions[metadata.get('ceph_version', "unknown")].append( daemon_info['name']) rank_table.add_row([ "{0}-s".format(daemon_info['rank']), "standby-replay", daemon_info['name'], activity, self.format_dimless(dns, 5), self.format_dimless(inos, 5) ]) df = self.get("df") pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) osdmap = self.get("osd_map") pools = dict([(p['pool'], p) for p in osdmap['pools']]) metadata_pool_id = mdsmap['metadata_pool'] data_pool_ids = mdsmap['data_pools'] pools_table = PrettyTable(["Pool", "type", "used", "avail"]) for pool_id in [metadata_pool_id] + data_pool_ids: pool_type = "metadata" if pool_id == metadata_pool_id else "data" stats = pool_stats[pool_id] pools_table.add_row([ pools[pool_id]['pool_name'], pool_type, self.format_bytes(stats['bytes_used'], 5), self.format_bytes(stats['max_avail'], 5) ]) output += "{0} - {1} clients\n".format(mdsmap['fs_name'], client_count) output += "=" * len(mdsmap['fs_name']) + "\n" output += rank_table.get_string() output += "\n" + pools_table.get_string() + "\n" if not output and fs_filter is not None: return errno.EINVAL, "", "Invalid filesystem: " + fs_filter standby_table = PrettyTable(["Standby MDS"]) for standby in fsmap['standbys']: metadata = self.get_metadata('mds', standby['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name']) standby_table.add_row([standby['name']]) output += "\n" + standby_table.get_string() + "\n" if len(mds_versions) == 1: output += "MDS version: {0}".format(mds_versions.keys()[0]) else: version_table = PrettyTable(["version", "daemons"]) for version, daemons in six.iteritems(mds_versions): version_table.add_row([version, ", ".join(daemons)]) output += version_table.get_string() + "\n" return 0, output, ""
def __init__(self, fname, label): self.name = fname #file name of input transcript self.label = label #init for updateWordCount() self.words = [] #sequenced word list self.wordCount = defaultdict(int) #dictionary of words and their count self.wordCountSort = defaultdict(int) #sorted dictionary self.transSize = 0 #words the transcript have self.transSizeNS = 0 #meaningful words the transcript have self.wordsetSize = 0 #all different words #init for updateKeywordStatistics() self.keywordStat = defaultdict( dict) #num(int) certain keyword appears in the transcript self.keywordStatSorted = defaultdict(dict) self.keywordLoc = defaultdict( dict) #locations(list) certain keyword appears in the transcript self.keywordPart = defaultdict( dict) #percentage per keyword from all keywords self.keywordPer = float(0) #percentage all keywords from all words #init for questionMark() self.questionStat = 0 self.questionLoc = [] self.questionPer = float(0) #init for emoVoice() self.emoTerm = [] self.emoLoc = [] #init for keywordQuestionmark() self.keywordQNum = defaultdict(dict) self.keywordQPer = defaultdict(dict) self.keywordQOutput = defaultdict(dict) #init for nonSpeech() self.nSpeech = [] #init for lengthByTurn() self.listTurn = [] print("Reading data...") from nltk.tokenize import sent_tokenize self.data = [] self.lines = [] self.sents = [] self.sentNum = 0 with open(fname, encoding='utf-8', errors='ignore') as f: for line in f: line = line.lower() if (line[0] == label): self.lines.append(sent_tokenize(line)) for sent in sent_tokenize(line): self.sents.append(sent) self.sentNum += len(sent_tokenize(line)) tokenizer = nltk.tokenize.TreebankWordTokenizer() self.data += tokenizer.tokenize(line) print("done") print("Initializing keyword dictionary...") # Dictionary of key-terms for CTS fidelity self.keywordCTS = defaultdict(set) self.keywordCTS['Agenda'] = {'agenda','priorities','priority','do first','most important','work on','focus on','talk about' \ ,'plan','todo list' ,'focus on first','focus on today','focus on during the session' \ ,'talk about today','talk about first','talk about during the session' \ ,'work on today','work on first','work on during the session' \ ,'you like to','you want to','add to the agenda','add anything to the agenda','last week' \ ,'evidence','mistake in thinking','what did you think','what did you want' \ ,'how did you feel'} self.keywordCTS['Feedback'] = {'feedback','reaction','advise','advice','suggestion','previous','last time','last week','last session','past session' \ ,'think about today','things go today','think about today\'s session','concern' \ ,'what question','what questions','unhelpful','helpful','least helpful','about today\'s session' \ ,'anything i can do better','anything we can do better','concerns about today\'s session','helpful about the session' \ ,'can help you','we can try','learn','take in','skill','learn skills','achieve','goals','goal' \ ,'if i understand you correctly','are you saying','do i have it right','work on your goals' \ ,'was this helpful','how am i doing today','am i explaining things clearly','do you understand' \ ,'do you follow me'} self.keywordCTS['Understanding'] = {'understand','understanding','recognize','observe','grasp','comprehend','know','understand why' \ ,'sounds like','you are saying','you were feeling','you felt','i felt','i was feeling' \ ,'see','makes sense','i see','feel that way','feel this way'} self.keywordCTS['Interpersonal Effectiveness'] = {'sorry','hard','difficult','tough' \ ,'dissappointing','stressful','stressed' \ ,'scary','frightening','upset','upsetting'\ ,'unfortunate'} self.keywordCTS['Collaboration'] = { 'choice', 'you want to do', 'good idea', 'because', 'will', 'help you get your goal' } self.keywordCTS['Guided Discovery'] = {'meaning','mean','self','how','why','evidence' \ ,'conclusion','conclude','decide','decision','decided' \ ,'know','proof','tell me more','assume','assumption' \ ,'hypothesis','disprove','facts','fact','solutions' \ ,'brainstorm','solve','alternative','other explanations' \ ,'another way','other way','to think about','to explain','reason'} self.keywordCTS['Focus on Key Cognitions'] = {'thinking','tell yourself','through your mind' \ ,'what did you tell yourself','what went through your mind' \ ,'thought','think','connection','lead to','connected' \ ,'connect','link','linked','make you','you do','feel about the thought'} self.keywordCTS['Choices of Intervention'] = {} self.keywordCTS['Homework'] = {'homework','review','homework review','at home','practice','assignment','assign','get in the way of','work around' \ ,'assigned','progress','learned','improve','learn','skills','skill','out of session','outside of session' \ ,'goal','better','barrier','in the way','expect','problems','problem','succeed','success'} self.keywordCTS['Social Skills Training'] = {'rational','help you learn this skill','help you with your goal' \ ,'demonstrate','to make your next role','play better','play even better' \ ,'try to focus on','do well','did well','did a good job' \ ,'for the next role play','recommend focusing on'}
try: from collections.abc import defaultdict except ImportError: from collections import defaultdict try: from inspect import getfullargspec except ImportError: # Python 2.7 hack from inspect import getargspec as getfullargspec from functools import wraps from six import iteritems, itervalues _instances = list() _methods = defaultdict(dict) _injectables = None class InjectionException(Exception): """ Raised when there is an error with injection. """ def supports_injection(injectable_class): """ Indicate that the class has methods on which objects can be injected. """ orig_init = injectable_class.__init__ def new_init(self, *args, **kwargs): # pylint: disable=protected-access orig_init(self, *args, **kwargs) for method in itervalues(injectable_class.__dict__):
def __init__(self, ctx): # Deliberately skip calling parent constructor self._ctx = ctx self.mon_manager = LocalCephManager() self._conf = defaultdict(dict)
def _determine_algorithm_order( self, inputs, required_outputs, algorithm_data, optional_algorithm_data, converter_algorithms_datas, tokens, required_output_tokens): """ Takes the algorithms and determines which order they need to be\ executed to generate the correct data objects :param inputs: list of input types :type inputs: iterable(str) :param required_outputs: \ the set of outputs that this workflow is meant to generate :param converter_algorithms_datas: the set of converter algorithms :param optional_algorithm_data: the set of optional algorithms :rtype: None """ # Go through the algorithms and get all possible outputs all_outputs = set(iterkeys(inputs)) for algorithms in (algorithm_data, optional_algorithm_data): for algorithm in algorithms: # Get the algorithm output types alg_outputs = { output.output_type for output in algorithm.outputs} # Remove from the outputs any optional input that is also an # output for alg_input in algorithm.optional_inputs: for matching in alg_input.get_matching_inputs(alg_outputs): alg_outputs.discard(matching) all_outputs.update(alg_outputs) # Set up the token tracking and make all specified tokens complete token_states = TokenStates() for token_name in tokens: token = Token(token_name) token_states.track_token(token) token_states.process_output_token(token) # Go through the algorithms and add in the tokens that can be completed # by any of the algorithms for algorithms in (algorithm_data, optional_algorithm_data): for algorithm in algorithms: for token in algorithm.generated_output_tokens: if not token_states.is_token_complete(token): token_states.track_token(token) # Go through the algorithms and add a fake token for any algorithm that # requires an optional token that can't be provided and a fake input # for any algorithm that requires an optional input that can't be # provided. This allows us to require the other optional inputs and # tokens so that algorithms that provide those items are run before # those that can make use of them. fake_inputs = set() fake_tokens = TokenStates() for algorithms in (algorithm_data, optional_algorithm_data): for algorithm in algorithms: for input_parameter in algorithm.optional_inputs: if not input_parameter.input_matches(all_outputs): fake_inputs.update( input_parameter.get_fake_inputs(all_outputs)) for token in algorithm.optional_input_tokens: if (not token_states.is_tracking_token(token) and not fake_tokens.is_token_complete(token)): fake_tokens.track_token(token) fake_tokens.process_output_token(token) input_types = set(iterkeys(inputs)) allocated_algorithms = list() generated_outputs = set() generated_outputs.union(input_types) algorithms_to_find = list(algorithm_data) optionals_to_use = list(optional_algorithm_data) outputs_to_find = self._remove_outputs_which_are_inputs( required_outputs, inputs) tokens_to_find = self._remove_complete_tokens( token_states, required_output_tokens) while algorithms_to_find or outputs_to_find or tokens_to_find: suitable_algorithm = None algorithm_list = None # Order of searching - each combination will be attempted in order; # the first matching algorithm will be used (and search will stop) # Elements are: # 1. Algorithm list to search, # 2. check generated outputs, # 3. require optional inputs) order = [ # Check required algorithms forcing optional inputs (algorithms_to_find, False, True), # Check optional algorithms forcing optional inputs (optionals_to_use, True, True), # Check required algorithms without optional inputs # - shouldn't need to do this, but might if an optional input # is also a generated output of the same algorithm (algorithms_to_find, False, False), # Check optional algorithms without optional inputs # - as above, it shouldn't be necessary but might be if an # optional input is also an output of the same algorithm (optionals_to_use, True, False), # Check converter algorithms # (only if they generate something new) (converter_algorithms_datas, True, False) ] for (algorithms, check_outputs, force_required) in order: suitable_algorithm, algorithm_list = \ self._locate_suitable_algorithm( algorithms, input_types, generated_outputs, token_states, fake_inputs, fake_tokens, check_outputs, force_required) if suitable_algorithm is not None: break if suitable_algorithm is not None: # Remove the value self._remove_algorithm_and_update_outputs( algorithm_list, suitable_algorithm, input_types, generated_outputs, outputs_to_find) # add the suitable algorithms to the list and take the outputs # as new inputs allocated_algorithms.append(suitable_algorithm) # Mark any tokens generated as complete for output_token in suitable_algorithm.generated_output_tokens: token_states.process_output_token(output_token) if token_states.is_token_complete( Token(output_token.name)): tokens_to_find.discard(output_token.name) else: # Failed to find an algorithm to run! algorithms_to_find_names = list() for algorithm in algorithms_to_find: algorithms_to_find_names.append(algorithm.algorithm_id) optional_algorithms_names = list() for algorithm in optional_algorithm_data: optional_algorithms_names.append(algorithm.algorithm_id) algorithms_used = list() for algorithm in allocated_algorithms: algorithms_used.append(algorithm.algorithm_id) algorithm_input_requirement_breakdown = "" for algorithm in algorithms_to_find: algorithm_input_requirement_breakdown += \ self._deduce_inputs_required_to_run( algorithm, input_types, token_states, fake_inputs, fake_tokens) for algorithm in optionals_to_use: algorithm_input_requirement_breakdown += \ self._deduce_inputs_required_to_run( algorithm, input_types, token_states, fake_inputs, fake_tokens) algorithms_by_output = defaultdict(list) algorithms_by_token = defaultdict(list) for algorithms in (algorithm_data, optional_algorithm_data): for algorithm in algorithms: for output in algorithm.outputs: algorithms_by_output[output.output_type].append( algorithm.algorithm_id) for token in algorithm.generated_output_tokens: algorithms_by_token[token.name].append( "{}: part={}".format( algorithm.algorithm_id, token.part)) raise PacmanConfigurationException( "Unable to deduce a future algorithm to use.\n" " Inputs: {}\n" " Fake Inputs: {}\n" " Outputs to find: {}\n" " Tokens complete: {}\n" " Fake tokens complete: {}\n" " Tokens to find: {}\n" " Required algorithms remaining to be used: {}\n" " Optional Algorithms unused: {}\n" " Functions used: {}\n" " Algorithm by outputs: {}\n" " Algorithm by tokens: {}\n" " Inputs required per function: \n{}\n".format( sorted(input_types), sorted(fake_inputs), outputs_to_find, token_states.get_completed_tokens(), fake_tokens.get_completed_tokens(), tokens_to_find, algorithms_to_find_names, optional_algorithms_names, algorithms_used, algorithms_by_output, algorithms_by_token, algorithm_input_requirement_breakdown)) # Test that the outputs are generated all_required_outputs_generated = True failed_to_generate_output_string = "" for output in outputs_to_find: if output not in generated_outputs: all_required_outputs_generated = False failed_to_generate_output_string += ":{}".format(output) if not all_required_outputs_generated: raise PacmanConfigurationException( "Unable to generate outputs {}".format( failed_to_generate_output_string)) self._algorithms = allocated_algorithms self._completed_tokens = token_states.get_completed_tokens()
def __init__(self, test_case): self._test_case = test_case self._n_cores_in_app = defaultdict(lambda: 0) self._executable_on_core = dict()
def __init__(self): self._sdram_usage = defaultdict(lambda: 0) self._region_sizes = dict() self._vertices_by_chip = defaultdict(list)
def __call__(self, machine_graph, n_keys_map, routing_tables): # check that this algorithm supports the constraints check_algorithm_can_support_constraints( constrained_vertices=machine_graph.outgoing_edge_partitions, supported_constraints=[ FixedMaskConstraint, FixedKeyAndMaskConstraint, ContiguousKeyRangeContraint], abstract_constraint_type=AbstractKeyAllocatorConstraint) # verify that no edge has more than 1 of a constraint ,and that # constraints are compatible check_types_of_edge_constraint(machine_graph) routing_infos = RoutingInfo() # Get the edges grouped by those that require the same key (fixed_keys, _shared_keys, fixed_masks, fixed_fields, flexi_fields, continuous, noncontinuous) = \ get_edge_groups(machine_graph, EdgeTrafficType.MULTICAST) if flexi_fields: raise PacmanConfigurationException( "MallocBasedRoutingInfoAllocator does not support FlexiField") # Even non-continuous keys will be continuous for group in noncontinuous: continuous.add(group) # Go through the groups and allocate keys progress = ProgressBar( machine_graph.n_outgoing_edge_partitions, "Allocating routing keys") # allocate the groups that have fixed keys for group in progress.over(fixed_keys, False): # Get any fixed keys and masks from the group and attempt to # allocate them fixed_mask = None fixed_key_and_mask_constraint = locate_constraints_of_type( group.constraints, FixedKeyAndMaskConstraint)[0] # attempt to allocate them self._allocate_fixed_keys_and_masks( fixed_key_and_mask_constraint.keys_and_masks, fixed_mask) # update the pacman data objects self._update_routing_objects( fixed_key_and_mask_constraint.keys_and_masks, routing_infos, group) continuous.remove(group) for group in progress.over(fixed_masks, False): # get mask and fields if need be fixed_mask = locate_constraints_of_type( group.constraints, FixedMaskConstraint)[0].mask fields = None if group in fixed_fields: fields = locate_constraints_of_type( group.constraints, FixedKeyFieldConstraint)[0].fields fixed_fields.remove(group) # try to allocate keys_and_masks = self._allocate_keys_and_masks( fixed_mask, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous.remove(group) for group in progress.over(fixed_fields, False): fields = locate_constraints_of_type( group.constraints, FixedKeyFieldConstraint)[0].fields # try to allocate keys_and_masks = self._allocate_keys_and_masks( None, fields, n_keys_map.n_keys_for_partition(group)) # update the pacman data objects self._update_routing_objects(keys_and_masks, routing_infos, group) continuous.remove(group) # Sort the rest of the groups, using the routing tables for guidance # Group partitions by those which share routes in any table partition_groups = OrderedDict() routers = reversed(sorted( routing_tables.get_routers(), key=lambda item: len(routing_tables.get_entries_for_router( item[0], item[1])))) for x, y in routers: # Find all partitions that share a route in this table partitions_by_route = defaultdict(OrderedSet) routing_table = routing_tables.get_entries_for_router(x, y) for partition, entry in iteritems(routing_table): if partition in continuous: entry_hash = sum( 1 << i for i in entry.link_ids) entry_hash += sum( 1 << (i + 6) for i in entry.processor_ids) partitions_by_route[entry_hash].add(partition) for entry_hash, partitions in iteritems(partitions_by_route): found_groups = list() for partition in partitions: if partition in partition_groups: found_groups.append(partition_groups[partition]) if not found_groups: # If no group was found, create a new one for partition in partitions: partition_groups[partition] = partitions elif len(found_groups) == 1: # If a single other group was found, merge it for partition in partitions: found_groups[0].add(partition) partition_groups[partition] = found_groups[0] else: # Merge the groups new_group = partitions for group in found_groups: for partition in group: new_group.add(partition) for partition in new_group: partition_groups[partition] = new_group # Sort partitions by largest group continuous = list(OrderedSet( tuple(group) for group in itervalues(partition_groups))) for group in reversed(sorted(continuous, key=len)): for partition in progress.over(group, False): keys_and_masks = self._allocate_keys_and_masks( None, None, n_keys_map.n_keys_for_partition(partition)) # update the pacman data objects self._update_routing_objects( keys_and_masks, routing_infos, partition) progress.end() return routing_infos
def fit(self, X, y=None, groups=None, callback=None): """Run fit on the estimator with randomly drawn parameters. Parameters ---------- X : array-like or sparse matrix, shape = [n_samples, n_features] The training input samples. y : array-like, shape = [n_samples] or [n_samples, n_output] Target relative to X for classification or regression (class labels should be integers or strings). groups : array-like, with shape (n_samples,), optional Group labels for the samples used while splitting the dataset into train/test set. callback: [callable, list of callables, optional] If callable then `callback(res)` is called after each parameter combination tested. If list of callables, then each callable in the list is called. """ # check if space is a single dict, convert to list if so search_spaces = self.search_spaces if isinstance(search_spaces, dict): search_spaces = [search_spaces] callbacks = check_callback(callback) if self.optimizer_kwargs is None: self.optimizer_kwargs_ = {} else: self.optimizer_kwargs_ = dict(self.optimizer_kwargs) random_state = check_random_state(self.random_state) self.optimizer_kwargs_['random_state'] = random_state # Instantiate optimizers for all the search spaces. optimizers = [] for search_space in search_spaces: if isinstance(search_space, tuple): search_space = search_space[0] optimizers.append(self._make_optimizer(search_space)) self.optimizers_ = optimizers # will save the states of the optimizers self.cv_results_ = defaultdict(list) self.best_index_ = None self.multimetric_ = False self.optimizer_results_ = {} n_points = self.n_points for search_space, optimizer in zip(search_spaces, optimizers): # if not provided with search subspace, n_iter is taken as # self.n_iter if isinstance(search_space, tuple): search_space, n_iter = search_space else: n_iter = self.n_iter # do the optimization for particular search space while n_iter > 0: # when n_iter < n_points points left for evaluation n_points_adjusted = min(n_iter, n_points) optim_result = self._step(X, y, search_space, optimizer, groups=groups, n_points=n_points_adjusted) n_iter -= n_points if eval_callbacks(callbacks, optim_result): break self.optimizer_results_[optimizer] = optim_result # Refit the best model on the the whole dataset if self.refit: self._fit_best_model(X, y) return self
def test_local_verts_go_to_local_lpgs(self): machine = VirtualMachine(width=12, height=12, with_wrap_arounds=True) graph = MachineGraph("Test") default_params = { 'use_prefix': False, 'key_prefix': None, 'prefix_type': None, 'message_type': EIEIOType.KEY_32_BIT, 'right_shift': 0, 'payload_as_time_stamps': True, 'use_payload_prefix': True, 'payload_prefix': None, 'payload_right_shift': 0, 'number_of_packets_sent_per_time_step': 0, 'hostname': None, 'port': None, 'strip_sdp': None, 'board_address': None, 'tag': None} # data stores needed by algorithm live_packet_gatherers = dict() extended = dict(default_params) extended.update({'partition_id': "EVENTS"}) default_params_holder = LivePacketGatherParameters(**extended) live_packet_gatherers[default_params_holder] = list() live_packet_gatherers_to_vertex_mapping = dict() live_packet_gatherers_to_vertex_mapping[default_params_holder] = dict() placements = Placements() # add LPG's (1 for each Ethernet connected chip) for chip in machine.ethernet_connected_chips: extended = dict(default_params) extended.update({'label': 'test'}) vertex = LivePacketGatherMachineVertex(**extended) graph.add_vertex(vertex) placements.add_placement( Placement(x=chip.x, y=chip.y, p=2, vertex=vertex)) live_packet_gatherers_to_vertex_mapping[ default_params_holder][chip.x, chip.y] = vertex # tracker of wirings verts_expected = defaultdict(list) positions = list() positions.append([0, 0, 0, 0]) positions.append([4, 4, 0, 0]) positions.append([1, 1, 0, 0]) positions.append([2, 2, 0, 0]) positions.append([8, 4, 8, 4]) positions.append([11, 4, 8, 4]) positions.append([4, 11, 4, 8]) positions.append([4, 8, 4, 8]) positions.append([0, 11, 8, 4]) positions.append([11, 11, 4, 8]) positions.append([8, 8, 4, 8]) positions.append([4, 0, 0, 0]) positions.append([7, 7, 0, 0]) # add graph vertices which reside on areas of the machine to ensure # spread over boards. for x, y, eth_x, eth_y in positions: vertex = SimpleMachineVertex(resources=ResourceContainer()) graph.add_vertex(vertex) live_packet_gatherers[default_params_holder].append(vertex) verts_expected[eth_x, eth_y].append(vertex) placements.add_placement(Placement(x=x, y=y, p=5, vertex=vertex)) # run edge inserter that should go boom edge_inserter = InsertEdgesToLivePacketGatherers() edge_inserter( live_packet_gatherer_parameters=live_packet_gatherers, placements=placements, live_packet_gatherers_to_vertex_mapping=( live_packet_gatherers_to_vertex_mapping), machine=machine, machine_graph=graph, application_graph=None, graph_mapper=None) # verify edges are in the right place for chip in machine.ethernet_connected_chips: edges = graph.get_edges_ending_at_vertex( live_packet_gatherers_to_vertex_mapping[ default_params_holder][chip.x, chip.y]) for edge in edges: self.assertIn(edge.pre_vertex, verts_expected[chip.x, chip.y])
def __init__(self): self.__data = defaultdict(dict)
def render_table(table, column_sep=" "): """ Render an ASCII table with optional ANSI escape codes. An example table:: Something Another thing Finally some value woah 1234 ace duuued -1 magic rather good 9001 Parameters ---------- table : [row, ...] A table to render. Each row contains an iterable of column values which may be either values or a tuples (f, value) where value is the string to print, or an integer to print right-aligned. If a column is a tuple, f is a formatting function which is applied to the string before the table is finally displayed. Columns are padded to have matching widths *before* any formatting functions are applied. column_sep : str String inserted between each column. Returns ------- str The formatted table. """ # Determine maximum column widths column_widths = defaultdict(lambda: 0) for row in table: for i, column in enumerate(row): if isinstance(column, string_types): string = column elif isinstance(column, int): string = str(column) else: _, string = column column_widths[i] = max(len(str(string)), column_widths[i]) # Render the table cells with padding [[str, ...], ...] out = [] for row in table: rendered_row = [] out.append(rendered_row) for i, column in enumerate(row): # Get string length and formatted string if isinstance(column, string_types): string = column length = len(string) right_align = False elif isinstance(column, int): string = str(column) length = len(string) right_align = True elif isinstance(column[1], string_types): f, string = column length = len(string) right_align = False string = f(string) elif isinstance(column[1], int): f, string = column length = len(str(string)) right_align = True string = f(string) padding = " " * (column_widths[i] - length) if right_align: rendered_row.append(padding + string) else: rendered_row.append(string + padding) # Render the final table return "\n".join(column_sep.join(row).rstrip() for row in out)
def __init__(self): self._sdram_usage = defaultdict(lambda: 0) self._region_sizes = dict() self._vertices_by_chip = defaultdict(list)