def group_exceptions(error_requests, exceptions, tracebacks): """ Groups exceptions into a form usable by an exception. :param error_requests: the error requests :param exceptions: the exceptions :param tracebacks: the tracebacks :return: a sorted exception pile :rtype: dict(Exception,_Group) """ data = OrderedDict() for error_request, exception, trace_back in zip( error_requests, exceptions, tracebacks): for stored_exception in data.keys(): if isinstance(exception, type(stored_exception)): found_exception = stored_exception break else: data[exception] = _Group(trace_back) found_exception = exception data[found_exception].add_coord(error_request.sdp_header) for exception in data: data[exception].finalise() return data.items()
class TestDataFrame(unittest.TestCase): def setUp(self): self.tmpdir = TempDir("dataframetest") self.testfilename = os.path.join(self.tmpdir.path, "dataframetest.nix") self.file = nix.File.open(self.testfilename, nix.FileMode.Overwrite) self.block = self.file.create_block("test block", "recordingsession") self.df1_dtype = OrderedDict([('name', np.int64), ('id', str), ('time', float), ('sig1', np.float64), ('sig2', np.int32)]) self.df1_data = [(1, "alpha", 20.18, 5.0, 100), (2, "beta", 20.09, 5.5, 101), (2, "gamma", 20.05, 5.1, 100), (1, "delta", 20.15, 5.3, 150), (2, "epsilon", 20.23, 5.7, 200), (2, "fi", 20.07, 5.2, 300), (1, "zeta", 20.12, 5.1, 39), (1, "eta", 20.27, 5.1, 600), (2, "theta", 20.15, 5.6, 400), (2, "iota", 20.08, 5.1, 200)] other_arr = np.arange(11101, 11200).reshape((33, 3)) other_di = OrderedDict({'name': np.int64, 'id': int, 'time': float}) self.df1 = self.block.create_data_frame("test df", "signal1", data=self.df1_data, col_dict=self.df1_dtype) self.df2 = self.block.create_data_frame("other df", "signal2", data=self.df1_data, col_dict=self.df1_dtype) self.df3 = self.block.create_data_frame("reference df", "signal3", data=other_arr, col_dict=other_di) self.dtype = self.df1._h5group.group["data"].dtype def tearDown(self): self.file.close() self.tmpdir.cleanup() def test_data_frame_eq(self): assert self.df1 == self.df1 assert not self.df1 == self.df2 assert self.df2 == self.df2 assert self.df1 is not None assert self.df2 is not None def test_create_with_list(self): arr = [(1, 'a', 20.18, 5.1, 100), (2, 'b', 20.09, 5.5, 101), (2, 'c', 20.05, 5.1, 100)] namelist = np.array(['name', 'id', 'time', 'sig1', 'sig2']) dtlist = np.array([np.int64, str, float, np.float64, np.int32]) df_li = self.block.create_data_frame("test_list", "make_of_list", data=arr, col_names=namelist, col_dtypes=dtlist) assert df_li.column_names == self.df1.column_names assert df_li.dtype == self.df1.dtype for i in df_li[:]: self.assertIsInstance(i['id'], string_types) self.assertIsInstance(i['sig2'], np.int32) def test_column_name_collision(self): arr = [(1, 'a', 20.18, 5.1, 100), (2, 'b', 20.09, 5.5, 101), (2, 'c', 20.05, 5.1, 100)] dtlist = np.array([np.int64, str, float, np.float64, np.int32]) namelist = np.array(['name', 'name', 'name', 'name', 'name']) self.assertRaises(nix.exceptions.DuplicateColumnName, self.block.create_data_frame, 'testerror', 'for_test', col_names=namelist, col_dtypes=dtlist, data=arr) def test_data_frame_type(self): assert self.df1.type == "signal1" self.df1.type = "test change" assert self.df1.type == "test change" def test_write_row(self): # test write single row row = ["1", 'abc', 3, 4.4556356242341, 5.1111111] assert list(self.df1[9]) == [2, 'iota', 20.08, 5.1, 200] self.df1.write_rows([row], [9]) assert list(self.df1[9]) == [1, 'abc', 3., 4.4556356242341, 5] self.assertIsInstance(self.df1[9]['name'], np.integer) self.assertIsInstance(self.df1[9]['sig2'], np.int32) assert self.df1[9]['sig2'] == int(5) # test write multiple rows multi_rows = [[1775, '12355', 1777, 1778, 1779], [1785, '12355', 1787, 1788, 1789]] self.df1.write_rows(multi_rows, [1, 2]) assert list(self.df1[1]) == [1775, '12355', 1777, 1778, 1779] assert list(self.df1[2]) == [1785, '12355', 1787, 1788, 1789] def test_write_column(self): # write by name column1 = np.arange(10000, 10010) self.df1.write_column(column1, name='sig1') assert list(self.df1[:]['sig1']) == list(column1) # write by index column2 = np.arange(20000, 20010) self.df1.write_column(column2, index=4) assert list(self.df1[:]['sig2']) == list(column2) def test_read_row(self): df1_array = np.array(self.df1_data, dtype=list(self.df1_dtype.items())) # read single row assert self.df1.read_rows(0) == df1_array[0] # read multiple multi_rows = self.df1.read_rows(np.arange(4, 9)) np.testing.assert_array_equal(multi_rows, df1_array[4:9]) multi_rows = self.df1.read_rows([3, 6]) np.testing.assert_array_equal(multi_rows, [df1_array[3], df1_array[6]]) def test_read_column(self): # read single column by index single_idx_col = self.df1.read_columns(index=[1]) data = np.array([row[1] for row in self.df1_data], dtype=nix.DataType.String) np.testing.assert_array_equal(single_idx_col, data) # read multiple columns by name multi_col = self.df1.read_columns(name=['sig1', 'sig2']) data = [(row[3], row[4]) for row in self.df1_data] assert len(multi_col) == 10 for data_row, df_row in zip(data, multi_col): assert data_row == tuple(df_row) # read columns with slices slice_cols = self.df1.read_columns(name=['sig1', 'sig2'], slc=slice(0, 6)) data = [(row[3], row[4]) for row in self.df1_data[:6]] assert len(slice_cols) == 6 for data_row, df_row in zip(data, slice_cols): assert data_row == tuple(df_row) # read single column by name single_idx_col = self.df1.read_columns(name=["sig2"]) data = np.array([100, 101, 100, 150, 200, 300, 39, 600, 400, 200], dtype=nix.DataType.Int32) np.testing.assert_array_equal(single_idx_col, data) # Read multiple columns where one is string slice_str_cols = self.df1.read_columns(name=['id', 'sig2'], slc=slice(3, 10)) data = [(row[1], row[4]) for row in self.df1_data[3:10]] assert len(slice_str_cols) == 7 for data_row, df_row in zip(data, slice_str_cols): assert data_row == tuple(df_row) def test_index_column_by_name(self): for colidx, colname in enumerate(self.df1_dtype.keys()): expdata = [row[colidx] for row in self.df1_data] assert all(self.df1[colname] == expdata) def test_read_cell(self): # read cell by position scell = self.df1.read_cell(position=[5, 3]) assert scell == 5.2 # read cell by row_idx + col_name crcell = self.df1.read_cell(col_name=['id'], row_idx=9) assert crcell == 'iota' # test error raise if only one param given self.assertRaises(ValueError, self.df1.read_cell, row_idx=10) self.assertRaises(ValueError, self.df1.read_cell, col_name='sig1') def test_write_cell(self): # write cell by position self.df1.write_cell(105, position=[8, 3]) assert self.df1[8]['sig1'] == 105 # write cell by rowid colname self.df1.write_cell('test', col_name='id', row_idx=3) assert self.df1[3]['id'] == 'test' # test error raise self.assertRaises(ValueError, self.df1.write_cell, 11, col_name='sig1') def test_append_column(self): col_data = np.arange(start=16000, stop=16010, step=1) self.df1.append_column(col_data, name='trial_col', datatype=int) assert self.df1.column_names == ('name', 'id', 'time', 'sig1', 'sig2', 'trial_col') assert len(self.df1.dtype) == 6 k = np.array(self.df1[0:10]["trial_col"], dtype=np.int64) np.testing.assert_almost_equal(k, col_data) # too short column sh_col = np.arange(start=16000, stop=16003, step=1) with self.assertRaises(ValueError): self.df1.append_column(sh_col, name='sh_col') # too long column long = np.arange(start=16000, stop=16500, step=1) with self.assertRaises(ValueError): self.df1.append_column(long, name='long') def test_append_rows(self): # append single row srow = (1, "test", 3, 4, 5) self.df1.append_rows([srow]) assert self.df1[10] == np.array(srow, dtype=list(self.df1_dtype.items())) # append multi-rows mrows = [(1, "2", 3, 4, 5), (6, "testing", 8, 9, 10)] self.df1.append_rows(mrows) assert all(self.df1[-2:] == np.array( mrows, dtype=list(self.df1_dtype.items()))) # append row with incorrect length errrow = [5, 6, 7, 8] self.assertRaises(ValueError, self.df1.append_rows, [errrow]) def test_unit(self): assert self.df1.units is None self.df1.units = ["s", 'A', 'ms', 'Hz', 'mA'] np.testing.assert_array_equal(self.df1.units, np.array(["s", 'A', 'ms', 'Hz', 'mA'])) assert self.df2.units is None def test_df_shape(self): assert tuple(self.df1.df_shape) == (10, 5) # create df with incorrect dimension to see if Error is raised arr = np.arange(1000).reshape(10, 10, 10) if sys.version_info[0] == 3: with self.assertRaises(ValueError): self.block.create_data_frame('err', 'err', {'name': np.int64}, data=arr) def test_data_type(self): assert self.df1.dtype[4] == np.int32 assert self.df1.dtype[0] != self.df1.dtype[4] assert self.df1.dtype[2] == self.df1.dtype[3] def test_create_without_dtypes(self): data = np.array([("a", 1, 2.2), ("b", 2, 3.3), ("c", 3, 4.4)], dtype=[('name', 'U10'), ("id", 'i4'), ('val', 'f4')]) df = self.block.create_data_frame("without_name", "test", data=data) assert sorted(list(df.column_names)) == sorted(["name", "id", "val"]) assert sorted(list(df["name"])) == ["a", "b", "c"] def test_timestamp_autoupdate(self): self.file.auto_update_timestamps = True df = self.block.create_data_frame("df.time", "test.time", col_dict=OrderedDict({"idx": int})) dftime = df.updated_at time.sleep(1) df.units = ("ly", ) self.assertNotEqual(dftime, df.updated_at) def test_timestamp_noautoupdate(self): self.file.auto_update_timestamps = False df = self.block.create_data_frame("df.time", "test.time", col_dict=OrderedDict({"idx": int})) dftime = df.updated_at time.sleep(1) df.units = ("ly", ) self.assertEqual(dftime, df.updated_at)
class Api(object): ''' The main entry point for the application. You need to initialize it with a Flask Application: :: >>> app = Flask(__name__) >>> api = Api(app) Alternatively, you can use :meth:`init_app` to set the Flask application after it has been constructed. The endpoint parameter prefix all views and resources: - The API root/documentation will be ``{endpoint}.root`` - A resource registered as 'resource' will be available as ``{endpoint}.resource`` :param flask.Flask|flask.Blueprint app: the Flask application object or a Blueprint :param str version: The API version (used in Swagger documentation) :param str title: The API title (used in Swagger documentation) :param str description: The API description (used in Swagger documentation) :param str terms_url: The API terms page URL (used in Swagger documentation) :param str contact: A contact email for the API (used in Swagger documentation) :param str license: The license associated to the API (used in Swagger documentation) :param str license_url: The license page URL (used in Swagger documentation) :param str endpoint: The API base endpoint (default to 'api). :param str default: The default namespace base name (default to 'default') :param str default_label: The default namespace label (used in Swagger documentation) :param str default_mediatype: The default media type to return :param bool validate: Whether or not the API should perform input payload validation. :param bool ordered: Whether or not preserve order models and marshalling. :param str doc: The documentation path. If set to a false value, documentation is disabled. (Default to '/') :param list decorators: Decorators to attach to every resource :param bool catch_all_404s: Use :meth:`handle_error` to handle 404 errors throughout your app :param dict authorizations: A Swagger Authorizations declaration as dictionary :param bool serve_challenge_on_401: Serve basic authentication challenge with 401 responses (default 'False') :param FormatChecker format_checker: A jsonschema.FormatChecker object that is hooked into the Model validator. A default or a custom FormatChecker can be provided (e.g., with custom checkers), otherwise the default action is to not enforce any format validation. ''' def __init__(self, app=None, version='1.0', title=None, description=None, terms_url=None, license=None, license_url=None, contact=None, contact_url=None, contact_email=None, authorizations=None, security=None, doc='/', default_id=default_id, default='default', default_label='Default namespace', validate=None, tags=None, prefix='', ordered=False, default_mediatype='application/json', decorators=None, catch_all_404s=False, serve_challenge_on_401=False, format_checker=None, **kwargs): self.version = version self.title = title or 'API' self.description = description self.terms_url = terms_url self.contact = contact self.contact_email = contact_email self.contact_url = contact_url self.license = license self.license_url = license_url self.authorizations = authorizations self.security = security self.default_id = default_id self.ordered = ordered self._validate = validate self._doc = doc self._doc_view = None self._default_error_handler = None self.tags = tags or [] self.error_handlers = { ParseError: mask_parse_error_handler, MaskError: mask_error_handler, } self._schema = None self.models = {} self._refresolver = None self.format_checker = format_checker self.namespaces = [] self.ns_paths = dict() self.representations = OrderedDict(DEFAULT_REPRESENTATIONS) self.urls = {} self.prefix = prefix self.default_mediatype = default_mediatype self.decorators = decorators if decorators else [] self.catch_all_404s = catch_all_404s self.serve_challenge_on_401 = serve_challenge_on_401 self.blueprint_setup = None self.endpoints = set() self.resources = [] self.app = None self.blueprint = None # must come after self.app initialisation to prevent __getattr__ recursion # in self._configure_namespace_logger self.default_namespace = self.namespace( default, default_label, endpoint='{0}-declaration'.format(default), validate=validate, api=self, path='/', ) if app is not None: self.app = app self.init_app(app) # super(Api, self).__init__(app, **kwargs) def init_app(self, app, **kwargs): ''' Allow to lazy register the API on a Flask application:: >>> app = Flask(__name__) >>> api = Api() >>> api.init_app(app) :param flask.Flask app: the Flask application object :param str title: The API title (used in Swagger documentation) :param str description: The API description (used in Swagger documentation) :param str terms_url: The API terms page URL (used in Swagger documentation) :param str contact: A contact email for the API (used in Swagger documentation) :param str license: The license associated to the API (used in Swagger documentation) :param str license_url: The license page URL (used in Swagger documentation) ''' self.app = app self.title = kwargs.get('title', self.title) self.description = kwargs.get('description', self.description) self.terms_url = kwargs.get('terms_url', self.terms_url) self.contact = kwargs.get('contact', self.contact) self.contact_url = kwargs.get('contact_url', self.contact_url) self.contact_email = kwargs.get('contact_email', self.contact_email) self.license = kwargs.get('license', self.license) self.license_url = kwargs.get('license_url', self.license_url) self._add_specs = kwargs.get('add_specs', True) # If app is a blueprint, defer the initialization try: app.record(self._deferred_blueprint_init) # Flask.Blueprint has a 'record' attribute, Flask.Api does not except AttributeError: self._init_app(app) else: self.blueprint = app def _init_app(self, app): ''' Perform initialization actions with the given :class:`flask.Flask` object. :param flask.Flask app: The flask application object ''' self._register_specs(self.blueprint or app) self._register_doc(self.blueprint or app) app.handle_exception = partial(self.error_router, app.handle_exception) app.handle_user_exception = partial(self.error_router, app.handle_user_exception) if len(self.resources) > 0: for resource, namespace, urls, kwargs in self.resources: self._register_view(app, resource, namespace, *urls, **kwargs) for ns in self.namespaces: self._configure_namespace_logger(app, ns) self._register_apidoc(app) self._validate = self._validate if self._validate is not None else app.config.get( 'RESTPLUS_VALIDATE', False) app.config.setdefault('RESTPLUS_MASK_HEADER', 'X-Fields') app.config.setdefault('RESTPLUS_MASK_SWAGGER', True) def __getattr__(self, name): try: return getattr(self.default_namespace, name) except AttributeError: raise AttributeError( 'Api does not have {0} attribute'.format(name)) def _complete_url(self, url_part, registration_prefix): ''' This method is used to defer the construction of the final url in the case that the Api is created with a Blueprint. :param url_part: The part of the url the endpoint is registered with :param registration_prefix: The part of the url contributed by the blueprint. Generally speaking, BlueprintSetupState.url_prefix ''' parts = (registration_prefix, self.prefix, url_part) return ''.join(part for part in parts if part) def _register_apidoc(self, app): conf = app.extensions.setdefault('restplus', {}) if not conf.get('apidoc_registered', False): app.register_blueprint(apidoc.apidoc) conf['apidoc_registered'] = True def _register_specs(self, app_or_blueprint): if self._add_specs: endpoint = str('specs') self._register_view(app_or_blueprint, SwaggerView, self.default_namespace, '/swagger.json', endpoint=endpoint, resource_class_args=(self, )) self.endpoints.add(endpoint) def _register_doc(self, app_or_blueprint): if self._add_specs and self._doc: # Register documentation before root if enabled app_or_blueprint.add_url_rule(self._doc, 'doc', self.render_doc) app_or_blueprint.add_url_rule(self.prefix or '/', 'root', self.render_root) def register_resource(self, namespace, resource, *urls, **kwargs): endpoint = kwargs.pop('endpoint', None) endpoint = str(endpoint or self.default_endpoint(resource, namespace)) kwargs['endpoint'] = endpoint self.endpoints.add(endpoint) if self.app is not None: self._register_view(self.app, resource, namespace, *urls, **kwargs) else: self.resources.append((resource, namespace, urls, kwargs)) return endpoint def _configure_namespace_logger(self, app, namespace): for handler in app.logger.handlers: namespace.logger.addHandler(handler) namespace.logger.setLevel(app.logger.level) def _register_view(self, app, resource, namespace, *urls, **kwargs): endpoint = kwargs.pop('endpoint', None) or camel_to_dash( resource.__name__) resource_class_args = kwargs.pop('resource_class_args', ()) resource_class_kwargs = kwargs.pop('resource_class_kwargs', {}) # NOTE: 'view_functions' is cleaned up from Blueprint class in Flask 1.0 if endpoint in getattr(app, 'view_functions', {}): previous_view_class = app.view_functions[endpoint].__dict__[ 'view_class'] # if you override the endpoint with a different class, avoid the # collision by raising an exception if previous_view_class != resource: msg = 'This endpoint (%s) is already set to the class %s.' raise ValueError(msg % (endpoint, previous_view_class.__name__)) resource.mediatypes = self.mediatypes_method() # Hacky resource.endpoint = endpoint resource_func = self.output( resource.as_view(endpoint, self, *resource_class_args, **resource_class_kwargs)) # Apply Namespace and Api decorators to a resource for decorator in chain(namespace.decorators, self.decorators): resource_func = decorator(resource_func) for url in urls: # If this Api has a blueprint if self.blueprint: # And this Api has been setup if self.blueprint_setup: # Set the rule to a string directly, as the blueprint is already # set up. self.blueprint_setup.add_url_rule(url, view_func=resource_func, **kwargs) continue else: # Set the rule to a function that expects the blueprint prefix # to construct the final url. Allows deferment of url finalization # in the case that the associated Blueprint has not yet been # registered to an application, so we can wait for the registration # prefix rule = partial(self._complete_url, url) else: # If we've got no Blueprint, just build a url with no prefix rule = self._complete_url(url, '') # Add the url to the application or blueprint app.add_url_rule(rule, view_func=resource_func, **kwargs) def output(self, resource): ''' Wraps a resource (as a flask view function), for cases where the resource does not directly return a response object :param resource: The resource as a flask view function ''' @wraps(resource) def wrapper(*args, **kwargs): resp = resource(*args, **kwargs) if isinstance(resp, BaseResponse): return resp data, code, headers = unpack(resp) return self.make_response(data, code, headers=headers) return wrapper def make_response(self, data, *args, **kwargs): ''' Looks up the representation transformer for the requested media type, invoking the transformer to create a response object. This defaults to default_mediatype if no transformer is found for the requested mediatype. If default_mediatype is None, a 406 Not Acceptable response will be sent as per RFC 2616 section 14.1 :param data: Python object containing response data to be transformed ''' default_mediatype = kwargs.pop('fallback_mediatype', None) or self.default_mediatype mediatype = request.accept_mimetypes.best_match( self.representations, default=default_mediatype, ) if mediatype is None: raise NotAcceptable() if mediatype in self.representations: resp = self.representations[mediatype](data, *args, **kwargs) resp.headers['Content-Type'] = mediatype return resp elif mediatype == 'text/plain': resp = original_flask_make_response(str(data), *args, **kwargs) resp.headers['Content-Type'] = 'text/plain' return resp else: raise InternalServerError() def documentation(self, func): '''A decorator to specify a view function for the documentation''' self._doc_view = func return func def render_root(self): self.abort(HTTPStatus.NOT_FOUND) def render_doc(self): '''Override this method to customize the documentation page''' if self._doc_view: return self._doc_view() elif not self._doc: self.abort(HTTPStatus.NOT_FOUND) return apidoc.ui_for(self) def default_endpoint(self, resource, namespace): ''' Provide a default endpoint for a resource on a given namespace. Endpoints are ensured not to collide. Override this method specify a custom algorithm for default endpoint. :param Resource resource: the resource for which we want an endpoint :param Namespace namespace: the namespace holding the resource :returns str: An endpoint name ''' endpoint = camel_to_dash(resource.__name__) if namespace is not self.default_namespace: endpoint = '{ns.name}_{endpoint}'.format(ns=namespace, endpoint=endpoint) if endpoint in self.endpoints: suffix = 2 while True: new_endpoint = '{base}_{suffix}'.format(base=endpoint, suffix=suffix) if new_endpoint not in self.endpoints: endpoint = new_endpoint break suffix += 1 return endpoint def get_ns_path(self, ns): return self.ns_paths.get(ns) def ns_urls(self, ns, urls): path = self.get_ns_path(ns) or ns.path return [path + url for url in urls] def add_namespace(self, ns, path=None): ''' This method registers resources from namespace for current instance of api. You can use argument path for definition custom prefix url for namespace. :param Namespace ns: the namespace :param path: registration prefix of namespace ''' if ns not in self.namespaces: self.namespaces.append(ns) if self not in ns.apis: ns.apis.append(self) # Associate ns with prefix-path if path is not None: self.ns_paths[ns] = path # Register resources for r in ns.resources: urls = self.ns_urls(ns, r.urls) self.register_resource(ns, r.resource, *urls, **r.kwargs) # Register models for name, definition in six.iteritems(ns.models): self.models[name] = definition if not self.blueprint and self.app is not None: self._configure_namespace_logger(self.app, ns) def namespace(self, *args, **kwargs): ''' A namespace factory. :returns Namespace: a new namespace instance ''' kwargs['ordered'] = kwargs.get('ordered', self.ordered) ns = Namespace(*args, **kwargs) self.add_namespace(ns) return ns def endpoint(self, name): if self.blueprint: return '{0}.{1}'.format(self.blueprint.name, name) else: return name @property def specs_url(self): ''' The Swagger specifications absolute url (ie. `swagger.json`) :rtype: str ''' return url_for(self.endpoint('specs'), _external=True) @property def base_url(self): ''' The API base absolute url :rtype: str ''' return url_for(self.endpoint('root'), _external=True) @property def base_path(self): ''' The API path :rtype: str ''' return url_for(self.endpoint('root'), _external=False) @cached_property def __schema__(self): ''' The Swagger specifications/schema for this API :returns dict: the schema as a serializable dict ''' if not self._schema: try: self._schema = Swagger(self).as_dict() except Exception: # Log the source exception for debugging purpose # and return an error message msg = 'Unable to render schema' log.exception(msg) # This will provide a full traceback return {'error': msg} return self._schema @property def _own_and_child_error_handlers(self): rv = {} rv.update(self.error_handlers) for ns in self.namespaces: for exception, handler in six.iteritems(ns.error_handlers): rv[exception] = handler return rv def errorhandler(self, exception): '''A decorator to register an error handler for a given exception''' if inspect.isclass(exception) and issubclass(exception, Exception): # Register an error handler for a given exception def wrapper(func): self.error_handlers[exception] = func return func return wrapper else: # Register the default error handler self._default_error_handler = exception return exception def owns_endpoint(self, endpoint): ''' Tests if an endpoint name (not path) belongs to this Api. Takes into account the Blueprint name part of the endpoint name. :param str endpoint: The name of the endpoint being checked :return: bool ''' if self.blueprint: if endpoint.startswith(self.blueprint.name): endpoint = endpoint.split(self.blueprint.name + '.', 1)[-1] else: return False return endpoint in self.endpoints def _should_use_fr_error_handler(self): ''' Determine if error should be handled with FR or default Flask The goal is to return Flask error handlers for non-FR-related routes, and FR errors (with the correct media type) for FR endpoints. This method currently handles 404 and 405 errors. :return: bool ''' adapter = current_app.create_url_adapter(request) try: adapter.match() except MethodNotAllowed as e: # Check if the other HTTP methods at this url would hit the Api valid_route_method = e.valid_methods[0] rule, _ = adapter.match(method=valid_route_method, return_rule=True) return self.owns_endpoint(rule.endpoint) except NotFound: return self.catch_all_404s except Exception: # Werkzeug throws other kinds of exceptions, such as Redirect pass def _has_fr_route(self): '''Encapsulating the rules for whether the request was to a Flask endpoint''' # 404's, 405's, which might not have a url_rule if self._should_use_fr_error_handler(): return True # for all other errors, just check if FR dispatched the route if not request.url_rule: return False return self.owns_endpoint(request.url_rule.endpoint) def error_router(self, original_handler, e): ''' This function decides whether the error occurred in a flask-restplus endpoint or not. If it happened in a flask-restplus endpoint, our handler will be dispatched. If it happened in an unrelated view, the app's original error handler will be dispatched. In the event that the error occurred in a flask-restplus endpoint but the local handler can't resolve the situation, the router will fall back onto the original_handler as last resort. :param function original_handler: the original Flask error handler for the app :param Exception e: the exception raised while handling the request ''' if self._has_fr_route(): try: return self.handle_error(e) except Exception as f: return original_handler(f) return original_handler(e) def handle_error(self, e): ''' Error handler for the API transforms a raised exception into a Flask response, with the appropriate HTTP status code and body. :param Exception e: the raised Exception object ''' got_request_exception.send(current_app._get_current_object(), exception=e) # When propagate_exceptions is set, do not return the exception to the # client if a handler is configured for the exception. if not isinstance(e, HTTPException) and \ current_app.propagate_exceptions and \ not isinstance(e, tuple(self.error_handlers.keys())): exc_type, exc_value, tb = sys.exc_info() if exc_value is e: raise else: raise e include_message_in_response = current_app.config.get( "ERROR_INCLUDE_MESSAGE", True) default_data = {} headers = Headers() for typecheck, handler in six.iteritems( self._own_and_child_error_handlers): if isinstance(e, typecheck): result = handler(e) default_data, code, headers = unpack( result, HTTPStatus.INTERNAL_SERVER_ERROR) break else: if isinstance(e, HTTPException): code = HTTPStatus(e.code) if include_message_in_response: default_data = { 'message': getattr(e, 'description', code.phrase) } headers = e.get_response().headers elif self._default_error_handler: result = self._default_error_handler(e) default_data, code, headers = unpack( result, HTTPStatus.INTERNAL_SERVER_ERROR) else: code = HTTPStatus.INTERNAL_SERVER_ERROR if include_message_in_response: default_data = { 'message': code.phrase, } if include_message_in_response: default_data['message'] = default_data.get('message', str(e)) data = getattr(e, 'data', default_data) fallback_mediatype = None if code >= HTTPStatus.INTERNAL_SERVER_ERROR: exc_info = sys.exc_info() if exc_info[1] is None: exc_info = None current_app.log_exception(exc_info) elif code == HTTPStatus.NOT_FOUND and current_app.config.get("ERROR_404_HELP", True) \ and include_message_in_response: data['message'] = self._help_on_404(data.get('message', None)) elif code == HTTPStatus.NOT_ACCEPTABLE and self.default_mediatype is None: # if we are handling NotAcceptable (406), make sure that # make_response uses a representation we support as the # default mediatype (so that make_response doesn't throw # another NotAcceptable error). supported_mediatypes = list(self.representations.keys()) fallback_mediatype = supported_mediatypes[ 0] if supported_mediatypes else "text/plain" # Remove blacklisted headers for header in HEADERS_BLACKLIST: headers.pop(header, None) resp = self.make_response(data, code, headers, fallback_mediatype=fallback_mediatype) if code == HTTPStatus.UNAUTHORIZED: resp = self.unauthorized(resp) return resp def _help_on_404(self, message=None): rules = dict([(RE_RULES.sub('', rule.rule), rule.rule) for rule in current_app.url_map.iter_rules()]) close_matches = difflib.get_close_matches(request.path, rules.keys()) if close_matches: # If we already have a message, add punctuation and continue it. message = ''.join(( (message.rstrip('.') + '. ') if message else '', 'You have requested this URI [', request.path, '] but did you mean ', ' or '.join((rules[match] for match in close_matches)), ' ?', )) return message def as_postman(self, urlvars=False, swagger=False): ''' Serialize the API as Postman collection (v1) :param bool urlvars: whether to include or not placeholders for query strings :param bool swagger: whether to include or not the swagger.json specifications ''' return PostmanCollectionV1(self, swagger=swagger).as_dict(urlvars=urlvars) @property def payload(self): '''Store the input payload in the current request context''' return request.get_json() @property def refresolver(self): if not self._refresolver: self._refresolver = RefResolver.from_schema(self.__schema__) return self._refresolver @staticmethod def _blueprint_setup_add_url_rule_patch(blueprint_setup, rule, endpoint=None, view_func=None, **options): ''' Method used to patch BlueprintSetupState.add_url_rule for setup state instance corresponding to this Api instance. Exists primarily to enable _complete_url's function. :param blueprint_setup: The BlueprintSetupState instance (self) :param rule: A string or callable that takes a string and returns a string(_complete_url) that is the url rule for the endpoint being registered :param endpoint: See BlueprintSetupState.add_url_rule :param view_func: See BlueprintSetupState.add_url_rule :param **options: See BlueprintSetupState.add_url_rule ''' if callable(rule): rule = rule(blueprint_setup.url_prefix) elif blueprint_setup.url_prefix: rule = blueprint_setup.url_prefix + rule options.setdefault('subdomain', blueprint_setup.subdomain) if endpoint is None: endpoint = _endpoint_from_view_func(view_func) defaults = blueprint_setup.url_defaults if 'defaults' in options: defaults = dict(defaults, **options.pop('defaults')) blueprint_setup.app.add_url_rule( rule, '%s.%s' % (blueprint_setup.blueprint.name, endpoint), view_func, defaults=defaults, **options) def _deferred_blueprint_init(self, setup_state): ''' Synchronize prefix between blueprint/api and registration options, then perform initialization with setup_state.app :class:`flask.Flask` object. When a :class:`flask_restplus.Api` object is initialized with a blueprint, this method is recorded on the blueprint to be run when the blueprint is later registered to a :class:`flask.Flask` object. This method also monkeypatches BlueprintSetupState.add_url_rule with _blueprint_setup_add_url_rule_patch. :param setup_state: The setup state object passed to deferred functions during blueprint registration :type setup_state: flask.blueprints.BlueprintSetupState ''' self.blueprint_setup = setup_state if setup_state.add_url_rule.__name__ != '_blueprint_setup_add_url_rule_patch': setup_state._original_add_url_rule = setup_state.add_url_rule setup_state.add_url_rule = MethodType( Api._blueprint_setup_add_url_rule_patch, setup_state) if not setup_state.first_registration: raise ValueError( 'flask-restplus blueprints can only be registered once.') self._init_app(setup_state.app) def mediatypes_method(self): '''Return a method that returns a list of mediatypes''' return lambda resource_cls: self.mediatypes( ) + [self.default_mediatype] def mediatypes(self): '''Returns a list of requested mediatypes sent in the Accept header''' return [ h for h, q in sorted(request.accept_mimetypes, key=operator.itemgetter(1), reverse=True) ] def representation(self, mediatype): ''' Allows additional representation transformers to be declared for the api. Transformers are functions that must be decorated with this method, passing the mediatype the transformer represents. Three arguments are passed to the transformer: * The data to be represented in the response body * The http status code * A dictionary of headers The transformer should convert the data appropriately for the mediatype and return a Flask response object. Ex:: @api.representation('application/xml') def xml(data, code, headers): resp = make_response(convert_data_to_xml(data), code) resp.headers.extend(headers) return resp ''' def wrapper(func): self.representations[mediatype] = func return func return wrapper def unauthorized(self, response): '''Given a response, change it to ask for credentials''' if self.serve_challenge_on_401: realm = current_app.config.get("HTTP_BASIC_AUTH_REALM", "flask-restplus") challenge = u"{0} realm=\"{1}\"".format("Basic", realm) response.headers['WWW-Authenticate'] = challenge return response def url_for(self, resource, **values): ''' Generates a URL to the given resource. Works like :func:`flask.url_for`. ''' endpoint = resource.endpoint if self.blueprint: endpoint = '{0}.{1}'.format(self.blueprint.name, endpoint) return url_for(endpoint, **values)
class CPUInfos(object): """ A set of CPU information objects. """ __slots__ = [ "_cpu_infos"] def __init__(self): self._cpu_infos = OrderedDict() def add_processor(self, x, y, processor_id, cpu_info): """ Add a processor on a given chip to the set. :param x: The x-coordinate of the chip :type x: int :param y: The y-coordinate of the chip :type y: int :param processor_id: A processor ID :type processor_id: int :param cpu_info: The CPU information for the core :type cpu_info: :py:class:`spinnman.model.enums.cpu_info.CPUInfo` """ self._cpu_infos[x, y, processor_id] = cpu_info @property def cpu_infos(self): """ The one per core core info. :return: iterable of x,y,p core info """ return iteritems(self._cpu_infos) def __iter__(self): return iter(self._cpu_infos) def iteritems(self): """ Get an iterable of (x, y, p), cpu_info """ return iteritems(self._cpu_infos) def items(self): return self._cpu_infos.items() def values(self): return self._cpu_infos.values() def itervalues(self): """ Get an iterable of cpu_info. """ return itervalues(self._cpu_infos) def keys(self): return self._cpu_infos.keys() def iterkeys(self): """ Get an iterable of (x, y, p). """ return iterkeys(self._cpu_infos) def __len__(self): """ The total number of processors that are in these core subsets. """ return len(self._cpu_infos)
class ExternalDeviceLifControlVertex( AbstractPopulationVertex, AbstractEthernetController, AbstractProvidesOutgoingPartitionConstraints, AbstractVertexWithEdgeToDependentVertices): """ Abstract control module for the pushbot, based on the LIF neuron,\ but without spikes, and using the voltage as the output to the various\ devices """ __slots__ = [ "__dependent_vertices", "__devices", "__message_translator", "__partition_id_to_atom", "__partition_id_to_key" ] # all commands will use this mask _DEFAULT_COMMAND_MASK = 0xFFFFFFFF def __init__(self, devices, create_edges, max_atoms_per_core, neuron_impl, pynn_model, translator=None, spikes_per_second=None, label=None, ring_buffer_sigma=None, incoming_spike_buffer_size=None, constraints=None): """ :param n_neurons: The number of neurons in the population :param devices:\ The AbstractMulticastControllableDevice instances to be controlled\ by the population :param create_edges:\ True if edges to the devices should be added by this dev (set\ to False if using the dev over Ethernet using a translator) :param translator:\ Translator to be used when used for Ethernet communication. Must\ be provided if the dev is to be controlled over Ethernet. """ # pylint: disable=too-many-arguments, too-many-locals if not devices: raise ConfigurationException("No devices specified") # Create a partition to key map self.__partition_id_to_key = OrderedDict( (str(dev.device_control_partition_id), dev.device_control_key) for dev in devices) # Create a partition to atom map self.__partition_id_to_atom = { partition: i for (i, partition) in enumerate(self.__partition_id_to_key.keys()) } self.__devices = devices self.__message_translator = translator # Add the edges to the devices if required self.__dependent_vertices = list() if create_edges: self.__dependent_vertices = devices super(ExternalDeviceLifControlVertex, self).__init__(len(devices), label, constraints, max_atoms_per_core, spikes_per_second, ring_buffer_sigma, incoming_spike_buffer_size, neuron_impl, pynn_model) def routing_key_partition_atom_mapping(self, routing_info, partition): # pylint: disable=arguments-differ key = self.__partition_id_to_key[partition.identifier] atom = self.__partition_id_to_atom[partition.identifier] return [(atom, key)] @overrides(AbstractProvidesOutgoingPartitionConstraints. get_outgoing_partition_constraints) def get_outgoing_partition_constraints(self, partition): return [ FixedKeyAndMaskConstraint([ BaseKeyAndMask( self.__partition_id_to_key[partition.identifier], self._DEFAULT_COMMAND_MASK) ]) ] @overrides(AbstractVertexWithEdgeToDependentVertices.dependent_vertices) def dependent_vertices(self): return self.__dependent_vertices @overrides(AbstractVertexWithEdgeToDependentVertices. edge_partition_identifiers_for_dependent_vertex) def edge_partition_identifiers_for_dependent_vertex(self, vertex): return [vertex.device_control_partition_id] @overrides(AbstractEthernetController.get_external_devices) def get_external_devices(self): return self.__devices @overrides(AbstractEthernetController.get_message_translator) def get_message_translator(self): if self.__message_translator is None: raise ConfigurationException( "This population was not given a translator, and so cannot be" "used for Ethernet communication. Please provide a " "translator for the population.") return self.__message_translator @overrides(AbstractEthernetController.get_outgoing_partition_ids) def get_outgoing_partition_ids(self): return self.__partition_id_to_key.keys()
class NeuronRecorder(object): __slots__ = ["__indexes", "__n_neurons", "__sampling_rates"] N_BYTES_FOR_TIMESTAMP = 4 N_BYTES_PER_VALUE = 4 N_BYTES_PER_RATE = 4 # uint32 N_BYTES_PER_INDEX = 1 # currently uint8 N_BYTES_PER_SIZE = 4 N_CPU_CYCLES_PER_NEURON = 8 N_BYTES_PER_WORD = 4 N_BYTES_PER_POINTER = 4 SARK_BLOCK_SIZE = 8 # Seen in sark.c MAX_RATE = 2**32 - 1 # To allow a unit32_t to be used to store the rate def __init__(self, allowed_variables, n_neurons): self.__sampling_rates = OrderedDict() self.__indexes = dict() self.__n_neurons = n_neurons for variable in allowed_variables: self.__sampling_rates[variable] = 0 self.__indexes[variable] = None def _count_recording_per_slice(self, variable, vertex_slice): if self.__sampling_rates[variable] == 0: return 0 if self.__indexes[variable] is None: return vertex_slice.n_atoms return sum(vertex_slice.lo_atom <= index <= vertex_slice.hi_atom for index in self.__indexes[variable]) def _neurons_recording(self, variable, vertex_slice): if self.__sampling_rates[variable] == 0: return [] if self.__indexes[variable] is None: return range(vertex_slice.lo_atom, vertex_slice.hi_atom + 1) recording = [] indexes = self.__indexes[variable] for index in xrange(vertex_slice.lo_atom, vertex_slice.hi_atom + 1): if index in indexes: recording.append(index) return recording def get_neuron_sampling_interval(self, variable): """ Return the current sampling interval for this variable :param variable: PyNN name of the variable :return: Sampling interval in micro seconds """ step = globals_variables.get_simulator().machine_time_step / 1000 return self.__sampling_rates[variable] * step def get_matrix_data(self, label, buffer_manager, region, placements, graph_mapper, application_vertex, variable, n_machine_time_steps): """ Read a uint32 mapped to time and neuron IDs from the SpiNNaker\ machine. :param label: vertex label :param buffer_manager: the manager for buffered data :param region: the DSG region ID used for this data :param placements: the placements object :param graph_mapper: \ the mapping between application and machine vertices :param application_vertex: :param variable: PyNN name for the variable (V, gsy_inh etc.) :type variable: str :param n_machine_time_steps: :return: """ if variable == SPIKES: msg = "Variable {} is not supported use get_spikes".format(SPIKES) raise ConfigurationException(msg) vertices = graph_mapper.get_machine_vertices(application_vertex) progress = ProgressBar(vertices, "Getting {} for {}".format(variable, label)) sampling_rate = self.__sampling_rates[variable] expected_rows = int(math.ceil(n_machine_time_steps / sampling_rate)) missing_str = "" data = None indexes = [] for vertex in progress.over(vertices): placement = placements.get_placement_of_vertex(vertex) vertex_slice = graph_mapper.get_slice(vertex) neurons = self._neurons_recording(variable, vertex_slice) n_neurons = len(neurons) if n_neurons == 0: continue indexes.extend(neurons) # for buffering output info is taken form the buffer manager record_raw, missing_data = buffer_manager.get_data_by_placement( placement, region) record_length = len(record_raw) row_length = self.N_BYTES_FOR_TIMESTAMP + \ n_neurons * self.N_BYTES_PER_VALUE # There is one column for time and one for each neuron recording n_rows = record_length // row_length if record_length > 0: # Converts bytes to ints and make a matrix record = (numpy.asarray( record_raw, dtype="uint8").view(dtype="<i4")).reshape( (n_rows, (n_neurons + 1))) else: record = numpy.empty((0, n_neurons)) # Check if you have the expected data if not missing_data and n_rows == expected_rows: # Just cut the timestamps off to get the fragment fragment = (record[:, 1:] / float(DataType.S1615.scale)) else: missing_str += "({}, {}, {}); ".format(placement.x, placement.y, placement.p) # Start the fragment for this slice empty fragment = numpy.empty((expected_rows, n_neurons)) for i in xrange(0, expected_rows): time = i * sampling_rate # Check if there is data for this timestep local_indexes = numpy.where(record[:, 0] == time) if len(local_indexes[0]) == 1: fragment[i] = (record[local_indexes[0], 1:] / float(DataType.S1615.scale)) elif len(local_indexes[0]) > 1: logger.warning( "Population {} on multiple recorded data for " "time {}".format(label, time)) else: # Set row to nan fragment[i] = numpy.full(n_neurons, numpy.nan) if data is None: data = fragment else: # Add the slice fragment on axis 1 which is IDs/channel_index data = numpy.append(data, fragment, axis=1) if len(missing_str) > 0: logger.warning( "Population {} is missing recorded data in region {} from the" " following cores: {}".format(label, region, missing_str)) sampling_interval = self.get_neuron_sampling_interval(variable) return (data, indexes, sampling_interval) def get_spikes(self, label, buffer_manager, region, placements, graph_mapper, application_vertex, machine_time_step): spike_times = list() spike_ids = list() ms_per_tick = machine_time_step / 1000.0 vertices = graph_mapper.get_machine_vertices(application_vertex) missing_str = "" progress = ProgressBar(vertices, "Getting spikes for {}".format(label)) for vertex in progress.over(vertices): placement = placements.get_placement_of_vertex(vertex) vertex_slice = graph_mapper.get_slice(vertex) if self.__indexes[SPIKES] is None: neurons_recording = vertex_slice.n_atoms else: neurons_recording = sum((index >= vertex_slice.lo_atom and index <= vertex_slice.hi_atom) for index in self.__indexes[SPIKES]) if neurons_recording == 0: continue # Read the spikes n_words = int(math.ceil(neurons_recording / 32.0)) n_bytes = n_words * self.N_BYTES_PER_WORD n_words_with_timestamp = n_words + 1 # for buffering output info is taken form the buffer manager record_raw, data_missing = buffer_manager.get_data_by_placement( placement, region) if data_missing: missing_str += "({}, {}, {}); ".format(placement.x, placement.y, placement.p) if len(record_raw) > 0: raw_data = (numpy.asarray(record_raw, dtype="uint8").view( dtype="<i4")).reshape([-1, n_words_with_timestamp]) else: raw_data = record_raw if len(raw_data) > 0: record_time = raw_data[:, 0] * float(ms_per_tick) spikes = raw_data[:, 1:].byteswap().view("uint8") bits = numpy.fliplr( numpy.unpackbits(spikes).reshape((-1, 32))).reshape( (-1, n_bytes * 8)) time_indices, local_indices = numpy.where(bits == 1) if self.__indexes[SPIKES] is None: indices = local_indices + vertex_slice.lo_atom times = record_time[time_indices].reshape((-1)) spike_ids.extend(indices) spike_times.extend(times) else: neurons = self._neurons_recording(SPIKES, vertex_slice) n_neurons = len(neurons) for time_indice, local in zip(time_indices, local_indices): if local < n_neurons: spike_ids.append(neurons[local]) spike_times.append(record_time[time_indice]) if len(missing_str) > 0: logger.warning( "Population {} is missing spike data in region {} from the" " following cores: {}".format(label, region, missing_str)) if len(spike_ids) == 0: return numpy.zeros((0, 2), dtype="float") result = numpy.column_stack((spike_ids, spike_times)) return result[numpy.lexsort((spike_times, spike_ids))] def get_recordable_variables(self): return self.__sampling_rates.keys() def is_recording(self, variable): try: return self.__sampling_rates[variable] > 0 except KeyError as e: msg = "Variable {} is not supported. Supported variables are {}" \ "".format(variable, self.get_recordable_variables()) raise_from(ConfigurationException(msg), e) @property def recording_variables(self): results = list() for region, rate in self.__sampling_rates.items(): if rate > 0: results.append(region) return results @property def recorded_region_ids(self): results = list() for id, rate in enumerate(self.__sampling_rates.values()): if rate > 0: results.append(id) return results def _compute_rate(self, sampling_interval): """ Convert a sampling interval into a rate. \ Remember, machine time step is in nanoseconds :param sampling_interval: interval between samples in microseconds :return: rate """ if sampling_interval is None: return 1 step = globals_variables.get_simulator().machine_time_step / 1000 rate = int(sampling_interval / step) if sampling_interval != rate * step: msg = "sampling_interval {} is not an an integer multiple of the "\ "simulation timestep {}".format(sampling_interval, step) raise ConfigurationException(msg) if rate > self.MAX_RATE: msg = "sampling_interval {} higher than max allowed which is {}" \ "".format(sampling_interval, step * self.MAX_RATE) raise ConfigurationException(msg) return rate def check_indexes(self, indexes): if indexes is None: return if len(indexes) == 0: raise ConfigurationException("Empty indexes list") found = False warning = None for index in indexes: if index < 0: raise ConfigurationException( "Negative indexes are not supported") elif index >= self.__n_neurons: warning = "Ignoring indexes greater than population size." else: found = True if warning is not None: logger.warning(warning) if not found: raise ConfigurationException( "All indexes larger than population size") def _turn_off_recording(self, variable, sampling_interval, remove_indexes): if self.__sampling_rates[variable] == 0: # Already off so ignore other parameters return if remove_indexes is None: # turning all off so ignoring sampling interval self.__sampling_rates[variable] = 0 self.__indexes[variable] = None return # No good reason to specify_interval when turning off if sampling_interval is not None: rate = self._compute_rate(sampling_interval) # But if they do make sure it is the same as before if rate != self.__sampling_rates[variable]: raise ConfigurationException( "Illegal sampling_interval parameter while turning " "off recording") if self.__indexes[variable] is None: # start with all indexes self.__indexes[variable] = range(self.__n_neurons) # remove the indexes not recording self.__indexes[variable] = \ [index for index in self.__indexes[variable] if index not in remove_indexes] # Check is at least one index still recording if len(self.__indexes[variable]) == 0: self.__sampling_rates[variable] = 0 self.__indexes[variable] = None def _check_complete_overwrite(self, variable, indexes): if indexes is None: # overwriting all OK! return if self.__indexes[variable] is None: if set(set(range(self.__n_neurons))).issubset(set(indexes)): # overwriting all previous so OK! return else: if set(self.__indexes[variable]).issubset(set(indexes)): # overwriting all previous so OK! return raise ConfigurationException( "Current implementation does not support multiple " "sampling_intervals for {} on one population.".format(variable)) def _turn_on_recording(self, variable, sampling_interval, indexes): rate = self._compute_rate(sampling_interval) if self.__sampling_rates[variable] == 0: # Previously not recording so OK self.__sampling_rates[variable] = rate elif rate != self.__sampling_rates[variable]: self._check_complete_overwrite(variable, indexes) # else rate not changed so no action if indexes is None: # previous recording indexes does not matter as now all (None) self.__indexes[variable] = None else: # make sure indexes is not a generator like range indexes = list(indexes) self.check_indexes(indexes) if self.__indexes[variable] is not None: # merge the two indexes indexes = self.__indexes[variable] + indexes # Avoid duplicates and keep in numerical order self.__indexes[variable] = list(set(indexes)) self.__indexes[variable].sort() def set_recording(self, variable, new_state, sampling_interval=None, indexes=None): if variable == "all": for key in self.__sampling_rates.keys(): self.set_recording(key, new_state, sampling_interval, indexes) elif variable in self.__sampling_rates: if new_state: self._turn_on_recording(variable, sampling_interval, indexes) else: self._turn_off_recording(variable, sampling_interval, indexes) else: raise ConfigurationException( "Variable {} is not supported".format(variable)) def get_buffered_sdram_per_record(self, variable, vertex_slice): """ Return the SDRAM used per record :param variable: :param vertex_slice: :return: """ n_neurons = self._count_recording_per_slice(variable, vertex_slice) if n_neurons == 0: return 0 if variable == SPIKES: # Overflow can be ignored as it is not save if in an extra word out_spike_words = int(math.ceil(n_neurons / 32.0)) out_spike_bytes = out_spike_words * self.N_BYTES_PER_WORD return self.N_BYTES_FOR_TIMESTAMP + out_spike_bytes else: return self.N_BYTES_FOR_TIMESTAMP + \ n_neurons * self.N_BYTES_PER_VALUE def get_buffered_sdram_per_timestep(self, variable, vertex_slice): """ Return the SDRAM used per timestep. In the case where sampling is used it returns the average\ for recording and none recording based on the recording rate :param variable: :param vertex_slice: :return: """ rate = self.__sampling_rates[variable] if rate == 0: return 0 data_size = self.get_buffered_sdram_per_record(variable, vertex_slice) if rate == 1: return data_size else: return data_size // rate def get_sampling_overflow_sdram(self, vertex_slice): """ Get the extra SDRAM that should be reserved if using per_timestep This is the extra that must be reserved if per_timestep is an average\ rather than fixed for every timestep. When sampling the average * time_steps may not be quite enough.\ This returns the extra space in the worst case\ where time_steps is a multiple of sampling rate + 1,\ and recording is done in the first and last time_step :param vertex_slice: :return: Highest possible overflow needed """ overflow = 0 for variable, rate in iteritems(self.__sampling_rates): # If rate is 0 no recording so no overflow # If rate is 1 there is no overflow as average is exact if rate > 1: data_size = self.get_buffered_sdram_per_record( variable, vertex_slice) overflow += data_size // rate * (rate - 1) return overflow def get_buffered_sdram(self, variable, vertex_slice, n_machine_time_steps): """ Returns the SDRAM used for this may timesteps If required the total is rounded up so the space will always fit :param variable: The :param vertex_slice: :return: """ rate = self.__sampling_rates[variable] if rate == 0: return 0 data_size = self.get_buffered_sdram_per_record(variable, vertex_slice) records = n_machine_time_steps // rate if n_machine_time_steps % rate > 0: records = records + 1 return data_size * records def get_sdram_usage_in_bytes(self, vertex_slice): n_words_for_n_neurons = (vertex_slice.n_atoms + 3) // 4 n_bytes_for_n_neurons = n_words_for_n_neurons * 4 return (8 + n_bytes_for_n_neurons) * len(self.__sampling_rates) def _get_fixed_sdram_usage(self, vertex_slice): total_neurons = vertex_slice.hi_atom - vertex_slice.lo_atom + 1 fixed_sdram = 0 # Recording rate for each neuron fixed_sdram += self.N_BYTES_PER_RATE # Number of recording neurons fixed_sdram += self.N_BYTES_PER_INDEX # index_parameters one per neuron # even if not recording as also act as a gate fixed_sdram += self.N_BYTES_PER_INDEX * total_neurons return fixed_sdram def get_variable_sdram_usage(self, vertex_slice): fixed_sdram = 0 per_timestep_sdram = 0 for variable in self.__sampling_rates: rate = self.__sampling_rates[variable] fixed_sdram += self._get_fixed_sdram_usage(vertex_slice) if rate > 0: fixed_sdram += self.SARK_BLOCK_SIZE per_record = self.get_buffered_sdram_per_record( variable, vertex_slice) if rate == 1: # Add size for one record as recording every timestep per_timestep_sdram += per_record else: # Get the average cost per timestep average_per_timestep = per_record / rate per_timestep_sdram += average_per_timestep # Add the rest once to fixed for worst case fixed_sdram += (per_record - average_per_timestep) return VariableSDRAM(fixed_sdram, per_timestep_sdram) def get_dtcm_usage_in_bytes(self, vertex_slice): # *_rate + n_neurons_recording_* + *_indexes usage = self.get_sdram_usage_in_bytes(vertex_slice) # *_count + *_increment usage += len(self.__sampling_rates) * self.N_BYTES_PER_POINTER * 2 # out_spikes, *_values for variable in self.__sampling_rates: if variable == SPIKES: out_spike_words = int(math.ceil(vertex_slice.n_atoms / 32.0)) out_spike_bytes = out_spike_words * self.N_BYTES_PER_WORD usage += self.N_BYTES_FOR_TIMESTAMP + out_spike_bytes else: usage += (self.N_BYTES_FOR_TIMESTAMP + vertex_slice.n_atoms * self.N_BYTES_PER_VALUE) # *_size usage += len(self.__sampling_rates) * self.N_BYTES_PER_SIZE # n_recordings_outstanding usage += self.N_BYTES_PER_WORD * 4 return usage def get_n_cpu_cycles(self, n_neurons): return n_neurons * self.N_CPU_CYCLES_PER_NEURON * \ len(self.recording_variables) def get_data(self, vertex_slice): data = list() n_words_for_n_neurons = (vertex_slice.n_atoms + 3) // 4 n_bytes_for_n_neurons = n_words_for_n_neurons * 4 for variable in self.__sampling_rates: rate = self.__sampling_rates[variable] n_recording = self._count_recording_per_slice( variable, vertex_slice) data.append(numpy.array([rate, n_recording], dtype="uint32")) if rate == 0: data.append(numpy.zeros(n_words_for_n_neurons, dtype="uint32")) elif self.__indexes[variable] is None: data.append( numpy.arange(n_bytes_for_n_neurons, dtype="uint8").view("uint32")) else: indexes = self.__indexes[variable] local_index = 0 local_indexes = list() for index in xrange(n_bytes_for_n_neurons): if index + vertex_slice.lo_atom in indexes: local_indexes.append(local_index) local_index += 1 else: # write to one beyond recording range local_indexes.append(n_recording) data.append( numpy.array(local_indexes, dtype="uint8").view("uint32")) return numpy.concatenate(data) def get_global_parameters(self, vertex_slice): params = [] for variable in self.__sampling_rates: params.append( NeuronParameter(self.__sampling_rates[variable], DataType.UINT32)) for variable in self.__sampling_rates: n_recording = self._count_recording_per_slice( variable, vertex_slice) params.append(NeuronParameter(n_recording, DataType.UINT8)) return params def get_index_parameters(self, vertex_slice): params = [] for variable in self.__sampling_rates: if self.__sampling_rates[variable] <= 0: local_indexes = 0 elif self.__indexes[variable] is None: local_indexes = IndexIsValue() else: local_indexes = [] n_recording = sum( vertex_slice.lo_atom <= index <= vertex_slice.hi_atom for index in self.__indexes[variable]) indexes = self.__indexes[variable] local_index = 0 for index in xrange(vertex_slice.lo_atom, vertex_slice.hi_atom + 1): if index in indexes: local_indexes.append(local_index) local_index += 1 else: # write to one beyond recording range local_indexes.append(n_recording) params.append(NeuronParameter(local_indexes, DataType.UINT8)) return params @property def _indexes(self): # for testing only return _ReadOnlyDict(self.__indexes)