def benchmark(storage: Callable[..., AbstractStorage]): if storage.__name__ == "PeeweeStorage": ds = Datastore(storage, testing=True, filepath="test.db") else: ds = Datastore(storage, testing=True) num_single_events = 50 num_replace_events = 50 num_bulk_events = 20_000 num_events = num_single_events + num_replace_events + num_bulk_events + 1 num_final_events = num_single_events + num_bulk_events + 1 events = create_test_events(num_events) single_events = events[:num_single_events] replace_events = events[num_single_events:num_single_events + num_replace_events] bulk_events = events[num_single_events + num_replace_events:-1] print(storage.__name__) with temporary_bucket(ds) as bucket: with ttt(" sum"): with ttt(f" single insert {num_single_events} events"): for event in single_events: bucket.insert(event) with ttt(f" bulk insert {num_bulk_events} events"): bucket.insert(bulk_events) with ttt(f" replace last {num_replace_events}"): for e in replace_events: bucket.replace_last(e) with ttt(" insert 1 event"): bucket.insert(events[-1]) with ttt(" get one"): events_tmp = bucket.get(limit=1) with ttt(" get all"): events_tmp = bucket.get(limit=-1) assert len(events_tmp) == num_final_events with ttt(" get range"): events_tmp = bucket.get( limit=-1, starttime=events[1].timestamp + 0.01 * td1s, endtime=events[-1].timestamp + events[-1].duration, ) assert len(events_tmp) == num_final_events - 1
def _start(storage_method, host, port, testing=False): # TODO: This should probably be more specific origins = "moz-extension://*" if testing: # CORS won't be supported in non-testing mode until we fix our authentication logger.warning( "CORS is enabled when ran in testing mode, don't store any sensitive data when running in testing mode!" ) origins = "*" # See: https://flask-cors.readthedocs.org/en/latest/ CORS(app, resources={r"/api/*": {"origins": origins}}) # Only pretty-print JSON if in testing mode (because of performance) app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing db = Datastore(storage_method, testing=testing) app.api = ServerAPI(db=db, testing=testing) try: app.run(debug=testing, host=host, port=port, request_handler=FlaskLogHandler, use_reloader=False) except OSError as e: logger.error(e) raise e
def create_app( host: str, testing=True, storage_method=None, cors_origins=[], custom_static=dict() ) -> AWFlask: app = AWFlask("aw-server", static_folder=static_folder, static_url_path="") if storage_method is None: storage_method = aw_datastore.get_storage_methods()["memory"] # Only pretty-print JSON if in testing mode (because of performance) app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing with app.app_context(): _config_cors(cors_origins, testing) app.json_encoder = rest.CustomJSONEncoder app.register_blueprint(root) app.register_blueprint(rest.blueprint) app.register_blueprint(get_custom_static_blueprint(custom_static)) db = Datastore(storage_method, testing=testing) app.api = ServerAPI(db=db, testing=testing) # needed for host-header check app.config["HOST"] = host return app
def _start(storage_method, port=5600, testing=False): if testing: # CORS won't be supported in non-testing mode until we fix our authentication CORS(app) # See: https://flask-cors.readthedocs.org/en/latest/ logger.warning("CORS is enabled when ran in testing mode") app.db = Datastore(storage_method, testing=testing) app.run(debug=testing, port=port, request_handler=FlaskLogHandler)
def q2_find_bucket(datastore: Datastore, filter_str: str, hostname: str = None): """Find bucket by using a filter_str (to avoid hardcoding bucket names)""" for bucket in datastore.buckets(): if filter_str in bucket: bucket_metadata = datastore[bucket].metadata() if hostname: if bucket_metadata["hostname"] == hostname: return bucket else: return bucket raise QueryFunctionException("Unable to find bucket matching '{}' (hostname filter set to '{}')".format(filter_str, hostname))
def q2_find_bucket(datastore: Datastore, filter_str: str, hostname: str = None): """Find bucket by using a filter_str (to avoid hardcoding bucket names)""" for bucket in datastore.buckets(): if filter_str in bucket: bucket_metadata = datastore[bucket].metadata() if hostname: if bucket_metadata["hostname"] == hostname: return bucket else: return bucket raise QueryFunctionException("Unable to find bucket matching '{}' (hostname filter set to '{}')".format(filter_str, hostname))
def _start(storage_method, host: str, port: int, testing: bool=False, cors_origins: List[str] = []): _config_cors(cors_origins, testing) # Only pretty-print JSON if in testing mode (because of performance) app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing db = Datastore(storage_method, testing=testing) app.api = ServerAPI(db=db, testing=testing) try: app.run(debug=testing, host=host, port=port, request_handler=FlaskLogHandler, use_reloader=False) except OSError as e: logger.error(str(e)) raise e
def create_app(testing=True, storage_method=None, cors_origins=[]) -> AWFlask: app = AWFlask("aw-server", static_folder=static_folder, static_url_path='') if storage_method is None: storage_method = aw_datastore.get_storage_methods()['memory'] # Only pretty-print JSON if in testing mode (because of performance) app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing with app.app_context(): _config_cors(cors_origins, testing) app.json_encoder = rest.CustomJSONEncoder app.register_blueprint(root) app.register_blueprint(rest.blueprint) db = Datastore(storage_method, testing=testing) app.api = ServerAPI(db=db, testing=testing) return app
def benchmark(storage: Callable[..., AbstractStorage]): ds = Datastore(storage, testing=True) num_single_events = 50 num_replace_events = 50 num_bulk_events = 2 * 10**3 num_events = num_single_events + num_replace_events + num_bulk_events + 1 num_final_events = num_single_events + num_bulk_events + 1 events = create_test_events(num_events) single_events = events[:num_single_events] replace_events = events[num_single_events:num_single_events + num_replace_events] bulk_events = events[num_single_events + num_replace_events:-1] print(storage.__name__) with temporary_bucket(ds) as bucket: with ttt(" sum"): with ttt(" single insert {} events".format(num_single_events)): for event in single_events: bucket.insert(event) with ttt(" bulk insert {} events".format(num_bulk_events)): bucket.insert(bulk_events) with ttt(" replace last {}".format(num_replace_events)): for e in replace_events: bucket.replace_last(e) with ttt(" insert 1 event"): bucket.insert(events[-1]) with ttt(" get one"): events_tmp = bucket.get(limit=1) with ttt(" get all"): events_tmp = bucket.get() assert len(events_tmp) == num_final_events
def param_testing_buckets_cm(): datastores = [ Datastore(storage_strategy=strategy, testing=True) for name, strategy in _storage_methods.items() ] return [TempTestBucket(ds) for ds in datastores]
def param_datastore_objects(): return [ Datastore(storage_strategy=strategy, testing=True) for name, strategy in _storage_methods.items() ]