def benchmark(storage: Callable[..., AbstractStorage]):
if storage.__name__ == "PeeweeStorage":
ds = Datastore(storage, testing=True, filepath="test.db")
else:
ds = Datastore(storage, testing=True)
num_single_events = 50
num_replace_events = 50
num_bulk_events = 20_000
num_events = num_single_events + num_replace_events + num_bulk_events + 1
num_final_events = num_single_events + num_bulk_events + 1
events = create_test_events(num_events)
single_events = events[:num_single_events]
replace_events = events[num_single_events:num_single_events +
num_replace_events]
bulk_events = events[num_single_events + num_replace_events:-1]
print(storage.__name__)
with temporary_bucket(ds) as bucket:
with ttt(" sum"):
with ttt(f" single insert {num_single_events} events"):
for event in single_events:
bucket.insert(event)
with ttt(f" bulk insert {num_bulk_events} events"):
bucket.insert(bulk_events)
with ttt(f" replace last {num_replace_events}"):
for e in replace_events:
bucket.replace_last(e)
with ttt(" insert 1 event"):
bucket.insert(events[-1])
with ttt(" get one"):
events_tmp = bucket.get(limit=1)
with ttt(" get all"):
events_tmp = bucket.get(limit=-1)
assert len(events_tmp) == num_final_events
with ttt(" get range"):
events_tmp = bucket.get(
limit=-1,
starttime=events[1].timestamp + 0.01 * td1s,
endtime=events[-1].timestamp + events[-1].duration,
)
assert len(events_tmp) == num_final_events - 1
def _start(storage_method, host, port, testing=False):
# TODO: This should probably be more specific
origins = "moz-extension://*"
if testing:
# CORS won't be supported in non-testing mode until we fix our authentication
logger.warning(
"CORS is enabled when ran in testing mode, don't store any sensitive data when running in testing mode!"
)
origins = "*"
# See: https://flask-cors.readthedocs.org/en/latest/
CORS(app, resources={r"/api/*": {"origins": origins}})
# Only pretty-print JSON if in testing mode (because of performance)
app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing
db = Datastore(storage_method, testing=testing)
app.api = ServerAPI(db=db, testing=testing)
try:
app.run(debug=testing,
host=host,
port=port,
request_handler=FlaskLogHandler,
use_reloader=False)
except OSError as e:
logger.error(e)
raise e
def create_app(
host: str, testing=True, storage_method=None, cors_origins=[], custom_static=dict()
) -> AWFlask:
app = AWFlask("aw-server", static_folder=static_folder, static_url_path="")
if storage_method is None:
storage_method = aw_datastore.get_storage_methods()["memory"]
# Only pretty-print JSON if in testing mode (because of performance)
app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing
with app.app_context():
_config_cors(cors_origins, testing)
app.json_encoder = rest.CustomJSONEncoder
app.register_blueprint(root)
app.register_blueprint(rest.blueprint)
app.register_blueprint(get_custom_static_blueprint(custom_static))
db = Datastore(storage_method, testing=testing)
app.api = ServerAPI(db=db, testing=testing)
# needed for host-header check
app.config["HOST"] = host
return app
def _start(storage_method, port=5600, testing=False):
if testing:
# CORS won't be supported in non-testing mode until we fix our authentication
CORS(app) # See: https://flask-cors.readthedocs.org/en/latest/
logger.warning("CORS is enabled when ran in testing mode")
app.db = Datastore(storage_method, testing=testing)
app.run(debug=testing, port=port, request_handler=FlaskLogHandler)
def q2_find_bucket(datastore: Datastore, filter_str: str, hostname: str = None):
"""Find bucket by using a filter_str (to avoid hardcoding bucket names)"""
for bucket in datastore.buckets():
if filter_str in bucket:
bucket_metadata = datastore[bucket].metadata()
if hostname:
if bucket_metadata["hostname"] == hostname:
return bucket
else:
return bucket
raise QueryFunctionException("Unable to find bucket matching '{}' (hostname filter set to '{}')".format(filter_str, hostname))
def q2_find_bucket(datastore: Datastore, filter_str: str, hostname: str = None):
"""Find bucket by using a filter_str (to avoid hardcoding bucket names)"""
for bucket in datastore.buckets():
if filter_str in bucket:
bucket_metadata = datastore[bucket].metadata()
if hostname:
if bucket_metadata["hostname"] == hostname:
return bucket
else:
return bucket
raise QueryFunctionException("Unable to find bucket matching '{}' (hostname filter set to '{}')".format(filter_str, hostname))
def _start(storage_method, host: str, port: int, testing: bool=False, cors_origins: List[str] = []):
_config_cors(cors_origins, testing)
# Only pretty-print JSON if in testing mode (because of performance)
app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing
db = Datastore(storage_method, testing=testing)
app.api = ServerAPI(db=db, testing=testing)
try:
app.run(debug=testing, host=host, port=port, request_handler=FlaskLogHandler, use_reloader=False)
except OSError as e:
logger.error(str(e))
raise e
def create_app(testing=True, storage_method=None, cors_origins=[]) -> AWFlask:
app = AWFlask("aw-server", static_folder=static_folder, static_url_path='')
if storage_method is None:
storage_method = aw_datastore.get_storage_methods()['memory']
# Only pretty-print JSON if in testing mode (because of performance)
app.config["JSONIFY_PRETTYPRINT_REGULAR"] = testing
with app.app_context():
_config_cors(cors_origins, testing)
app.json_encoder = rest.CustomJSONEncoder
app.register_blueprint(root)
app.register_blueprint(rest.blueprint)
db = Datastore(storage_method, testing=testing)
app.api = ServerAPI(db=db, testing=testing)
return app
def benchmark(storage: Callable[..., AbstractStorage]):
ds = Datastore(storage, testing=True)
num_single_events = 50
num_replace_events = 50
num_bulk_events = 2 * 10**3
num_events = num_single_events + num_replace_events + num_bulk_events + 1
num_final_events = num_single_events + num_bulk_events + 1
events = create_test_events(num_events)
single_events = events[:num_single_events]
replace_events = events[num_single_events:num_single_events +
num_replace_events]
bulk_events = events[num_single_events + num_replace_events:-1]
print(storage.__name__)
with temporary_bucket(ds) as bucket:
with ttt(" sum"):
with ttt(" single insert {} events".format(num_single_events)):
for event in single_events:
bucket.insert(event)
with ttt(" bulk insert {} events".format(num_bulk_events)):
bucket.insert(bulk_events)
with ttt(" replace last {}".format(num_replace_events)):
for e in replace_events:
bucket.replace_last(e)
with ttt(" insert 1 event"):
bucket.insert(events[-1])
with ttt(" get one"):
events_tmp = bucket.get(limit=1)
with ttt(" get all"):
events_tmp = bucket.get()
assert len(events_tmp) == num_final_events
def param_testing_buckets_cm():
datastores = [
Datastore(storage_strategy=strategy, testing=True)
for name, strategy in _storage_methods.items()
]
return [TempTestBucket(ds) for ds in datastores]
def param_datastore_objects():
return [
Datastore(storage_strategy=strategy, testing=True)
for name, strategy in _storage_methods.items()
]
