def datastore(datastore_connection, filestore):
try:
create_users(datastore_connection)
create_submission(datastore_connection, filestore)
yield datastore_connection
finally:
wipe_users(datastore_connection)
wipe_submissions(datastore_connection, filestore)
def datastore(datastore_connection, filestore):
try:
create_users(datastore_connection)
for _ in range(NUM_SUBMISSIONS):
create_submission(datastore_connection, filestore)
yield datastore_connection
finally:
wipe_users(datastore_connection)
wipe_submissions(datastore_connection, filestore)
def datastore(request, datastore_connection, fs):
purge_data(datastore_connection, fs)
create_alerts(datastore_connection, alert_count=1)
create_heuristics(datastore_connection)
create_services(datastore_connection)
create_signatures(datastore_connection)
create_submission(datastore_connection, fs)
create_users(datastore_connection)
create_workflows(datastore_connection)
request.addfinalizer(lambda: purge_data(datastore_connection, fs))
return datastore_connection
def datastore(datastore_connection, filestore):
ds = datastore_connection
try:
random_data.create_heuristics(ds, heuristics_count=10)
random_data.create_services(ds)
random_data.create_signatures(ds)
random_data.create_users(ds)
random_data.create_workflows(ds)
submissions = []
for _ in range(2):
submissions.append(random_data.create_submission(
ds, filestore))
random_data.create_alerts(ds,
alert_count=10,
submission_list=submissions)
yield ds
finally:
# Cleanup test data
random_data.wipe_alerts(ds)
random_data.wipe_heuristics(ds)
random_data.wipe_services(ds)
random_data.wipe_signatures(ds)
random_data.wipe_submissions(ds, filestore)
random_data.wipe_users(ds)
random_data.wipe_workflows(ds)
def datastore(request, datastore_connection, fs):
for _ in range(NUM_SUBMISSIONS):
all_submissions.append(create_submission(datastore_connection, fs))
try:
yield datastore_connection
finally:
wipe_submissions(datastore_connection, fs)
datastore_connection.alert.wipe()
def datastore(datastore_connection, filestore):
global submission
try:
create_users(datastore_connection)
submission = create_submission(datastore_connection, filestore)
yield datastore_connection
finally:
wipe_users(datastore_connection)
wipe_submissions(datastore_connection, filestore)
sq.delete()
def create_extra_data(log=None, ds=None, fs=None):
ds = ds or forge.get_datastore()
fs = fs or forge.get_filestore()
log.info("\nCreating 10 Submissions...")
submissions = []
for _ in range(10):
s = create_submission(ds, fs, log=log)
submissions.append(s)
log.info("\nCreating 50 Alerts...")
create_alerts(ds, submission_list=submissions, log=log)
def test_delete(datastore, cli, fs):
try:
# delete all users and check if there are still users
cli.do_delete("user force id:*")
datastore.user.commit()
assert datastore.user.search("id:*")['total'] == 0
finally:
# Restore users ...
create_users(datastore)
try:
# Delete random submission and check if still there
sub_id = datastore.submission.search("id:*",
fl="id",
rows=1,
as_obj=False)['items'][0]['id']
cli.do_delete(f"submission full force id:{sub_id}")
datastore.submission.commit()
assert datastore.user.search(f"id:{sub_id}")['total'] == 0
finally:
# Re-create a submission
create_submission(datastore, fs)
def datastore(datastore_connection, filestore):
try:
create_users(datastore_connection)
submission = create_submission(datastore_connection, filestore)
alert = random_model_obj(Alert)
alert.alert_id = ALERT_ID
alert.sid = submission.sid
datastore_connection.alert.save(ALERT_ID, alert)
yield datastore_connection
finally:
wipe_users(datastore_connection)
wipe_submissions(datastore_connection, filestore)
datastore_connection.alert.delete(ALERT_ID)
def datastore(request, datastore_connection, filestore):
global test_alert, test_submission
ds = datastore_connection
create_users(ds)
test_submission = create_submission(ds, filestore)
test_alert = random_model_obj(Alert)
test_alert.sid = test_submission.sid
test_alert.file.sha256 = test_submission.files[0].sha256
ds.alert.save(test_alert.alert_id, test_alert)
ds.alert.commit()
request.addfinalizer(lambda: cleanup(ds, filestore))
return ds
def create_extra_data(log=None, ds=None, fs=None):
ds = ds or forge.get_datastore()
fs = fs or forge.get_filestore()
log.info("\nCreating 10 Submissions...")
submissions = []
for _ in range(10):
s = create_submission(ds, fs, log=log)
submissions.append(s)
log.info("\nCreating 50 Alerts...")
create_alerts(ds, submission_list=submissions, log=log)
log.info("\nGenerating statistics for signatures and heuristics...")
ds.calculate_signature_stats()
ds.calculate_heuristic_stats()
def test_alert_bundle(datastore_connection, filestore, config):
# Cleanup previous runs
datastore_connection.alert.delete(ALERT_ID)
# Create a temporary submission
submission = create_submission(datastore_connection, filestore)
sid = submission['sid']
# Create a random alert
alert = random_model_obj(Alert)
alert.alert_id = ALERT_ID
alert.sid = sid
datastore_connection.alert.save(ALERT_ID, alert)
# Create the submission's bundle
path = create_bundle(ALERT_ID, use_alert=True)
# Test if the bundle
assert os.path.exists(path)
with open(path, 'rb') as fh:
assert is_cart(fh.read(256))
# Remove alert and submission from DB
datastore_connection.alert.delete(alert.alert_id)
datastore_connection.delete_submission_tree(sid, transport=filestore)
assert datastore_connection.alert.get_if_exists(alert.alert_id) is None
assert datastore_connection.submission.get_if_exists(sid) is None
# Restore bundle
new_submission = import_bundle(path)
# Validate restored submission
assert new_submission['sid'] == sid
assert new_submission['metadata']['bundle.source'] == config.ui.fqdn
# Validate restored alert
new_alert = datastore_connection.alert.get_if_exists(alert.alert_id, as_obj=False)
assert new_alert['alert_id'] == ALERT_ID
assert new_alert['sid'] == sid
assert new_alert['metadata']['bundle.source'] == config.ui.fqdn
# Cleanup
assert not os.path.exists(path)
datastore_connection.alert.delete(alert.alert_id)
datastore_connection.delete_submission_tree(sid, transport=filestore)
assert datastore_connection.alert.get_if_exists(alert.alert_id) is None
assert datastore_connection.submission.get_if_exists(sid) is None
def datastore(request, datastore_connection, fs):
wipe_alerts(datastore_connection)
wipe_submissions(datastore_connection, fs)
for _ in range(NUM_SUBMISSIONS):
all_submissions.append(create_submission(datastore_connection, fs))
create_alerts(datastore_connection,
alert_count=NUM_ALERTS,
submission_list=all_submissions)
for alert in datastore_connection.alert.stream_search("id:*", fl="*"):
all_alerts.append(alert)
try:
yield datastore_connection
finally:
wipe_alerts(datastore_connection)
wipe_submissions(datastore_connection, fs)
datastore_connection.alert.wipe()
def datastore(request, datastore_connection, filestore):
global test_alert
ds = datastore_connection
create_users(ds)
submission = create_submission(ds, filestore)
for _ in range(NUM_ALERTS):
a = random_model_obj(Alert)
if test_alert is None:
test_alert = a
a.owner = None
a.sid = submission.sid
ds.alert.save(a.alert_id, a)
ds.alert.commit()
request.addfinalizer(lambda: purge_alert(ds, filestore))
return ds
def test_submission_bundle(datastore_connection, filestore, config):
# Create a temporary submission
submission = create_submission(datastore_connection, filestore)
sid = submission['sid']
# Create the submission's bundle
path = create_bundle(sid)
# Test if the bundle
assert os.path.exists(path)
with open(path, 'rb') as fh:
assert is_cart(fh.read(256))
# Remove submission from DB
datastore_connection.delete_submission_tree(sid, transport=filestore)
assert datastore_connection.submission.get_if_exists(sid) is None
# Restore bundle
new_submission = import_bundle(path, cleanup=False)
# Validate restored submission
assert new_submission['sid'] == sid
assert new_submission['metadata']['bundle.source'] == config.ui.fqdn
# Test inserting failure
with pytest.raises(SubmissionAlreadyExist):
import_bundle(path, cleanup=False)
# Test skip failure on exist
new_submission = import_bundle(path, exist_ok=True)
# Validate restored submission
assert new_submission['sid'] == sid
assert new_submission['metadata']['bundle.source'] == config.ui.fqdn
# Cleanup
assert not os.path.exists(path)
datastore_connection.delete_submission_tree(sid, transport=filestore)
assert datastore_connection.submission.get_if_exists(sid) is None
def datastore(request, datastore_connection, fs):
create_alerts(datastore_connection, alert_count=1)
create_submission(datastore_connection, fs)
request.addfinalizer(lambda: purge_alert(datastore_connection, fs))
return datastore_connection
