def test_dispatch_extracted(clean_redis, clean_datastore):
redis = clean_redis
ds = clean_datastore
# def service_queue(name): return get_service_queue(name, redis)
# Setup the fake datastore
file_hash = get_random_hash(64)
second_file_hash = get_random_hash(64)
for fh in [file_hash, second_file_hash]:
obj = random_model_obj(models.file.File)
obj.sha256 = fh
ds.file.save(fh, obj)
# Inject the fake submission
submission = random_model_obj(models.submission.Submission)
submission.files = [dict(name='./file', sha256=file_hash)]
sid = submission.sid = 'first-submission'
disp = Dispatcher(ds, redis, redis)
disp.running = ToggleTrue()
client = DispatchClient(ds, redis, redis)
client.dispatcher_data_age = time.time()
client.dispatcher_data.append(disp.instance_id)
# Launch the submission
client.dispatch_submission(submission)
disp.pull_submissions()
disp.service_worker(disp.process_queue_index(sid))
# Finish one service extracting a file
job = client.request_work('0', 'extract', '0')
assert job.fileinfo.sha256 == file_hash
assert job.filename == './file'
new_result: Result = random_minimal_obj(Result)
new_result.sha256 = file_hash
new_result.response.service_name = 'extract'
new_result.response.extracted = [
dict(sha256=second_file_hash,
name='second-*',
description='abc',
classification='U')
]
client.service_finished(sid, 'extracted-done', new_result)
# process the result
disp.pull_service_results()
disp.service_worker(disp.process_queue_index(sid))
disp.service_worker(disp.process_queue_index(sid))
#
job = client.request_work('0', 'extract', '0')
assert job.fileinfo.sha256 == second_file_hash
assert job.filename == 'second-*'
def test_safelist_missing(client, storage):
invalid_hash = randomizer.get_random_hash(64)
storage.safelist.get_if_exists.return_value = None
resp = client.get(f'/api/v1/safelist/{invalid_hash}/', headers=headers)
assert resp.status_code == 404
assert resp.json['api_response'] is None
def test_safelist_exist(client, storage):
valid_hash = randomizer.get_random_hash(64)
valid_resp = randomizer.random_model_obj(Safelist, as_json=True)
valid_resp['hashes']['sha256'] = valid_hash
storage.safelist.get_if_exists.return_value = valid_resp
resp = client.get(f'/api/v1/safelist/{valid_hash}/', headers=headers)
assert resp.status_code == 200
assert resp.json['api_response'] == valid_resp
def create_safelists(ds, log=None):
for _ in range(20):
sl = random_model_obj(Safelist, as_json=True)
if sl['type'] == 'file':
sl.pop('tag', None)
elif sl['type'] == 'tag':
sl.pop('file', None)
sl['hashes']['sha256'] = "0" + get_random_hash(63)
ds.safelist.save(sl['hashes']['sha256'], sl)
if log:
log.info(f"\t{sl['hashes']['sha256']}")
ds.safelist.commit()
def test_dispatch_submission(clean_redis):
ds = MockDatastore(
collections=['submission', 'result', 'service', 'error', 'file'])
file_hash = get_random_hash(64)
ds.file.save(file_hash, random_model_obj(models.file.File))
ds.file.get(file_hash).sha256 = file_hash
# ds.file.get(file_hash).sha256 = ''
submission = random_model_obj(models.submission.Submission)
submission.files.clear()
submission.files.append(dict(name='./file', sha256=file_hash))
submission.sid = 'first-submission'
disp = Dispatcher(ds,
logger=logging,
redis=clean_redis,
redis_persist=clean_redis)
# Submit a problem, and check that it gets added to the dispatch hash
# and the right service queues
task = SubmissionTask(dict(submission=submission))
disp.dispatch_submission(task)
file_task = FileTask(disp.file_queue.pop())
assert file_task.sid == submission.sid
assert file_task.file_info.sha256 == file_hash
assert file_task.depth == 0
assert file_task.file_info.type == ds.file.get(file_hash).type
dh = DispatchHash(submission.sid, clean_redis)
for service_name in disp.scheduler.services.keys():
dh.fail_nonrecoverable(file_hash, service_name, 'error-code')
disp.dispatch_submission(task)
assert ds.submission.get(submission.sid).state == 'completed'
assert ds.submission.get(
submission.sid).errors == ['error-code'] * len(disp.scheduler.services)
import pytest
from unittest.mock import MagicMock, patch
from assemblyline.odm import randomizer
from assemblyline.odm.models.safelist import Safelist
from assemblyline_service_server import app
from assemblyline_service_server.config import AUTH_KEY
headers = {
'Container-Id': randomizer.get_random_hash(12),
'X-APIKey': AUTH_KEY,
'Service-Name': 'Safelist',
'Service-Version': randomizer.get_random_service_version(),
'Service-Tool-Version': randomizer.get_random_hash(64),
'Timeout': 1,
'X-Forwarded-For': '127.0.0.1',
}
@pytest.fixture(scope='function')
def storage():
ds = MagicMock()
with patch('assemblyline_service_server.config.SAFELIST_CLIENT.datastore',
ds):
yield ds
@pytest.fixture()
def client():
client = app.app.test_client()
def test_dispatch_file(clean_redis):
service_queue = lambda name: get_service_queue(name, clean_redis)
ds = MockDatastore(collections=[
'submission', 'result', 'service', 'error', 'file', 'filescore'
])
file_hash = get_random_hash(64)
sub = random_model_obj(models.submission.Submission)
sub.sid = sid = 'first-submission'
sub.params.ignore_cache = False
disp = Dispatcher(ds, clean_redis, clean_redis, logging)
disp.active_submissions.add(
sid,
SubmissionTask(dict(submission=sub)).as_primitives())
dh = DispatchHash(sid=sid, client=clean_redis)
print('==== first dispatch')
# Submit a problem, and check that it gets added to the dispatch hash
# and the right service queues
file_task = FileTask({
'sid':
'first-submission',
'min_classification':
get_classification().UNRESTRICTED,
'file_info':
dict(sha256=file_hash,
type='unknown',
magic='a',
md5=get_random_hash(32),
mime='a',
sha1=get_random_hash(40),
size=10),
'depth':
0,
'max_files':
5
})
disp.dispatch_file(file_task)
assert dh.dispatch_time(file_hash, 'extract') > 0
assert dh.dispatch_time(file_hash, 'wrench') > 0
assert service_queue('extract').length() == 1
assert service_queue('wrench').length() == 1
# Making the same call again will queue it up again
print('==== second dispatch')
disp.dispatch_file(file_task)
assert dh.dispatch_time(file_hash, 'extract') > 0
assert dh.dispatch_time(file_hash, 'wrench') > 0
assert service_queue('extract').length() == 2
assert service_queue('wrench').length() == 2
# assert len(mq) == 4
# Push back the timestamp in the dispatch hash to simulate a timeout,
# make sure it gets pushed into that service queue again
print('==== third dispatch')
[service_queue(name).delete() for name in disp.scheduler.services]
dh.fail_recoverable(file_hash, 'extract')
disp.dispatch_file(file_task)
assert dh.dispatch_time(file_hash, 'extract') > 0
assert dh.dispatch_time(file_hash, 'wrench') > 0
assert service_queue('extract').length() == 1
# assert len(mq) == 1
# Mark extract as finished, wrench as failed
print('==== fourth dispatch')
[service_queue(name).delete() for name in disp.scheduler.services]
dh.finish(file_hash, 'extract', 'result-key', 0, 'U')
dh.fail_nonrecoverable(file_hash, 'wrench', 'error-key')
disp.dispatch_file(file_task)
assert dh.finished(file_hash, 'extract')
assert dh.finished(file_hash, 'wrench')
assert service_queue('av-a').length() == 1
assert service_queue('av-b').length() == 1
assert service_queue('frankenstrings').length() == 1
# Have the AVs fail, frankenstrings finishes
print('==== fifth dispatch')
[service_queue(name).delete() for name in disp.scheduler.services]
dh.fail_nonrecoverable(file_hash, 'av-a', 'error-a')
dh.fail_nonrecoverable(file_hash, 'av-b', 'error-b')
dh.finish(file_hash, 'frankenstrings', 'result-key', 0, 'U')
disp.dispatch_file(file_task)
assert dh.finished(file_hash, 'av-a')
assert dh.finished(file_hash, 'av-b')
assert dh.finished(file_hash, 'frankenstrings')
assert service_queue('xerox').length() == 1
# Finish the xerox service and check if the submission completion got checked
print('==== sixth dispatch')
[service_queue(name).delete() for name in disp.scheduler.services]
dh.finish(file_hash, 'xerox', 'result-key', 0, 'U')
disp.dispatch_file(file_task)
assert dh.finished(file_hash, 'xerox')
assert len(disp.submission_queue) == 1
def test_dispatch_extracted(clean_redis):
# Setup the fake datastore
ds = MockDatastore(
collections=['submission', 'result', 'service', 'error', 'file'])
file_hash = get_random_hash(64)
second_file_hash = get_random_hash(64)
for fh in [file_hash, second_file_hash]:
ds.file.save(fh, random_model_obj(models.file.File))
ds.file.get(fh).sha256 = fh
# Inject the fake submission
submission = random_model_obj(models.submission.Submission)
submission.files.clear()
submission.files.append(dict(name='./file', sha256=file_hash))
submission.sid = 'first-submission'
# Launch the dispatcher
disp = Dispatcher(ds,
logger=logging,
redis=clean_redis,
redis_persist=clean_redis)
# Launch the submission
task = SubmissionTask(dict(submission=submission))
disp.dispatch_submission(task)
# Check that the right values were sent to the
file_task = FileTask(disp.file_queue.pop(timeout=1))
assert file_task.sid == submission.sid
assert file_task.file_info.sha256 == file_hash
assert file_task.depth == 0
assert file_task.file_info.type == ds.file.get(file_hash).type
# Finish the services
dh = DispatchHash(submission.sid, clean_redis)
for service_name in disp.scheduler.services.keys():
dh.finish(file_hash, service_name, 'error-code', 0, 'U')
# But one of the services extracted a file
dh.add_file(second_file_hash, 10, file_hash)
# But meanwhile, dispatch_submission has been recalled on the submission
disp.dispatch_submission(task)
# It should see the missing file, and we should get a new file dispatch message for it
# to make sure it is getting processed properly, this should be at depth 1, the first layer of
# extracted files
file_task = disp.file_queue.pop(timeout=1)
assert file_task is not None
file_task = FileTask(file_task)
assert file_task.sid == submission.sid
assert file_task.file_info.sha256 == second_file_hash
assert file_task.depth == 1
assert file_task.file_info.type == ds.file.get(second_file_hash).type
# Finish the second file
for service_name in disp.scheduler.services.keys():
dh.finish(second_file_hash, service_name, 'error-code', 0, 'U')
# And now we should get the finished submission
disp.dispatch_submission(task)
submission = ds.submission.get(submission.sid)
assert submission.state == 'completed'
assert submission.errors == []
assert len(submission.results) == 2 * len(disp.scheduler.services)
def datastore(datastore_connection):
ds = datastore_connection
try:
create_users(ds)
signatures.extend(create_signatures(ds))
ds.signature.commit()
for _ in range(TEST_SIZE):
f = random_model_obj(File)
ds.file.save(f.sha256, f)
file_list.append(f.sha256)
ds.file.commit()
for x in range(TEST_SIZE):
a = random_model_obj(Alert)
a.file.sha256 = file_list[x]
ds.alert.save(a.alert_id, a)
ds.alert.commit()
for x in range(TEST_SIZE):
r = random_model_obj(Result)
r.sha256 = file_list[x]
ds.result.save(r.build_key(), r)
ds.result.commit()
for x in range(TEST_SIZE):
s = random_model_obj(Submission)
for f in s.files:
f.sha256 = file_list[x]
ds.submission.save(s.sid, s)
ds.submission.commit()
for x in range(TEST_SIZE):
h = random_model_obj(Heuristic)
h.heur_id = f"AL_TEST_{x}"
ds.heuristic.save(h.heur_id, h)
ds.heuristic.commit()
for _ in range(TEST_SIZE):
w_id = "0"+get_random_hash(63)
w = random_model_obj(Safelist)
ds.safelist.save(w_id, w)
ds.safelist.commit()
for _ in range(TEST_SIZE):
w_id = get_random_id()
w = random_model_obj(Workflow)
ds.workflow.save(w_id, w)
ds.workflow.commit()
yield ds
finally:
ds.alert.wipe()
ds.file.wipe()
ds.result.wipe()
ds.signature.wipe()
ds.submission.wipe()
ds.heuristic.wipe()
ds.safelist.wipe()
ds.workflow.wipe()
wipe_users(ds)