async def test_get_all_containers_date_filtered():
ts = time.perf_counter()
bprint('Test: Get All Containers Date Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_record_count(
ContainerQuery(filter={'_filter_tenant': 2}))
try:
unfiltered = results.success[0]['count']
except IndexError:
unfiltered = 0
print(f'Found {unfiltered} unfiltered containers.')
results = await pac.get_records(
query=ContainerQuery(date_filter_start='2019-10-01',
date_filter_field='create_time',
filter={'_filter_tenant': 2}))
filtered = len(results.success)
print(f'Found {filtered} filtered containers.')
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
assert unfiltered > filtered
tprint(results, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_containers_filtered():
ts = time.perf_counter()
bprint('Test: Get All Containers Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client_prod.toml') as pac:
# f = {'_filter_container_type': '"case"'}
f = {'_filter_tenant': 0}
# results = await pac.get_container_count(query=ContainerQuery(filter=f))
# count = results.success[0]['count']
# print('count:', count)
results = await pac.get_containers(query=ContainerQuery(filter=f))
# print(results)
ids = list(set([r['id'] for r in results.success]))
print('unique_ids:', len(ids))
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
tprint(results, top=5)
# dedup = frozenset(item.items()), item) for item in results.sucess)
# dedup = (frozenset(item.items()),item for item in results.success)
# dedup = dict((frozenset(item.items()), item) for item in results.success).values()
# print('tdedsup', type(dedup))
# print('dedup0', dedup[0])
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_containers_performance_analysis():
ts = time.perf_counter()
bprint('Test: Get Containers Performance Analysis')
# Semaphore, PageSize
semaphores = range(1, 100, 5)
page_sizes = range(1, 100, 10)
stats = []
for semaphore in semaphores:
for page_size in range(1, 100, 10):
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml',
sem=semaphore) as pac:
tts = time.perf_counter()
results = await pac.get_containers(
ContainerFilter(page_size=page_size))
recs = len(results.success)
dur = time.perf_counter() - tts
stat = f'| {semaphore}\t| {page_size}\t| {recs}\t| {dur:f}\t| {recs / dur:f}\t|'
stats.append(stat)
print(stat)
with open(realpath('./data/get_containers_stats_2019-07-30.txt')) as sfile:
sfile.writelines(stats)
bprint(f'-> Completed in {(time.perf_counter() - ts) / 60:f} minutes.')
async def test_create_many_artifacts():
ts = time.perf_counter()
bprint('Test: Create Many Artifacts')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client_prod.toml') as pac:
container = generate_container(artifact_count=2)
results = await pac.get_records(
ContainerQuery(filter={'_filter_tenant': 2}))
ids = [c['id'] for c in results.success]
cid = choice(ids)
print(f'Container: {cid}')
container[0].update_id(cid)
response_results, request_results = await pac.create_artifacts(
container)
# print(response_results)
assert type(response_results) is Results
assert len(request_results) == 1
assert len(response_results.success) == 2
assert not response_results.failure
assert response_results.success[0]['success']
tprint(response_results, request_results, top=5)
# todo: check for creation within phantom
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_delete_one_artifact():
ts = time.perf_counter()
bprint('Test: Delete One Artifact')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(
ArtifactQuery(filter={'_filter_type': '"test"'}))
ids = [c['id'] for c in results.success]
aid = choice(ids)
results = await pac.delete_records(query=[ArtifactQuery(id=aid)])
# print(results)
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
assert results.success
tprint(results)
print('\nVerify Results from Phantom')
results = await pac.get_records(query=ArtifactQuery(
filter={'_filter_id': aid}))
assert not results.success
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_update_one_artifact():
ts = time.perf_counter()
bprint('Test: Update One Artifact')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
container = generate_container(artifact_count=1)[0]
artifact = container.artifacts[0]
rid = artifact.data['request_id']
artifact.clear()
artifact.data = {'request_id': rid}
artifact.name = 'Update Test'
results = await pac.get_records(
ArtifactQuery(filter={'_filter_type': '"test"'}))
ids = [c['id'] for c in results.success]
aid = choice(ids)
artifact.update_id(aid)
print(f'Artifact: {aid}\n\t->{artifact}')
results = await pac.update_records(requests=[artifact])
# print(response_results)
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
assert results.success
assert results.success[0]['success']
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_delete_one_container():
ts = time.perf_counter()
bprint('Test: Delete One Container')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_tenant': 2}))
ids = [c['id'] for c in results.success]
assert ids # Need containers to test
cid = choice(ids)
results = await pac.delete_records(query=ContainerQuery(id=cid))
# print(results)
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
assert results.success
tprint(results)
print('\nVerify Results from Phantom')
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_id': cid}))
assert not results.success
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_delete_many_artifacts():
ts = time.perf_counter()
bprint('Test: Delete Many Artifacts')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(
ArtifactQuery(filter={'_filter_type': '"test"'}))
ids = [c['id'] for c in results.success]
aids = [choice(ids), choice(ids)]
# print(results)
while aids[0] == aids[1]:
aids[1] = choice(ids)
results = await pac.delete_records(
query=[ArtifactQuery(id=aid) for aid in aids])
assert type(results) is Results
assert len(results.success) == 2
assert not results.failure
assert results.success[0]['success']
tprint(results)
print('\nVerify Results from Phantom')
results = await pac.get_records(query=ArtifactQuery(
filter={'_filter_id__in': str(aids)}))
assert not results.success
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_update_one_container():
ts = time.perf_counter()
bprint('Test: Update One Container')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=ContainerQuery(
page=0, page_size=20, filter={'_filter_tenant': 2}))
old_container = choice(results.success)
print(f'Container Prior Update\n\t-> {old_container}')
container = generate_container()[0]
container.clear()
container.name = 'Update Test'
container.update_id(old_container['id'])
results = await pac.update_records(container)
# print(response_results)
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
assert results.success
assert results.success[0]['success']
tprint(results)
results = await pac.get_records(query=ContainerQuery(
id=old_container['id']))
print(f'Container After Update\n\t-> {results.success[0]}')
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_create_many_containers():
ts = time.perf_counter()
bprint('Test: Create Many Containers')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
response_results, request_results = await pac.create_containers(
generate_container(container_count=2))
# print(response_results)
assert type(response_results) is Results
assert len(request_results) == 2
assert len(response_results.success) == 2
assert not response_results.failure
assert response_results.success[0]['success']
tprint(response_results, request_results)
print('\nVerify Results from Phantom')
ids = [r['id'] for r in response_results.success]
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_id__in': str(ids)}))
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_container_artifacts_date_filtered():
# TODO: This is hard to test since we don't know what artifacts we'll encounter.
ts = time.perf_counter()
bprint('Test: Get All Container Artifacts Date Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
f = {'_filter_type': '"test"'}
results = await pac.get_record_count(ArtifactQuery(filter=f))
unfiltered = results.success[0]['count']
assert unfiltered
print(f'Found {unfiltered} unfiltered containers.')
results = await pac.get_records(
ArtifactQuery(date_filter_start='2019-09-10',
date_filter_field='create_time',
filter=f))
filtered = len(results.success)
print(f'Found {filtered} filtered containers.')
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
assert unfiltered > filtered
tprint(results, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_one_container_audit_data():
ts = time.perf_counter()
bprint('Test: Get One Container Audit Data')
async with PhantomApiClient(cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=AuditQuery(container=130416))
# print(results)
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
tprint(results, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_artifacts_count():
ts = time.perf_counter()
bprint('Test: Get All Artifacts Count')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_record_count(ArtifactQuery())
# print(results)
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
tprint(results) # Should return all artifacts across all containers
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_one_container():
ts = time.perf_counter()
bprint('Test: Get One Container')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=ContainerQuery(
page=0, page_size=50, filter={'_filter_tenant': 2}))
ids = [c['id'] for c in results.success]
results = await pac.get_records(query=ContainerQuery(id=choice(ids)))
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_delete_all_container_artifacts():
ts = time.perf_counter()
bprint('Test: Delete All Container Artifacts')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(
query=ContainerQuery(page=0, page_size=50))
ids = [c['id'] for c in results.success if c['artifact_count'] > 1]
cid = choice(ids)
print(f'Container: {cid}')
results = await pac.get_record_count(ArtifactQuery(container_id=cid))
count = results.success[0]['count']
results = await pac.get_records(ArtifactQuery(container_id=cid))
# print(results)
assert type(results) is Results
assert len(results.success) == count
assert not results.failure
tprint(results, top=5)
aids = [a['id'] for a in results.success]
results = await pac.delete_records(
query=[ArtifactQuery(id=aid) for aid in aids])
tprint(results, top=5)
print('\nVerify Results from Phantom')
results = await pac.get_records(query=ArtifactQuery(
filter={'_filter_id__in': str(aids)}))
assert not results.success
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_artifacts_filtered():
ts = time.perf_counter()
bprint('Test: Get All Artifacts Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
f = {'_filter_type': '"test"'}
results = await pac.get_record_count(ArtifactQuery(filter=f))
count = results.success[0]['count']
assert count
results = await pac.get_records(ArtifactQuery(filter=f))
# print(results)
assert type(results) is Results
assert len(results.success) == count
assert not results.failure
tprint(results, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_containers_filtered():
ts = time.perf_counter()
bprint('Test: Get All Containers Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_record_count(query=ContainerQuery(
filter={'_filter_tenant': 2}))
count = results.success[0]['count']
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_tenant': 2}))
# print(results)
assert type(results) is Results
assert len(results.success) == count
assert not results.failure
tprint(results, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_one_artifact():
ts = time.perf_counter()
bprint('Test: Get One Artifact')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=ArtifactQuery(
page=0, page_size=50, filter={'_filter_type': '"test"'}))
ids = [c['id'] for c in results.success]
aid = choice(ids)
print(f'Artifact: {aid}')
results = await pac.get_records(ArtifactQuery(id=6143278))
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_delete_containers() -> NoReturn:
ts = time.perf_counter()
bprint('Test: Delete Containers')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client_prod.toml') as pac:
# Get containers
# f = {'_filter_name__icontains': '"bricata"', '_filter_tenant': 0}
f = {'_filter_tenant': 1}
results = await pac.get_containers(query=ContainerQuery(filter=f))
# print('results:', len(results.success))
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
tprint(results, top=5)
print('Deleting containers...')
filtered_ids = filter_by_date(results)
# results1 = await pac.delete_records(record_ids=filtered_ids, query=Query(type='container'))
#
# assert type(results1) is Results
# assert not results1.failure
#
# tprint(results1, top=5)
# # Verify test containers have been deleted
# results2 = await pac.get_containers(Query(type='container', filter=f))
# record_ids = filter_by_date(results2.success)
# # print('results2', results2)
#
# assert type(results2) is Results
# assert len(record_ids) == 0
# assert not results2.failure
#
# tprint(results2, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_one_container_artifacts_count():
ts = time.perf_counter()
bprint('Test: Get One Container Artifacts Count')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(
query=ContainerQuery(page=0, page_size=50))
ids = [c['id'] for c in results.success if c['artifact_count'] > 1]
cid = choice(ids)
print(f'Container: {cid}')
results = await pac.get_record_count(ArtifactQuery(container_id=cid))
# print(results)
assert type(results) is Results
assert len(results.success) >= 1
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_artifacts_count_filtered():
ts = time.perf_counter()
bprint('Test: Get All Artifacts Count Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_record_count(ArtifactQuery())
unfiltered = results.success[0]['count']
results = await pac.get_record_count(
ArtifactQuery(filter={'_filter_type': '"test"'}))
filtered = results.success[0]['count']
# print(results)
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
assert unfiltered > filtered
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_update_many_containers():
ts = time.perf_counter()
bprint('Test: Update Many Containers')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=ContainerQuery(
page=0, page_size=50, filter={'_filter_tenant': 2}))
old_containers = [choice(results.success), choice(results.success)]
while old_containers[0] == old_containers[1]:
old_containers[1] = choice(results.success)
ocp = '\n\t-> '.join([str(d) for d in old_containers])
print(f'Containers Prior Update\n\t-> {ocp}')
containers = generate_container(container_count=2)
[a.clear() for a in containers]
for i, a in enumerate(containers):
a.name = f'Update Test: {i}'
a.update_id(old_containers[i]['id'])
results = await pac.update_records(containers)
assert type(results) is Results
assert len(results.success) == 2
assert not results.failure
assert results.success
assert results.success[0]['success']
tprint(results)
results = await pac.get_records(query=ContainerQuery(
id=[c['id'] for c in old_containers]))
ucp = '\n\t-> '.join([str(d) for d in results.success])
print(f'Containers After Update\n\t-> {ucp}')
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_containers_count_filtered():
ts = time.perf_counter()
bprint('Test: Get All Containers Count Filtered')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_record_count(query=ContainerQuery())
unfiltered = results.success[0]['count']
results = await pac.get_record_count(query=ContainerQuery(
filter={'_filter_tenant': 2}))
filtered = results.success[0]['count']
# print(results)
assert type(results) is Results
assert len(results.success) == 1
assert not results.failure
assert unfiltered > filtered
tprint(results) # Should return all containers for tenant '2'
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_get_all_containers():
ts = time.perf_counter()
bprint('Test: Get All Containers')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client_prod.toml') as pac:
# results = await pac.get_record_count(query=ContainerQuery(filter={'_filter_container_type': '"case"'}))
# count = results.success[0]['count']
# print(f'Count: {count}')
# results = await pac.get_records(ContainerQuery())
results = await pac.get_records(query=ContainerQuery(
page_size=100000, filter={'_filter_container_type': '"case"'}))
# print(results)
assert type(results) is Results
# assert len(results.success) == count
assert not results.failure
tprint(results, top=5)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_update_many_artifacts():
ts = time.perf_counter()
bprint('Test: Update Many Artifacts')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
container = generate_container(artifact_count=2)[0]
artifacts = container.artifacts
rids = [a.data['request_id'] for a in artifacts]
[a.clear() for a in artifacts]
results = await pac.get_records(
ArtifactQuery(filter={'_filter_type': '"test"'}))
ids = [c['id'] for c in results.success]
aids = [choice(ids), choice(ids)]
while aids[0] == aids[1]:
aids[1] = choice(ids)
print(f'Artifacts: {aids}')
for i, a in enumerate(artifacts):
a.data = {'request_id': rids[i]}
a.name = f'Update Test: {i}'
a.update_id(aids[i])
results = await pac.update_records(artifacts)
# print(response_results)
assert type(results) is Results
assert len(results.success) == 2
assert not results.failure
assert results.success
assert results.success[0]['success']
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_delete_many_containers():
ts = time.perf_counter()
bprint('Test: Delete Many Containers')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client.toml') as pac:
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_tenant': 2}))
ids = [c['id'] for c in results.success]
assert ids # Need containers to test
cids = [choice(ids), choice(ids)]
while cids[0] == cids[1]:
cids[1] = choice(results.success)
results = await pac.delete_records(
query=[ContainerQuery(id=cid) for cid in cids])
# print(results)
assert type(results) is Results
assert len(results.success) == 2
assert not results.failure
assert results.success[0]['success']
tprint(results)
print('\nVerify Results from Phantom')
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_id__in': str(cids)}))
assert not results.success
assert not results.failure
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')
async def test_create_one_container():
ts = time.perf_counter()
bprint('Test: Create One Container')
async with PhantomApiClient(
cfg=f'{getenv("CFG_HOME")}/phantom_api_client_prod.toml') as pac:
response_results, request_results = await pac.create_containers(
generate_container())
# print(response_results)
assert type(response_results) is Results
assert len(request_results) == 1
assert len(response_results.success) == 1
assert not response_results.failure
assert response_results.success[0]['success']
tprint(response_results, request_results)
print('\nVerify Results from Phantom')
results = await pac.get_records(query=ContainerQuery(
filter={'_filter_id': response_results.success[0]['id']}))
tprint(results)
bprint(f'-> Completed in {(time.perf_counter() - ts):f} seconds.')