def test_process_scc_query_two_steps_no_final_result_2():
# given
uuid = '74b8f5a8982948fb9acc6377ecf5149a'
step1 = Step(
uuid=uuid,
order=1,
kind='ASSET',
filter_=
'securityCenterProperties.resourceType = "google.cloud.resourcemanager.Project" AND '
'securityCenterProperties.resourceOwners : "blabla"',
out_join='securityCenterProperties.resourceName',
threshold=Threshold(operator="ge", value="0"))
step2 = Step(uuid=uuid,
order=2,
kind='FINDING',
in_join='resourceName',
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid,
name='query_test',
description='testing',
steps=[step2, step1])
# when
result, _, _, _ = process_scc_query(
query, {'sccquery{}'.format(uuid): 'working{}'.format(uuid)})
# then
assert result == 0, 'Should has empty return.'
def test_process_scc_query_two_steps_no_final_result():
# given
uuid = '386fa8353a0c4840a727fb13e013601a'
read_time = ReadTime("FROM_NOW", "1h", "são paulo")
step1 = Step(
uuid=uuid,
order=1,
kind='ASSET',
compare_duration='40w',
filter_='securityCenterProperties.resourceType = "INSTANCE" AND '
'securityCenterProperties.resourceName = "marine-physics-196005/instance/6515504379959957375"',
read_time=read_time,
out_join='securityCenterProperties.resourceName',
threshold=Threshold(operator="ge", value="0"))
step2 = Step(uuid=uuid,
order=2,
kind='FINDING',
filter_='category = "FOO"',
read_time=read_time,
in_join='resourceName',
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid,
name='Find projects on organization',
steps=[step2, step1])
# when
result, _, _, _ = process_scc_query(
query, {'sccquery{}'.format(uuid): 'working{}'.format(uuid)})
# then
assert result == 0, 'Should has empty return.'
def test_process_scc_query_three_steps():
# given
uuid = '74b8f5a8123450fb9acc6377ecf5159b'
step1 = Step(uuid=uuid,
order=1,
kind='FINDING',
filter_='category : "PROJECT_ACCESS"',
out_join='resourceName',
threshold=Threshold(operator="ge", value="0"))
step2 = Step(uuid=uuid,
order=2,
kind='ASSET',
in_join='name',
out_join='name',
threshold=Threshold(operator="ge", value="0"))
step3 = Step(uuid=uuid,
order=3,
kind='FINDING',
in_join='resourceName',
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid,
name='Find projects on organization',
steps=[step3, step2, step1])
# when
result, result_kind, step_result, _ = process_scc_query(
query, {'sccquerytest{}'.format(uuid): 'workingtest{}'.format(uuid)})
# then
assert result > 0, 'Should return at least one item.'
assert 'FINDING' == result_kind
assert 3 == len(step_result)
assert 'SUCCESS' == step_result[1]['status']
assert 'SUCCESS' == step_result[2]['status']
assert 'SUCCESS' == step_result[3]['status']
def test_process_scc_query_two_steps():
# given
uuid = '293f42e2089c4f4491db881e961ca0ca'
read_time = ReadTime("FROM_NOW", "1h", "são paulo")
step1 = Step(uuid=uuid,
order=1,
kind='ASSET',
compare_duration='40w',
filter_='resourceProperties.name : "notifier"',
read_time=read_time,
out_join='securityCenterProperties.resourceParent',
threshold=Threshold(operator="ge", value="0"))
step2 = Step(uuid=uuid,
order=2,
kind='FINDING',
read_time=read_time,
in_join='resourceName',
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid,
name='Find projects on organization',
steps=[step2, step1])
# when
result, result_kind, step_result, _ = process_scc_query(
query, {'sccquery2step{}'.format(uuid): 'working2step{}'.format(uuid)})
# then
assert 2 == len(step_result)
assert 'FINDING' == result_kind
assert 'SUCCESS' == step_result[1]['status']
assert 'SUCCESS' == step_result[2]['status']
def mock_queries(queries):
steps = []
ref_time = ReadTime(_type='FROM_NOW', value="1w", zone=None)
steps.append(
stepDto(uuid.uuid4(),
order="1",
kind="ASSET",
out_join="attribute.parent_id",
threshold=Threshold(operator="ge", value="15"),
read_time=ref_time,
in_join=None,
filter_="attribute.asset_type = \"Firewall\" AND "
"property.allowed : \"0-65535\"",
last_execution_result=5,
last_execution_status="fail"))
steps.append(
stepDto(uuid.uuid4(),
order="2",
kind="ASSET",
in_join="attribute.parent_id",
compare_duration="1w",
threshold=Threshold(operator="le", value="15"),
read_time=None,
out_join=None,
filter_="attribute.asset_type = \"Instance\"",
last_execution_result=None,
last_execution_status="not executed"))
queries.append(
queryDto(uuid=uuid.uuid4(),
name='number_1',
description='some description for first query',
owner='developer',
topic='topic',
steps=steps))
queries.append(
queryDto(uuid=uuid.uuid4(),
name='number_2',
description='another description, but for second',
owner='manager',
steps=steps))
queries.append(
queryDto(uuid=uuid.uuid4(),
name='number_3',
description='this is the third query',
owner='intern',
steps=steps))
queries.append(
queryDto(uuid=uuid.uuid4(),
name='number_4',
description='4th is the last query that we have',
owner='admin',
steps=steps))
def test_process_scc_query_single_step():
# given
uuid = '6f04f63232764ff3b32a92f794fb0f2f'
read_time = ReadTime("TIMESTAMP", "2018-11-01T01:00:00-0200", "são paulo")
step = Step(
uuid=uuid,
order=1,
kind='ASSET',
compare_duration='2w',
filter_=
'securityCenterProperties.resourceType = "google.cloud.resourcemanager.Project" AND '
'securityCenterProperties.resourceOwners : "ciandt" AND '
'resourceProperties.name : "tools"',
read_time=read_time,
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid,
name='Find projects on organization',
steps=[step])
# when
result, result_kind, step_result, _ = process_scc_query(
query, {'sccquerycit{}'.format(uuid): 'workingcit{}'.format(uuid)})
# then
assert 1 == len(step_result)
assert result == step_result[1]['responseSize']
assert 'ASSET' == result_kind
assert 'SUCCESS' == step_result[1]['status']
def test_validate_runnable_query_with_threshold_operator(threshold_operator):
# given
step = Step(uuid='uuid',
order=1,
kind='ASSET',
threshold=Threshold(operator=threshold_operator, value="10"))
query = Query(uuid='uuid', steps=[step])
# when / then
validate_runnable_query(query)
def test_validate_runnable_query_duration_valid(duration):
# given
step = Step(uuid='uuid',
order=1,
kind='ASSET',
compare_duration=duration,
threshold=Threshold(operator="le", value="10"))
query = Query(uuid='uuid', steps=[step])
# when / then
validate_runnable_query(query)
def test_process_scc_query_three_steps_with_all_ports_allowed_firewall_rule_fail_threshold(
):
# given
uuid = '74b8f5a8982948fb9acc63775485149f'
step1 = Step(
uuid=uuid,
order=1,
kind='ASSET',
filter_=
'securityCenterProperties.resourceType = "google.compute.Firewall" AND '
'resourceProperties.allowed : "0-65535" AND '
'securityCenterProperties.resourceOwners : "an"',
out_join='securityCenterProperties.resourceParent',
threshold=Threshold(operator="ge", value="0"))
step2 = Step(
uuid=uuid,
order=2,
kind='ASSET',
filter_=
'securityCenterProperties.resourceType = "google.compute.Instance" AND '
'resourceProperties.zone : "central"',
in_join='securityCenterProperties.resourceParent',
out_join='name',
threshold=Threshold(operator="ge", value="0"))
step3 = Step(uuid=uuid,
order=3,
kind='FINDING',
filter_='category : "audit_log"',
in_join='resourceName',
threshold=Threshold(operator="ge", value="100"))
query = Query(uuid=uuid,
name='Find projects on organization',
steps=[step3, step2, step1])
# when
result, _, step_results, _ = process_scc_query(
query, {'sccquery{}'.format(uuid): 'working{}'.format(uuid)})
# then
assert result == 0, 'Should return no items.'
assert len(step_results) == 3, 'Should have executed all steps.'
assert int(
step_results[2]['responseSize']) > 0, 'Last step must not be empty.'
def test_validate_runnable_query_ref_time_timestamp_valid(timestamp):
# given
ref_time = ReadTime(_type='TIMESTAMP', value=timestamp, zone=None)
step = Step(uuid='uuid',
order=1,
kind='ASSET',
read_time=ref_time,
threshold=Threshold(operator="ge", value="10"))
query = Query(uuid='uuid', steps=[step])
# when / then
validate_runnable_query(query)
def test_validate_runnable_query_ref_time_from_now_valid(duration):
# given
ref_time = ReadTime(_type='FROM_NOW', value=duration, zone=None)
step = Step(uuid='uuid',
order=1,
kind='ASSET',
read_time=ref_time,
threshold=Threshold(operator="le", value="10"))
query = Query(uuid='uuid', steps=[step])
# when / then
validate_runnable_query(query)
def test_process_scc_query_with_invalid_marks():
# given
uuid = '6f04f632-3276-4ff3-b32a-92f794fb0f2f'
step = Step(uuid=uuid,
order=1,
kind='FINDING',
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid, name='All findings', steps=[step])
# when
result, result_kind, step_result, _ = process_scc_query(
query, {'sccquerycit{}'.format(uuid): 'workingcit{}'.format(uuid)})
def test_validate_save_query_ok():
# given
uuid = '57e43773-7890-4530-a667-443089a90adc'
step = Step(uuid=uuid,
order=1,
kind='ASSET',
threshold=Threshold(operator='gt', value='0'))
query = Query(uuid=uuid,
steps=[step],
name='Name',
description='Description')
# when / then
validate_save_query(query)
def test_process_scc_query_three_steps_with_all_ports_allowed_firewall_rule():
# given
uuid = '74b8f5a8982948fb9acc63775485149f'
step1 = Step(
uuid=uuid,
order=1,
kind='ASSET',
filter_=
'securityCenterProperties.resourceType = "google.compute.Firewall" AND '
'resourceProperties.allowed : "0-65535" AND '
'securityCenterProperties.resourceOwners : "an"',
out_join='securityCenterProperties.resourceParent',
threshold=Threshold(operator="ge", value="0"))
step2 = Step(
uuid=uuid,
order=2,
kind='ASSET',
filter_=
'securityCenterProperties.resourceType = "google.compute.Instance"',
in_join='securityCenterProperties.resourceParent',
out_join='name',
threshold=Threshold(operator="ge", value="0"))
step3 = Step(uuid=uuid,
order=3,
kind='FINDING',
filter_='category : "audit_log"',
in_join='resourceName',
threshold=Threshold(operator="ge", value="0"))
query = Query(uuid=uuid,
name='Find projects on organization',
steps=[step3, step2, step1])
# when
result, result_kind, step_result, _ = process_scc_query(
query, {'sccquery{}'.format(uuid): 'working{}'.format(uuid)})
# then
assert result > 0, 'Should return at least one item.'
def test_run_scheduled_queries_each_minute(mocker):
# given
step = stepDto(uuid.uuid4(),
order="1",
kind="ASSET",
threshold=Threshold(operator="ge", value="15"),
read_time=None,
in_join=None,
out_join=None)
queries = []
queries.append(
queryDto(uuid=uuid.uuid4(),
name='even',
schedule='*/2 * * * *',
description='even minute',
owner='manager',
steps=[step]))
each_uuid = uuid.uuid4()
queries.append(
queryDto(uuid=each_uuid,
name='each',
schedule='* * * * *',
description='each minute',
owner='manager',
steps=[step]))
# Needed to force next run date on queries to near from mocked execution date
with mocker.patch(
'query_builder.domain_model.services.query_service.__now',
return_value=datetime(year=2018,
month=6,
day=20,
hour=12,
minute=0,
second=59)):
save_mocked_queries(queries)
with mocker.patch(
'query_builder.domain_model.services.query_service.__now',
return_value=datetime(year=2018,
month=6,
day=20,
hour=12,
minute=1)):
with mocker.mock_module.patch(
'query_builder.domain_model.services.query_service.publish_queries_uuid'
) as mock:
# when
run_scheduled_queries()
# then
mock.assert_called_with([each_uuid])
def test_validate_save_query_empty_description():
# given
uuid = '57e43773-7890-4530-a667-443089a90adc'
step = Step(uuid=uuid,
order=1,
kind='ASSET',
threshold=Threshold(operator='gt', value='0'))
query = Query(uuid=uuid, steps=[step], name='Name')
expected_error_key = 'description'
expected_error_value = {'message': 'Field description required'}
with pytest.raises(QBValidationError) as ex:
# when
validate_save_query(query)
# then
assert len(ex.value.errors) == 1, 'Should return exactly 1 error'
assert ex.value.errors[expected_error_key] == expected_error_value
def test_validate_runnable_query_ref_time_from_now_invalid(duration):
# given
ref_time = ReadTime(_type='FROM_NOW', value=duration, zone=None)
step = Step(uuid='uuid',
order=1,
kind='ASSET',
read_time=ref_time,
threshold=Threshold(operator="le", value="10"))
query = Query(uuid='uuid', steps=[step])
expected_error_key = 'readTimeValue1'
expected_error_value = {'message': 'Read time field invalid on step #1'}
with pytest.raises(QBValidationError) as ex:
# when
validate_runnable_query(query)
# then
assert len(ex.value.errors) == 1, 'Should return exactly 1 error'
assert ex.value.errors[expected_error_key] == expected_error_value
def test_validate_runnable_query_duration_invalid(duration):
# given
step = Step(uuid='uuid',
order=1,
kind='ASSET',
compare_duration=duration,
threshold=Threshold(operator="le", value="5"))
query = Query(uuid='uuid', steps=[step])
expected_error_key = 'compareDuration1'
expected_error_value = {
'message': 'Compare duration field invalid on step #1'
}
with pytest.raises(QBValidationError) as ex:
# when
validate_runnable_query(query)
# then
assert len(ex.value.errors) == 1, 'Should return exactly 1 error'
assert ex.value.errors[expected_error_key] == expected_error_value
def test_validate_runnable_query_invalid_threshold_value(value):
# given
uuid = '57e43773-7890-4530-a667-443089a90adc'
step = Step(uuid=uuid,
order=1,
kind='ASSET',
threshold=Threshold(operator='gt', value=value))
query = Query(uuid=uuid, steps=[step])
expected_error_key = 'thresholdValue1'
expected_error_value = {
'message': 'Threshold value field invalid on step #1'
}
with pytest.raises(QBValidationError) as ex:
# when
validate_runnable_query(query)
# then
assert len(ex.value.errors) == 1, 'Should return exactly 1 error'
assert ex.value.errors[expected_error_key] == expected_error_value
import json
import os
import uuid
from datetime import datetime as dt
from query_builder.domain_model.dtos import Query as QueryDTO, Step as StepDTO, \
Threshold as Threshold, ReadTime
from query_builder.domain_model.models import Query as QueryModel, \
Step as StepModel
STEPS_MODEL = []
REF_TIME = ReadTime(_type='TIMESTAMP', value="1w", zone=None)
THRESHOLD_STEP_1 = Threshold(operator='ge', value='15')
THRESHOLD_STEP_2 = Threshold(operator='le', value='15')
STEPS_MODEL_1 = StepModel(
uuid.uuid4(),
order=1,
scc_resource_type="ASSET",
out_value="attribute.parent_id",
threshold_operator=THRESHOLD_STEP_1.operator,
threshold_value=THRESHOLD_STEP_1.value,
read_time_type=REF_TIME._type,
read_time_value=REF_TIME.value,
read_time_zone=REF_TIME.zone,
in_value=None,
filter=
"attribute.asset_type = \"Firewall\" AND property.allowed : \"0-65535\"",
last_execution_status="fail",
last_execution_result=5)
def build_step(operator, value):
threshold = Threshold(operator, value)
return Step('FAKE-UUID-001', 1, "ASSET", threshold=threshold)