def run(compile, fuzz, sample_host):
cli = raft.RaftCLI()
subs = {
'{sample.host}': sample_host,
'{defaults.deploymentName}': cli.definitions.deployment
}
compile_job_config = raft.RaftJobConfig(file_path=compile,
substitutions=subs)
print('Compile')
compile_job = cli.new_job(compile_job_config)
cli.poll(compile_job['jobId'])
subs['{compile.jobId}'] = compile_job['jobId']
fuzz_job_config = raft.RaftJobConfig(file_path=fuzz, substitutions=subs)
print('Fuzz')
fuzz_job = cli.new_job(fuzz_job_config)
cli.poll(fuzz_job['jobId'])
def bvt(cli, definitions, bvt_host):
print('Getting available wehook events')
webhook_events = cli.list_available_webhooks_events()
try:
test_url = webhooks_test_url(definitions.subscription,
definitions.resource_group,
definitions.test_infra)
for event in webhook_events:
print(f'Setting webhook for {event}')
compile_webhook = cli.set_webhooks_subscription(
'sample-compile', event, test_url)
fuzz_webhook = cli.set_webhooks_subscription(
'sample-fuzz', event, test_url)
added_compile = cli.list_webhooks('sample-compile', event)
if len(added_compile) == 0:
raise Exception(
'Expected sample-compile webhooks not to be empty after creation'
)
added_fuzz = cli.list_webhooks('sample-fuzz', event)
if len(added_fuzz) == 0:
raise Exception(
'Expected sample-fuzz webhooks not to be empty after creation')
t_pre_compile = time.time()
print('Compile')
compile_config_path = os.path.abspath(
os.path.join(cli_path, 'samples', 'restler', 'no-authentication',
'sample.restler.compile.json'))
subs = {'{sample.host}': bvt_host}
compile_config = raft.RaftJobConfig(file_path=compile_config_path,
substitutions=subs)
compile_job = cli.new_job(compile_config)
cli.poll(compile_job['jobId'], 10)
#calculate compile duration
t_pre_fuzz = time.time()
timespan_pre_fuzz = time_span(t_pre_compile, t_pre_fuzz)
after_compile_pre_fuzz = cli.list_jobs(timespan_pre_fuzz)
n = 0
for x in after_compile_pre_fuzz:
if x['jobId'] == compile_job['jobId']:
n += 1
if x['agentName'] == compile_job['jobId']:
if x['state'] != 'Completed':
raise Exception(
'Expected job to be in completed state when retrieved job list.'
f'{after_compile_pre_fuzz}')
if n != 2:
raise Exception('Expected 2 after compile job step'
f' for job {compile_job["jobId"]}'
f' got {n}'
f' {after_compile_pre_fuzz}')
print('Fuzz')
fuzz_config_path = os.path.abspath(
os.path.join(cli_path, 'samples', 'restler', 'no-authentication',
'sample.restler.fuzz.json'))
subs['{compile.jobId}'] = compile_job['jobId']
fuzz_config = raft.RaftJobConfig(file_path=fuzz_config_path,
substitutions=subs)
fuzz_config.config['duration'] = '00:20:00'
fuzz_job = cli.new_job(fuzz_config)
cli.poll(fuzz_job['jobId'], 10)
#calculate fuzz duration
timespan_post_fuzz = time_span(t_pre_fuzz, time.time())
after_fuzz = cli.list_jobs(timespan_post_fuzz)
#validate list jobs
m = 0
for x in after_fuzz:
if x['jobId'] == fuzz_job['jobId']:
m += 1
if x['agentName'] == fuzz_job['jobId']:
if x['state'] != 'Completed':
raise Exception(
'Expected job to be in completed state when retrieved job list.'
f'{after_fuzz}')
if m != 3:
raise Exception('Expected 3 after compile job step'
f' for job {fuzz_job["jobId"]}'
f' got {m}'
f' {after_fuzz}')
print('Validating webhook posted triggers')
compile_webhook_triggers = webhook_triggers_results(
compile_job['jobId'], test_url)
for r in compile_webhook_triggers:
if r['EventType'] == 'BugFound':
raise Exception(f'Compile step produced BugFound event')
fuzz_webhook_triggers = webhook_triggers_results(
fuzz_job['jobId'], test_url)
bug_found_events = []
job_status_events = []
for r in fuzz_webhook_triggers:
if r['EventType'] == 'BugFound':
bug_found_events.append(r)
elif r['EventType'] == 'JobStatus':
job_status_events.append(r)
else:
raise Exception(
f'Unhandled webhook trigger event type {r["EventType"]} : {r}'
)
if len(job_status_events) == 0:
raise Exception('Job did not post any job status events webhooks')
print(
'Validating that bugs posted events matches total bugs found in job status'
)
total_bugs_found = 0
for r in job_status_events:
if r['Data']['State'] == 'Completed' and r['Data'][
'AgentName'] != r['Data']['JobId']:
total_bugs_found += r['Data']['Metrics'][
'TotalBugBucketsCount']
print(f'Total bugs found: {total_bugs_found}')
print(f'Number of Bug found events: {len(bug_found_events)}')
if total_bugs_found != len(bug_found_events):
raise Exception(
f"Total bugs found does not match number of bug found webhook triggered events {total_bugs_found} and {len(bug_found_events)}"
)
except Exception as ex:
print(f"FAIL: {ex}")
raise ex
finally:
for event in webhook_events:
print(f'Cleaning up webhook {event}')
cli.delete_webhook('sample-compile', event)
cli.delete_webhook('sample-fuzz', event)
deleted_compile = cli.list_webhooks('sample-compile', event)
if len(deleted_compile) > 0:
raise Exception(
'Expected sample-compile webhooks to be empty after deletion, instead got %s',
deleted_compile)
deleted_fuzz = cli.list_webhooks('sample-fuzz', event)
if len(deleted_fuzz) > 0:
raise Exception(
'Expected sample-fuzz webhooks to be empty after deletion, instead got %s',
deleted_compile)
def fuzz(cli, fuzz, subs):
fuzz_job_config = raft.RaftJobConfig(file_path=fuzz, substitutions=subs)
print('Fuzz')
return cli.new_job(fuzz_job_config)
def compile(cli, compile, subs):
compile_job_config = raft.RaftJobConfig(file_path=compile, substitutions=subs)
print('Compile')
return cli.new_job(compile_job_config)