def test_preprocess_tasks():
conf = PoolConfiguration.from_file(POOL_FIXTURES / "pre-pool.yml")
task_ids, tasks = zip(*conf.build_tasks("someTaskId"))
# Check we have 2 valid generated task ids
assert len(task_ids) == 2
assert all(map(slugid.decode, task_ids))
# Check we have 2 valid task definitions
assert len(tasks) == 2
def _check_date(task, *keys):
# Dates can not be checked directly as they are generated
assert keys, "must specify at least one key"
value = task
for key in keys:
obj = value
assert isinstance(obj, dict)
value = obj[key]
assert isinstance(value, str)
date = datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ")
del obj[key]
return date
expected = [
{
"name": "preprocess",
"deps": ["someTaskId"],
"extra_env": {
"TASKCLUSTER_FUZZING_PREPROCESS": "1"
},
},
{
"name": "1/1",
"deps": ["someTaskId", task_ids[0]],
"extra_env": {}
},
]
for task, expect in zip(tasks, expected):
created = _check_date(task, "created")
deadline = _check_date(task, "deadline")
expires = _check_date(task, "expires")
assert expires >= deadline > created
expected_env = {
"TASKCLUSTER_FUZZING_POOL": "pre-pool",
"TASKCLUSTER_SECRET": "project/fuzzing/decision",
}
expected_env.update(expect["extra_env"])
log_expires = _check_date(task, "payload", "artifacts",
"project/fuzzing/private/logs", "expires")
assert log_expires == expires
assert set(task["scopes"]) == set(
["secrets:get:project/fuzzing/decision"])
# scopes are already asserted above
# - read the value for comparison instead of deleting the key, so the object is
# printed in full on failure
scopes = task["scopes"]
assert task == {
"dependencies": expect["deps"],
"extra": {},
"metadata": {
"description":
"*DO NOT EDIT* - This resource is configured "
"automatically.\n"
"\n"
"Fuzzing workers generated by decision task",
"name":
f"Fuzzing task linux-pre-pool - {expect['name']}",
"owner":
"*****@*****.**",
"source":
"https://github.com/MozillaSecurity/fuzzing-tc",
},
"payload": {
"artifacts": {
"project/fuzzing/private/logs": {
"path": "/logs/",
"type": "directory",
}
},
"cache": {},
"capabilities": {},
"env": expected_env,
"features": {
"taskclusterProxy": True
},
"image": "MozillaSecurity/fuzzer:latest",
"maxRunTime": 3600,
},
"priority": "high",
"provisionerId": "proj-fuzzing",
"retries": 5,
"routes": [],
"schedulerId": "-",
"scopes": scopes,
"tags": {},
"taskGroupId": "someTaskId",
"workerType": "linux-pre-pool",
}
def test_tasks(env, scope_caps):
scopes, expected_capabilities = scope_caps
conf = PoolConfiguration(
"test",
{
"cloud": "gcp",
"scopes": scopes,
"disk_size": "10g",
"cycle_time": "1h",
"max_run_time": "30s",
"schedule_start": None,
"cores_per_task": 1,
"metal": False,
"name": "Amazing fuzzing pool",
"tasks": 2,
"command": ["run-fuzzing.sh"],
"container": "MozillaSecurity/fuzzer:latest",
"minimum_memory_per_core": "1g",
"imageset": "anything",
"parents": [],
"cpu": "x64",
"platform": "linux",
"preprocess": None,
"macros": {},
"artifacts": {
"/some-file.txt": {
"type": "file",
"url": "project/fuzzing/private/file.txt",
},
"/var/log/": {
"type": "directory",
"url": "project/fuzzing/private/var-log",
},
},
},
)
task_ids, tasks = zip(*conf.build_tasks("someTaskId", env=env))
# Check we have 2 valid generated task ids
assert len(task_ids) == 2
assert all(map(slugid.decode, task_ids))
# Check we have 2 valid task definitions
assert len(tasks) == 2
def _check_date(task, *keys):
# Dates can not be checked directly as they are generated
assert keys, "must specify at least one key"
value = task
for key in keys:
obj = value
assert isinstance(obj, dict)
value = obj[key]
assert isinstance(value, str)
date = datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ")
del obj[key]
return date
for i, task in enumerate(tasks):
created = _check_date(task, "created")
deadline = _check_date(task, "deadline")
expires = _check_date(task, "expires")
assert expires >= deadline > created
expected_env = {
"TASKCLUSTER_FUZZING_POOL": "test",
"TASKCLUSTER_SECRET": "project/fuzzing/decision",
}
if env is not None:
expected_env.update(env)
log_expires = _check_date(task, "payload", "artifacts",
"project/fuzzing/private/logs", "expires")
assert log_expires == expires
log_expires = _check_date(task, "payload", "artifacts",
"project/fuzzing/private/file.txt",
"expires")
assert log_expires == expires
log_expires = _check_date(task, "payload", "artifacts",
"project/fuzzing/private/var-log", "expires")
assert log_expires == expires
assert set(
task["scopes"]) == set(["secrets:get:project/fuzzing/decision"] +
scopes)
# scopes are already asserted above
# - read the value for comparison instead of deleting the key, so the object is
# printed in full on failure
scopes = task["scopes"]
assert task == {
"dependencies": ["someTaskId"],
"extra": {},
"metadata": {
"description":
"*DO NOT EDIT* - This resource is configured "
"automatically.\n"
"\n"
"Fuzzing workers generated by decision task",
"name":
f"Fuzzing task linux-test - {i+1}/2",
"owner":
"*****@*****.**",
"source":
"https://github.com/MozillaSecurity/fuzzing-tc",
},
"payload": {
"artifacts": {
"project/fuzzing/private/logs": {
"path": "/logs/",
"type": "directory",
},
"project/fuzzing/private/file.txt": {
"path": "/some-file.txt",
"type": "file",
},
"project/fuzzing/private/var-log": {
"path": "/var/log/",
"type": "directory",
},
},
"cache": {},
"capabilities": expected_capabilities,
"env": expected_env,
"features": {
"taskclusterProxy": True
},
"image": "MozillaSecurity/fuzzer:latest",
"maxRunTime": 30,
},
"priority": "high",
"provisionerId": "proj-fuzzing",
"retries": 5,
"routes": [],
"schedulerId": "-",
"scopes": scopes,
"tags": {},
"taskGroupId": "someTaskId",
"workerType": "linux-test",
}
def test_aws_resources(env, mock_clouds, mock_machines):
conf = PoolConfiguration(
"test",
{
"cloud": "aws",
"scopes": [],
"disk_size": "120g",
"schedule_start": "1970-01-01T00:00:00Z",
"cycle_time": "12h",
"max_run_time": "12h",
"cores_per_task": 2,
"metal": False,
"name": "Amazing fuzzing pool",
"tasks": 3,
"command": ["run-fuzzing.sh"],
"container": "MozillaSecurity/fuzzer:latest",
"minimum_memory_per_core": "1g",
"imageset": "generic-worker-A",
"parents": [],
"cpu": "arm64",
"platform": "linux",
"preprocess": None,
"macros": {},
},
)
resources = conf.build_resources(mock_clouds, mock_machines, env=env)
assert len(resources) == 3
pool, hook, role = resources
assert pool.to_json() == {
"kind":
"WorkerPool",
"config": {
"launchConfigs": [{
"capacityPerInstance": 1,
"launchConfig": {
"ImageId": "ami-1234",
"InstanceMarketOptions": {
"MarketType": "spot"
},
"InstanceType": "a2",
"Placement": {
"AvailabilityZone": "us-west-1a"
},
"SecurityGroupIds": ["sg-A"],
"SubnetId": "subnet-XXX",
},
"region": "us-west-1",
"workerConfig": {
"dockerConfig": {
"allowPrivileged": True,
"allowDisableSeccomp": True,
},
"genericWorker": {
"config": {
"anyKey":
"anyValue",
"deploymentId":
"d9465de45bdee4be",
"os":
"linux",
"wstAudience":
"communitytc",
"wstServerURL":
"https://community-websocktunnel.services.mozilla.com",
},
},
"shutdown": {
"afterIdleSeconds": 15,
"enabled": True
},
},
}],
"maxCapacity":
7,
"minCapacity":
0,
"lifecycle": {
"registrationTimeout": 900,
"reregistrationTimeout": 43200
},
},
"emailOnError":
True,
"owner":
"*****@*****.**",
"providerId":
"community-tc-workers-aws",
"workerPoolId":
"proj-fuzzing/linux-test",
"description":
"*DO NOT EDIT* - This resource is configured automatically.\n"
"\n"
"Fuzzing workers generated by decision task",
}
# Update env in valid hook
valid_hook = copy.deepcopy(VALID_HOOK)
if env is not None:
valid_hook["task"]["payload"]["env"].update(env)
assert hook.to_json() == valid_hook
assert role.to_json() == VALID_ROLE
def test_gcp_resources(env, mock_clouds, mock_machines):
conf = PoolConfiguration(
"test",
{
"cloud": "gcp",
"scopes": [],
"disk_size": "120g",
"schedule_start": "1970-01-01T00:00:00Z",
"cycle_time": "12h",
"max_run_time": "12h",
"cores_per_task": 2,
"metal": False,
"name": "Amazing fuzzing pool",
"tasks": 3,
"command": ["run-fuzzing.sh"],
"container": "MozillaSecurity/fuzzer:latest",
"minimum_memory_per_core": "1g",
"imageset": "docker-worker",
"parents": [],
"cpu": "x64",
"platform": "linux",
"preprocess": None,
"macros": {},
},
)
resources = conf.build_resources(mock_clouds, mock_machines, env=env)
assert len(resources) == 3
pool, hook, role = resources
assert pool.to_json() == {
"kind":
"WorkerPool",
"config": {
"launchConfigs": [
{
"capacityPerInstance":
1,
"disks": [{
"autoDelete": True,
"boot": True,
"initializeParams": {
"diskSizeGb": 120,
"sourceImage": "path/to/image",
},
"type": "PERSISTENT",
}],
"machineType":
"zones/us-west1-a/machineTypes/2-cpus",
"networkInterfaces": [{
"accessConfigs": [{
"type": "ONE_TO_ONE_NAT"
}]
}],
"region":
"us-west1",
"scheduling": {
"onHostMaintenance": "terminate"
},
"workerConfig": {
"dockerConfig": {
"allowPrivileged": True,
"allowDisableSeccomp": True,
},
"genericWorker": {
"config": {
"deploymentId":
"ff173b0660475975",
"wstAudience":
"communitytc",
"wstServerURL":
"https://community-websocktunnel.services.mozilla.com",
},
},
"shutdown": {
"afterIdleSeconds": 15,
"enabled": True
},
},
"zone":
"us-west1-a",
},
{
"capacityPerInstance":
1,
"disks": [{
"autoDelete": True,
"boot": True,
"initializeParams": {
"diskSizeGb": 120,
"sourceImage": "path/to/image",
},
"type": "PERSISTENT",
}],
"machineType":
"zones/us-west1-b/machineTypes/2-cpus",
"networkInterfaces": [{
"accessConfigs": [{
"type": "ONE_TO_ONE_NAT"
}]
}],
"region":
"us-west1",
"scheduling": {
"onHostMaintenance": "terminate"
},
"workerConfig": {
"dockerConfig": {
"allowPrivileged": True,
"allowDisableSeccomp": True,
},
"genericWorker": {
"config": {
"deploymentId":
"ff173b0660475975",
"wstAudience":
"communitytc",
"wstServerURL":
"https://community-websocktunnel.services.mozilla.com",
},
},
"shutdown": {
"afterIdleSeconds": 15,
"enabled": True
},
},
"zone":
"us-west1-b",
},
{
"capacityPerInstance":
1,
"disks": [{
"autoDelete": True,
"boot": True,
"initializeParams": {
"diskSizeGb": 120,
"sourceImage": "path/to/image",
},
"type": "PERSISTENT",
}],
"machineType":
"zones/us-west1-a/machineTypes/more-ram",
"networkInterfaces": [{
"accessConfigs": [{
"type": "ONE_TO_ONE_NAT"
}]
}],
"region":
"us-west1",
"scheduling": {
"onHostMaintenance": "terminate"
},
"workerConfig": {
"dockerConfig": {
"allowPrivileged": True,
"allowDisableSeccomp": True,
},
"genericWorker": {
"config": {
"deploymentId":
"ff173b0660475975",
"wstAudience":
"communitytc",
"wstServerURL":
"https://community-websocktunnel.services.mozilla.com",
},
},
"shutdown": {
"afterIdleSeconds": 15,
"enabled": True
},
},
"zone":
"us-west1-a",
},
],
"maxCapacity":
7,
"minCapacity":
0,
"lifecycle": {
"registrationTimeout": 900,
"reregistrationTimeout": 43200
},
},
"emailOnError":
True,
"owner":
"*****@*****.**",
"providerId":
"community-tc-workers-google",
"workerPoolId":
"proj-fuzzing/linux-test",
"description":
"*DO NOT EDIT* - This resource is configured automatically.\n"
"\n"
"Fuzzing workers generated by decision task",
}
# Update env in valid hook
valid_hook = copy.deepcopy(VALID_HOOK)
if env is not None:
valid_hook["task"]["payload"]["env"].update(env)
assert hook.to_json() == valid_hook
assert role.to_json() == VALID_ROLE
def test_parse_size(size, divisor, result):
if isinstance(divisor, str):
divisor = PoolConfiguration.parse_size(divisor)
assert PoolConfiguration.parse_size(size, divisor) == result