def worker_create(name): worker = request.get_json() or {} required = ( "api_key", "distro", "mem_total", "cpu_total", "cpu_type", "concurrent_runs", "host_tags", ) missing = [] for x in required: if x not in worker: missing.append(x) if missing: raise ApiError(400, "Missing required field(s): " + ", ".join(missing)) w = Worker( name, worker["distro"], worker["mem_total"], worker["cpu_total"], worker["cpu_type"], worker["api_key"], worker["concurrent_runs"], worker["host_tags"], ) w.surges_only = worker.get("surges_only", False) db.session.add(w) db.session.commit() return jsendify({}, 201)
def test_worker_log_event(self): w = Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, []) w.enlisted = True db.session.add(w) db.session.commit() headers = [ ('Content-type', 'application/json'), ('Authorization', 'Token key'), ] event = '{"key": "val"}' resp = self.client.post('/workers/w1/events/', headers=headers, data=event) self.assertEqual(201, resp.status_code, resp.data) p = os.path.join(jobserv.models.WORKER_DIR, 'w1/events.log') with open(p) as f: buf = f.read() self.assertEqual(event, buf) w.deleted = True db.session.commit() resp = self.client.post('/workers/w1/events/', headers=headers, data=event) self.assertEqual(404, resp.status_code, resp.data)
def test_worker_log_event(self): w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, []) w.enlisted = True db.session.add(w) db.session.commit() headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] event = '{"key": "val"}' resp = self.client.post("/workers/w1/events/", headers=headers, data=event) self.assertEqual(201, resp.status_code, resp.data) p = os.path.join(jobserv.models.WORKER_DIR, "w1/events.log") with open(p) as f: buf = f.read() self.assertEqual(event, buf) w.deleted = True db.session.commit() resp = self.client.post("/workers/w1/events/", headers=headers, data=event) self.assertEqual(404, resp.status_code, resp.data)
def test_worker_get(self): db.session.add(Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, [])) db.session.add(Worker('w2', 'fedora', 14, 4, 'amd64', 'key', 1, [])) db.session.commit() data = self.get_json('/workers/w2/') self.assertEqual('w2', data['worker']['name']) self.assertEqual('fedora', data['worker']['distro']) self.assertEqual(1, data['worker']['concurrent_runs'])
def test_worker_get(self): db.session.add(Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, [])) db.session.add(Worker("w2", "fedora", 14, 4, "amd64", "key", 1, [])) db.session.commit() headers = [("Authorization", "Token key")] data = self.get_json("/workers/w2/", headers=headers) self.assertEqual("w2", data["worker"]["name"]) self.assertEqual("fedora", data["worker"]["distro"]) self.assertEqual(1, data["worker"]["concurrent_runs"])
def setUp(self): super().setUp() jobserv.models.WORKER_DIR = tempfile.mkdtemp() jobserv.worker.SURGE_FILE = os.path.join(jobserv.models.WORKER_DIR, "enable_surges") self.addCleanup(shutil.rmtree, jobserv.models.WORKER_DIR) self.worker = Worker("w1", "d", 1, 1, "amd64", "k", 1, "amd64") self.worker.enlisted = True self.worker.online = True db.session.add(self.worker) db.session.commit()
def setUp(self): super().setUp() jobserv.models.WORKER_DIR = tempfile.mkdtemp() jobserv.worker.SURGE_FILE = os.path.join(jobserv.models.WORKER_DIR, 'enable_surges') self.addCleanup(shutil.rmtree, jobserv.models.WORKER_DIR) self.worker = Worker('w1', 'd', 1, 1, 'amd64', 'k', 1, 'amd64') self.worker.enlisted = True self.worker.online = True db.session.add(self.worker) db.session.commit()
def test_worker_sync_builds_uploading(self, storage): """Make sure scheduler takes into account runs that are UPLOADING. 1. Create a "synchronous" Project 2. Add an UPLOADING build and and QUEUED build Make sure the QUEUED build is not assigned """ if db.engine.dialect.name == "sqlite": self.skipTest("Test requires MySQL") rundef = {"run_url": "foo", "runner_url": "foo", "env": {}} storage().get_run_definition.return_value = json.dumps(rundef) w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, ["aarch96"]) w.enlisted = True w.online = True db.session.add(w) self.create_projects("job-1") (p1, ) = Project.query.all() p1.synchronous_builds = True db.session.commit() # add active build b = Build.create(p1) r = Run(b, "p1b1r1") r.status = BuildStatus.UPLOADING r.host_tag = "aarch96" db.session.add(r) b = Build.create(p1) r = Run(b, "p1b2r1") r.host_tag = "aarch96" db.session.add(r) db.session.commit() headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] qs = "available_runners=1&foo=2" # There should be no work available resp = self.client.get("/workers/w1/", headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertNotIn("run-defs", data["data"]["worker"], data["data"]["worker"])
def test_worker_list(self): db.session.add(Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, [])) w = Worker("w2", "fedora", 14, 4, "amd64", "key", 2, []) db.session.add(w) db.session.commit() data = self.get_json("/workers/") self.assertEqual(2, len(data["workers"])) self.assertEqual("w1", data["workers"][0]["name"]) self.assertEqual(False, data["workers"][0]["enlisted"]) self.assertEqual("w2", data["workers"][1]["name"]) self.assertEqual(False, data["workers"][1]["enlisted"]) w.deleted = True db.session.commit() data = self.get_json("/workers/") self.assertEqual(1, len(data["workers"])) self.assertEqual("w1", data["workers"][0]["name"])
def test_worker_list(self): db.session.add(Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, [])) w = Worker('w2', 'fedora', 14, 4, 'amd64', 'key', 2, []) db.session.add(w) db.session.commit() data = self.get_json('/workers/') self.assertEqual(2, len(data['workers'])) self.assertEqual('w1', data['workers'][0]['name']) self.assertEqual(False, data['workers'][0]['enlisted']) self.assertEqual('w2', data['workers'][1]['name']) self.assertEqual(False, data['workers'][1]['enlisted']) w.deleted = True db.session.commit() data = self.get_json('/workers/') self.assertEqual(1, len(data['workers'])) self.assertEqual('w1', data['workers'][0]['name'])
def test_worker_sync_builds_regression(self, storage): """Make sure scheduler takes into account other active projects for sync builds. """ if db.engine.dialect.name == "sqlite": self.skipTest("Test requires MySQL") rundef = {"run_url": "foo", "runner_url": "foo", "env": {}} storage().get_run_definition.return_value = json.dumps(rundef) w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, ["aarch96"]) w.enlisted = True w.online = True db.session.add(w) self.create_projects("job-1") self.create_projects("job-2") p1, p2 = Project.query.all() p1.synchronous_builds = True db.session.commit() # add active build b = Build.create(p2) r = Run(b, "p2b1r1") r.status = BuildStatus.RUNNING r.host_tag = "aarch96" db.session.add(r) b = Build.create(p1) r = Run(b, "p1b1r1") r.host_tag = "aarch96" db.session.add(r) db.session.commit() headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] qs = "available_runners=1&foo=2" # This should make the p1b1r2 run running resp = self.client.get("/workers/w1/", headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data["data"]["worker"]["run-defs"]))
def test_worker_needs_auth(self): headers = [("Content-type", "application/json")] db.session.add(Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, [])) db.session.commit() data = {"distro": "ArchLinux"} r = self.client.patch("/workers/w1/", headers=headers, data=json.dumps(data)) self.assertEqual(401, r.status_code)
def test_worker_needs_auth(self): headers = [('Content-type', 'application/json')] db.session.add(Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, [])) db.session.commit() data = {'distro': 'ArchLinux'} r = self.client.patch('/workers/w1/', headers=headers, data=json.dumps(data)) self.assertEqual(401, r.status_code)
def worker_create(name): worker = request.get_json() or {} required = ('api_key', 'distro', 'mem_total', 'cpu_total', 'cpu_type', 'concurrent_runs', 'host_tags') missing = [] for x in required: if x not in worker: missing.append(x) if missing: raise ApiError(400, 'Missing required field(s): ' + ', '.join(missing)) w = Worker(name, worker['distro'], worker['mem_total'], worker['cpu_total'], worker['cpu_type'], worker['api_key'], worker['concurrent_runs'], worker['host_tags']) w.surges_only = worker.get('surges_only', False) db.session.add(w) db.session.commit() return jsendify({}, 201)
def test_run_health(self): self.create_projects('proj-1') p = Project.query.first() b = Build.create(p) db.session.add(Run(b, 'queued-1')) db.session.add(Run(b, 'queued-2')) w1 = Worker('worker1', 'distro', 12, 12, 'amd', 'key', 2, 'tag') db.session.add(w1) w2 = Worker('worker2', 'distro', 12, 12, 'amd', 'key', 2, 'tag') db.session.add(w2) db.session.flush() r = Run(b, 'run1-worker1') r.worker = w1 r.status = BuildStatus.UPLOADING db.session.add(r) r = Run(b, 'run1-worker2') r.worker = w2 r.status = BuildStatus.RUNNING db.session.add(r) r = Run(b, 'run2-worker2') r.worker = w2 r.status = BuildStatus.RUNNING db.session.add(r) r = Run(b, 'run3rworker2') r.worker = w2 r.status = BuildStatus.CANCELLING db.session.add(r) db.session.commit() r = self.client.get('/health/runs/') self.assertEqual(200, r.status_code) d = json.loads(r.data.decode())['data'] self.assertEqual(2, d['health']['statuses']['QUEUED']) self.assertEqual(2, d['health']['statuses']['RUNNING']) self.assertEqual(1, d['health']['statuses']['UPLOADING']) self.assertEqual(1, len(d['health']['RUNNING']['worker1'])) self.assertEqual(3, len(d['health']['RUNNING']['worker2'])) self.assertEqual(2, len(d['health']['QUEUED']))
def test_worker_queue_priority(self, storage): """Validate queue priorities for Runs are honored. 1. Create a normal project with 2 QUEUED builds. 2. Set the priority of the newer build higher than the older build 3. Verify queue priority is done properly. """ if db.engine.dialect.name == 'sqlite': self.skipTest('Test requires MySQL') rundef = {'run_url': 'foo', 'runner_url': 'foo', 'env': {}} storage().get_run_definition.return_value = json.dumps(rundef) w = Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, ['aarch96']) w.enlisted = True w.online = True db.session.add(w) self.create_projects('job-1') p = Project.query.all()[0] p.synchronous_builds = True db.session.commit() b = Build.create(p) r = Run(b, 'r1') r.host_tag = 'aarch96' db.session.add(r) r = Run(b, 'r2') r.host_tag = 'aarch96' r.queue_priority = 2 # this is *newer* build but *higher* priority db.session.add(r) db.session.commit() headers = [ ('Content-type', 'application/json'), ('Authorization', 'Token key'), ] qs = 'available_runners=1&foo=2' resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data['data']['worker']['run-defs'])) self.assertEqual([BuildStatus.QUEUED, BuildStatus.RUNNING], [x.status for x in Run.query])
def test_worker_queue_priority(self, storage): """Validate queue priorities for Runs are honored. 1. Create a normal project with 2 QUEUED builds. 2. Set the priority of the newer build higher than the older build 3. Verify queue priority is done properly. """ if db.engine.dialect.name == "sqlite": self.skipTest("Test requires MySQL") rundef = {"run_url": "foo", "runner_url": "foo", "env": {}} storage().get_run_definition.return_value = json.dumps(rundef) w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, ["aarch96"]) w.enlisted = True w.online = True db.session.add(w) self.create_projects("job-1") p = Project.query.all()[0] p.synchronous_builds = True db.session.commit() b = Build.create(p) r = Run(b, "r1") r.host_tag = "aarch96" db.session.add(r) r = Run(b, "r2") r.host_tag = "aarch96" r.queue_priority = 2 # this is *newer* build but *higher* priority db.session.add(r) db.session.commit() headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] qs = "available_runners=1&foo=2" resp = self.client.get("/workers/w1/", headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data["data"]["worker"]["run-defs"])) self.assertEqual([BuildStatus.QUEUED, BuildStatus.RUNNING], [x.status for x in Run.query])
def test_run_health(self): self.create_projects("proj-1") p = Project.query.first() b = Build.create(p) db.session.add(Run(b, "queued-1")) db.session.add(Run(b, "queued-2")) w1 = Worker("worker1", "distro", 12, 12, "amd", "key", 2, "tag") db.session.add(w1) w2 = Worker("worker2", "distro", 12, 12, "amd", "key", 2, "tag") db.session.add(w2) db.session.flush() r = Run(b, "run1-worker1") r.worker = w1 r.status = BuildStatus.UPLOADING db.session.add(r) r = Run(b, "run1-worker2") r.worker = w2 r.status = BuildStatus.RUNNING db.session.add(r) r = Run(b, "run2-worker2") r.worker = w2 r.status = BuildStatus.RUNNING db.session.add(r) r = Run(b, "run3rworker2") r.worker = w2 r.status = BuildStatus.CANCELLING db.session.add(r) db.session.commit() r = self.client.get("/health/runs/") self.assertEqual(200, r.status_code) d = json.loads(r.data.decode())["data"] self.assertEqual(2, d["health"]["statuses"]["QUEUED"]) self.assertEqual(2, d["health"]["statuses"]["RUNNING"]) self.assertEqual(1, d["health"]["statuses"]["UPLOADING"]) self.assertEqual(1, len(d["health"]["RUNNING"]["worker1"])) self.assertEqual(3, len(d["health"]["RUNNING"]["worker2"])) self.assertEqual(2, len(d["health"]["QUEUED"]))
def test_surge_complex(self): # we'll have two amd64 workers and one armhf worker = Worker('w2', 'd', 1, 1, 'amd64', 'k', 1, 'amd64') worker.enlisted = True worker.online = True db.session.add(worker) worker = Worker('w3', 'd', 1, 1, 'armhf', 'k', 1, 'armhf') worker.enlisted = True worker.online = True db.session.add(worker) db.session.commit() self.create_projects('proj1') b = Build.create(Project.query.all()[0]) for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, 'amd%d' % x) r.host_tag = 'amd64' db.session.add(r) r = Run(b, 'armhf%d' % x) r.host_tag = 'armhf' db.session.add(r) db.session.commit() _check_queue() self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64')) self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf')) # get us under surge for armhf db.session.delete(Run.query.filter(Run.host_tag == 'armhf').first()) # and over surge for amd64 for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, 'run%d' % x) r.host_tag = 'amd64' db.session.add(r) db.session.commit() worker_module.DETECT_FLAPPING = False _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64')) self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf')) # make sure we know about deleted workers worker.deleted = True db.session.commit() _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf'))
def test_surge_complex(self): # we'll have two amd64 workers and one armhf worker = Worker("w2", "d", 1, 1, "amd64", "k", 1, "amd64") worker.enlisted = True worker.online = True db.session.add(worker) worker = Worker("w3", "d", 1, 1, "armhf", "k", 1, "armhf") worker.enlisted = True worker.online = True db.session.add(worker) db.session.commit() self.create_projects("proj1") b = Build.create(Project.query.all()[0]) for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, "amd%d" % x) r.host_tag = "amd64" db.session.add(r) r = Run(b, "armhf%d" % x) r.host_tag = "armhf" db.session.add(r) db.session.commit() _check_queue() self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + "-amd64")) self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + "-armhf")) # get us under surge for armhf db.session.delete(Run.query.filter(Run.host_tag == "armhf").first()) # and over surge for amd64 for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, "run%d" % x) r.host_tag = "amd64" db.session.add(r) db.session.commit() worker_module.DETECT_FLAPPING = False _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + "-amd64")) self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + "-armhf")) # make sure we know about deleted workers worker.deleted = True db.session.commit() _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + "-armhf"))
def test_worker_ping(self): w = Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, []) w.enlisted = True w.online = False db.session.add(w) db.session.commit() headers = [ ('Content-type', 'application/json'), ('Authorization', 'Token key'), ] qs = 'num_available=1&foo=40' resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code) self.assertTrue(Worker.query.all()[0].online) p = os.path.join(jobserv.models.WORKER_DIR, 'w1/pings.log') with open(p) as f: buf = f.read() self.assertIn('num_available=1', buf) self.assertIn('foo=40', buf)
def test_worker_ping(self): w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, []) w.enlisted = True w.online = False db.session.add(w) db.session.commit() headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] qs = "num_available=1&foo=40" resp = self.client.get("/workers/w1/", headers=headers, query_string=qs) self.assertEqual(200, resp.status_code) self.assertTrue(Worker.query.all()[0].online) p = os.path.join(jobserv.models.WORKER_DIR, "w1/pings.log") with open(p) as f: buf = f.read() self.assertIn("num_available=1", buf) self.assertIn("foo=40", buf)
def test_worker_update(self): headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, []) db.session.add(w) db.session.commit() data = {"distro": "ArchLinux"} r = self.client.patch("/workers/w1/", headers=headers, data=json.dumps(data)) self.assertEqual(200, r.status_code) data = self.get_json("/workers/w1/", headers=headers) self.assertEqual("ArchLinux", data["worker"]["distro"]) w.deleted = True db.session.commit() r = self.client.patch("/workers/w1/", headers=headers, data=json.dumps(data)) self.assertEqual(404, r.status_code)
def test_worker_update(self): headers = [ ('Content-type', 'application/json'), ('Authorization', 'Token key'), ] w = Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, []) db.session.add(w) db.session.commit() data = {'distro': 'ArchLinux'} r = self.client.patch('/workers/w1/', headers=headers, data=json.dumps(data)) self.assertEqual(200, r.status_code) data = self.get_json('/workers/w1/') self.assertEqual('ArchLinux', data['worker']['distro']) w.deleted = True db.session.commit() r = self.client.patch('/workers/w1/', headers=headers, data=json.dumps(data)) self.assertEqual(404, r.status_code)
def test_worker_delete(self): w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, ["aarch96"]) db.session.add(w) db.session.commit() headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] r = self.client.delete("/workers/w1/", headers=headers, data="") self.assertEqual(200, r.status_code) # make sure it doesn't get access to runs r = self.client.get("/workers/w1/", headers=headers) self.assertNotIn("version", r.json)
def test_deleted_project(self): headers = [ ("Content-type", "application/json"), ("Authorization", "Token key"), ] w = Worker("w1", "ubuntu", 12, 2, "aarch64", "key", 2, []) w.enlisted = True db.session.add(w) self.create_projects("proj1") self.create_projects("proj2/lmp") self.create_projects("proj3/foo") db.session.commit() # we'll say the worker has found these directories under # /srv/jobserv/volumes: dirs_on_disk = {"directories": ["proj1", "proj2", "proj4"]} resp = self.client.get("/workers/w1/volumes-deleted/", headers=headers, json=dirs_on_disk) self.assertEqual(200, resp.status_code, resp.data) deletes = resp.json["data"]["volumes"] self.assertEqual(["proj4"], deletes)
def wrapper(*args, **kwargs): key = request.headers.get("Authorization", None) if not key: return jsendify("No Authorization header provided", 401) parts = key.split(" ") if len(parts) != 2 or parts[0] not in ("Token", "Bearer"): return jsendify("Invalid Authorization header", 401) if parts[0] == "Bearer": try: w = worker_from_jwt(parts[1]) except PyJWTError as e: return jsendify(str(e), 401) if w.name != kwargs["name"]: # worker can only access its self return jsendify("Not found", 404) worker = Worker.query.filter(Worker.name == w.name).first() if worker is None: # This looks a little nutty - constructing this object with # basically "I have no idea" data. But the worker will call # us with `worker_update` on its first connection which will # fill these handy but not mission-cricital fields out. worker = Worker(w.name, "?", 1, 1, "?", "", 1, w.allowed_tags) worker.enlisted = True db.session.add(worker) db.session.commit() elif worker.deleted: return jsendify("Not found", 404) worker.allowed_tags = w.allowed_tags else: worker = get_or_404( Worker.query.filter_by(name=kwargs["name"], deleted=False)) if not worker.validate_api_key(parts[1]): return jsendify("Incorrect API key for host", 401) worker.allowed_tags = [] request.worker = worker return f(*args, **kwargs)
def test_worker_get_run(self, storage): if db.engine.dialect.name == 'sqlite': self.skipTest('Test requires MySQL') rundef = {'run_url': 'foo', 'runner_url': 'foo', 'env': {}} storage().get_run_definition.return_value = json.dumps(rundef) w = Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, ['aarch96']) w.enlisted = True w.online = True db.session.add(w) self.create_projects('job-1') p = Project.query.all()[0] b = Build.create(p) r = Run(b, 'run0') r.host_tag = 'aarch96' db.session.add(r) db.session.commit() headers = [ ('Content-type', 'application/json'), ('Authorization', 'Token key'), ] qs = 'available_runners=1&foo=2' resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data['data']['worker']['run-defs'])) # now put a tag on the worker that doesn't match r.status = BuildStatus.QUEUED w.host_tags = 'amd64, foo' db.session.commit() resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code) data = json.loads(resp.data.decode()) self.assertNotIn('run-defs', data['data']['worker']) # now tag the run with the worker's host name r.host_tag = 'w1' w.host_tags = '' db.session.commit() resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data['data']['worker']['run-defs'])) # now do a pattern match w.host_tags = 'aarch96' r.host_tag = 'aa?c*' r.status = BuildStatus.QUEUED db.session.commit() resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data['data']['worker']['run-defs'])) # now mark it only for surges w.surges_only = True r.status = BuildStatus.QUEUED r.host_tag = 'aarch96' db.session.commit() resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code) data = json.loads(resp.data.decode()) self.assertNotIn('run-defs', data['data']['worker'])
def test_worker_sync_builds(self, storage): """Ensure Projects with "synchronous_builds" are assigned properly. 1. Create a "synchronous" Project 2. Add a RUNNING build and and QUEUED build 3. Create a regular Project with a QUEUED build Make sure the QUEUED build from the second Project is assigned rather than the *older* but blocked build from the first Project. """ if db.engine.dialect.name == 'sqlite': self.skipTest('Test requires MySQL') rundef = {'run_url': 'foo', 'runner_url': 'foo', 'env': {}} storage().get_run_definition.return_value = json.dumps(rundef) w = Worker('w1', 'ubuntu', 12, 2, 'aarch64', 'key', 2, ['aarch96']) w.enlisted = True w.online = True db.session.add(w) # create a "synchronous" builds project self.create_projects('job-1') p = Project.query.all()[0] p.synchronous_builds = True db.session.commit() # add an active build b = Build.create(p) r = Run(b, 'p1b1r1') r.host_tag = 'aarch96' r.status = BuildStatus.RUNNING db.session.add(r) # Queue up another run on this build. The project is sync, but the # runs in a single build can go in parallel r = Run(b, 'p1b1r2') r.host_tag = 'aarch96' db.session.add(r) # now queue a build up b = Build.create(p) r = Run(b, 'p1b2r1') r.host_tag = 'aarch97' # different host-tag, but should be blocked db.session.add(r) # create a normal project self.create_projects('job-2') p = Project.query.all()[1] db.session.commit() # queue up a build. This is "older" than the queued build for # the synchronous project, but should get selected below b = Build.create(p) r = Run(b, 'p2b1r1') r.host_tag = 'aarch97' db.session.add(r) db.session.commit() headers = [ ('Content-type', 'application/json'), ('Authorization', 'Token key'), ] qs = 'available_runners=1&foo=2' # This should make the p1b1r2 run running resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data['data']['worker']['run-defs'])) self.assertEqual( [ BuildStatus.RUNNING, BuildStatus.RUNNING, BuildStatus.QUEUED, BuildStatus.QUEUED ], # NOQA [x.status for x in Run.query]) # now job-1 should get blocked and job-2's run will get popped # lets change the host-tag to ensure this does *all* runs w.host_tags = ['aarch97'] db.session.commit() resp = self.client.get('/workers/w1/', headers=headers, query_string=qs) self.assertEqual(200, resp.status_code, resp.data) data = json.loads(resp.data.decode()) self.assertEqual(1, len(data['data']['worker']['run-defs'])) self.assertEqual( [ BuildStatus.RUNNING, BuildStatus.RUNNING, BuildStatus.QUEUED, BuildStatus.RUNNING ], # NOQA [x.status for x in Run.query])
class TestWorkerMonitor(JobServTest): def setUp(self): super().setUp() jobserv.models.WORKER_DIR = tempfile.mkdtemp() jobserv.worker.SURGE_FILE = os.path.join(jobserv.models.WORKER_DIR, 'enable_surges') self.addCleanup(shutil.rmtree, jobserv.models.WORKER_DIR) self.worker = Worker('w1', 'd', 1, 1, 'amd64', 'k', 1, 'amd64') self.worker.enlisted = True self.worker.online = True db.session.add(self.worker) db.session.commit() def test_offline_no_pings(self): _check_workers() db.session.refresh(self.worker) self.assertFalse(self.worker.online) def test_offline(self): self.worker.ping() offline = time.time() - 81 # 81 seconds old os.utime(self.worker.pings_log, (offline, offline)) _check_workers() db.session.refresh(self.worker) self.assertFalse(self.worker.online) @patch('jobserv.worker.WORKER_ROTATE_PINGS_LOG') def test_rotate(self, rotate): # enable rotation rotate.return_value = True # create a big file self.worker.ping() with open(self.worker.pings_log, 'a') as f: f.write('1' * 1024 * 1024) _check_workers() self.assertEqual(0, os.stat(self.worker.pings_log).st_size) # there should be two files now self.assertEqual( 2, len(os.listdir(os.path.dirname(self.worker.pings_log)))) # we should still be online db.session.refresh(self.worker) self.assertTrue(self.worker.online) def test_truncate(self): # rotation is disabled by default: # create a big file self.worker.ping() with open(self.worker.pings_log, 'a') as f: f.write('1' * 1024 * 1024) _check_workers() self.assertEqual(0, os.stat(self.worker.pings_log).st_size) # there should be two files now self.assertEqual( 1, len(os.listdir(os.path.dirname(self.worker.pings_log)))) # we should still be online db.session.refresh(self.worker) self.assertTrue(self.worker.online) def test_surge_simple(self): self.create_projects('proj1') b = Build.create(Project.query.all()[0]) for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, 'run%d' % x) r.host_tag = 'amd64' db.session.add(r) db.session.commit() _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64')) db.session.delete(Run.query.all()[0]) db.session.commit() worker_module.DETECT_FLAPPING = False _check_queue() self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64')) def test_surge_complex(self): # we'll have two amd64 workers and one armhf worker = Worker('w2', 'd', 1, 1, 'amd64', 'k', 1, 'amd64') worker.enlisted = True worker.online = True db.session.add(worker) worker = Worker('w3', 'd', 1, 1, 'armhf', 'k', 1, 'armhf') worker.enlisted = True worker.online = True db.session.add(worker) db.session.commit() self.create_projects('proj1') b = Build.create(Project.query.all()[0]) for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, 'amd%d' % x) r.host_tag = 'amd64' db.session.add(r) r = Run(b, 'armhf%d' % x) r.host_tag = 'armhf' db.session.add(r) db.session.commit() _check_queue() self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64')) self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf')) # get us under surge for armhf db.session.delete(Run.query.filter(Run.host_tag == 'armhf').first()) # and over surge for amd64 for x in range(SURGE_SUPPORT_RATIO + 1): r = Run(b, 'run%d' % x) r.host_tag = 'amd64' db.session.add(r) db.session.commit() worker_module.DETECT_FLAPPING = False _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-amd64')) self.assertFalse(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf')) # make sure we know about deleted workers worker.deleted = True db.session.commit() _check_queue() self.assertTrue(os.path.exists(jobserv.worker.SURGE_FILE + '-armhf')) @patch('jobserv.worker.notify_run_terminated') @patch('jobserv.worker._update_run') def test_stuck(self, update_run, notify): """Ensure stuck runs are failed.""" self.create_projects('proj1') b = Build.create(Project.query.all()[0]) r = Run(b, 'bla') r.status = BuildStatus.RUNNING db.session.add(r) db.session.commit() e = RunEvents(r, BuildStatus.RUNNING) e.time = datetime.datetime.utcnow() - datetime.timedelta(hours=13) db.session.add(e) db.session.commit() _check_stuck() self.assertEqual('bla', notify.call_args[0][0].name) notify.rest_mock() r.status = BuildStatus.CANCELLING e = RunEvents(r, BuildStatus.RUNNING) e.time = datetime.datetime.utcnow() - datetime.timedelta(hours=13) db.session.add(e) db.session.commit() _check_stuck() self.assertEqual('bla', notify.call_args[0][0].name) self.assertEqual('bla', update_run.call_args[0][0].name) @patch('jobserv.worker._update_run') def test_cancelled(self, update): """Ensure runs that were cancelled before they were assigned to a worker are failed.""" self.create_projects('proj1') b = Build.create(Project.query.all()[0]) r = Run(b, 'bla') r.status = BuildStatus.CANCELLING db.session.add(r) db.session.commit() _check_cancelled() self.assertEqual('FAILED', update.call_args[1]['status'])