def _register_realm(self, realm_spec): logger.debug("Got call to register realm %s with manager", realm_spec["realm"]) # Create the build information block for the registered realm. build_job = BuildJob(AttrDict(realm_spec["job_queue_item"])) execution_id = realm_spec.get("execution_id", None) executor_name = realm_spec.get("executor_name", "EC2Executor") logger.debug("Registering realm %s with manager: %s", realm_spec["realm"], realm_spec) component = self.register_component(realm_spec["realm"], BuildComponent, token=realm_spec["token"]) build_info = BuildInfo( component=component, build_job=build_job, execution_id=execution_id, executor_name=executor_name, ) self._component_to_job[component] = build_job self._build_uuid_to_info[build_job.build_uuid] = build_info logger.debug("Registered realm %s with manager", realm_spec["realm"]) return component
def get(self, processing_time=300, ordering_required=False): """ Get an available item and mark it as unavailable for the default of five minutes. The result of this method must always be composed of simple python objects which are JSON serializable for network portability reasons. """ now = datetime.utcnow() # Select an available queue item. db_item = self._select_available_item(ordering_required, now) if db_item is None: self._currently_processing = False return None # Attempt to claim the item for this instance. was_claimed = self._attempt_to_claim_item(db_item, now, processing_time) if not was_claimed: self._currently_processing = False return None self._currently_processing = True # Return a view of the queue item rather than an active db object return AttrDict({ "id": db_item.id, "body": db_item.body, "retries_remaining": db_item.retries_remaining - 1, })
def log_action( self, event_name, namespace_name, repo_name=None, analytics_name=None, analytics_sample=1, metadata=None, ): metadata = {} if metadata is None else metadata repo = None if repo_name is not None: db_repo = data.model.repository.get_repository( namespace_name, repo_name, kind_filter="application") repo = AttrDict({ "id": db_repo.id, "name": db_repo.name, "namespace_name": db_repo.namespace_user.username, "is_free_namespace": db_repo.namespace_user.stripe_id is None, }) track_and_log(event_name, repo, analytics_name=analytics_name, analytics_sample=analytics_sample, **metadata)
def test_vulnerability_notification_nolevel(): notification_data = AttrDict({ "event_config_dict": {}, }) # No level specified. assert VulnerabilityFoundEvent().should_perform({}, notification_data)
def get_gitlab_trigger(dockerfile_path='', add_permissions=True, missing_avatar_url=False): handlers = [ user_handler, users_handler, project_branches_handler, project_tree_handler, project_handler, get_projects_handler(add_permissions), tag_handler, project_branch_handler, get_group_handler(missing_avatar_url), dockerfile_handler, sub_dockerfile_handler, namespace_handler, user_namespace_handler, namespaces_handler, commit_handler, create_deploykey_handler, delete_deploykey_handker, create_hook_handler, delete_hook_handler, project_tags_handler, user_projects_list_handler, catchall_handler ] with HTTMock(*handlers): trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger')) trigger = GitLabBuildTrigger( trigger_obj, { 'build_source': 'foo/bar', 'dockerfile_path': dockerfile_path, 'username': '******' }) client = gitlab.Gitlab('http://fakegitlab', oauth_token='foobar', timeout=20, api_version=4) client.auth() trigger._get_authorized_client = lambda: client yield trigger
def test_build_withfilter(): notification_data = AttrDict({ 'event_config_dict': { "ref-regex": "refs/heads/master" }, }) # No build data at all. assert not BuildSuccessEvent().should_perform({}, notification_data) # With trigger metadata but no ref. assert not BuildSuccessEvent().should_perform({ 'trigger_metadata': {}, }, notification_data) # With trigger metadata and a not-matching ref. assert not BuildSuccessEvent().should_perform( { 'trigger_metadata': { 'ref': 'refs/heads/somebranch', }, }, notification_data) # With trigger metadata and a matching ref. assert BuildSuccessEvent().should_perform( { 'trigger_metadata': { 'ref': 'refs/heads/master', }, }, notification_data)
def _register_realm(self, realm_spec): logger.debug('Got call to register realm %s with manager', realm_spec['realm']) # Create the build information block for the registered realm. build_job = BuildJob(AttrDict(realm_spec['job_queue_item'])) execution_id = realm_spec.get('execution_id', None) executor_name = realm_spec.get('executor_name', 'EC2Executor') logger.debug('Registering realm %s with manager: %s', realm_spec['realm'], realm_spec) component = self.register_component(realm_spec['realm'], BuildComponent, token=realm_spec['token']) build_info = BuildInfo(component=component, build_job=build_job, execution_id=execution_id, executor_name=executor_name) self._component_to_job[component] = build_job self._build_uuid_to_info[build_job.build_uuid] = build_info logger.debug('Registered realm %s with manager', realm_spec['realm']) return component
def test_build_invalidfilter(): notification_data = AttrDict({ "event_config_dict": { "ref-regex": "][" }, }) # No build data at all. assert not BuildSuccessEvent().should_perform({}, notification_data) # With trigger metadata but no ref. assert not BuildSuccessEvent().should_perform( { "trigger_metadata": {}, }, notification_data, ) # With trigger metadata and a ref. assert not BuildSuccessEvent().should_perform( { "trigger_metadata": { "ref": "refs/heads/somebranch", }, }, notification_data, )
def test_build_emptyjson(): notification_data = AttrDict({ "event_config_dict": None, }) # No build data at all. assert BuildSuccessEvent().should_perform({}, notification_data)
def get_github_trigger(dockerfile_path=""): trigger_obj = AttrDict(dict(auth_token="foobar", id="sometrigger")) trigger = GithubBuildTrigger(trigger_obj, { "build_source": "foo", "dockerfile_path": dockerfile_path }) trigger._get_client = get_mock_github return trigger
def get_github_trigger(dockerfile_path=''): trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger')) trigger = GithubBuildTrigger(trigger_obj, { 'build_source': 'foo', 'dockerfile_path': dockerfile_path }) trigger._get_client = get_mock_github return trigger
def test_vulnerability_notification_normal(): notification_data = AttrDict({ "event_config_dict": { "level": 3 }, }) info = {"vulnerability": {"priority": "Critical"}} assert VulnerabilityFoundEvent().should_perform(info, notification_data)
def test_vulnerability_notification_nopvulninfo(): notification_data = AttrDict({ "event_config_dict": { "level": 3 }, }) # No vuln info. assert not VulnerabilityFoundEvent().should_perform({}, notification_data)
def notification_tuple(self, notification): # TODO: Replace this with a method once we refactor the notification stuff into its # own module. return AttrDict({ "event_config_dict": json.loads(notification.event_config_json), "method_config_dict": json.loads(notification.config_json), })
def job_scheduled(self, job_id, control_plane, execution_id, max_startup_time): """Mark the given job as scheduled with execution id, with max_startup_time. A job is considered scheduled once a worker is started with a given registration token. """ # Get job to schedule try: job_data = self._orchestrator.get_key(job_id) job_data_json = json.loads(job_data) except KeyError: logger.warning( "Failed to mark job %s as scheduled. Job no longer exists in the orchestrator", job_id, ) return False except Exception as e: logger.warning("Exception loading job %s from orchestrator: %s", job_id, e) return False # Update build context job_data_json["executor_name"] = control_plane job_data_json["execution_id"] = execution_id try: self._orchestrator.set_key(job_id, json.dumps(job_data_json), overwrite=True, expiration=max_startup_time) except Exception as e: logger.warning("Exception updating job %s in orchestrator: %s", job_id, e) return False build_job = BuildJob(AttrDict(job_data_json["job_queue_item"])) updated = self.update_job_phase(job_id, BUILD_PHASE.BUILD_SCHEDULED) if updated: self._queue.extend_processing( build_job.job_item, seconds_from_now=max_startup_time + 60, # Add some leeway to allow the expiry event to complete minimum_extension=MINIMUM_JOB_EXTENSION, ) logger.debug( "Job scheduled for job %s with execution with ID %s on control plane %s with max startup time of %s", job_id, execution_id, control_plane, max_startup_time, ) else: logger.warning( "Job %s not scheduled. Unable update build phase to SCHEDULED", job_id) return updated
async def _job_callback(self, key_change): """ This is the callback invoked when keys related to jobs are changed. It ignores all events related to the creation of new jobs. Deletes or expirations cause checks to ensure they've been properly marked as completed. :param key_change: the event and value produced by a key changing in the orchestrator :type key_change: :class:`KeyChange` """ if key_change.event in (KeyEvent.CREATE, KeyEvent.SET): return elif key_change.event in (KeyEvent.DELETE, KeyEvent.EXPIRE): # Handle the expiration/deletion. job_metadata = json.loads(key_change.value) build_job = BuildJob(AttrDict(job_metadata["job_queue_item"])) logger.debug('Got "%s" of job %s', key_change.event, build_job.build_uuid) # Get the build info. build_info = self._build_uuid_to_info.get(build_job.build_uuid, None) if build_info is None: logger.debug( 'No build info for "%s" job %s (%s); probably already deleted by this manager', key_change.event, build_job.build_uuid, job_metadata, ) return if key_change.event != KeyEvent.EXPIRE: # If the etcd action was not an expiration, then it was already deleted by some manager and # the execution was therefore already shutdown. All that's left is to remove the build info. self._build_uuid_to_info.pop(build_job.build_uuid, None) return logger.debug("got expiration for job %s with metadata: %s", build_job.build_uuid, job_metadata) if not job_metadata.get("had_heartbeat", False): # If we have not yet received a heartbeat, then the node failed to boot in some way. # We mark the job as incomplete here. await self._mark_job_incomplete(build_job, build_info) # Finally, we terminate the build execution for the job. We don't do this under a lock as # terminating a node is an atomic operation; better to make sure it is terminated than not. logger.debug( "Terminating expired build executor for job %s with execution id %s", build_job.build_uuid, build_info.execution_id, ) await self.kill_builder_executor(build_job.build_uuid) else: logger.warning("Unexpected KeyEvent (%s) on job key: %s", key_change.event, key_change.key)
def job_heartbeat(self, job_id): """Extend the processing time in the queue and updates the ttl of the job in the orchestrator. """ try: job_data = self._orchestrator.get_key(job_id) job_data_json = json.loads(job_data) build_job = BuildJob(AttrDict(job_data_json["job_queue_item"])) except KeyError: logger.warning( "Job %s no longer exists in the orchestrator, likely expired", job_id) return False except Exception as e: logger.error("Exception loading job %s from orchestrator: %s", job_id, e) return False max_expiration = datetime.utcfromtimestamp( job_data_json["max_expiration"]) max_expiration_remaining = max_expiration - datetime.utcnow() max_expiration_sec = max(1, int(max_expiration_remaining.total_seconds())) ttl = min(HEARTBEAT_PERIOD_SECONDS * 2, max_expiration_sec) # Update job expirations if (job_data_json["last_heartbeat"] and dateutil.parser.isoparse(job_data_json["last_heartbeat"]) < datetime.utcnow() - HEARTBEAT_DELTA): logger.warning( "Heartbeat expired for job %s. Marking job as expired. Last heartbeat received at %s", job_data_json["last_heartbeat"], ) self.update_job_phase(job_id, BUILD_PHASE.INTERNAL_ERROR) return False job_data_json["last_heartbeat"] = str(datetime.utcnow()) self._queue.extend_processing( build_job.job_item, seconds_from_now=JOB_TIMEOUT_SECONDS, minimum_extension=MINIMUM_JOB_EXTENSION, ) try: self._orchestrator.set_key(job_id, json.dumps(job_data_json), overwrite=True, expiration=ttl) except OrchestratorConnectionError: logger.error( "Could not update heartbeat for job %s. Orchestrator is not available", job_id) return False return True
def update_job_phase(self, job_id, phase, phase_metadata=None): """Updates the given job's phase and append the phase change to the buildlogs, with the given phase metadata. If the job reaches a completed state, update_job_phase also update the queue and cleanups any existing state and executors. """ try: job_data = self._orchestrator.get_key(job_id) job_data_json = json.loads(job_data) build_job = BuildJob(AttrDict(job_data_json["job_queue_item"])) except KeyError: logger.warning( "Job %s no longer exists in the orchestrator, likely expired", job_id) return False except Exception as e: logger.error("Exception loading job %s from orchestrator: %s", job_id, e) return False # Check if the build has not already reached a final phase if build_job.repo_build.phase in EphemeralBuilderManager.ARCHIVABLE_BUILD_PHASES: logger.warning( "Job %s is already in a final completed phase (%s), cannot update to %s", job_id, build_job.repo_build.phase, phase, ) return False # Update the build phase phase_metadata = phase_metadata or {} updated = model.build.update_phase_then_close(build_job.build_uuid, phase) if updated: self.append_log_message(build_job.build_uuid, phase, self._build_logs.PHASE, phase_metadata) # Check if on_job_complete needs to be called if updated and phase in EphemeralBuilderManager.COMPLETED_PHASES: executor_name = job_data_json.get("executor_name") execution_id = job_data_json.get("execution_id") if phase == BUILD_PHASE.ERROR: self.on_job_complete(build_job, BuildJobResult.ERROR, executor_name, execution_id) elif phase == BUILD_PHASE.COMPLETE: self.on_job_complete(build_job, BuildJobResult.COMPLETE, executor_name, execution_id) elif phase == BUILD_PHASE.INTERNAL_ERROR: self.on_job_complete(build_job, BuildJobResult.INCOMPLETE, executor_name, execution_id) elif phase == BUILD_PHASE.CANCELLED: self.on_job_complete(build_job, BuildJobResult.CANCELLED, executor_name, execution_id) return updated
def test_handle_trigger_request(payload, expected_error, expected_message): trigger = CustomBuildTrigger(None, {'build_source': 'foo'}) request = AttrDict(dict(data=payload)) if expected_error is not None: with pytest.raises(expected_error) as ipe: trigger.handle_trigger_request(request) assert str(ipe.value) == expected_message else: assert isinstance(trigger.handle_trigger_request(request), PreparedBuild)
def _realm_callback(self, key_change): logger.debug("realm callback for key: %s", key_change.key) if key_change.event == KeyEvent.CREATE: # Listen on the realm created by ourselves or another worker. realm_spec = json.loads(key_change.value) self._register_realm(realm_spec) elif key_change.event in (KeyEvent.DELETE, KeyEvent.EXPIRE): # Stop listening for new connections on the realm, if we did not get the connection. realm_spec = json.loads(key_change.value) realm_id = realm_spec["realm"] build_job = BuildJob(AttrDict(realm_spec["job_queue_item"])) build_uuid = build_job.build_uuid logger.debug("Realm key %s for build %s was %s", realm_id, build_uuid, key_change.event) build_info = self._build_uuid_to_info.get(build_uuid, None) if build_info is not None: # Pop off the component and if we find one, then the build has not connected to this # manager, so we can safely unregister its component. component = self._component_to_job.pop(build_info.component, None) if component is not None: # We were not the manager which the worker connected to, remove the bookkeeping for it logger.debug("Unregistering unused component for build %s", build_uuid) self.unregister_component(build_info.component) # If the realm has expired, then perform cleanup of the executor. if key_change.event == KeyEvent.EXPIRE: execution_id = realm_spec.get("execution_id", None) executor_name = realm_spec.get("executor_name", "EC2Executor") # Cleanup the job, since it never started. logger.debug("Job %s for incomplete marking: %s", build_uuid, build_info) if build_info is not None: yield From(self._mark_job_incomplete( build_job, build_info)) # Cleanup the executor. logger.info( "Realm %s expired for job %s, terminating executor %s with execution id %s", realm_id, build_uuid, executor_name, execution_id, ) yield From(self.terminate_executor(executor_name, execution_id)) else: logger.warning("Unexpected action (%s) on realm key: %s", key_change.event, key_change.key)
def get_bitbucket_trigger(dockerfile_path=''): trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger')) trigger = BitbucketBuildTrigger(trigger_obj, { 'build_source': 'foo/bar', 'dockerfile_path': dockerfile_path, 'nickname': 'knownuser', 'account_id': 'foo', }) trigger._get_client = get_mock_bitbucket return trigger
def _job_cancelled_callback(self, key_change): if key_change.event not in (KeyEvent.CREATE, KeyEvent.SET): return job_metadata = json.loads(key_change.value) build_job = BuildJob(AttrDict(job_metadata["job_queue_item"])) executor_name = job_metadata.get("executor_name") execution_id = job_metadata.get("execution_id") job_result = BuildJobResult.CANCELLED self.on_job_complete(build_job, job_result, executor_name, execution_id)
def test_handle_trigger_request(bitbucket_trigger, payload, expected_error, expected_message): def get_payload(): return json.loads(payload) request = AttrDict(dict(get_json=get_payload)) if expected_error is not None: with pytest.raises(expected_error) as ipe: bitbucket_trigger.handle_trigger_request(request) assert str(ipe.value) == expected_message else: assert isinstance(bitbucket_trigger.handle_trigger_request(request), PreparedBuild)
def _build_job_from_job_id(self, job_id): """Return the BuildJob from the job id.""" try: job_data = self._orchestrator.get_key(job_id) except KeyError: raise BuildJobDoesNotExistsError(job_id) except (OrchestratorConnectionError, OrchestratorError) as oe: raise BuildJobError(oe) job_metadata = json.loads(job_data) build_job = BuildJob(AttrDict(job_metadata["job_queue_item"])) return build_job
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app): with patch("redis.StrictRedis" if use_mock else "redis.None", mock_strict_redis_client): validator = RedisValidator() unvalidated_config = ValidatorContext(unvalidated_config) unvalidated_config.user = AttrDict(dict(username=user)) unvalidated_config.user_password = user_password if expected is not None: with pytest.raises(expected): validator.validate(unvalidated_config) else: validator.validate(unvalidated_config)
def test_refresh_service_key(initialized_db): # Create a service key for testing. original_expiration = datetime.utcnow() + timedelta(minutes=10) test_key_kid = model.create_service_key_for_testing(original_expiration) assert model.get_service_key_expiration(test_key_kid) instance_keys = AttrDict(dict(local_key_id=test_key_kid, service_key_expiration=30)) with patch("workers.servicekeyworker.servicekeyworker.instance_keys", instance_keys): worker = ServiceKeyWorker() worker._refresh_service_key() # Ensure the key's expiration was changed. assert model.get_service_key_expiration(test_key_kid) > original_expiration
def to_dict(self, avatar, include_namespace=False): view = { 'kind': _kinds()[self.kind_id], 'metadata': json.loads(self.metadata_json), 'ip': self.ip, 'datetime': _format_date(self.datetime), } if self.performer_username: performer = AttrDict({ 'username': self.performer_username, 'email': self.performer_email }) performer.robot = None if self.performer_robot: performer.robot = self.performer_robot view['performer'] = { 'kind': 'user', 'name': self.performer_username, 'is_robot': self.performer_robot, 'avatar': avatar.get_data_for_user(performer), } if include_namespace: if self.account_username: account = AttrDict({ 'username': self.account_username, 'email': self.account_email }) if self.account_organization: view['namespace'] = { 'kind': 'org', 'name': self.account_username, 'avatar': avatar.get_data_for_org(account), } else: account.robot = None if self.account_robot: account.robot = self.account_robot view['namespace'] = { 'kind': 'user', 'name': self.account_username, 'avatar': avatar.get_data_for_user(account), } return view
def to_dict(self, avatar, include_namespace=False): view = { "kind": _kinds()[self.kind_id], "metadata": json.loads(self.metadata_json), "ip": self.ip, "datetime": _format_date(self.datetime), } if self.performer_username: performer = AttrDict({ "username": self.performer_username, "email": self.performer_email }) performer.robot = None if self.performer_robot: performer.robot = self.performer_robot view["performer"] = { "kind": "user", "name": self.performer_username, "is_robot": self.performer_robot, "avatar": avatar.get_data_for_user(performer), } if include_namespace: if self.account_username: account = AttrDict({ "username": self.account_username, "email": self.account_email }) if self.account_organization: view["namespace"] = { "kind": "org", "name": self.account_username, "avatar": avatar.get_data_for_org(account), } else: account.robot = None if self.account_robot: account.robot = self.account_robot view["namespace"] = { "kind": "user", "name": self.account_username, "avatar": avatar.get_data_for_user(account), } return view
def get_bitbucket_trigger(dockerfile_path=""): trigger_obj = AttrDict(dict(auth_token="foobar", id="sometrigger")) trigger = BitbucketBuildTrigger( trigger_obj, { "build_source": "foo/bar", "dockerfile_path": dockerfile_path, "nickname": "knownuser", "account_id": "foo", }, ) trigger._get_client = get_mock_bitbucket return trigger
def determine_cached_tag(self, build_id, base_image_id): job_id = self._job_key(build_id) try: job_data = self._orchestrator.get_key(job_id) job_data_json = json.loads(job_data) build_job = BuildJob(AttrDict(job_data_json["job_queue_item"])) except KeyError: logger.warning("Job %s does not exist in orchestrator", job_id) return None except Exception as e: logger.warning("Exception loading job from orchestrator: %s", e) return None return build_job.determine_cached_tag(base_image_id)