def process_cloud_build_event(attr, msg): event_type = "build" e_id = attr["buildId"] # Unique hash for the event signature = shared.create_unique_id(msg) # Payload metadata = json.loads( base64.b64decode(msg["data"]).decode("utf-8").strip()) # Most up to date timestamp for the event if "finishTime" in metadata: time_created = metadata["finishTime"] if "startTime" in metadata: time_created = metadata["startTime"] if "createTime" in metadata: time_created = metadata["createTime"] build_event = { "event_type": event_type, "id": e_id, "metadata": json.dumps(metadata), "time_created": time_created, "signature": signature, "msg_id": msg["message_id"], "source": "cloud_build", } return build_event
def process_gitlab_event(headers, msg): # Unique hash for the event signature = shared.create_unique_id(msg) source = "gitlab" if "Mock" in headers: source += "mock" types = { "push", "merge_request", "note", "tag_push", "issue", "pipeline", "job" } metadata = json.loads( base64.b64decode(msg["data"]).decode("utf-8").strip()) event_type = metadata["object_kind"] if event_type not in types: raise Exception("Unsupported Gitlab event: '%s'" % event_type) if event_type in ("push", "tag_push"): e_id = metadata["checkout_sha"] for commit in metadata["commits"]: if commit["id"] == e_id: time_created = commit["timestamp"] if event_type in ("merge_request", "note", "issue", "pipeline"): event_object = metadata["object_attributes"] e_id = event_object["id"] time_created = (event_object.get("updated_at") or event_object.get("finished_at") or event_object.get("created_at")) if event_type in ("job"): e_id = metadata["build_id"] time_created = (event_object.get("finished_at") or event_object.get("started_at")) gitlab_event = { "event_type": event_type, "id": e_id, "metadata": json.dumps(metadata), # If time_created not supplied by event, default to pub/sub publishTime "time_created": time_created or msg["publishTime"], "signature": signature, "msg_id": msg["message_id"], "source": source, } return gitlab_event
def process_argocd_event(msg): metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip()) # Unique hash for the event signature = shared.create_unique_id(msg) argocd_event = { "event_type": "deployment", # Event type, eg "push", "pull_reqest", etc "id": metadata["id"], # Object ID, eg pull request ID "metadata": json.dumps(metadata), # The body of the msg "time_created": metadata["time"], # The timestamp of with the event "signature": signature, # The unique event signature "msg_id": msg["message_id"], # The pubsub message id "source": "argocd", # The name of the source, eg "github" } print(argocd_event) return argocd_event
def test_gitlab_event_processed(client): headers = {"X-Gitlab-Event": "push", "X-Gitlab-Token": "foo"} data = json.dumps({ "object_kind": "push", "checkout_sha": "foo", "commits": [{ "id": "foo", "timestamp": 2 }], }).encode("utf-8") pubsub_msg = { "message": { "data": base64.b64encode(data).decode("utf-8"), "attributes": { "headers": json.dumps(headers) }, "message_id": "foobar", "publishTime": 1, }, } event = { "event_type": "push", "id": "foo", "metadata": data.decode(), "time_created": 2, "signature": shared.create_unique_id(pubsub_msg["message"]), "msg_id": "foobar", "source": "gitlab", } shared.insert_row_into_bigquery = mock.MagicMock() r = client.post( "/", data=json.dumps(pubsub_msg), headers={"Content-Type": "application/json"}, ) shared.insert_row_into_bigquery.assert_called_with(event) assert r.status_code == 204
def test_deployment_event_processed(client): headers = {"X-Gitlab-Event": "deployment", "X-Gitlab-Token": "foo"} data = json.dumps({ "object_kind": "deployment", "short_sha": "279484c0", "status_changed_at": "2021-04-28 21:50:00 +0200", "deployment_id": 15, }).encode("utf-8") pubsub_msg = { "message": { "data": base64.b64encode(data).decode("utf-8"), "attributes": { "headers": json.dumps(headers) }, "message_id": "foobar", "publishTime": 1, }, } event = { "event_type": "deployment", "id": 15, "metadata": data.decode(), "time_created": "2021-04-28 21:50:00", "signature": shared.create_unique_id(pubsub_msg["message"]), "msg_id": "foobar", "source": "gitlab", } shared.insert_row_into_bigquery = mock.MagicMock() r = client.post( "/", data=json.dumps(pubsub_msg), headers={"Content-Type": "application/json"}, ) shared.insert_row_into_bigquery.assert_called_with(event) assert r.status_code == 204
def test_cloud_build_event_processed(client): data = json.dumps({ "createTime": 1, "startTime": 2, "finishTime": 3 }).encode("utf-8") pubsub_msg = { "message": { "data": base64.b64encode(data).decode("utf-8"), "attributes": { "buildId": "foo", "id": "foo" }, "message_id": "foobar", }, } build_event = { "event_type": "build", "id": "foo", "metadata": '{"createTime": 1, "startTime": 2, "finishTime": 3}', "time_created": 3, "signature": shared.create_unique_id(pubsub_msg["message"]), "msg_id": "foobar", "source": "cloud_build", } shared.insert_row_into_bigquery = mock.MagicMock() r = client.post( "/", data=json.dumps(pubsub_msg), headers={"Content-Type": "application/json"}, ) shared.insert_row_into_bigquery.assert_called_with(build_event) assert r.status_code == 204
def process_gitlab_event(headers, msg): # Unique hash for the event signature = shared.create_unique_id(msg) source = "gitlab" if "Mock" in headers: source += "mock" types = { "push", "merge_request", "note", "tag_push", "issue", "pipeline", "job", "deployment", "build" } metadata = json.loads( base64.b64decode(msg["data"]).decode("utf-8").strip()) event_type = metadata["object_kind"] if event_type not in types: raise Exception("Unsupported Gitlab event: '%s'" % event_type) if event_type in ("push", "tag_push"): e_id = metadata["checkout_sha"] for commit in metadata["commits"]: if commit["id"] == e_id: time_created = commit["timestamp"] if event_type in ("merge_request", "note", "issue", "pipeline"): event_object = metadata["object_attributes"] e_id = event_object["id"] time_created = (event_object.get("updated_at") or event_object.get("finished_at") or event_object.get("created_at")) if event_type in ("job"): e_id = metadata["build_id"] time_created = (event_object.get("finished_at") or event_object.get("started_at")) if event_type in ("deployment"): e_id = metadata["deployment_id"] # Deployment timestamps come in a format like "2021-04-28 21:50:00 +0200" # BigQuery does not accept this as a valid format # Removing the extra timezone information below time_created = metadata["status_changed_at"][:-6] if event_type in ("build"): e_id = metadata["build_id"] time_created = (metadata.get("build_finished_at") or metadata.get("build_started_at") or metadata.get("build_created_at")) gitlab_event = { "event_type": event_type, "id": e_id, "metadata": json.dumps(metadata), # If time_created not supplied by event, default to pub/sub publishTime "time_created": time_created or msg["publishTime"], "signature": signature, "msg_id": msg["message_id"], "source": source, } return gitlab_event