def process_cloud_build_event(attr, msg):
event_type = "build"
e_id = attr["buildId"]
# Unique hash for the event
signature = shared.create_unique_id(msg)
# Payload
metadata = json.loads(
base64.b64decode(msg["data"]).decode("utf-8").strip())
# Most up to date timestamp for the event
if "finishTime" in metadata:
time_created = metadata["finishTime"]
if "startTime" in metadata:
time_created = metadata["startTime"]
if "createTime" in metadata:
time_created = metadata["createTime"]
build_event = {
"event_type": event_type,
"id": e_id,
"metadata": json.dumps(metadata),
"time_created": time_created,
"signature": signature,
"msg_id": msg["message_id"],
"source": "cloud_build",
}
return build_event
def process_gitlab_event(headers, msg):
# Unique hash for the event
signature = shared.create_unique_id(msg)
source = "gitlab"
if "Mock" in headers:
source += "mock"
types = {
"push", "merge_request", "note", "tag_push", "issue", "pipeline", "job"
}
metadata = json.loads(
base64.b64decode(msg["data"]).decode("utf-8").strip())
event_type = metadata["object_kind"]
if event_type not in types:
raise Exception("Unsupported Gitlab event: '%s'" % event_type)
if event_type in ("push", "tag_push"):
e_id = metadata["checkout_sha"]
for commit in metadata["commits"]:
if commit["id"] == e_id:
time_created = commit["timestamp"]
if event_type in ("merge_request", "note", "issue", "pipeline"):
event_object = metadata["object_attributes"]
e_id = event_object["id"]
time_created = (event_object.get("updated_at")
or event_object.get("finished_at")
or event_object.get("created_at"))
if event_type in ("job"):
e_id = metadata["build_id"]
time_created = (event_object.get("finished_at")
or event_object.get("started_at"))
gitlab_event = {
"event_type": event_type,
"id": e_id,
"metadata": json.dumps(metadata),
# If time_created not supplied by event, default to pub/sub publishTime
"time_created": time_created or msg["publishTime"],
"signature": signature,
"msg_id": msg["message_id"],
"source": source,
}
return gitlab_event
def process_argocd_event(msg):
metadata = json.loads(base64.b64decode(msg["data"]).decode("utf-8").strip())
# Unique hash for the event
signature = shared.create_unique_id(msg)
argocd_event = {
"event_type": "deployment", # Event type, eg "push", "pull_reqest", etc
"id": metadata["id"], # Object ID, eg pull request ID
"metadata": json.dumps(metadata), # The body of the msg
"time_created": metadata["time"], # The timestamp of with the event
"signature": signature, # The unique event signature
"msg_id": msg["message_id"], # The pubsub message id
"source": "argocd", # The name of the source, eg "github"
}
print(argocd_event)
return argocd_event
def test_gitlab_event_processed(client):
headers = {"X-Gitlab-Event": "push", "X-Gitlab-Token": "foo"}
data = json.dumps({
"object_kind": "push",
"checkout_sha": "foo",
"commits": [{
"id": "foo",
"timestamp": 2
}],
}).encode("utf-8")
pubsub_msg = {
"message": {
"data": base64.b64encode(data).decode("utf-8"),
"attributes": {
"headers": json.dumps(headers)
},
"message_id": "foobar",
"publishTime": 1,
},
}
event = {
"event_type": "push",
"id": "foo",
"metadata": data.decode(),
"time_created": 2,
"signature": shared.create_unique_id(pubsub_msg["message"]),
"msg_id": "foobar",
"source": "gitlab",
}
shared.insert_row_into_bigquery = mock.MagicMock()
r = client.post(
"/",
data=json.dumps(pubsub_msg),
headers={"Content-Type": "application/json"},
)
shared.insert_row_into_bigquery.assert_called_with(event)
assert r.status_code == 204
def test_deployment_event_processed(client):
headers = {"X-Gitlab-Event": "deployment", "X-Gitlab-Token": "foo"}
data = json.dumps({
"object_kind": "deployment",
"short_sha": "279484c0",
"status_changed_at": "2021-04-28 21:50:00 +0200",
"deployment_id": 15,
}).encode("utf-8")
pubsub_msg = {
"message": {
"data": base64.b64encode(data).decode("utf-8"),
"attributes": {
"headers": json.dumps(headers)
},
"message_id": "foobar",
"publishTime": 1,
},
}
event = {
"event_type": "deployment",
"id": 15,
"metadata": data.decode(),
"time_created": "2021-04-28 21:50:00",
"signature": shared.create_unique_id(pubsub_msg["message"]),
"msg_id": "foobar",
"source": "gitlab",
}
shared.insert_row_into_bigquery = mock.MagicMock()
r = client.post(
"/",
data=json.dumps(pubsub_msg),
headers={"Content-Type": "application/json"},
)
shared.insert_row_into_bigquery.assert_called_with(event)
assert r.status_code == 204
def test_cloud_build_event_processed(client):
data = json.dumps({
"createTime": 1,
"startTime": 2,
"finishTime": 3
}).encode("utf-8")
pubsub_msg = {
"message": {
"data": base64.b64encode(data).decode("utf-8"),
"attributes": {
"buildId": "foo",
"id": "foo"
},
"message_id": "foobar",
},
}
build_event = {
"event_type": "build",
"id": "foo",
"metadata": '{"createTime": 1, "startTime": 2, "finishTime": 3}',
"time_created": 3,
"signature": shared.create_unique_id(pubsub_msg["message"]),
"msg_id": "foobar",
"source": "cloud_build",
}
shared.insert_row_into_bigquery = mock.MagicMock()
r = client.post(
"/",
data=json.dumps(pubsub_msg),
headers={"Content-Type": "application/json"},
)
shared.insert_row_into_bigquery.assert_called_with(build_event)
assert r.status_code == 204
def process_gitlab_event(headers, msg):
# Unique hash for the event
signature = shared.create_unique_id(msg)
source = "gitlab"
if "Mock" in headers:
source += "mock"
types = {
"push", "merge_request", "note", "tag_push", "issue", "pipeline",
"job", "deployment", "build"
}
metadata = json.loads(
base64.b64decode(msg["data"]).decode("utf-8").strip())
event_type = metadata["object_kind"]
if event_type not in types:
raise Exception("Unsupported Gitlab event: '%s'" % event_type)
if event_type in ("push", "tag_push"):
e_id = metadata["checkout_sha"]
for commit in metadata["commits"]:
if commit["id"] == e_id:
time_created = commit["timestamp"]
if event_type in ("merge_request", "note", "issue", "pipeline"):
event_object = metadata["object_attributes"]
e_id = event_object["id"]
time_created = (event_object.get("updated_at")
or event_object.get("finished_at")
or event_object.get("created_at"))
if event_type in ("job"):
e_id = metadata["build_id"]
time_created = (event_object.get("finished_at")
or event_object.get("started_at"))
if event_type in ("deployment"):
e_id = metadata["deployment_id"]
# Deployment timestamps come in a format like "2021-04-28 21:50:00 +0200"
# BigQuery does not accept this as a valid format
# Removing the extra timezone information below
time_created = metadata["status_changed_at"][:-6]
if event_type in ("build"):
e_id = metadata["build_id"]
time_created = (metadata.get("build_finished_at")
or metadata.get("build_started_at")
or metadata.get("build_created_at"))
gitlab_event = {
"event_type": event_type,
"id": e_id,
"metadata": json.dumps(metadata),
# If time_created not supplied by event, default to pub/sub publishTime
"time_created": time_created or msg["publishTime"],
"signature": signature,
"msg_id": msg["message_id"],
"source": source,
}
return gitlab_event
