def test_convert_date_time_to_timestamp(start_time, end_time, format):
start_timestamp = 1593227828
end_timestamp = 1593238628
three_hours = 10800
calc_start = pelorus.convert_date_time_to_timestamp(start_time, format)
assert calc_start == start_timestamp
calc_end = pelorus.convert_date_time_to_timestamp(end_time, format)
assert calc_end == end_timestamp
assert calc_end - calc_start == three_hours
def collect(self):
logging.info("collect: start")
metric = GaugeMetricFamily('deploy_timestamp',
'Deployment timestamp',
labels=['namespace', 'app', 'image_sha'])
metrics = generate_metrics(self._namespaces)
for m in metrics:
logging.info(
"Collected deploy_timestamp{namespace=%s, app=%s, image=%s} %s"
% (m.namespace, m.name, m.image_sha,
pelorus.convert_date_time_to_timestamp(m.deploy_time)))
metric.add_metric(
[m.namespace, m.name, m.image_sha, m.deploy_time],
pelorus.convert_date_time_to_timestamp(m.deploy_time))
yield (metric)
def get_commit_time(self, metric):
"""Method called to collect data and send to Prometheus"""
git_server = metric.git_fqdn
# check for gitlab or bitbucket
if "gitlab" in git_server or "bitbucket" in git_server:
logging.warn("Skipping non GitHub server, found %s" % (git_server))
return None
url = self._prefix + metric.repo_group + "/" + metric.repo_project + self._suffix + metric.commit_hash
response = requests.get(url, auth=(self._username, self._token))
if response.status_code != 200:
# This will occur when trying to make an API call to non-Github
logging.warning(
"Unable to retrieve commit time for build: %s, hash: %s, url: %s. Got http code: %s"
% (metric.build_name, metric.commit_hash, metric.repo_fqdn,
str(response.status_code)))
else:
commit = response.json()
try:
metric.commit_time = commit['commit']['committer']['date']
metric.commit_timestamp = pelorus.convert_date_time_to_timestamp(
metric.commit_time, self._timedate_format)
except Exception:
logging.error("Failed processing commit time for build %s" %
metric.build_name,
exc_info=True)
logging.debug(commit)
raise
return metric
def collect(self):
metric = GaugeMetricFamily('deploy_timestamp', 'Deployment timestamp', labels=['namespace', 'app', 'image_sha'])
metrics = generate_metrics(self._namespaces)
for m in metrics:
print("Namespace: ", m.namespace, ", App: ", m.name, ", Image: ", m.image_sha)
metric.add_metric([m.namespace, m.name, m.image_sha, m.deploy_time],
pelorus.convert_date_time_to_timestamp(m.deploy_time))
yield(metric)
def convert_jira_timestamp(self, date_time):
"""Convert a Jira datetime with TZ to UTC """
# The time retunred by Jira has a TZ, so convert to UTC
utc = datetime.strptime(date_time,
'%Y-%m-%dT%H:%M:%S.%f%z').astimezone(pytz.utc)
# Change the datetime to a string
utc_string = utc.strftime('%Y-%m-%dT%H:%M:%SZ')
# convert to timestamp
return pelorus.convert_date_time_to_timestamp(utc_string)
def get_commit_time(self, metric):
"""Method called to collect data and send to Prometheus"""
session = requests.Session()
session.verify = False
project_name = metric.repo_project
git_server = metric.git_server
if "github" in git_server or "bitbucket" in git_server:
logging.warn("Skipping non GitLab server, found %s" % (git_server))
return None
# Private or personal token
gl = gitlab.Gitlab(git_server,
private_token=self._token,
api_version=4,
session=session)
# oauth token authentication
# gl = gitlab.Gitlab(git_server, oauth_token='my_long_token_here', api_version=4, session=session)
# get the project id from the map by search for the project_name
project = None
try:
logging.debug("Searching for project: %s" % project_name)
project = self._get_next_results(gl, project_name, metric.repo_url,
0)
logging.debug("Setting project to %s : %s" %
(project.name, str(project.id)))
except Exception:
logging.error(
"Failed to find project: %s, repo: %s for build %s" %
(metric.repo_url, project_name, metric.build_name),
exc_info=True,
)
raise
# Using the project id, get the project
if project is None:
raise TypeError("Failed to find repo project: %s, for build %s" %
(metric.repo_url, metric.build_name))
try:
# get the commit from the project using the hash
short_hash = metric.commit_hash[:8]
commit = project.commits.get(short_hash)
# get the commit date/time
metric.commit_time = commit.committed_date
# set the timestamp after conversion
metric.commit_timestamp = pelorus.convert_date_time_to_timestamp(
metric.commit_time, self._timedate_format)
except Exception:
logging.error(
"Failed processing commit time for build %s" %
metric.build_name,
exc_info=True,
)
raise
return metric
def collect(self) -> Iterable[GaugeMetricFamily]:
logging.info("collect: start")
metric = GaugeMetricFamily(
"deploy_timestamp",
"Deployment timestamp",
labels=["namespace", "app", "image_sha"],
)
metrics = generate_metrics(self._namespaces, self.client)
for m in metrics:
logging.info(
"Collected deploy_timestamp{namespace=%s, app=%s, image=%s} %s"
% (
m.namespace,
m.name,
m.image_sha,
pelorus.convert_date_time_to_timestamp(m.deploy_time),
)
)
metric.add_metric(
[m.namespace, m.name, m.image_sha, m.deploy_time],
pelorus.convert_date_time_to_timestamp(m.deploy_time),
)
yield (metric)
def get_commit_time(self, metric):
"""Method called to collect data and send to Prometheus"""
session = requests.Session()
session.verify = False
git_server = metric.git_server
if ("github" in git_server or "bitbucket" in git_server
or "gitlab" in git_server or "azure" in git_server):
logging.warn("Skipping non Gitea server, found %s" % (git_server))
return None
url = (self._prefix + metric.repo_group + "/" + metric.repo_project +
self._suffix + metric.commit_hash)
logging.info("URL %s" % (url))
response = requests.get(url, auth=(self._username, self._token))
logging.info("response %s" %
(requests.get(url, auth=(self._username, self._token))))
if response.status_code != 200:
# This will occur when trying to make an API call to non-Github
logging.warning(
"Unable to retrieve commit time for build: %s, hash: %s, url: %s. Got http code: %s"
% (
metric.build_name,
metric.commit_hash,
metric.repo_url,
str(response.status_code),
))
else:
commit = response.json()
try:
metric.commit_time = commit["commit"]["committer"]["date"]
logging.debug("metric.commit_time %s" %
(str(metric.commit_time)[:19]))
logging.debug("self._timedate_format %s" %
(self._timedate_format))
metric.commit_timestamp = pelorus.convert_date_time_to_timestamp(
(str(metric.commit_time)[:19]), self._timedate_format)
except Exception:
logging.error(
"Failed processing commit time for build %s" %
metric.build_name,
exc_info=True,
)
logging.debug(commit)
raise
return metric
def getCommitTime(self):
myurl = self.repo_url
url_tokens = myurl.split("/")
url = self._prefix + url_tokens[3] + "/" + url_tokens[4].split(".")[0] + self._suffix + self.commit_hash
response = requests.get(url, auth=(username, token))
if response.status_code != 200:
# This will occur when trying to make an API call to non-Github
logging.warning("Unable to retrieve commit time for build: %s, hash: %s, url: %s. Got http code: %s" % (
self.build_name, self.commit_hash, url_tokens[2], str(response.status_code)))
else:
commit = response.json()
try:
self.commit_time = commit['commit']['committer']['date']
self.commit_timestamp = pelorus.convert_date_time_to_timestamp(self.commit_time)
except Exception:
logging.error("Failed processing commit time for build %s" % self.build_name, exc_info=True)
logging.debug(commit)
raise
def get_commit_time(self, metric):
"""Method called to collect data and send to Prometheus"""
git_server = metric.git_server
# do a simple check for hosted Git services.
if "github" in git_server or "gitlab" in git_server:
logging.warn("Skipping non BitBucket server, found %s" %
(git_server))
return None
# Set the session auth
self.__session.auth = (self._username, self._token)
# Get or figure out the BB API version
# check the cache
api_version = self.get_api_version(git_server)
if api_version is None:
return metric
# get the project, group, and commit sha properties from the existing metric
project_name = metric.repo_project
sha = metric.commit_hash
group = metric.repo_group
# set API variables depending on the version
if api_version == '1.0':
# start with the V1 globals
api_root = self.V1_API_ROOT
api_pattern = self.V1_API_PATTERN
# Due to the BB V1 git pattern differences, need remove '/scm' and parse again.
old_url = metric.repo_url
# Parse out the V1 /scm, for whatever reason why it is present.
new_url = old_url.replace('/scm', '')
# set the new url, so the parsing will happen
metric.repo_url = new_url
# set the new project name
project_name = metric.repo_project
# set the new group
group = metric.repo_group
# set the URL back to the original
metric.repo_url = old_url
elif api_version == '2.0':
# Just set the V2 globals
api_root = self.V2_API_ROOT
api_pattern = self.V2_API_PATTERN
# Create the API server from the Git server and API Root
api_server = pelorus.url_joiner(git_server, api_root)
# Finally, make the API call
api_response = self.get_commit_information(api_pattern, api_server,
group, project_name, sha,
metric)
# Check for a valid response and continue if none is found
if api_response is None or api_response.status_code != 200 or api_response.text is None:
logging.warning(
"Unable to retrieve commit time for build: %s, hash: %s, url: %s. Got http code: %s"
% (metric.build_name, metric.commit_hash, metric.repo_fqdn,
str(api_response.status_code)))
return metric
try:
logging.debug("API call returned: %s" % (api_response.text))
api_j = json.loads(api_response.text)
if api_version == '2.0':
# API V2 only has the commit time, which needs to be converted.
# get the commit date/time
commit_time = api_j["date"]
logging.debug("API v2 returned sha: %s, commit date: %s" %
(sha, str(commit_time)))
# set the commit time from the API
metric.commit_time = commit_time
# set the timestamp after conversion
metric.commit_timestamp = pelorus.convert_date_time_to_timestamp(
metric.commit_time, self._timedate_format)
else:
# API V1 has the commit timestamp, which does not need to be converted
commit_timestamp = api_j["committerTimestamp"]
logging.debug("API v1 returned sha: %s, timestamp: %s" %
(sha, str(commit_timestamp)))
# Convert timestamp from miliseconds to seconds
converted_timestamp = commit_timestamp / 1000
# set the timestamp in the metric
metric.commit_timestamp = converted_timestamp
# convert the time stamp to datetime and set in metric
metric.commit_time = pelorus.convert_timestamp_to_date_time_str(
converted_timestamp)
except Exception:
logging.error("Failed processing commit time for build %s" %
metric.build_name,
exc_info=True)
raise
return metric
def test_datetime__as_str_conversion_type(date_time, timestamp, date_format):
ts = pelorus.convert_date_time_to_timestamp(date_time, date_format)
assert ts is not None
assert timestamp == ts
def test_datetime_conversion_type(date_format):
d = datetime.now()
myts = d.replace(tzinfo=timezone.utc).timestamp()
ts = pelorus.convert_date_time_to_timestamp(d, date_format)
assert ts is not None
assert myts == ts
