class DatadogMetrics(object):
"""DataDog Metric backend"""
def __init__(self, api_key, app_key, flush_interval=10,
namespace="aplt"):
datadog.initialize(api_key=api_key, app_key=app_key)
self._client = ThreadStats()
self._flush_interval = flush_interval
self._host = get_hostname()
self._namespace = namespace
def _prefix_name(self, name):
return "%s.%s" % (self._namespace, name)
def start(self):
self._client.start(flush_interval=self._flush_interval,
roll_up_interval=self._flush_interval)
def increment(self, name, count=1, **kwargs):
self._client.increment(self._prefix_name(name), count, host=self._host,
**kwargs)
def timing(self, name, duration, **kwargs):
self._client.timing(self._prefix_name(name), value=duration,
host=self._host, **kwargs)
class DatadogMetrics(object):
"""DataDog Metric backend"""
def __init__(self, api_key, app_key, flush_interval=10, namespace="aplt"):
datadog.initialize(api_key=api_key, app_key=app_key)
self._client = ThreadStats()
self._flush_interval = flush_interval
self._host = get_hostname()
self._namespace = namespace
def _prefix_name(self, name):
return "%s.%s" % (self._namespace, name)
def start(self):
self._client.start(flush_interval=self._flush_interval,
roll_up_interval=self._flush_interval)
def increment(self, name, count=1, **kwargs):
self._client.increment(self._prefix_name(name),
count,
host=self._host,
**kwargs)
def timing(self, name, duration, **kwargs):
self._client.timing(self._prefix_name(name),
value=duration,
host=self._host,
**kwargs)
class DatadogMetricsBackend(MetricsBackend):
def __init__(self, prefix=None, **kwargs):
# TODO(dcramer): it'd be nice if the initialize call wasn't a global
initialize(**kwargs)
self._stats = ThreadStats()
self._stats.start()
super(DatadogMetricsBackend, self).__init__(prefix=prefix)
def __del__(self):
self._stats.stop()
def incr(self, key, amount=1, sample_rate=1):
self._stats.increment(self._get_key(key), amount, sample_rate=sample_rate)
def timing(self, key, value, sample_rate=1):
self._stats.timing(self._get_key(key), value, sample_rate=sample_rate)
class DatadogMetricsBackend(MetricsBackend):
def __init__(self, prefix=None, **kwargs):
self._stats = ThreadStats()
self._stats.start()
# TODO(dcramer): it'd be nice if the initialize call wasn't a global
initialize(**kwargs)
super(DatadogMetricsBackend, self).__init__(prefix=prefix)
def __del__(self):
self._stats.stop()
def incr(self, key, amount=1, sample_rate=1):
self._stats.increment(self._get_key(key),
amount,
sample_rate=sample_rate)
def timing(self, key, value, sample_rate=1):
self._stats.timing(self._get_key(key), value, sample_rate=sample_rate)
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam",
action="store_true",
default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert (get_keysize(signing_certs['sha1']) == 2048)
assert (get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(
os.environ.get('DATADOG_API_SECRET')).get('key')
# Create this even when not sending metrics, so the context manager
# statements work.
ddstats = ThreadStats(namespace='releng.releases.partials')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
use_old_format = False
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
with ddstats.timer('mar.download.time'):
download(f, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, signing_certs)
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
complete_mars["%s_hash" % mar_type] = get_hash(dest)
with ddstats.timer('mar.unpack.time'):
unpack(work_env, dest, unpack_dir)
if mar_type == 'from':
version = get_option(unpack_dir,
filename="application.ini",
section="App",
option="Version")
major = int(version.split(".")[0])
# The updater for versions less than 56.0 requires BZ2
# compressed MAR files
if major < 56:
use_old_format = True
log.info("Forcing BZ2 compression for %s", f)
log.info("AV-scanning %s ...", unpack_dir)
metric_tags = [
"platform:{}".format(e['platform']),
]
with ddstats.timer('mar.clamscan.time', tags=metric_tags):
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
path = os.path.join(work_env.workdir, "to")
from_path = os.path.join(work_env.workdir, "from")
mar_data = {
"ACCEPTED_MAR_CHANNEL_IDS":
get_option(path,
filename="update-settings.ini",
section="Settings",
option="ACCEPTED_MAR_CHANNEL_IDS"),
"version":
get_option(path,
filename="application.ini",
section="App",
option="Version"),
"to_buildid":
get_option(path,
filename="application.ini",
section="App",
option="BuildID"),
"from_buildid":
get_option(from_path,
filename="application.ini",
section="App",
option="BuildID"),
"appName":
get_option(from_path,
filename="application.ini",
section="App",
option="Name"),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceRepository"),
"revision":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceStamp"),
"from_mar":
e["from_mar"],
"to_mar":
e["to_mar"],
"platform":
e["platform"],
"locale":
e["locale"],
}
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ[
"ACCEPTED_MAR_CHANNEL_IDS"]
for field in ("update_number", "previousVersion",
"previousBuildNumber", "toVersion", "toBuildNumber"):
if field in e:
mar_data[field] = e[field]
mar_data.update(complete_mars)
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
if 'dest_mar' in e:
mar_name = e['dest_mar']
else:
# default to formatted name if not specified
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once
work_env.download_buildsystem_bits(repo=mar_data["repo"],
revision=mar_data["revision"])
metric_tags = [
"branch:{}".format(mar_data['branch']),
"platform:{}".format(mar_data['platform']),
# If required. Shouldn't add much useful info, but increases
# cardinality of metrics substantially, so avoided.
# "locale:{}".format(mar_data['locale']),
]
with ddstats.timer('generate_partial.time', tags=metric_tags):
generate_partial(work_env, from_path, path, dest_mar,
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
mar_data["version"], use_old_format)
mar_data["size"] = os.path.getsize(dest_mar)
metric_tags.append("unit:bytes")
# Allows us to find out how many releases there were between the two,
# making buckets of the file sizes easier.
metric_tags.append("update_number:{}".format(
mar_data.get('update_number', 0)))
ddstats.gauge('partial_mar_size', mar_data['size'], tags=metric_tags)
mar_data["hash"] = get_hash(dest_mar)
shutil.copy(dest_mar, args.artifacts_dir)
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(mar_data['branch']),
]
ddstats.timing('task_duration',
time.time() - start,
start,
tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
time.sleep(10)
