def refresh_antivirus(self):
self.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
except sh.ErrorReturnCode:
self.warning("Freshclam failed, skipping DB update")
def refresh_antivirus(self):
self.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
except sh.ErrorReturnCode:
self.warning("Freshclam failed, skipping DB update")
def _scan_files(self):
"""Scan the files we've collected. We do the download and scan concurrently to make
it easier to have a coherent log afterwards. Uses the venv python."""
self.info("Refreshing clamav db...")
redo.retry(lambda:
sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
'-j{}'.format(self.config['scan_parallelization']),
'clamscan', '--no-summary', '--', self.dest_dir])
def _scan_files(self):
"""Scan the files we've collected. We do the download and scan concurrently to make
it easier to have a coherent log afterwards. Uses the venv python."""
self.info("Refreshing clamav db...")
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
external_tools_path = os.path.join(
os.path.abspath(
os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools')
self.run_command([
self.query_python_path(),
os.path.join(external_tools_path, 'extract_and_run_command.py'),
'-j{}'.format(self.config['scan_parallelization']), 'clamscan',
'--no-summary', '--', self.dest_dir
])
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition", required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam", action="store_true", default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q", "--quiet", dest="log_level",
action="store_const", const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(os.environ.get('DATADOG_API_SECRET')).get('key')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
# For use in shell scripts.
write_dogrc(dd_api_key)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
_timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_certs))
loop.close()
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(manifest[0]['branch']),
"platform:{}".format(manifest[0]['platform']),
]
ddstats.timing('task_duration', time.time() - start,
start, tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
if dd_api_key:
time.sleep(10)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--signing-cert", required=True)
parser.add_argument("--task-definition", required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("-q", "--quiet", dest="log_level",
action="store_const", const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.log_level)
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
log.info("Refreshing clamav db...")
try:
redo.retry(lambda:
sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
download(f, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, args.signing_cert)
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
complete_mars["%s_hash" % mar_type] = get_hash(dest)
unpack(work_env, dest, unpack_dir)
log.info("AV-scanning %s ...", unpack_dir)
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
path = os.path.join(work_env.workdir, "to")
from_path = os.path.join(work_env.workdir, "from")
mar_data = {
"ACCEPTED_MAR_CHANNEL_IDS": get_option(
path, filename="update-settings.ini", section="Settings",
option="ACCEPTED_MAR_CHANNEL_IDS"),
"version": get_option(path, filename="application.ini",
section="App", option="Version"),
"to_buildid": get_option(path, filename="application.ini",
section="App", option="BuildID"),
"from_buildid": get_option(from_path, filename="application.ini",
section="App", option="BuildID"),
"appName": get_option(from_path, filename="application.ini",
section="App", option="Name"),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo": get_option(path, filename="platform.ini", section="Build",
option="SourceRepository"),
"revision": get_option(path, filename="platform.ini",
section="Build", option="SourceStamp"),
"from_mar": e["from_mar"],
"to_mar": e["to_mar"],
"platform": e["platform"],
"locale": e["locale"],
}
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ["ACCEPTED_MAR_CHANNEL_IDS"]
for field in ("update_number", "previousVersion",
"previousBuildNumber", "toVersion",
"toBuildNumber"):
if field in e:
mar_data[field] = e[field]
mar_data.update(complete_mars)
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once
work_env.download_buildsystem_bits(repo=mar_data["repo"],
revision=mar_data["revision"])
generate_partial(work_env, from_path, path, dest_mar,
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
mar_data["version"])
mar_data["size"] = os.path.getsize(dest_mar)
mar_data["hash"] = get_hash(dest_mar)
shutil.copy(dest_mar, args.artifacts_dir)
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam",
action="store_true",
default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert (get_keysize(signing_certs['sha1']) == 2048)
assert (get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(
os.environ.get('DATADOG_API_SECRET')).get('key')
# Create this even when not sending metrics, so the context manager
# statements work.
ddstats = ThreadStats(namespace='releng.releases.partials')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
use_old_format = False
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
with ddstats.timer('mar.download.time'):
download(f, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, signing_certs)
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
complete_mars["%s_hash" % mar_type] = get_hash(dest)
with ddstats.timer('mar.unpack.time'):
unpack(work_env, dest, unpack_dir)
if mar_type == 'from':
version = get_option(unpack_dir,
filename="application.ini",
section="App",
option="Version")
major = int(version.split(".")[0])
# The updater for versions less than 56.0 requires BZ2
# compressed MAR files
if major < 56:
use_old_format = True
log.info("Forcing BZ2 compression for %s", f)
log.info("AV-scanning %s ...", unpack_dir)
metric_tags = [
"platform:{}".format(e['platform']),
]
with ddstats.timer('mar.clamscan.time', tags=metric_tags):
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
path = os.path.join(work_env.workdir, "to")
from_path = os.path.join(work_env.workdir, "from")
mar_data = {
"ACCEPTED_MAR_CHANNEL_IDS":
get_option(path,
filename="update-settings.ini",
section="Settings",
option="ACCEPTED_MAR_CHANNEL_IDS"),
"version":
get_option(path,
filename="application.ini",
section="App",
option="Version"),
"to_buildid":
get_option(path,
filename="application.ini",
section="App",
option="BuildID"),
"from_buildid":
get_option(from_path,
filename="application.ini",
section="App",
option="BuildID"),
"appName":
get_option(from_path,
filename="application.ini",
section="App",
option="Name"),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceRepository"),
"revision":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceStamp"),
"from_mar":
e["from_mar"],
"to_mar":
e["to_mar"],
"platform":
e["platform"],
"locale":
e["locale"],
}
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ[
"ACCEPTED_MAR_CHANNEL_IDS"]
for field in ("update_number", "previousVersion",
"previousBuildNumber", "toVersion", "toBuildNumber"):
if field in e:
mar_data[field] = e[field]
mar_data.update(complete_mars)
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
if 'dest_mar' in e:
mar_name = e['dest_mar']
else:
# default to formatted name if not specified
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once
work_env.download_buildsystem_bits(repo=mar_data["repo"],
revision=mar_data["revision"])
metric_tags = [
"branch:{}".format(mar_data['branch']),
"platform:{}".format(mar_data['platform']),
# If required. Shouldn't add much useful info, but increases
# cardinality of metrics substantially, so avoided.
# "locale:{}".format(mar_data['locale']),
]
with ddstats.timer('generate_partial.time', tags=metric_tags):
generate_partial(work_env, from_path, path, dest_mar,
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
mar_data["version"], use_old_format)
mar_data["size"] = os.path.getsize(dest_mar)
metric_tags.append("unit:bytes")
# Allows us to find out how many releases there were between the two,
# making buckets of the file sizes easier.
metric_tags.append("update_number:{}".format(
mar_data.get('update_number', 0)))
ddstats.gauge('partial_mar_size', mar_data['size'], tags=metric_tags)
mar_data["hash"] = get_hash(dest_mar)
shutil.copy(dest_mar, args.artifacts_dir)
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(mar_data['branch']),
]
ddstats.timing('task_duration',
time.time() - start,
start,
tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
time.sleep(10)
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition", required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam", action="store_true", default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q", "--quiet", dest="log_level",
action="store_const", const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(os.environ.get('DATADOG_API_SECRET')).get('key')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
_timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_certs))
loop.close()
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(manifest[0]['branch']),
]
ddstats.timing('task_duration', time.time() - start,
start, tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
if dd_api_key:
time.sleep(10)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.log_level)
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
download(f, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, args.signing_cert)
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
complete_mars["%s_hash" % mar_type] = get_hash(dest)
unpack(work_env, dest, unpack_dir)
log.info("AV-scanning %s ...", unpack_dir)
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
path = os.path.join(work_env.workdir, "to")
from_path = os.path.join(work_env.workdir, "from")
mar_data = {
"ACCEPTED_MAR_CHANNEL_IDS":
get_option(path,
filename="update-settings.ini",
section="Settings",
option="ACCEPTED_MAR_CHANNEL_IDS"),
"version":
get_option(path,
filename="application.ini",
section="App",
option="Version"),
"to_buildid":
get_option(path,
filename="application.ini",
section="App",
option="BuildID"),
"from_buildid":
get_option(from_path,
filename="application.ini",
section="App",
option="BuildID"),
"appName":
get_option(from_path,
filename="application.ini",
section="App",
option="Name"),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceRepository"),
"revision":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceStamp"),
"from_mar":
e["from_mar"],
"to_mar":
e["to_mar"],
"platform":
e["platform"],
"locale":
e["locale"],
}
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ[
"ACCEPTED_MAR_CHANNEL_IDS"]
for field in ("update_number", "previousVersion",
"previousBuildNumber", "toVersion", "toBuildNumber"):
if field in e:
mar_data[field] = e[field]
mar_data.update(complete_mars)
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once
work_env.download_buildsystem_bits(repo=mar_data["repo"],
revision=mar_data["revision"])
generate_partial(work_env, from_path, path, dest_mar,
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
mar_data["version"])
mar_data["size"] = os.path.getsize(dest_mar)
mar_data["hash"] = get_hash(dest_mar)
shutil.copy(dest_mar, args.artifacts_dir)
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)