def fetch_secret_from_taskcluster(name):
try:
secrets = taskcluster.Secrets({
# BaseUrl is still needed for tasks that haven't migrated to taskgraph yet.
'baseUrl': 'http://taskcluster/secrets/v1',
})
except taskcluster.exceptions.TaskclusterFailure:
# taskcluster library >=5 errors out when `baseUrl` is used
secrets = taskcluster.Secrets({
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
})
return secrets.get(name)
def get_secrets_service(self):
"""
Configured Secrets Service
"""
return taskcluster.Secrets(
self.build_options('secrets/v1')
)
def ext_pillar(minion_id, _pillar, *_args):
if not CACHE:
secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
CACHE["credentials"] = secrets.get(
"project/servo/tc-client/worker/macos/1")["secret"]
url = "https://raw.githubusercontent.com/servo/saltfs/master/admin/files/ssh/%s.pub"
CACHE["ssh_keys"] = [
urllib.urlopen(url % name).read() for name in [
"jdm",
"manishearth",
"simonsapin",
]
]
CACHE["workers"] = {
worker: (pool_name, config)
for pool_name, pool in read_yaml("worker-pools.yml").items()
if pool["kind"] == "static"
for worker, config in pool["workers"].items()
}
pool_name, config = CACHE["workers"][minion_id]
disabled = {"disabled": True, None: False}[config]
return dict(disabled=disabled, worker_type=pool_name, **CACHE)
def main(name, argv):
secret_path = ''
output_file = ''
data_name = ''
decode = False;
try:
opts, args = getopt.getopt(argv,"hs:o:n:d")
except getopt.GetoptError:
print name + '-s <secret> -o <filename> -d'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print name + '-s <secret> -o <filename> -n <data name> -d (decode base64)'
sys.exit()
elif opt in ("-s"):
secret_path = arg
elif opt in ("-o"):
output_file = arg
elif opt in ("-n"):
data_name = arg
elif opt in ("-d"):
decode = True
if data_name == '':
data_name = os.path.basename(secret_path)
secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
data = secrets.get(secret_path)
data = data['secret'][data_name]
if decode:
data = base64.b64decode(data)
with open(output_file, 'w') as output:
output.write(data)
def get_secret(secret):
# use proxy if configured, otherwise local credentials from env vars
if "TASKCLUSTER_PROXY_URL" in os.environ:
secrets_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
else:
secrets_options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(secrets_options)
return secrets.get(secret)["secret"]
def get_secret(self, name):
"""
Helper to read a Taskcluster secret
"""
secrets = taskcluster.Secrets(self.taskcluster_options())
secret = secrets.get(name)
if not secret:
raise Exception('Missing TC secret {}'.format(name))
return secret['secret']
def main():
# Use proxy if configured (within a task), otherwise, use local credentials from env vars
if 'TASKCLUSTER_PROXY_URL' in os.environ:
options = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
else:
options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(options)
secret = secrets.get("project/cia/garbage/foo")
print(secret["secret"])
def load_secrets(self, secrets_path, client_id=None, access_token=None):
"""
Load Taskcluster secrets
"""
if client_id and access_token:
# Use provided credentials
tc = taskcluster.Secrets({
'credentials': {
'clientId': client_id,
'accessToken': access_token,
}
})
else:
# Get taskcluster proxy host
# as /etc/hosts is not used in the Nix image (?)
hosts = read_hosts()
if 'taskcluster' not in hosts:
raise Exception('Missing taskcluster in /etc/hosts')
# Load secrets from TC task context
# with taskclusterProxy
base_url = 'http://{}/secrets/v1'.format(hosts['taskcluster'])
logger.info('Taskcluster Proxy enabled', url=base_url)
tc = taskcluster.Secrets({'baseUrl': base_url})
# Check mandatory keys in secrets
secrets = tc.get(secrets_path)
secrets = secrets['secret']
required = ('BUGZILLA_URL', 'BUGZILLA_TOKEN', 'API_URL')
for req in required:
if req not in secrets:
raise Exception(
'Missing value {} in Taskcluster secret value {}'.format(
req, secrets_path)) # noqa
# Add credentials too
if 'TASKCLUSTER_CLIENT_ID' not in secrets:
secrets['TASKCLUSTER_CLIENT_ID'] = client_id
if 'TASKCLUSTER_ACCESS_TOKEN' not in secrets:
secrets['TASKCLUSTER_ACCESS_TOKEN'] = access_token
return secrets
def main():
with open("/etc/ssl/openssl.cnf") as f:
config = f.read()
config += ("[v3_req]\n"
"basicConstraints = critical,CA:FALSE\n"
"keyUsage = digitalSignature\n"
"extendedKeyUsage = critical,codeSigning")
with tempfile.TemporaryDirectory() as tmp:
os.chdir(tmp)
with open("config", "w") as f:
f.write(config)
now = datetime.datetime.now()
run("""
openssl req
-x509
-sha256
-nodes
-days 730
-newkey rsa:4096
-subj /CN=Allizom
-extensions v3_req
-batch
-config config
-keyout key.pem
-out cert.pem
""")
run("""
openssl pkcs12
-export
-passout pass:
-inkey key.pem
-in cert.pem
-out servo.pfx
""")
with open("servo.pfx", "rb") as f:
pfx = f.read()
value = {"pfx": {"base64": base64.b64encode(pfx)}, "created": now}
tc_options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(tc_options)
for name in [now.strftime("%Y-%m-%d_%H-%M-%S"), "latest"]:
payload = {
"secret": value,
"expires": datetime.datetime(3000, 1, 1, 0, 0, 0)
}
secrets.set("project/servo/windows-codesign-cert/" + name, payload)
print("https://community-tc.services.mozilla.com/secrets/"
"project%2Fservo%2Fwindows-codesign-cert%2Flatest")
def load_from_secret(self):
"""
Load configuration from a Taskcluster secret
"""
assert self.TASKCLUSTER_CONFIG_SECRET, "Missing TASKCLUSTER_CONFIG_SECRET"
logger.info(
f"Loading configuration from secret {self.TASKCLUSTER_CONFIG_SECRET}"
)
return load_secrets(
secrets_service=taskcluster.Secrets(get_taskcluster_options()),
secret_name=self.TASKCLUSTER_CONFIG_SECRET,
)
def main(client_id):
assert client_id.startswith("project/servo/")
print("Client ID: `%s`" % client_id)
print("Creating a new access token will invalidate the current one.")
if input("Continue? [y/n] ") != "y":
return 1
options = taskcluster.optionsFromEnvironment()
result = taskcluster.Auth(options).resetAccessToken(client_id)
key = "project/servo/tc-client/" + client_id[len("project/servo/"):]
secret = {"client_id": client_id, "access_token": result["accessToken"]}
payload = {"secret": secret, "expires": result["expires"]}
taskcluster.Secrets(options).set(key, payload)
def load_secrets(self, tc_options, secrets_path):
"""
Load Taskcluster secrets
"""
# Check mandatory keys in secrets
secrets = taskcluster.Secrets(tc_options).get(secrets_path)
secrets = secrets['secret']
required = ('BUGZILLA_URL', 'BUGZILLA_TOKEN', 'API_URL')
for req in required:
if req not in secrets:
raise Exception(
'Missing value {} in Taskcluster secret value {}'.format(
req, secrets_path)) # noqa
return secrets
def get_secret(name):
secret = None
if "TASK_ID" in os.environ:
secrets_url = (
"http://taskcluster/secrets/v1/secret/project/updatebot/" +
("3" if OPERATING_MODE == "prod" else "2") + "/" + name)
res = requests.get(secrets_url)
res.raise_for_status()
secret = res.json()
else:
secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" +
name)
secret = secret["secret"] if "secret" in secret else None
secret = secret["value"] if "value" in secret else None
return secret
def inject_secrets(config):
"""
INJECT THE SECRETS INTO THE CONFIGURATION
:param config: CONFIG DATA
************************************************************************
** ENSURE YOU HAVE AN ENVIRONMENT VARIABLE SET:
** TASKCLUSTER_ROOT_URL = https://community-tc.services.mozilla.com
************************************************************************
"""
with Timer("get secrets"):
options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(options)
acc = Data()
for s in listwrap(SECRET_NAMES):
acc[s] = secrets.get(concat_field(SECRET_PREFIX, s))['secret']
set_default(config, acc)
def inject_secrets(config):
"""
INJECT THE SECRETS INTO THE CONFIGURATION
:param config: CONFIG DATA
************************************************************************
** ENSURE YOU HAVE AN ENVIRONMENT VARIABLE SET:
** TASKCLUSTER_ROOT_URL = https://community-tc.services.mozilla.com
************************************************************************
"""
with Timer("get secrets"):
secrets = taskcluster.Secrets(config.taskcluster)
acc = Data()
for s in listwrap(SECRET_NAMES):
secret_name = concat_field(SECRET_PREFIX, s)
Log.note("get secret named {{name|quote}}", name=secret_name)
acc[s] = secrets.get(secret_name)["secret"]
set_default(config, acc)
def get_secret(secret_id: str) -> Any:
"""Return the secret value"""
env_variable_name = f"BUGBUG_{secret_id}"
# Try in the environment first
secret_from_env = os.environ.get(env_variable_name)
if secret_from_env:
return secret_from_env
# If not in env, try with TC if we have the secret id
tc_secret_id = os.environ.get("TC_SECRET_ID")
if tc_secret_id:
secrets = taskcluster.Secrets(get_taskcluster_options())
secret_bucket = secrets.get(tc_secret_id)
return secret_bucket["secret"][secret_id]
else:
raise ValueError("Failed to find secret {}".format(secret_id))
def fetch_secret_from_taskcluster(name):
secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
return secrets.get(name)
import json
import os
import re
import string
import taskcluster
import urllib.request
import yaml
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import ComputeManagementClient
from cib import updateWorkerPool
from datetime import datetime
taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
taskclusterSecretsClient = taskcluster.Secrets(taskclusterOptions)
secret = taskclusterSecretsClient.get(
'project/relops/image-builder/dev')['secret']
currentEnvironment = 'staging' if 'stage.taskcluster.nonprod' in os.environ[
'TASKCLUSTER_ROOT_URL'] else 'production'
taskclusterWorkerManagerClient = taskcluster.WorkerManager(taskclusterOptions)
azureComputeManagementClient = ComputeManagementClient(
ServicePrincipalCredentials(client_id=secret['azure']['id'],
secret=secret['azure']['key'],
tenant=secret['azure']['account']),
secret['azure']['subscription'])
def getLatestImage(resourceGroup, key):
pattern = re.compile('^{}-{}-([a-z0-9]{{7}})-([a-z0-9]{{7}})$'.format(
import json
import os
import re
import string
import taskcluster
import urllib.request
import yaml
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import ComputeManagementClient
from cib import updateWorkerPool
from datetime import datetime
taskclusterProductionOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
taskclusterProductionSecretsClient = taskcluster.Secrets(
taskclusterProductionOptions)
secret = taskclusterProductionSecretsClient.get(
'project/relops/image-builder/dev')['secret']
taskclusterStagingOptions = {
'rootUrl': 'https://stage.taskcluster.nonprod.cloudops.mozgcp.net',
'credentials': {
'clientId':
'project/relops/image-builder/dev',
'accessToken':
secret['accessToken']['staging']['relops']['image-builder']['dev']
}
}
taskclusterStagingWorkerManagerClient = taskcluster.WorkerManager(
taskclusterStagingOptions)
taskclusterProductionWorkerManagerClient = taskcluster.WorkerManager(
taskclusterProductionOptions)
def load_secret(self, name):
secrets = taskcluster.Secrets(self.get_taskcluster_options())
logging.info('Loading Taskcluster secret {}'.format(name))
payload = secrets.get(name)
assert 'secret' in payload, 'Missing secret value'
self.config = payload['secret']
def load_secret(self, name):
secrets = taskcluster.Secrets(self.get_taskcluster_options())
logging.info("Loading Taskcluster secret {}".format(name))
payload = secrets.get(name)
assert "secret" in payload, "Missing secret value"
self.config = payload["secret"]
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
This script talks to the taskcluster secrets service to obtain the
Google Firebase service account token and write it to the .firebase_token
file in the root directory.
"""
import os
import taskcluster
import json
# Get JSON data from taskcluster secrets service
secrets = taskcluster.Secrets({'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']})
data = secrets.get('project/mobile/firefox-tv/tokens')
with open(
os.path.join(os.path.dirname(__file__), '../../.firebase_token.json'),
'w') as file:
json.dump(data['secret']['firebaseToken'], file)
print("Imported google firebase token from secrets service.")
if response.status_code != 201:
print('Status:', response.status_code, 'Headers:', response.headers,
'Error Response:', response.json())
exit(1)
# Print Response Details
print 'Response Status Code:', response.status_code
print ''
print('Reponse Payload:')
print json.dumps(response.json(), indent=4)
def uploadNightlyFenixApk(apk_url, key):
apk_data = urllib2.urlopen(apk_url).read()
with open('./fenix_example_nd.apk', 'wb') as f:
f.write(apk_data)
uploadApk({'apk': open('fenix_example_nd.apk')}, key)
# Get JSON data from taskcluster secrets service
secrets = taskcluster.Secrets({
'rootUrl':
os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
})
data = secrets.get('project/mobile/fenix/nimbledroid')
# upload the nightly build to Nimbledroid
apk_url = sys.argv[1]
uploadNightlyFenixApk(apk_url, data['secret']['api_key'])
def new_ami(tmp, tc_options):
secrets = taskcluster.Secrets(tc_options)
def set_secret(name, value):
payload = {"secret": value, "expires": datetime.datetime(3000, 1, 1, 0, 0, 0)}
secrets.set("project/servo/windows-administrator/" + name, payload)
# Ensure we have appropriate credentials for writing secrets now
# rather than at the end of the lengthy bootstrap process.
set_secret("dummy", {})
result = ec2(
"describe-images", "--owners", "amazon",
"--filters", "Name=platform,Values=windows", "Name=name,Values=" + BASE_AMI_PATTERN,
)
# Find the most recent
base_ami = max(result["Images"], key=lambda x: x["CreationDate"])
log("Found base image:", base_ami["Name"])
key_name = "ami-bootstrap"
ec2("delete-key-pair", "--key-name", key_name)
result = ec2("create-key-pair", "--key-name", key_name)
key_filename = os.path.join(tmp, "key")
with open(key_filename, "w") as f:
f.write(result["KeyMaterial"])
ps1 = os.path.join(os.path.dirname(__file__), "..", "config", "windows-first-boot.ps1")
with open(ps1) as f:
user_data = "<powershell>\n%s\n</powershell>" % f.read()
result = ec2(
"run-instances", "--image-id", base_ami["ImageId"],
"--key-name", key_name,
"--user-data", user_data,
"--instance-type", "c4.xlarge",
"--block-device-mappings",
"DeviceName=/dev/sda1,Ebs={VolumeSize=75,DeleteOnTermination=true,VolumeType=gp2}",
"--instance-initiated-shutdown-behavior", "stop"
)
assert len(result["Instances"]) == 1
instance_id = result["Instances"][0]["InstanceId"]
ec2_wait("password-data-available", "--instance-id", instance_id)
password_result = ec2("get-password-data", "--instance-id", instance_id,
"--priv-launch-key", key_filename)
log("Waiting for the instance to finish `first-boot.ps1` and shut down…")
ec2_wait("instance-stopped", "--instance-id", instance_id)
now = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H.%M.%S")
image_id = ec2("create-image", "--instance-id", instance_id,
"--name", "win2016 bootstrap " + now)["ImageId"]
set_secret(image_id, {"Administrator": password_result["PasswordData"]})
log("Password available at https://community-tc.services.mozilla.com/secrets/"
"project%2Fservo%2Fwindows-administrator%2F" + image_id)
log("Creating image with ID %s …" % image_id)
ec2_wait("image-available", "--image-ids", image_id)
ec2("modify-image-attribute", "--image-id", image_id,
"--launch-permission", "Add=[{UserId=%s}]" % TASKCLUSTER_AWS_USER_ID)
log("Image available.")
ec2("terminate-instances", "--instance-ids", instance_id)
return image_id
import requests
import json
url = "https://nimbledroid.com/api/v2/apks"
def uploadApk(apk,key):
headers = {"Accept":"*/*"}
payload = {'auto_scenarios':'false'}
response = requests.post(url, auth=(key, ''), headers=headers, files=apk, data=payload)
if response.status_code != 201:
print('Status:', response.status_code, 'Headers:', response.headers, 'Error Response:',response.json())
exit(1)
# Print Response Details
print 'Response Status Code:', response.status_code
print ''
print('Reponse Payload:')
print json.dumps(response.json(), indent=4)
# Get JSON data from taskcluster secrets service
secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
data = secrets.get('project/focus/nimbledroid')
klar_file = {'apk': open('app/build/outputs/apk/klarArm/nightly/app-klar-arm-nightly-unsigned.apk')}
focus_file = {'apk': open('app/build/outputs/apk/focusArm/release/app-focus-arm-release-unsigned.apk')}
uploadApk(klar_file, data['secret']['api_key'])
uploadApk(focus_file, data['secret']['api_key'])
def fetch_publish_secrets(secret_name):
"""Fetch and return secrets from taskcluster's secret service"""
secrets = taskcluster.Secrets({'baseUrl': TASKCLUSTER_BASE_URL})
return secrets.get(secret_name)
import os
import slugid
import taskcluster
import yaml
from cib import createTask, diskImageManifestHasChanged, machineImageManifestHasChanged, machineImageExists
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import ComputeManagementClient
taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
auth = taskcluster.Auth(taskclusterOptions)
queue = taskcluster.Queue(taskclusterOptions)
index = taskcluster.Index(taskclusterOptions)
secrets = taskcluster.Secrets(taskclusterOptions)
secret = secrets.get('project/relops/image-builder/dev')['secret']
platformClient = {
'azure':
ComputeManagementClient(
ServicePrincipalCredentials(client_id=secret['azure']['id'],
secret=secret['azure']['key'],
tenant=secret['azure']['account']),
secret['azure']['subscription'])
}
commitSha = os.getenv('GITHUB_HEAD_SHA')
taskGroupId = os.getenv('TASK_ID')
print('debug: auth.currentScopes')
print(auth.currentScopes())
azurePurgeTaskId = slugid.nice()
if resource.__class__.__name__ == 'ResourceGroup':
return (resource.name.startswith('rg-') and '-us-' in resource.name
and (resource.name.endswith('-gecko-1')
or resource.name.endswith('-gecko-3')
or resource.name.endswith('-gecko-t')
or resource.name.endswith('-mpd001-1')
or resource.name.endswith('-mpd001-3')
or resource.name.endswith('-relops')))
else:
print('no filter mechanism identified for {}'.format(
resource.__class__.__name__))
return False
if 'TASKCLUSTER_PROXY_URL' in os.environ:
secretsClient = taskcluster.Secrets(
{'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']})
secret = secretsClient.get(
'project/relops/image-builder/dev')['secret']['azure']
print('secrets fetched using taskcluster proxy')
elif 'TASKCLUSTER_ROOT_URL' in os.environ and 'TASKCLUSTER_CLIENT_ID' in os.environ and 'TASKCLUSTER_ACCESS_TOKEN' in os.environ:
secretsClient = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
secret = secretsClient.get(
'project/relops/image-builder/dev')['secret']['azure']
print('secrets fetched using taskcluster environment credentials')
elif os.path.isfile('{}/.cloud-image-builder-secrets.yml'.format(
os.environ['HOME'])):
secret = yaml.safe_load(
open('{}/.cloud-image-builder-secrets.yml'.format(os.environ['HOME']),
'r'))['azure']
print('secrets obtained from local filesystem')
else:
