def fetch_secret_from_taskcluster(name): try: secrets = taskcluster.Secrets({ # BaseUrl is still needed for tasks that haven't migrated to taskgraph yet. 'baseUrl': 'http://taskcluster/secrets/v1', }) except taskcluster.exceptions.TaskclusterFailure: # taskcluster library >=5 errors out when `baseUrl` is used secrets = taskcluster.Secrets({ 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), }) return secrets.get(name)
def get_secrets_service(self): """ Configured Secrets Service """ return taskcluster.Secrets( self.build_options('secrets/v1') )
def ext_pillar(minion_id, _pillar, *_args): if not CACHE: secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment()) CACHE["credentials"] = secrets.get( "project/servo/tc-client/worker/macos/1")["secret"] url = "https://raw.githubusercontent.com/servo/saltfs/master/admin/files/ssh/%s.pub" CACHE["ssh_keys"] = [ urllib.urlopen(url % name).read() for name in [ "jdm", "manishearth", "simonsapin", ] ] CACHE["workers"] = { worker: (pool_name, config) for pool_name, pool in read_yaml("worker-pools.yml").items() if pool["kind"] == "static" for worker, config in pool["workers"].items() } pool_name, config = CACHE["workers"][minion_id] disabled = {"disabled": True, None: False}[config] return dict(disabled=disabled, worker_type=pool_name, **CACHE)
def main(name, argv): secret_path = '' output_file = '' data_name = '' decode = False; try: opts, args = getopt.getopt(argv,"hs:o:n:d") except getopt.GetoptError: print name + '-s <secret> -o <filename> -d' sys.exit(2) for opt, arg in opts: if opt == '-h': print name + '-s <secret> -o <filename> -n <data name> -d (decode base64)' sys.exit() elif opt in ("-s"): secret_path = arg elif opt in ("-o"): output_file = arg elif opt in ("-n"): data_name = arg elif opt in ("-d"): decode = True if data_name == '': data_name = os.path.basename(secret_path) secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'}) data = secrets.get(secret_path) data = data['secret'][data_name] if decode: data = base64.b64decode(data) with open(output_file, 'w') as output: output.write(data)
def get_secret(secret): # use proxy if configured, otherwise local credentials from env vars if "TASKCLUSTER_PROXY_URL" in os.environ: secrets_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]} else: secrets_options = taskcluster.optionsFromEnvironment() secrets = taskcluster.Secrets(secrets_options) return secrets.get(secret)["secret"]
def get_secret(self, name): """ Helper to read a Taskcluster secret """ secrets = taskcluster.Secrets(self.taskcluster_options()) secret = secrets.get(name) if not secret: raise Exception('Missing TC secret {}'.format(name)) return secret['secret']
def main(): # Use proxy if configured (within a task), otherwise, use local credentials from env vars if 'TASKCLUSTER_PROXY_URL' in os.environ: options = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']} else: options = taskcluster.optionsFromEnvironment() secrets = taskcluster.Secrets(options) secret = secrets.get("project/cia/garbage/foo") print(secret["secret"])
def load_secrets(self, secrets_path, client_id=None, access_token=None): """ Load Taskcluster secrets """ if client_id and access_token: # Use provided credentials tc = taskcluster.Secrets({ 'credentials': { 'clientId': client_id, 'accessToken': access_token, } }) else: # Get taskcluster proxy host # as /etc/hosts is not used in the Nix image (?) hosts = read_hosts() if 'taskcluster' not in hosts: raise Exception('Missing taskcluster in /etc/hosts') # Load secrets from TC task context # with taskclusterProxy base_url = 'http://{}/secrets/v1'.format(hosts['taskcluster']) logger.info('Taskcluster Proxy enabled', url=base_url) tc = taskcluster.Secrets({'baseUrl': base_url}) # Check mandatory keys in secrets secrets = tc.get(secrets_path) secrets = secrets['secret'] required = ('BUGZILLA_URL', 'BUGZILLA_TOKEN', 'API_URL') for req in required: if req not in secrets: raise Exception( 'Missing value {} in Taskcluster secret value {}'.format( req, secrets_path)) # noqa # Add credentials too if 'TASKCLUSTER_CLIENT_ID' not in secrets: secrets['TASKCLUSTER_CLIENT_ID'] = client_id if 'TASKCLUSTER_ACCESS_TOKEN' not in secrets: secrets['TASKCLUSTER_ACCESS_TOKEN'] = access_token return secrets
def main(): with open("/etc/ssl/openssl.cnf") as f: config = f.read() config += ("[v3_req]\n" "basicConstraints = critical,CA:FALSE\n" "keyUsage = digitalSignature\n" "extendedKeyUsage = critical,codeSigning") with tempfile.TemporaryDirectory() as tmp: os.chdir(tmp) with open("config", "w") as f: f.write(config) now = datetime.datetime.now() run(""" openssl req -x509 -sha256 -nodes -days 730 -newkey rsa:4096 -subj /CN=Allizom -extensions v3_req -batch -config config -keyout key.pem -out cert.pem """) run(""" openssl pkcs12 -export -passout pass: -inkey key.pem -in cert.pem -out servo.pfx """) with open("servo.pfx", "rb") as f: pfx = f.read() value = {"pfx": {"base64": base64.b64encode(pfx)}, "created": now} tc_options = taskcluster.optionsFromEnvironment() secrets = taskcluster.Secrets(tc_options) for name in [now.strftime("%Y-%m-%d_%H-%M-%S"), "latest"]: payload = { "secret": value, "expires": datetime.datetime(3000, 1, 1, 0, 0, 0) } secrets.set("project/servo/windows-codesign-cert/" + name, payload) print("https://community-tc.services.mozilla.com/secrets/" "project%2Fservo%2Fwindows-codesign-cert%2Flatest")
def load_from_secret(self): """ Load configuration from a Taskcluster secret """ assert self.TASKCLUSTER_CONFIG_SECRET, "Missing TASKCLUSTER_CONFIG_SECRET" logger.info( f"Loading configuration from secret {self.TASKCLUSTER_CONFIG_SECRET}" ) return load_secrets( secrets_service=taskcluster.Secrets(get_taskcluster_options()), secret_name=self.TASKCLUSTER_CONFIG_SECRET, )
def main(client_id): assert client_id.startswith("project/servo/") print("Client ID: `%s`" % client_id) print("Creating a new access token will invalidate the current one.") if input("Continue? [y/n] ") != "y": return 1 options = taskcluster.optionsFromEnvironment() result = taskcluster.Auth(options).resetAccessToken(client_id) key = "project/servo/tc-client/" + client_id[len("project/servo/"):] secret = {"client_id": client_id, "access_token": result["accessToken"]} payload = {"secret": secret, "expires": result["expires"]} taskcluster.Secrets(options).set(key, payload)
def load_secrets(self, tc_options, secrets_path): """ Load Taskcluster secrets """ # Check mandatory keys in secrets secrets = taskcluster.Secrets(tc_options).get(secrets_path) secrets = secrets['secret'] required = ('BUGZILLA_URL', 'BUGZILLA_TOKEN', 'API_URL') for req in required: if req not in secrets: raise Exception( 'Missing value {} in Taskcluster secret value {}'.format( req, secrets_path)) # noqa return secrets
def get_secret(name): secret = None if "TASK_ID" in os.environ: secrets_url = ( "http://taskcluster/secrets/v1/secret/project/updatebot/" + ("3" if OPERATING_MODE == "prod" else "2") + "/" + name) res = requests.get(secrets_url) res.raise_for_status() secret = res.json() else: secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment()) secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name) secret = secret["secret"] if "secret" in secret else None secret = secret["value"] if "value" in secret else None return secret
def inject_secrets(config): """ INJECT THE SECRETS INTO THE CONFIGURATION :param config: CONFIG DATA ************************************************************************ ** ENSURE YOU HAVE AN ENVIRONMENT VARIABLE SET: ** TASKCLUSTER_ROOT_URL = https://community-tc.services.mozilla.com ************************************************************************ """ with Timer("get secrets"): options = taskcluster.optionsFromEnvironment() secrets = taskcluster.Secrets(options) acc = Data() for s in listwrap(SECRET_NAMES): acc[s] = secrets.get(concat_field(SECRET_PREFIX, s))['secret'] set_default(config, acc)
def inject_secrets(config): """ INJECT THE SECRETS INTO THE CONFIGURATION :param config: CONFIG DATA ************************************************************************ ** ENSURE YOU HAVE AN ENVIRONMENT VARIABLE SET: ** TASKCLUSTER_ROOT_URL = https://community-tc.services.mozilla.com ************************************************************************ """ with Timer("get secrets"): secrets = taskcluster.Secrets(config.taskcluster) acc = Data() for s in listwrap(SECRET_NAMES): secret_name = concat_field(SECRET_PREFIX, s) Log.note("get secret named {{name|quote}}", name=secret_name) acc[s] = secrets.get(secret_name)["secret"] set_default(config, acc)
def get_secret(secret_id: str) -> Any: """Return the secret value""" env_variable_name = f"BUGBUG_{secret_id}" # Try in the environment first secret_from_env = os.environ.get(env_variable_name) if secret_from_env: return secret_from_env # If not in env, try with TC if we have the secret id tc_secret_id = os.environ.get("TC_SECRET_ID") if tc_secret_id: secrets = taskcluster.Secrets(get_taskcluster_options()) secret_bucket = secrets.get(tc_secret_id) return secret_bucket["secret"][secret_id] else: raise ValueError("Failed to find secret {}".format(secret_id))
def fetch_secret_from_taskcluster(name): secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'}) return secrets.get(name)
import json import os import re import string import taskcluster import urllib.request import yaml from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.compute import ComputeManagementClient from cib import updateWorkerPool from datetime import datetime taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']} taskclusterSecretsClient = taskcluster.Secrets(taskclusterOptions) secret = taskclusterSecretsClient.get( 'project/relops/image-builder/dev')['secret'] currentEnvironment = 'staging' if 'stage.taskcluster.nonprod' in os.environ[ 'TASKCLUSTER_ROOT_URL'] else 'production' taskclusterWorkerManagerClient = taskcluster.WorkerManager(taskclusterOptions) azureComputeManagementClient = ComputeManagementClient( ServicePrincipalCredentials(client_id=secret['azure']['id'], secret=secret['azure']['key'], tenant=secret['azure']['account']), secret['azure']['subscription']) def getLatestImage(resourceGroup, key): pattern = re.compile('^{}-{}-([a-z0-9]{{7}})-([a-z0-9]{{7}})$'.format(
import json import os import re import string import taskcluster import urllib.request import yaml from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.compute import ComputeManagementClient from cib import updateWorkerPool from datetime import datetime taskclusterProductionOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']} taskclusterProductionSecretsClient = taskcluster.Secrets( taskclusterProductionOptions) secret = taskclusterProductionSecretsClient.get( 'project/relops/image-builder/dev')['secret'] taskclusterStagingOptions = { 'rootUrl': 'https://stage.taskcluster.nonprod.cloudops.mozgcp.net', 'credentials': { 'clientId': 'project/relops/image-builder/dev', 'accessToken': secret['accessToken']['staging']['relops']['image-builder']['dev'] } } taskclusterStagingWorkerManagerClient = taskcluster.WorkerManager( taskclusterStagingOptions) taskclusterProductionWorkerManagerClient = taskcluster.WorkerManager( taskclusterProductionOptions)
def load_secret(self, name): secrets = taskcluster.Secrets(self.get_taskcluster_options()) logging.info('Loading Taskcluster secret {}'.format(name)) payload = secrets.get(name) assert 'secret' in payload, 'Missing secret value' self.config = payload['secret']
def load_secret(self, name): secrets = taskcluster.Secrets(self.get_taskcluster_options()) logging.info("Loading Taskcluster secret {}".format(name)) payload = secrets.get(name) assert "secret" in payload, "Missing secret value" self.config = payload["secret"]
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ This script talks to the taskcluster secrets service to obtain the Google Firebase service account token and write it to the .firebase_token file in the root directory. """ import os import taskcluster import json # Get JSON data from taskcluster secrets service secrets = taskcluster.Secrets({'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}) data = secrets.get('project/mobile/firefox-tv/tokens') with open( os.path.join(os.path.dirname(__file__), '../../.firebase_token.json'), 'w') as file: json.dump(data['secret']['firebaseToken'], file) print("Imported google firebase token from secrets service.")
if response.status_code != 201: print('Status:', response.status_code, 'Headers:', response.headers, 'Error Response:', response.json()) exit(1) # Print Response Details print 'Response Status Code:', response.status_code print '' print('Reponse Payload:') print json.dumps(response.json(), indent=4) def uploadNightlyFenixApk(apk_url, key): apk_data = urllib2.urlopen(apk_url).read() with open('./fenix_example_nd.apk', 'wb') as f: f.write(apk_data) uploadApk({'apk': open('fenix_example_nd.apk')}, key) # Get JSON data from taskcluster secrets service secrets = taskcluster.Secrets({ 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), }) data = secrets.get('project/mobile/fenix/nimbledroid') # upload the nightly build to Nimbledroid apk_url = sys.argv[1] uploadNightlyFenixApk(apk_url, data['secret']['api_key'])
def new_ami(tmp, tc_options): secrets = taskcluster.Secrets(tc_options) def set_secret(name, value): payload = {"secret": value, "expires": datetime.datetime(3000, 1, 1, 0, 0, 0)} secrets.set("project/servo/windows-administrator/" + name, payload) # Ensure we have appropriate credentials for writing secrets now # rather than at the end of the lengthy bootstrap process. set_secret("dummy", {}) result = ec2( "describe-images", "--owners", "amazon", "--filters", "Name=platform,Values=windows", "Name=name,Values=" + BASE_AMI_PATTERN, ) # Find the most recent base_ami = max(result["Images"], key=lambda x: x["CreationDate"]) log("Found base image:", base_ami["Name"]) key_name = "ami-bootstrap" ec2("delete-key-pair", "--key-name", key_name) result = ec2("create-key-pair", "--key-name", key_name) key_filename = os.path.join(tmp, "key") with open(key_filename, "w") as f: f.write(result["KeyMaterial"]) ps1 = os.path.join(os.path.dirname(__file__), "..", "config", "windows-first-boot.ps1") with open(ps1) as f: user_data = "<powershell>\n%s\n</powershell>" % f.read() result = ec2( "run-instances", "--image-id", base_ami["ImageId"], "--key-name", key_name, "--user-data", user_data, "--instance-type", "c4.xlarge", "--block-device-mappings", "DeviceName=/dev/sda1,Ebs={VolumeSize=75,DeleteOnTermination=true,VolumeType=gp2}", "--instance-initiated-shutdown-behavior", "stop" ) assert len(result["Instances"]) == 1 instance_id = result["Instances"][0]["InstanceId"] ec2_wait("password-data-available", "--instance-id", instance_id) password_result = ec2("get-password-data", "--instance-id", instance_id, "--priv-launch-key", key_filename) log("Waiting for the instance to finish `first-boot.ps1` and shut down…") ec2_wait("instance-stopped", "--instance-id", instance_id) now = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H.%M.%S") image_id = ec2("create-image", "--instance-id", instance_id, "--name", "win2016 bootstrap " + now)["ImageId"] set_secret(image_id, {"Administrator": password_result["PasswordData"]}) log("Password available at https://community-tc.services.mozilla.com/secrets/" "project%2Fservo%2Fwindows-administrator%2F" + image_id) log("Creating image with ID %s …" % image_id) ec2_wait("image-available", "--image-ids", image_id) ec2("modify-image-attribute", "--image-id", image_id, "--launch-permission", "Add=[{UserId=%s}]" % TASKCLUSTER_AWS_USER_ID) log("Image available.") ec2("terminate-instances", "--instance-ids", instance_id) return image_id
import requests import json url = "https://nimbledroid.com/api/v2/apks" def uploadApk(apk,key): headers = {"Accept":"*/*"} payload = {'auto_scenarios':'false'} response = requests.post(url, auth=(key, ''), headers=headers, files=apk, data=payload) if response.status_code != 201: print('Status:', response.status_code, 'Headers:', response.headers, 'Error Response:',response.json()) exit(1) # Print Response Details print 'Response Status Code:', response.status_code print '' print('Reponse Payload:') print json.dumps(response.json(), indent=4) # Get JSON data from taskcluster secrets service secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'}) data = secrets.get('project/focus/nimbledroid') klar_file = {'apk': open('app/build/outputs/apk/klarArm/nightly/app-klar-arm-nightly-unsigned.apk')} focus_file = {'apk': open('app/build/outputs/apk/focusArm/release/app-focus-arm-release-unsigned.apk')} uploadApk(klar_file, data['secret']['api_key']) uploadApk(focus_file, data['secret']['api_key'])
def fetch_publish_secrets(secret_name): """Fetch and return secrets from taskcluster's secret service""" secrets = taskcluster.Secrets({'baseUrl': TASKCLUSTER_BASE_URL}) return secrets.get(secret_name)
import os import slugid import taskcluster import yaml from cib import createTask, diskImageManifestHasChanged, machineImageManifestHasChanged, machineImageExists from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.compute import ComputeManagementClient taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']} auth = taskcluster.Auth(taskclusterOptions) queue = taskcluster.Queue(taskclusterOptions) index = taskcluster.Index(taskclusterOptions) secrets = taskcluster.Secrets(taskclusterOptions) secret = secrets.get('project/relops/image-builder/dev')['secret'] platformClient = { 'azure': ComputeManagementClient( ServicePrincipalCredentials(client_id=secret['azure']['id'], secret=secret['azure']['key'], tenant=secret['azure']['account']), secret['azure']['subscription']) } commitSha = os.getenv('GITHUB_HEAD_SHA') taskGroupId = os.getenv('TASK_ID') print('debug: auth.currentScopes') print(auth.currentScopes()) azurePurgeTaskId = slugid.nice()
if resource.__class__.__name__ == 'ResourceGroup': return (resource.name.startswith('rg-') and '-us-' in resource.name and (resource.name.endswith('-gecko-1') or resource.name.endswith('-gecko-3') or resource.name.endswith('-gecko-t') or resource.name.endswith('-mpd001-1') or resource.name.endswith('-mpd001-3') or resource.name.endswith('-relops'))) else: print('no filter mechanism identified for {}'.format( resource.__class__.__name__)) return False if 'TASKCLUSTER_PROXY_URL' in os.environ: secretsClient = taskcluster.Secrets( {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}) secret = secretsClient.get( 'project/relops/image-builder/dev')['secret']['azure'] print('secrets fetched using taskcluster proxy') elif 'TASKCLUSTER_ROOT_URL' in os.environ and 'TASKCLUSTER_CLIENT_ID' in os.environ and 'TASKCLUSTER_ACCESS_TOKEN' in os.environ: secretsClient = taskcluster.Secrets(taskcluster.optionsFromEnvironment()) secret = secretsClient.get( 'project/relops/image-builder/dev')['secret']['azure'] print('secrets fetched using taskcluster environment credentials') elif os.path.isfile('{}/.cloud-image-builder-secrets.yml'.format( os.environ['HOME'])): secret = yaml.safe_load( open('{}/.cloud-image-builder-secrets.yml'.format(os.environ['HOME']), 'r'))['azure'] print('secrets obtained from local filesystem') else: