def _recreate_task(task_id): one_year = 365 queue = taskcluster_client.Queue() task = queue.task(task_id) LOG.debug("Original task: (Limit 1024 char)") LOG.debug(str(json.dumps(task))[:1024]) # Start updating the task task["taskId"] = taskcluster_client.slugId() artifacts = task["payload"].get("artifacts", {}) for artifact, definition in artifacts.iteritems(): definition["expires"] = taskcluster_client.fromNow("%s days" % one_year) # https://bugzilla.mozilla.org/show_bug.cgi?id=1190660 # TC workers create public logs which are 365 days; if the task expiration # date is the same or less than that we won't have logs for the task task["expires"] = taskcluster_client.fromNow("%s days" % (one_year + 1)) now = datetime.datetime.utcnow() tomorrow = now + datetime.timedelta(hours=24) task["created"] = taskcluster_client.stringDate(now) task["deadline"] = taskcluster_client.stringDate(tomorrow) LOG.debug("Contents of new task: (Limit 1024 char)") LOG.debug(str(task)[:1024]) return task
def refresh_timestamps(task): ''' It refreshes the timestamps of the task. ''' # XXX split this function LOG.debug("Updating timestamps of task.") LOG.debug("Original task: (Limit 1024 char)") LOG.debug(str(json.dumps(task))[:1024]) artifacts = task['payload'].get('artifacts', {}) for artifact, definition in artifacts.iteritems(): definition['expires'] = taskcluster_client.fromNow('%s days' % 365) # https://bugzilla.mozilla.org/show_bug.cgi?id=1190660 # TC workers create public logs which are 365 days; if the task expiration # date is the same or less than that we won't have logs for the task task['expires'] = taskcluster_client.fromNow('%s days' % (365 + 1)) now = datetime.datetime.utcnow() tomorrow = now + datetime.timedelta(hours=24) task['created'] = taskcluster_client.stringDate(now) task['deadline'] = taskcluster_client.stringDate(tomorrow) LOG.debug("Contents of new task: (Limit 1024 char)") LOG.debug(str(task)[:1024]) return task
def generate_task_payload(self, flavor, properties): """Generate the task payload data for the given type of test and properties. :param flavor: Type of test to run (functional or update). :param properties: Task properties for template rendering """ template_file = os.path.join(os.path.dirname(__file__), 'tasks', '{}.yml'.format(flavor)) if not os.path.isfile(template_file): raise errors.NotSupportedException('Test type "{}" not supported.'.format(flavor)) with open(template_file) as f: template = jinja2.Template(f.read(), undefined=jinja2.StrictUndefined) template_vars = copy.deepcopy(properties) template_vars.update({ 'stableSlugId': taskcluster.stableSlugId(), 'now': taskcluster.stringDate(datetime.datetime.utcnow()), 'fromNow': taskcluster.fromNow, 'docker_task_id': self.get_docker_task_id(properties), }) rendered = template.render(**template_vars) return yaml.safe_load(rendered)
def build_task(self, name, description, command, artifacts={}, scopes=[], features={}): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": 'gecko-focus', "taskGroupId": self.task_id, "schedulerId": self.scheduler_id, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "deadline": taskcluster.stringDate(deadline), "dependencies": [self.task_id], "routes": [], "scopes": scopes, "requires": "all-completed", "payload": { "features": features, "maxRunTime": 7200, "image": "mozillamobile/android-components:1.15", "command": ["/bin/bash", "--login", "-cx", command], "artifacts": artifacts, "deadline": taskcluster.stringDate(deadline) }, "provisionerId": "aws-provisioner-v1", "metadata": { "name": name, "description": description, "owner": self.owner, "source": self.source } }
def _craft_artifacts_from_variant(variant): return { apk.taskcluster_path: { 'type': 'file', 'path': apk.absolute_path(variant.build_type), 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), } for apk in variant.apks }
def _craft_artifacts_from_variant(variant): return { 'public/target.apk': { 'type': 'file', 'path': _craft_apk_full_path_from_variant(variant), 'expires': taskcluster.stringDate( taskcluster.fromNow(lib.tasks.DEFAULT_EXPIRES_IN)), } }
def create_task(name, description, command): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": 'github-worker', "taskGroupId": TASK_ID, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "schedulerId": "taskcluster-github", "deadline": taskcluster.stringDate(deadline), "dependencies": [ TASK_ID ], "routes": [], "scopes": [], "requires": "all-completed", "payload": { "features": {}, "maxRunTime": 7200, "image": "mozillamobile/android-components:1.4", "command": [ "/bin/bash", "--login", "-cx", "export TERM=dumb && git fetch %s %s && git config advice.detachedHead false && git checkout %s && ./gradlew --no-daemon clean %s" % (REPO_URL, BRANCH, COMMIT, command) ], "artifacts": {}, "deadline": taskcluster.stringDate(deadline) }, "provisionerId": "aws-provisioner-v1", "metadata": { "name": name, "description": description, "owner": "*****@*****.**", "source": "https://github.com/mozilla-mobile/android-components" } }
def create_task(name, description, command): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": 'github-worker', "taskGroupId": TASK_ID, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "schedulerId": "taskcluster-github", "deadline": taskcluster.stringDate(deadline), "dependencies": [ TASK_ID ], "routes": [], "scopes": [], "requires": "all-completed", "payload": { "features": {}, "maxRunTime": 7200, "image": "mozillamobile/mentat:1.1", "command": [ "/bin/bash", "--login", "-cx", "export TERM=dumb && git fetch %s %s && git config advice.detachedHead false && git checkout %s && cd sdks/android/Mentat && ./gradlew --no-daemon clean %s" % (REPO_URL, BRANCH, COMMIT, command) ], "artifacts": {}, "deadline": taskcluster.stringDate(deadline) }, "provisionerId": "aws-provisioner-v1", "metadata": { "name": name, "description": description, "owner": "*****@*****.**", "source": "https://github.com/mozilla/mentat" } }
def build_signing_task(self, build_task_id, name, description, apks=[], scopes=[], routes=[]): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": 'mobile-signing-v1', "taskGroupId": self.task_id, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "schedulerId": "taskcluster-github", "deadline": taskcluster.stringDate(deadline), "dependencies": [ self.task_id, build_task_id], "routes": routes, "scopes": scopes, "requires": "all-completed", "payload": { "maxRunTime": 3600, "upstreamArtifacts": [ { "paths": apks, "formats": [ "focus-jar" ], "taskId": build_task_id, "taskType": "build" } ] }, "provisionerId": "scriptworker-prov-v1", "metadata": { "name": name, "description": description, "owner": self.owner, "source": self.source } }
def craft_push_task(self, signing_task_id, name, description, is_staging, apks, scopes, commit): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": 'mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1', "taskGroupId": self.task_id, "schedulerId": self.scheduler_id, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "deadline": taskcluster.stringDate(deadline), "dependencies": [self.task_id, signing_task_id], "routes": [], "scopes": scopes, "requires": "all-completed", "payload": { "commit": commit, "google_play_track": 'nightly', "upstreamArtifacts": [{ "paths": apks, "taskId": signing_task_id, "taskType": "signing" }] }, "provisionerId": "scriptworker-prov-v1", "metadata": { "name": name, "description": description, "owner": self.owner, "source": self.source } }
def craft_build_task(self, module_name, gradle_tasks, build_docker_image_task_id, subtitle='', run_coverage=False, is_snapshot=False, artifact_info=None): artifacts = {} if artifact_info is None else { artifact_info['artifact']: { 'type': 'file', 'expires': taskcluster.stringDate( taskcluster.fromNow(DEFAULT_EXPIRES_IN)), 'path': artifact_info['path'] } } scopes = [ "secrets:get:project/mobile/android-components/public-tokens" ] if run_coverage else [] snapshot_flag = '-Psnapshot ' if is_snapshot else '' coverage_flag = '-Pcoverage ' if run_coverage else '' gradle_command = ('./gradlew --no-daemon clean ' + coverage_flag + snapshot_flag + gradle_tasks) post_gradle_command = 'automation/taskcluster/action/upload_coverage_report.sh' if run_coverage else '' command = ' && '.join(cmd for cmd in (gradle_command, post_gradle_command) if cmd) features = {} if artifact_info is not None: features['chainOfTrust'] = True elif any(scope.startswith('secrets:') for scope in scopes): features['taskclusterProxy'] = True return self._craft_build_ish_task( name='Android Components - Module {} {}'.format( module_name, subtitle), description='Execure Gradle tasks for module {}'.format( module_name), command=command, features=features, scopes=scopes, artifacts=artifacts, build_docker_image_task_id=build_docker_image_task_id, )
def craft_assemble_release_task(self, is_staging=False): artifacts = { _ARCH_APK_LOCATION_PATTERN.format(arch): { "type": 'file', "path": "/build/reference-browser/app/build/outputs/apk/geckoNightly{}/release/" "app-geckoNightly-{}-release-unsigned.apk".format( arch.capitalize(), arch), "expires": taskcluster.stringDate( taskcluster.fromNow(DEFAULT_EXPIRES_IN)), } for arch in _SUPPORTED_ARCHITECTURES } sentry_secret = '{}project/mobile/reference-browser/sentry'.format( 'garbage/staging/' if is_staging else '') pre_gradle_commands = ( 'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}' .format(sentry_secret, 'dsn', '.sentry_token'), ) gradle_commands = ( './gradlew --no-daemon -PcrashReportEnabled=true -Ptelemetry=true clean test assembleRelease', ) command = ' && '.join(cmd for commands in (pre_gradle_commands, gradle_commands) for cmd in commands if cmd) routes = [] if is_staging else [ "[email protected]" ] return self._craft_build_ish_task( name='Build task', description='Build Reference-Browser from source code', command=command, scopes=["secrets:get:{}".format(sentry_secret)], artifacts=artifacts, routes=routes, treeherder={ 'jobKind': 'build', 'machine': { 'platform': 'android-all', }, 'symbol': 'NA', 'tier': 1, }, )
def _craft_default_task_definition( self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload, treeherder=None ): treeherder = {} if treeherder is None else treeherder created = datetime.datetime.now() deadline = taskcluster.fromNow('1 day') expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN) return { "provisionerId": provisioner_id, "workerType": worker_type, "taskGroupId": self.task_id, "schedulerId": self.scheduler_id, "created": taskcluster.stringDate(created), "deadline": taskcluster.stringDate(deadline), "expires": taskcluster.stringDate(expires), "retries": 5, "tags": {}, "priority": self.tasks_priority, "dependencies": [self.task_id] + dependencies, "requires": "all-completed", "routes": routes + [ "tc-treeherder.v2.fenix.{}".format(self.commit) ], "scopes": scopes, "payload": payload, "extra": { "treeherder": treeherder, }, "metadata": { "name": "Fenix - {}".format(name), "description": description, "owner": self.owner, "source": self.source, }, }
def beetmover_task(self, name, description, version, artifact_id, dependencies=[], upstreamArtifacts=[], scopes=[]): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": "mobile-beetmover-v1", "taskGroupId": self.task_id, "schedulerId": self.scheduler_id, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "schedulerId": "taskcluster-github", "deadline": taskcluster.stringDate(deadline), "dependencies": [self.task_id] + dependencies, "routes": [], "scopes": scopes, "requires": "all-completed", "payload": { "maxRunTime": 600, "upstreamArtifacts": upstreamArtifacts, "releaseProperties": { "appName": "components", }, "version": version, "artifact_id": artifact_id }, "provisionerId": "scriptworker-prov-v1", "metadata": { "name": name, "description": description, "owner": self.owner, "source": self.source } }
def build_push_task(self, signing_task_id, name, description, apks=[], scopes=[], track='internal', commit=False): created = datetime.datetime.now() expires = taskcluster.fromNow('1 year') deadline = taskcluster.fromNow('1 day') return { "workerType": 'mobile-pushapk-v1', "taskGroupId": self.task_id, "schedulerId": self.scheduler_id, "expires": taskcluster.stringDate(expires), "retries": 5, "created": taskcluster.stringDate(created), "tags": {}, "priority": "lowest", "deadline": taskcluster.stringDate(deadline), "dependencies": [ self.task_id, signing_task_id], "routes": [], "scopes": scopes, "requires": "all-completed", "payload": { "commit": commit, "google_play_track": track, "upstreamArtifacts": [ { "paths": apks, "taskId": signing_task_id, "taskType": "signing" } ] }, "provisionerId": "scriptworker-prov-v1", "metadata": { "name": name, "description": description, "owner": "*****@*****.**", "source": "https://github.com/mozilla-mobile/focus-android/tree/master/tools/taskcluster" } }
def craft_ui_tests_task(self): artifacts = { "public": { "type": "directory", "path": "/build/fenix/results", "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)) } } env_vars = { "GOOGLE_PROJECT": "moz-fenix", "GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json" } gradle_commands = ( './gradlew --no-daemon clean assemble assembleAndroidTest', ) test_commands = ( 'automation/taskcluster/androidTest/ui-test.sh arm64-v8a -1', 'automation/taskcluster/androidTest/ui-test.sh armeabi-v7a -1', ) command = ' && '.join( cmd for commands in (gradle_commands, test_commands) for cmd in commands if cmd ) treeherder = { 'jobKind': 'test', 'machine': { 'platform': 'ui-test', }, 'symbol': 'ui-test', 'tier': 2, } return self._craft_build_ish_task( name='Fenix - UI test', description='Execute Gradle tasks for UI tests', command=command, scopes=[ 'secrets:get:project/mobile/fenix/firebase' ], artifacts=artifacts, env_vars=env_vars, treeherder=treeherder, )
def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage=False, is_snapshot=False, component=None, artifacts=None, timestamp=None): taskcluster_artifacts = {} # component is not None when this is a release build, in which case artifacts is defined too if component is not None: taskcluster_artifacts = { artifact['taskcluster_path']: { 'type': 'file', 'expires': taskcluster.stringDate( taskcluster.fromNow(DEFAULT_EXPIRES_IN)), 'path': artifact['build_fs_path'], } for artifact in artifacts } scopes = [ "secrets:get:project/mobile/android-components/public-tokens" ] if run_coverage else [] snapshot_flag = '-Psnapshot -Ptimestamp={} '.format( timestamp) if is_snapshot else '' coverage_flag = '-Pcoverage ' if run_coverage else '' gradle_command = ('./gradlew --no-daemon clean ' + coverage_flag + snapshot_flag + gradle_tasks) post_gradle_command = 'automation/taskcluster/action/upload_coverage_report.sh' if run_coverage else '' command = ' && '.join(cmd for cmd in (gradle_command, post_gradle_command) if cmd) return self._craft_build_ish_task( name='Android Components - Module {} {}'.format( module_name, subtitle), description='Execute Gradle tasks for module {}'.format( module_name), command=command, scopes=scopes, artifacts=taskcluster_artifacts)
def generate_compare_locales_task(): return taskcluster.slugId(), generate_task( name="(Focus for Android) String validation", description="Check Focus/Klar for Android for errors in en-US and l10n.", command=('pip install "compare-locales>=5.0.2,<6.0"' ' && mkdir -p /opt/focus-android/test_artifacts' ' && compare-locales --validate l10n.toml .' ' && compare-locales --json=/opt/focus-android/test_artifacts/data.json l10n.toml .'), artifacts={ "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def _craft_default_task_definition(self, worker_type, provisioner_id, dependencies, routes, scopes, name, description, payload): created = datetime.datetime.now() deadline = taskcluster.fromNow('1 day') expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN) routes.append('checks') return { "attributes": {}, "dependencies": dependencies, "label": name, "task": { "provisionerId": provisioner_id, "workerType": worker_type, "taskGroupId": self.task_id, "schedulerId": self.scheduler_id, "created": taskcluster.stringDate(created), "deadline": taskcluster.stringDate(deadline), "expires": taskcluster.stringDate(expires), "retries": 5, "tags": {}, "priority": self.tasks_priority, "requires": "all-completed", "routes": routes, "scopes": scopes, "payload": payload, "metadata": { "name": name, "description": description, "owner": self.owner, "source": self.source, }, } }
def from_template(self, platform, revision, branch, update_number, chunk_name, subchunk, extra): """Reads and populates graph template. :param platform: buildbot platform (linux, macosx64) :param locale: en-US, de, ka, etc. :param from_mar: "from" MAR URL :param to_mar: "to" MAR URL :return: graph definition dictionary """ template_file = os.path.join(os.path.dirname(__file__), "tasks", "funsize.yml") extra_balrog_submitter_params = None if branch in STAGING_BRANCHES: extra_balrog_submitter_params = "--dummy" template_vars = { # Stable slugId "stableSlugId": stableSlugId(), # Now in ISO format "now": stringDate(datetime.datetime.utcnow()), # Now in ms "now_ms": time.time() * 1000, "fromNow": fromNow, "platform": platform, "s3_bucket": self.s3_info["s3_bucket"], "aws_access_key_id": self.s3_info["aws_access_key_id"], "aws_secret_access_key": self.s3_info["aws_secret_access_key"], "balrog_api_root": self.balrog_worker_api_root, "balrog_username": self.balrog_client.auth[0], "balrog_password": self.balrog_client.auth[1], "encryptEnvVar": encryptEnvVar_wrapper, "revision": revision, "branch": branch, "treeherder_platform": buildbot_to_treeherder(platform), "revision_hash": revision_to_revision_hash(self.th_api_root, branch, revision), "update_number": update_number, "extra_balrog_submitter_params": extra_balrog_submitter_params, "extra": extra, "chunk_name": chunk_name, "subchunk": subchunk, "sign_task": partial(sign_task, pvt_key=self.pvt_key), } with open(template_file) as f: template = Template(f.read(), undefined=StrictUndefined) rendered = template.render(**template_vars) return yaml.safe_load(rendered)
def generate_ui_test_task(dependencies): return taskcluster.slugId(), generate_task( name = "(Focus for Android) UI tests", description = "Run UI tests for Focus/Klar for Android.", command = ('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean assembleFocusWebviewUniversalDebug assembleFocusWebviewUniversalDebugAndroidTest' ' && tools/taskcluster/execute-firebase-test.sh'), dependencies = dependencies, scopes = [ 'secrets:get:project/focus/firebase' ], artifacts = { "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def generate_gecko_X86_ui_test_task(dependencies): return taskcluster.slugId(), generate_task( name = "(Focus for Android) UI tests - Gecko X86", description = "Run UI tests for Klar Gecko X86 for Android.", command = ('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean assembleKlarX86Debug assembleKlarX86DebugAndroidTest' ' && ./tools/taskcluster/google-firebase-testlab-login.sh' ' && tools/taskcluster/execute-firebase-test.sh klarX86 app-klar-x86-debug model=Nexus9,version=25'), dependencies = dependencies, scopes = [ 'secrets:get:project/focus/firebase' ], artifacts = { "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def generate_ui_test_task(dependencies, engine="Klar", device="arm"): ''' :param str engine: Klar, Webview :param str device: ARM, X86 :return: uiWebviewARMTestTaskId, uiWebviewARMTestTask ''' if engine is "Klar": engine = "geckoview" assemble_engine = engine elif engine is "Webview": engine = "webview" assemble_engine = "Focus" else: raise Exception("ERROR: unknown engine type --> Aborting!") task_name = "(Focus for Android) UI tests - {0} {1}".format(engine, device) task_description = "Run UI tests for {0} build for Android.".format( engine, device) build_dir = "assemble{0}{1}Debug".format(assemble_engine, device.capitalize()) build_dir_test = "assemble{0}{1}DebugAndroidTest".format( assemble_engine, device.capitalize()) print('BUILD_DIR: {0}'.format(build_dir)) print('BUILD_DIR_TEST: {0}'.format(build_dir_test)) device = device.lower() return taskcluster.slugId(), generate_task( name=task_name, description=task_description, command=('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean ' + build_dir + ' ' + build_dir_test + ' ' ' && ./tools/taskcluster/google-firebase-testlab-login.sh' ' && tools/taskcluster/execute-firebase-tests.sh ' + device + ' ' + engine), dependencies=dependencies, scopes=['secrets:get:project/focus/firebase'], routes=['notify.irc-channel.#android-ci.on-any'], artifacts={ "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def generate_webview_ARM_ui_test_task(dependencies): return taskcluster.slugId(), generate_task( name = "(Focus for Android) UI tests - Webview ARM", description = "Run UI tests for Focus/Klar for Android.", command = ('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean assembleFocusArmDebug assembleFocusArmDebugAndroidTest' ' && ./tools/taskcluster/google-firebase-testlab-login.sh' ' && tools/taskcluster/execute-firebase-test.sh focusArm app-focus-arm-debug model=walleye,version=26 model=shamu,version=23'), dependencies = dependencies, scopes = [ 'secrets:get:project/focus/firebase' ], artifacts = { "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def generate_demo_test_task(): slug_id = taskcluster.slugId() task_json = generate_task( name = "(Demo Taskcluster) Dummy tests", description = "Run a demo for the heck of it.", command = ('./tools/demo.sh'), routes=['notify.irc-channel.#demo-ci.on-any', '/repository/kglazko/demo-taskcluster/issues/1/comments'], #dependencies = dependencies, scopes = ['github:create-comment:kglazko/*', 'queue:route:notify.irc-channel.*'], artifacts = { "public": { "type": "directory", "path": "/opt/demo-taskcluster/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } }) print(task_json) post_github_comment('1', payload_source) return slug_id, task_json
def generate_release_task(dependencies): return taskcluster.slugId(), generate_task( name = "(Focus for Android) Preview release", description = "Build preview versions for testing Focus/Klar for Android.", command = ('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean assembleBeta' ' && python tools/taskcluster/sign-preview-builds.py' ' && touch /opt/focus-android/builds/`date +"%Y-%m-%d-%H-%M"`' ' && touch /opt/focus-android/builds/' + COMMIT), dependencies = dependencies, scopes = [ "secrets:get:project/focus/preview-key-store", "queue:route:index.project.focus.android.preview-builds"], routes = [ "index.project.focus.android.preview-builds" ], artifacts = { "public": { "type": "directory", "path": "/opt/focus-android/builds", "expires": taskcluster.stringDate(taskcluster.fromNow('1 month')) } })
def craft_ui_tests_task(self): artifacts = { "public": { "type": "directory", "path": "/build/reference-browser/results", "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)) } } env_vars = { "GOOGLE_PROJECT": "moz-reference-browser-230023", "GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json" } gradle_commands = ( './gradlew --no-daemon clean assembleDebug assembleAndroidTest', ) test_commands = ( 'automation/taskcluster/androidTest/ui-test.sh aarch64 -1', 'automation/taskcluster/androidTest/ui-test.sh arm -1', ) command = ' && '.join( cmd for commands in (gradle_commands, test_commands) for cmd in commands if cmd ) return self._craft_build_ish_task( name='UI tests', description='Execute Gradle tasks for UI tests', command=command, scopes=[ 'secrets:get:project/mobile/reference-browser/firebase' ], artifacts=artifacts, env_vars=env_vars, )
def generate_massager_task(build_task_id, massager_task_id, artifacts_info, version): command = ( "git fetch origin --tags && " "git config advice.detachedHead false && " "git checkout {} && " "apt-get install -y python3-pip && " "pip3 install scriptworker && " "python3 automation/taskcluster/release/convert_group_and_artifact_ids.py {}" .format(version, massager_task_id)) scopes = [] upstreamZip = [] for artifact, info in artifacts_info.items(): new_artifact_id = None if info['name'] == info[ 'old_artifact_id'] else info['name'] new_group_id = info['new_group_id'] if new_artifact_id else None upstreamZip.append({ "path": artifact, "taskId": build_task_id, "newGroupId": new_group_id, "newArtifactId": new_artifact_id, }) return BUILDER.massager_task( name="Android Components - Massager", description="Task to massage the group/artifact ids", dependencies=[build_task_id], command=command, scopes=scopes, features={"chainOfTrust": True}, artifacts={ "public/build": { "expires": taskcluster.stringDate(taskcluster.fromNow('1 year')), "path": "/build/android-components/work_dir/public/build", "type": "directory" } }, upstreamZip=upstreamZip, )
def generate_build_task(apks, is_staging): artifacts = { 'public/{}'.format(os.path.basename(apk)): { "type": 'file', "path": apk, "expires": taskcluster.stringDate(taskcluster.fromNow('1 year')), } for apk in apks } checkout = ("export TERM=dumb && git fetch {} {} --tags && " "git config advice.detachedHead false && " "git checkout {}".format(GITHUB_HTTP_REPOSITORY, HEAD_BRANCH, HEAD_REV)) sentry_secret = '{}project/mobile/fenix/sentry'.format( 'garbage/staging/' if is_staging else '') leanplum_secret = '{}project/mobile/fenix/leanplum'.format( 'garbage/staging/' if is_staging else '') return taskcluster.slugId(), BUILDER.build_task( name="(Fenix) Build task", description="Build Fenix from source code.", command= (checkout + ' && python automation/taskcluster/helper/get-secret.py' ' -s {} -k dsn -f .sentry_token'.format(sentry_secret) + ' && python automation/taskcluster/helper/get-secret.py' ' -s {} -k production -f .leanplum_token'.format(leanplum_secret) + ' && ./gradlew --no-daemon -PcrashReports=true clean test assembleRelease' ), features={ "chainOfTrust": True, "taskclusterProxy": True }, artifacts=artifacts, scopes=[ "secrets:get:{}".format(sentry_secret), "secrets:get:{}".format(leanplum_secret) ], routes=["[email protected]"])
def generate_ui_test_task(dependencies): task_name = "(Focus for Android) UI tests" task_description = "Run UI tests for Focus for Android." build_dir = "assembleFocusDebug" build_dir_test = "assembleFocusDebugAndroidTest" return taskcluster.slugId(), generate_task( name=task_name, description=task_description, command=('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean ' + build_dir + ' ' + build_dir_test + ' ' ' && ./tools/taskcluster/google-firebase-testlab-login.sh' ' && ./tools/taskcluster/execute-firebase-tests.sh'), dependencies=dependencies, scopes=['secrets:get:project/focus/firebase'], routes=['notify.irc-channel.#android-ci.on-any'], artifacts={ "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def generate_build_task(apks, tag, is_staging): artifacts = {} for apk in apks: artifact = { "type": 'file', "path": apk, "expires": taskcluster.stringDate(taskcluster.fromNow('1 year')) } artifacts["public/%s" % os.path.basename(apk)] = artifact checkout = "cd .. && git clone {} repository && cd repository".format( GITHUB_HTTP_REPOSITORY) if tag is not None: checkout += " && git checkout {}".format(tag) assemble_task = 'assembleNightly' if tag: # Non-tagged (nightly) builds should contain all languages checkout = checkout + ' && python tools/l10n/filter-release-translations.py' assemble_task = 'assembleRelease' return taskcluster.slugId(), BUILDER.build_task( name="(Focus for Android) Build task", description="Build Focus/Klar from source code.", command=(checkout + ' && python tools/taskcluster/get-adjust-token.py {}'.format( '--staging' if is_staging else '') + ' && python tools/taskcluster/get-sentry-token.py {}'.format( '--staging' if is_staging else '') + ' && ./gradlew --no-daemon clean test ' + assemble_task), features={"chainOfTrust": True}, artifacts=artifacts, worker_type='gecko-focus', scopes=[ "secrets:get:garbage/staging/project/focus/tokens" if is_staging else "secrets:get:project/focus/tokens" ])
def generate_build_task(apks, tag): artifacts = {} for apk in apks: artifact = { "type": 'file', "path": apk, "expires": taskcluster.stringDate(taskcluster.fromNow('1 year')) } artifacts["public/%s" % os.path.basename(apk)] = artifact checkout = "git fetch origin && git reset --hard origin/master" if tag is None else "git fetch origin && git checkout %s" % (tag) checkout = checkout + ' && tools/taskcluster/accept-license.sh' assemble_task = 'assembleNightly' if tag: # Non-tagged (nightly) builds should contain all languages checkout = checkout + ' && python tools/l10n/filter-release-translations.py' assemble_task = 'assembleRelease' return taskcluster.slugId(), BUILDER.build_task( name="(Focus for Android) Build task", description="Build Focus/Klar from source code.", command=(checkout + ' && python tools/taskcluster/get-adjust-token.py' ' && python tools/taskcluster/get-sentry-token.py' ' && ./gradlew --no-daemon clean test ' + assemble_task), features = { "chainOfTrust": True }, artifacts = artifacts, worker_type='gecko-focus', scopes=[ "secrets:get:project/focus/tokens" ])
def retrigger_task(task_id, dry_run=False): """ Given a task id (our uuid) we query it and build a new task based on the old one which we schedule on TaskCluster. We don't call the rerun API since we can't rerun a task past its deadline, instead we create a new task with a new taskGroupId, expiration, creation and deadline values. task_id (int) - ID that identifies a task on Taskcluster dry_run (bool) - Default to False. If True, it won't trigger a task. returns - 0 for dry_run case, -1 for any failure or the task id (int) of a succesful retriggered task. http://docs.taskcluster.net/queue/api-docs/#createTask """ one_year = 365 new_task_id = 0 try: queue = taskcluster_client.Queue() task = queue.task(task_id) LOG.debug("Original task: (Limit 1024 char)") LOG.debug(str(json.dumps(task))[:1024]) new_task_id = taskcluster_client.slugId() artifacts = task['payload'].get('artifacts', {}) for artifact, definition in artifacts.iteritems(): definition['expires'] = taskcluster_client.fromNow('%s days' % one_year) # The task group will be identified by the ID of the only # task in the group task['taskGroupId'] = new_task_id # https://bugzilla.mozilla.org/show_bug.cgi?id=1190660 # TC workers create public logs which are 365 days; if the task expiration # date is the same or less than that we won't have logs for the task task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1)) now = datetime.datetime.utcnow() tomorrow = now + datetime.timedelta(hours=24) task['created'] = taskcluster_client.stringDate(now) task['deadline'] = taskcluster_client.stringDate(tomorrow) LOG.debug("Contents of new task: (Limit 1024 char)") LOG.debug(str(task)[:1024]) if not dry_run: LOG.info("Attempting to schedule new task with task_id: {}".format(new_task_id)) result = queue.createTask(new_task_id, task) LOG.debug(json.dumps(result)) LOG.info("{}/task-inspector/#{}".format(TASKCLUSTER_TOOLS_HOST, new_task_id)) else: LOG.info("Dry-run mode: Nothing was retriggered.") except taskcluster_client.exceptions.TaskclusterRestFailure as e: traceback.print_exc() new_task_id = -1 except taskcluster_client.exceptions.TaskclusterAuthFailure as e: # Hack until we fix it in the issue if str(e) == "Authorization Failed": LOG.error("The taskclaster client that you specified is lacking " "the right set of scopes.") LOG.error("Run this same command with --debug and you will see " "the missing scopes (the output comes from the " "taskcluster python client)") elif str(e) == "Authentication Error": LOG.error("Make sure that you create permanent credentials and you " "set these environment variables: TASKCLUSTER_CLIENT_ID & " "TASKCLUSTER_ACCESS_TOKEN") new_task_id = -1 return new_task_id
def _craft_raptor_task( self, signing_task_id, mozharness_task_id, variant, gecko_revision, name_prefix, description, test_name, job_symbol, group_symbol=None, extra_test_args=None, force_run_on_64_bit_device=False, ): extra_test_args = [] if extra_test_args is None else extra_test_args apk_location = '{}/{}/artifacts/{}'.format( _DEFAULT_TASK_URL, signing_task_id, DEFAULT_APK_ARTIFACT_LOCATION) worker_type = 'gecko-t-bitbar-gw-perf-p2' if force_run_on_64_bit_device or variant.abi == 'aarch64' else 'gecko-t-bitbar-gw-perf-g5' if force_run_on_64_bit_device: treeherder_platform = 'android-hw-p2-8-0-arm7-api-16' elif variant.abi == 'arm': treeherder_platform = 'android-hw-g5-7-0-arm7-api-16' elif variant.abi == 'aarch64': treeherder_platform = 'android-hw-p2-8-0-android-aarch64' else: raise ValueError('Unsupported architecture "{}"'.format( variant.abi)) task_name = '{}: {} {}'.format( name_prefix, variant.raw, '(on 64-bit-device)' if force_run_on_64_bit_device else '') apk_url = '{}/{}/artifacts/{}'.format(_DEFAULT_TASK_URL, signing_task_id, DEFAULT_APK_ARTIFACT_LOCATION) return self._craft_default_task_definition( worker_type=worker_type, provisioner_id='proj-autophone', dependencies=[signing_task_id], name=task_name, description=description, payload={ "maxRunTime": 2700, "artifacts": [{ 'path': worker_path, 'expires': taskcluster.stringDate( taskcluster.fromNow(DEFAULT_EXPIRES_IN)), 'type': 'directory', 'name': 'public/{}/'.format(public_folder) } for worker_path, public_folder in ( ('artifacts/public', 'test'), ('workspace/logs', 'logs'), ('workspace/build/blobber_upload_dir', 'test_info'), )], "command": [[ "/builds/taskcluster/script.py", "bash", "./test-linux.sh", "--cfg=mozharness/configs/raptor/android_hw_config.py", "--test={}".format(test_name), "--app=refbrow", "--binary=org.mozilla.reference.browser.raptor", "--activity=org.mozilla.reference.browser.GeckoViewActivity", "--download-symbols=ondemand" ] + extra_test_args], "env": { "EXTRA_MOZHARNESS_CONFIG": json.dumps({ "test_packages_url": "{}/{}/artifacts/public/build/en-US/target.test_packages.json" .format(_DEFAULT_TASK_URL, mozharness_task_id), "installer_url": apk_url, }), "GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/mozilla-central", "GECKO_HEAD_REV": gecko_revision, "MOZ_AUTOMATION": "1", "MOZ_HIDE_RESULTS_TABLE": "1", "MOZ_NO_REMOTE": "1", "MOZ_NODE_PATH": "/usr/local/bin/node", "MOZHARNESS_CONFIG": "raptor/android_hw_config.py", "MOZHARNESS_SCRIPT": "raptor_script.py", "MOZHARNESS_URL": "{}/{}/artifacts/public/build/en-US/mozharness.zip".format( _DEFAULT_TASK_URL, mozharness_task_id), "MOZILLA_BUILD_URL": apk_location, "NEED_XVFB": "false", "NO_FAIL_ON_TEST_ERRORS": "1", "SCCACHE_DISABLE": "1", "TASKCLUSTER_WORKER_TYPE": worker_type[len('gecko-'):], "XPCOM_DEBUG_BREAK": "warn", }, "mounts": [{ "content": { "url": "https://hg.mozilla.org/mozilla-central/raw-file/{}/taskcluster/scripts/tester/test-linux.sh" .format(gecko_revision), }, "file": "test-linux.sh", }] }, treeherder={ 'jobKind': 'test', 'groupSymbol': 'Rap' if group_symbol is None else group_symbol, 'machine': { 'platform': treeherder_platform, }, 'symbol': job_symbol, 'tier': 2, })
def from_now_json(self, offset): """ Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”. """ return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))
def from_now_json(self, offset): """ Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”. """ return taskcluster.stringDate( taskcluster.fromNow(offset, dateObj=self.now))
"iat": iat, "exp": exp, "taskId": task_id, "version": "1", } return jws.sign(claims, pvt_key, algorithm=algorithm) config = json.load(open("secrets.json")) config["credentials"]["certificate"] = json.dumps(config["credentials"]["certificate"]) scheduler = taskcluster.Scheduler(config) # funsize scheduler key pvt_key = open("id_rsa").read() template_vars = { "stableSlugId": stableSlugId(), "now": stringDate(datetime.datetime.utcnow()), "now_ms": time.time() * 1000, "fromNow": fromNow, "sign_task": partial(sign_task, pvt_key=pvt_key), ### TODO: change these "url": "http://people.mozilla.org/~raliiev/bouncer.apk", "filename": "bouncer.apk", "signign_format": "jar", "hash": "8f4210c62cf533322b09237a3741bc3e9bb52582b8d0b88c52a0d78a6eabe08e29b636d5c9668e8db721c9dead36736db643c53231e966c86dbc28d86b9eb699", } template_file = os.path.join(os.path.dirname(__file__), "graph.yml") with open(template_file) as f: template = Template(f.read(), undefined=StrictUndefined) rendered = template.render(**template_vars) graph = yaml.safe_load(rendered)