def test_action(self): context = { "tasks_for": "action", "repository": { "url": "https://hg.mozilla.org/mozilla-central", "project": "mozilla-central", "level": 3, }, "push": { "revision": "e8d2d9aff5026ef1f1777b781b47fdcbdb9d8f20", "owner": "*****@*****.**", "pushlog_id": 1556565286, "pushdate": 112957, }, "action": { "name": "test-action", "title": "Test Action", "description": "Just testing", "taskGroupId": slugid.nice().encode("ascii"), "symbol": "t", "repo_scope": "assume:repo:hg.mozilla.org/try:action:generic", "cb_name": "test_action", }, "input": {}, "parameters": {}, "now": current_json_time(), "taskId": slugid.nice().encode("ascii"), "ownTaskId": slugid.nice().encode("ascii"), "clientId": "testing/testing/testing", } rendered = jsone.render(self.taskcluster_yml, context) pprint.pprint(rendered) self.assertEqual(rendered["tasks"][0]["metadata"]["name"], "Action: Test Action")
async def bigwig(watcher, event): uuid = slugid.nice() await watcher.tilesets.update({ slugid.nice(): TileSet(uuid=uuid, datafile=str(event.path), datatype="vector", filetype="bigwig").todict() })
async def test_run_successful_task(context_function): task_id = slugid.nice() task_group_id = slugid.nice() async with context_function(None) as context: result = await create_task(context, task_id, task_group_id) assert result['status']['state'] == 'pending' async with remember_cwd(): os.chdir(os.path.dirname(context.config['work_dir'])) status = await worker.run_tasks(context) assert status == 1 result = await task_status(context, task_id) assert result['status']['state'] == 'failed'
async def test_run_successful_task(self, event_loop, context_function): task_id = slugid.nice().decode('utf-8') task_group_id = slugid.nice().decode('utf-8') with context_function(None) as context: result = await create_task(context, task_id, task_group_id) assert result['status']['state'] == 'pending' with remember_cwd(): os.chdir("integration") status = await worker.run_loop(context, creds_key="integration_credentials") assert status == 1 result = await task_status(context, task_id) assert result['status']['state'] == 'failed'
async def test_run_successful_task(self, event_loop, context_function): task_id = slugid.nice().decode('utf-8') task_group_id = slugid.nice().decode('utf-8') with context_function(None) as context: result = await create_task(context, task_id, task_group_id) assert result['status']['state'] == 'pending' with remember_cwd(): os.chdir("integration") status = await worker.run_loop( context, creds_key="integration_credentials") assert status == 1 result = await task_status(context, task_id) assert result['status']['state'] == 'failed'
def test_run_successful_task(event_loop, context_function): task_id = slugid.nice().decode('utf-8') task_group_id = slugid.nice().decode('utf-8') with context_function(None) as context: result = event_loop.run_until_complete( create_task(context, task_id, task_group_id)) assert result['status']['state'] == 'pending' with remember_cwd(): os.chdir(os.path.dirname(context.config['work_dir'])) status = event_loop.run_until_complete( worker.run_loop(context, creds_key="integration_credentials")) assert status == 1 result = event_loop.run_until_complete(task_status(context, task_id)) assert result['status']['state'] == 'failed'
def build_config(override, basedir): randstring = slugid.nice()[0:6] config = get_unfrozen_copy(DEFAULT_CONFIG) ED25519_DIR = os.path.join(os.path.dirname(__file__), "data", "ed25519") config.update({ 'log_dir': os.path.join(basedir, "log"), 'artifact_dir': os.path.join(basedir, "artifact"), 'task_log_dir': os.path.join(basedir, "artifact", "public", "logs"), 'work_dir': os.path.join(basedir, "work"), "worker_type": "dummy-worker-{}".format(randstring), "worker_id": "dummy-worker-{}".format(randstring), 'artifact_upload_timeout': 60 * 2, 'poll_interval': 5, 'reclaim_interval': 5, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 9 && exit 1'), 'task_max_timeout': 60, 'cot_product': 'firefox', 'ed25519_private_key_path': os.path.join(ED25519_DIR, 'scriptworker_private_key'), 'ed25519_public_keys': { 'docker-worker': ['8dBv4bbnZ3RsDzQiPKTJ18uo3hq5Rjm94JG6HXzAcBM='], 'generic-worker': ['PkI5NslA78wSsYaKNzKq7iD7MLQy7W6wYO/0WFd4tWM='], 'scriptworker': ['KxYrV3XAJ3uOyAUX0Wcl1Oeu6GSMrI/5hOn39q8Lf0I='], }, }) creds = read_integration_creds() del(config['credentials']) if isinstance(override, dict): config.update(override) with open(os.path.join(basedir, "config.json"), "w") as fh: json.dump(config, fh, indent=2, sort_keys=True) config = apply_product_config(config) return config, creds
async def test_private_artifacts(context_function): task_group_id = task_id = slugid.nice() override = { 'task_script': ( 'bash', '-c', '>&2 echo' ), } async with context_function(override) as context: result = await create_task(context, task_id, task_group_id) assert result['status']['state'] == 'pending' path = os.path.join(context.config['artifact_dir'], 'SampleArtifacts/_/X.txt') utils.makedirs(os.path.dirname(path)) with open(path, "w") as fh: fh.write("bar") async with remember_cwd(): os.chdir(os.path.dirname(context.config['work_dir'])) status = await worker.run_tasks(context) assert status == 0 result = await task_status(context, task_id) assert result['status']['state'] == 'completed' url = artifacts.get_artifact_url(context, task_id, 'SampleArtifacts/_/X.txt') path2 = os.path.join(context.config['work_dir'], 'downloaded_file') await utils.download_file(context, url, path2) with open(path2, "r") as fh: contents = fh.read().strip() assert contents == 'bar'
def test_cron(self): context = { "tasks_for": "cron", "repository": { "url": "https://hg.mozilla.org/mozilla-central", "project": "mozilla-central", "level": 3, }, "push": { "revision": "e8aebe488b2f2e567940577de25013d00e818f7c", "pushlog_id": -1, "pushdate": 0, "owner": "cron", }, "cron": { "task_id": "<cron task id>", "job_name": "test", "job_symbol": "T", "quoted_args": "abc def", }, "now": current_json_time(), "ownTaskId": slugid.nice().encode("ascii"), } rendered = jsone.render(self.taskcluster_yml, context) pprint.pprint(rendered) self.assertEqual(rendered["tasks"][0]["metadata"]["name"], "Decision Task for cron job test")
async def test_cancel_task(): task_id = slugid.nice() partial_config = { 'invalid_reclaim_status': 19, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 30 && exit 1'), } # Don't use temporary credentials from claimTask, since they don't allow us # to cancel the created task. async with get_context(partial_config) as context: result = await create_task(context, task_id, task_id) assert result['status']['state'] == 'pending' cancel_fut = asyncio.ensure_future(do_cancel(context, task_id)) task_fut = asyncio.ensure_future(run_task_until_stopped(context)) await utils.raise_future_exceptions([cancel_fut, task_fut]) status = await context.queue.status(task_id) assert len(status['status']['runs']) == 1 assert status['status']['state'] == 'exception' assert status['status']['runs'][0]['reasonResolved'] == 'canceled' log_url = context.queue.buildUrl( 'getLatestArtifact', task_id, 'public/logs/live_backing.log' ) log_path = os.path.join(context.config['work_dir'], 'log') await utils.download_file(context, log_url, log_path) with open(log_path) as fh: contents = fh.read() assert contents.rstrip() == "bar\nfoo\nAutomation Error: python exited with signal -15"
def retrigger(self, task_id, count=1, retries=5): payload = self.queue.task(task_id) now = taskcluster.fromNow("0 days") created = datetime.strptime(payload["created"], _DATE_FMT) deadline = datetime.strptime(payload["deadline"], _DATE_FMT) expiration = datetime.strptime(payload["expires"], _DATE_FMT) to_dead = deadline - created to_expire = expiration - created payload["deadline"] = taskcluster.stringDate( taskcluster.fromNow( "%d days %d seconds" % (to_dead.days, to_dead.seconds), now)) payload["expires"] = taskcluster.stringDate( taskcluster.fromNow( "%d days %d seconds" % (to_expire.days, to_expire.seconds), now)) payload["created"] = taskcluster.stringDate(now) payload["retries"] = 0 rv = [] while count > 0: new_id = slugid.nice() r = retries while r > 0: try: rv.append(self.queue.createTask(new_id, payload)) break except Exception as e: r -= 1 logger.warning(traceback.format_exc(e)) count -= 1 return rv or None
def viewconfs(request): ''' Retrieve a viewconfs with a given uid Args: request (django.http.HTTPRequest): The request object containing the uid (e.g. d=hg45ksdjfds) that identifies the viewconf. Return: ''' if request.method == 'POST': uuid = slugid.nice() viewconf = request.body serializer = tss.ViewConfSerializer(data={'viewconf': request.body}) if not serializer.is_valid(): return JsonResponse( {'error': 'Serializer not valid'}, status=rfs.HTTP_400_BAD_REQUEST ) serializer.save(uuid=uuid, viewconf=viewconf) return JsonResponse({'uid': uuid}) uuid = request.GET.get('d') obj = tm.ViewConf.objects.get(uuid=uuid) return JsonResponse(json.loads(obj.viewconf))
async def test_private_artifacts(context_function): task_group_id = task_id = slugid.nice() override = { 'task_script': ('bash', '-c', '>&2 echo'), } async with context_function(override) as context: result = await create_task(context, task_id, task_group_id) assert result['status']['state'] == 'pending' path = os.path.join(context.config['artifact_dir'], 'SampleArtifacts/_/X.txt') utils.makedirs(os.path.dirname(path)) with open(path, "w") as fh: fh.write("bar") async with remember_cwd(): os.chdir(os.path.dirname(context.config['work_dir'])) status = await worker.run_tasks(context) assert status == 0 result = await task_status(context, task_id) assert result['status']['state'] == 'completed' url = artifacts.get_artifact_url(context, task_id, 'SampleArtifacts/_/X.txt') path2 = os.path.join(context.config['work_dir'], 'downloaded_file') await utils.download_file(context, url, path2) with open(path2, "r") as fh: contents = fh.read().strip() assert contents == 'bar'
def __init__(self, tileset_info=None, tiles=None, chromsizes=lambda: None, uuid=None, private=False, name='', datatype=''): ''' Parameters ---------- tileset_info: function A function returning the information (min_pos, max_pos, max_width, max_zoom), for this tileset. tiles: function A function returning tile data for this tileset ''' self.name = name self.tileset_info_fn = tileset_info self.tiles_fn = tiles self.chromsizes_fn = chromsizes self.private = private if uuid is not None: self.uuid = uuid else: self.uuid = slugid.nice().decode('utf-8')
def show(self): # credit for this code goes to the higlass-python team: https://github.com/higlass/higlass-python for p in list(self.processes.keys()): if self.port == p: print("delete ", self.processes[p]) self.processes[p].kill() del self.processes[p] time.sleep(0.5) #print(self.processes) uuid = slugid.nice() target = partial(eventlet.wsgi.server, sock=eventlet.listen(('localhost', self.port)), site=fApp) self.processes[self.port] = mp.Process( target=target) #self.startServer, args=(q,)) self.processes[self.port].start() self.connected = False while not self.connected: try: url = "http://{}:{}/".format('localhost', self.port) r = requests.head(url) if r.ok: self.connected = True except requests.ConnectionError: time.sleep(0.2)
def start(self, log_file='/tmp/hgserver.log', log_level=logging.INFO): for puid in list(self.processes.keys()): print("terminating:", puid) self.processes[puid].terminate() del self.processes[puid] self.app = create_app( self.tilesets, __name__, log_file=log_file, log_level=log_level) # we're going to assign a uuid to each server process so that if anything # goes wrong, the variable referencing the process doesn't get lost uuid = slugid.nice().decode('utf8') if self.port is None: self.port = get_open_port() target = partial(self.app.run, threaded=True, debug=True, host='0.0.0.0', port=self.port, use_reloader=False) self.processes[uuid] = mp.Process(target=target) self.processes[uuid].start() self.connected = False while not self.connected: try: url = 'http://{}:{}/api/v1'.format(self.host, self.port) r = requests.head(url) if r.ok: self.connected = True except requests.ConnectionError as err: time.sleep(.2)
class Tileset(models.Model): created = models.DateTimeField(auto_now_add=True) uuid = models.CharField(max_length=100, unique=True, default=lambda: slugid.nice().decode('utf-8')) # processed_file = models.TextField() datafile = models.FileField(upload_to='uploads') filetype = models.TextField() datatype = models.TextField(default='unknown') coordSystem = models.TextField() coordSystem2 = models.TextField(default='') owner = models.ForeignKey( 'auth.User', related_name='tilesets', on_delete=models.CASCADE, blank=True, null=True # Allow anonymous owner ) private = models.BooleanField(default=False) name = models.TextField(blank=True) class Meta: ordering = ('created', ) permissions = (('view_tileset', "View tileset"), ) def __str__(self): ''' Get a string representation of this model. Hopefully useful for the admin interface. ''' return "Tileset [name: " + self.name + '] [ft: ' + self.filetype + ']'
def build_config(override, basedir): randstring = slugid.nice()[0:6].decode('utf-8') config = dict(deepcopy(DEFAULT_CONFIG)) GPG_HOME = os.path.join(os.path.basename(__file__), "data", "gpg") config.update({ 'log_dir': os.path.join(basedir, "log"), 'artifact_dir': os.path.join(basedir, "artifact"), 'task_log_dir': os.path.join(basedir, "artifact", "public", "logs"), 'work_dir': os.path.join(basedir, "work"), "worker_type": "dummy-worker-{}".format(randstring), "worker_id": "dummy-worker-{}".format(randstring), 'artifact_upload_timeout': 60 * 2, 'artifact_expiration_hours': 1, 'gpg_home': GPG_HOME, "gpg_encoding": 'utf-8', "gpg_options": None, "gpg_path": os.environ.get("GPG_PATH", None), "gpg_public_keyring": os.path.join(GPG_HOME, "pubring.gpg"), "gpg_secret_keyring": os.path.join(GPG_HOME, "secring.gpg"), "gpg_use_agent": None, 'reclaim_interval': 5, 'credential_update_interval': .1, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 9 && exit 1'), 'task_max_timeout': 60, }) creds = read_integration_creds() del(config['credentials']) if isinstance(override, dict): config.update(override) with open(os.path.join(basedir, "config.json"), "w") as fh: json.dump(config, fh, indent=2, sort_keys=True) return config, creds
async def test_cancel_task(): task_id = slugid.nice() partial_config = { "invalid_reclaim_status": 19, "task_script": ("bash", "-c", ">&2 echo bar && echo foo && sleep 30 && exit 1") } # Don't use temporary credentials from claimTask, since they don't allow us # to cancel the created task. async with get_context(partial_config) as context: result = await create_task(context, task_id, task_id) assert result["status"]["state"] == "pending" cancel_fut = asyncio.ensure_future(do_cancel(context, task_id)) task_fut = asyncio.ensure_future(run_task_until_stopped(context)) await utils.raise_future_exceptions([cancel_fut, task_fut]) status = await context.queue.status(task_id) assert len(status["status"]["runs"]) == 1 assert status["status"]["state"] == "exception" assert status["status"]["runs"][0]["reasonResolved"] == "canceled" log_url = context.queue.buildUrl("getLatestArtifact", task_id, "public/logs/live_backing.log") log_path = os.path.join(context.config["work_dir"], "log") await utils.download_file(context, log_url, log_path) with open(log_path) as fh: contents = fh.read() assert contents.rstrip( ) == "bar\nfoo\nAutomation Error: python exited with signal -15"
def mozharness_on_buildbot_bridge(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] branch = config.params['project'] product = run.get('index', {}).get('product', 'firefox') worker.pop('env', None) if 'devedition' in job['attributes']['build_platform']: buildername = 'OS X 10.7 {} devedition build'.format(branch) else: buildername = 'OS X 10.7 {} build'.format(branch) worker.update({ 'buildername': buildername, 'sourcestamp': { 'branch': branch, 'repository': config.params['head_repository'], 'revision': config.params['head_rev'], }, 'properties': { 'product': product, 'who': config.params['owner'], 'upload_to_task_id': slugid.nice(), } })
async def test_cancel_task(): task_id = slugid.nice() partial_config = { 'invalid_reclaim_status': 19, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 30 && exit 1'), } # Don't use temporary credentials from claimTask, since they don't allow us # to cancel the created task. async with get_context(partial_config) as context: result = await create_task(context, task_id, task_id) assert result['status']['state'] == 'pending' cancel_fut = asyncio.ensure_future(do_cancel(context, task_id)) task_fut = asyncio.ensure_future(run_task_until_stopped(context)) await utils.raise_future_exceptions([cancel_fut, task_fut]) status = await context.queue.status(task_id) assert len(status['status']['runs']) == 1 assert status['status']['state'] == 'exception' assert status['status']['runs'][0]['reasonResolved'] == 'canceled' log_url = context.queue.buildUrl('getLatestArtifact', task_id, 'public/logs/live_backing.log') log_path = os.path.join(context.config['work_dir'], 'log') await utils.download_file(context, log_url, log_path) with open(log_path) as fh: contents = fh.read() assert contents.rstrip( ) == "bar\nfoo\nAutomation Error: python exited with signal -15"
def __init__( self, tileset_info=None, tiles=None, chromsizes=lambda: None, uuid=None, private=False, name="", datatype="", track_type=None, track_position=None, ): """ Parameters ---------- tileset_info: function A function returning the information (min_pos, max_pos, max_width, max_zoom), for this tileset. tiles: function A function returning tile data for this tileset """ self.name = name self.datatype = datatype self.tileset_info_fn = tileset_info self.tiles_fn = tiles self.chromsizes_fn = chromsizes self.private = private self.track_type = None self.track_position = None if uuid is not None: self.uuid = uuid else: self.uuid = slugid.nice()
def build_config(override): cwd = os.getcwd() basedir = os.path.join(cwd, "integration") if not os.path.exists(basedir): os.makedirs(basedir) randstring = slugid.nice()[0:6].decode('utf-8') config = deepcopy(DEFAULT_CONFIG) config.update({ 'log_dir': os.path.join(basedir, "log"), 'artifact_dir': os.path.join(basedir, "artifact"), 'work_dir': os.path.join(basedir, "work"), "worker_type": "dummy-worker-{}".format(randstring), "worker_id": "dummy-worker-{}".format(randstring), 'artifact_upload_timeout': 60 * 2, 'artifact_expiration_hours': 1, 'reclaim_interval': 5, 'credential_update_interval': .1, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 9 && exit 1'), 'task_max_timeout': 60, }) creds = read_integration_creds() del(config['credentials']) if isinstance(override, dict): config.update(override) with open(os.path.join(basedir, "config.json"), "w") as fh: json.dump(config, fh, indent=2, sort_keys=True) return config, creds
def invalidate_cloudfront_cache(paths): sanitized_paths = [ path if path.startswith('/') else '/{}'.format(path) for path in paths ] number_of_paths = len(sanitized_paths) print('Invalidating {} CloudFront paths: {}'.format( number_of_paths, sanitized_paths)) distribution_id = os.environ.get('CLOUDFRONT_DISTRIBUTION_ID', None) if distribution_id: request_id = slugid.nice() try: cloudfront.create_invalidation(DistributionId=distribution_id, InvalidationBatch={ 'Paths': { 'Quantity': number_of_paths, 'Items': sanitized_paths, }, 'CallerReference': request_id, }) except ClientError as e: print('WARN: Could not invalidate cache. Reason: {}'.format(e)) else: print('CLOUDFRONT_DISTRIBUTION_ID not set. No cache to invalidate.')
def build_config(override, basedir): randstring = slugid.nice()[0:6] config = get_unfrozen_copy(DEFAULT_CONFIG) GPG_HOME = os.path.join(os.path.dirname(__file__), "data", "gpg") ED25519_DIR = os.path.join(os.path.dirname(__file__), "data", "ed25519") config.update({ 'log_dir': os.path.join(basedir, "log"), 'artifact_dir': os.path.join(basedir, "artifact"), 'task_log_dir': os.path.join(basedir, "artifact", "public", "logs"), 'work_dir': os.path.join(basedir, "work"), "worker_type": "dummy-worker-{}".format(randstring), "worker_id": "dummy-worker-{}".format(randstring), 'artifact_upload_timeout': 60 * 2, 'gpg_home': GPG_HOME, "gpg_encoding": 'utf-8', "gpg_options": None, "gpg_path": os.environ.get("GPG_PATH", None), "gpg_public_keyring": os.path.join(GPG_HOME, "pubring.gpg"), "gpg_secret_keyring": os.path.join(GPG_HOME, "secring.gpg"), "gpg_use_agent": None, 'poll_interval': 5, 'reclaim_interval': 5, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 9 && exit 1'), 'task_max_timeout': 60, 'cot_product': 'firefox', 'ed25519_private_key_path': os.path.join(ED25519_DIR, 'scriptworker_private_key'), 'ed25519_public_keys': { 'docker-worker': ['8dBv4bbnZ3RsDzQiPKTJ18uo3hq5Rjm94JG6HXzAcBM='], 'generic-worker': ['PkI5NslA78wSsYaKNzKq7iD7MLQy7W6wYO/0WFd4tWM='], 'scriptworker': ['KxYrV3XAJ3uOyAUX0Wcl1Oeu6GSMrI/5hOn39q8Lf0I='], }, }) creds = read_integration_creds() del (config['credentials']) if isinstance(override, dict): config.update(override) with open(os.path.join(basedir, "config.json"), "w") as fh: json.dump(config, fh, indent=2, sort_keys=True) config = apply_product_config(config) return config, creds
def slugid_nice(): """ Returns a new, random utf-8 slug (based on uuid4). :return: slug representation of a new uuid4, as a utf-8 string :rtype: str """ return slugid.nice().decode('utf-8')
def as_slugid(name): if name not in slugids: slugids[name] = slugid.nice().decode() print('cache miss', name, slugids[name]) else: print('cache hit', name, slugids[name]) return slugids[name]
def __init__( self, tilesets, host="localhost", port=None, name=None, fuse=True, tmp_dir=OS_TEMPDIR, log_level=logging.INFO, log_file=None, ): self.name = name or __name__.split(".")[0] + '-' + slugid.nice()[:8] self.tilesets = tilesets self.host = host self.port = port if fuse: self.fuse_process = FuseProcess(op.join(tmp_dir, 'higlass-python')) self.fuse_process.setup() else: self.fuse_process = None self.app = create_app(self.name, self.tilesets, fuse=self.fuse_process) if log_file: self.log = None handler = logging.handlers.RotatingFileHandler(log_file, maxBytes=100000, backupCount=1) else: self.log = StringIO() handler = logging.StreamHandler(self.log) handler.setLevel(log_level) self.app.logger.addHandler(handler)
def ingest(filename=None, datatype=None, filetype=None, coordSystem='', coordSystem2='', uid=None, name=None, no_upload=False, project_name='', temporary=False, **ignored): uid = uid or slugid.nice().decode('utf-8') name = name or op.split(filename)[1] if not filetype: raise CommandError('Filetype has to be specified') django_file = None # if we're ingesting a url, place it relative to the httpfs directories # and append two dots at the end if filename[:7] == 'http://': filename = "{}..".format(filename.replace('http:/', 'http')) no_upload=True if filename[:8] == 'https://': filename = "{}..".format(filename.replace('https:/', 'https')) no_upload=True if filename[:6] == 'ftp://': filename = "{}..".format(filename.replace('ftp:/', 'ftp')) no_upload=True # it's a regular file on the filesystem, not a file being entered as a url if no_upload: if (not op.isfile(op.join(settings.MEDIA_ROOT, filename)) and not op.islink(op.join(settings.MEDIA_ROOT, filename)) and not any([filename.startswith('http/'), filename.startswith('https/'), filename.startswith('ftp/')])): raise CommandError('File does not exist under media root') filename = op.join(settings.MEDIA_ROOT, filename) django_file = filename else: if os.path.islink(filename): django_file = File(open(os.readlink(filename),'rb')) else: django_file = File(open(filename,'rb')) # remove the filepath of the filename django_file.name = op.split(django_file.name)[1] if filetype.lower() == 'bigwig': coordSystem = check_for_chromsizes(filename, coordSystem) try: project_obj = tm.Project.objects.get(name=project_name) except dce.ObjectDoesNotExist: project_obj = tm.Project.objects.create( name=project_name ) return tm.Tileset.objects.create( datafile=django_file, filetype=filetype, datatype=datatype, coordSystem=coordSystem, coordSystem2=coordSystem2, owner=None, project=project_obj, uuid=uid, temporary=temporary, name=name)
def __init__(self, tracks=[], x=0, y=0, width=12, height=6, initialXDomain=None, initialYDomain=None, uid=None): ''' Add a new view Parameters -------- tracks: [] A list of Tracks to include in this view x: int The position of this view on the grid y: int The position of this view on the grid width: int The width of this of view on a 12 unit grid height: int The height of the this view. The height is proportional to the height of all the views present. initialXDoamin: [int, int] The initial x range of the view initialYDomain: [int, int] The initial y range of the view uid: string The uid of new view ''' if uid is None: uid = slugid.nice().decode('utf8') self.tracks = tracks self.uid = uid self.viewconf = { 'uid': uid, 'tracks': { 'top': [], 'center': [], 'left': [], 'right': [], 'bottom': [] }, "layout": { "w": width, "h": height, "x": x, "y": y }, } if initialXDomain is not None: self.viewconf['initialXDomain'] = initialXDomain if initialYDomain is not None: self.viewconf['initialYDomain'] = initialYDomain
def phase_signoffs(self, phase): return [ Signoff(uid=slugid.nice(), name=req["name"], description=req["description"], permissions=req["permissions"]) for req in SIGNOFFS.get( self.branch, {}).get(self.product, {}).get(phase, []) ]
def add_sync(self, locks_name, views_to_sync): lock_id = slugid.nice() for view_uid in [v.uid for v in views_to_sync]: if lock_id not in self.viewconf[locks_name]['locksDict']: self.viewconf[locks_name]['locksDict'][lock_id] = {} self.viewconf[locks_name]["locksDict"][lock_id][view_uid] = (1, 1, 1) self.viewconf[locks_name]["locksByViewUid"][view_uid] = lock_id
def phase_signoffs(self, phase): return [ XPISignoff(uid=slugid.nice(), name=req["name"], description=req["description"], permissions=req["permissions"]) for req in SIGNOFFS.get( "xpi", {}).get(self.xpi_type, {}).get(phase, []) ]
def phase_signoffs(branch, product, phase): return [ Signoff(uid=slugid.nice(), name=req['name'], description=req['description'], permissions=req['permissions']) for req in shipit_api.config.SIGNOFFS.get(branch, {}).get( product, {}).get(phase, []) ]
async def bam_register(watcher, event, bam_index_path): if bam_index_path.exists(): uuid = slugid.nice() await watcher.tilesets.update({ uuid: TileSet(uuid=uuid, datafile=str(event.path), datatype="reads", filetype="bam", indexfile=str(bam_index_path)).todict() })
def phase_signoffs(branch, product, phase): return [ Signoff( uid=slugid.nice(), name=req['name'], description=req['description'], permissions=req['permissions'] ) for req in shipit_api.config.SIGNOFFS.get(branch, {}).get(product, {}).get(phase, []) ]
async def test_run_maxtimeout(context_function): task_id = slugid.nice() partial_config = { 'task_max_timeout': 2, 'task_script': ('bash', '-c', '>&2 echo bar && echo foo && sleep 30 && exit 1'), } async with context_function(partial_config) as context: result = await create_task(context, task_id, task_id) assert result['status']['state'] == 'pending' async with remember_cwd(): os.chdir(os.path.dirname(context.config['work_dir'])) status = await worker.run_tasks(context) assert status == context.config['task_max_timeout_status']
def test_run_maxtimeout(self, event_loop, context_function): task_id = slugid.nice().decode('utf-8') task_group_id = slugid.nice().decode('utf-8') partial_config = { 'task_max_timeout': 2, } with context_function(partial_config) as context: result = event_loop.run_until_complete( create_task(context, task_id, task_group_id) ) assert result['status']['state'] == 'pending' with remember_cwd(): os.chdir("integration") with pytest.raises(RuntimeError): event_loop.run_until_complete( worker.run_loop(context, creds_key="integration_credentials") ) # Because we're using asyncio to kill tasks in the loop, # we're going to hit a RuntimeError result = event_loop.run_until_complete(task_status(context, task_id)) # TODO We need to be able to ensure this is 'failed'. assert result['status']['state'] in ('failed', 'running')
def generate_action_task(decision_task_id, action_name, input_, actions): target_action = find_action(action_name, actions) context = copy.deepcopy(actions['variables']) # parameters action_task_id = slugid.nice() context.update({ 'input': input_, 'taskGroupId': decision_task_id, 'ownTaskId': action_task_id, 'taskId': None, 'task': None, }) action_task = copy.deepcopy(target_action['task']) log.info('TASK: %s', action_task) return action_task_id, action_task, context
def generate_action_task(decision_task_id, action_task_input): actions = fetch_actions_json(decision_task_id) relpro = find_action("release-promotion", actions) context = copy.deepcopy(actions["variables"]) # parameters action_task_id = slugid.nice() context.update({ "input": action_task_input, "ownTaskId": action_task_id, "taskId": None, "task": None, "taskGroupId": decision_task_id, }) action_task = jsone.render(relpro["task"], context) return action_task_id, action_task
async def test_shutdown(): task_id = slugid.nice() partial_config = { 'task_script': ('bash', '-c', '>&2 echo running task script && sleep 30 && exit 1'), } # Don't use temporary credentials from claimTask, since they don't allow us # to cancel the created task. async with get_context(partial_config) as context: result = await create_task(context, task_id, task_id) assert result['status']['state'] == 'pending' fake_cot_log = os.path.join(context.config['artifact_dir'], 'public', 'logs', 'chain_of_trust.log') fake_other_artifact = os.path.join(context.config['artifact_dir'], 'public', 'artifact.apk') with open(fake_cot_log, 'w') as file: file.write('CoT logs') with open(fake_other_artifact, 'w') as file: file.write('unrelated artifact') cancel_fut = asyncio.ensure_future(do_shutdown(context)) task_fut = asyncio.ensure_future(run_task_until_stopped(context)) await utils.raise_future_exceptions([cancel_fut, task_fut]) status = await context.queue.status(task_id) assert len(status['status']['runs']) == 2 # Taskcluster should create a replacement task assert status['status']['runs'][0]['state'] == 'exception' assert status['status']['runs'][0]['reasonResolved'] == 'worker-shutdown' log_url = context.queue.buildUrl( 'getArtifact', task_id, 0, 'public/logs/live_backing.log' ) cot_log_url = context.queue.buildUrl( 'getArtifact', task_id, 0, 'public/logs/chain_of_trust.log' ) other_artifact_url = context.queue.buildUrl( 'getArtifact', task_id, 0, 'public/artifact.apk' ) log_path = os.path.join(context.config['work_dir'], 'log') cot_log_path = os.path.join(context.config['work_dir'], 'cot_log') other_artifact_path = os.path.join(context.config['work_dir'], 'artifact.apk') await utils.download_file(context, log_url, log_path) await utils.download_file(context, cot_log_url, cot_log_path) with pytest.raises(Download404): await utils.download_file(context, other_artifact_url, other_artifact_path) with open(log_path) as fh: contents = fh.read() assert contents.rstrip() == "running task script\nAutomation Error: python exited with signal -15" with open(cot_log_path) as fh: contents = fh.read() assert contents.rstrip() == "CoT logs"
def mozharness_on_buildbot_bridge(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] branch = config.params['project'] product = run.get('index', {}).get('product', 'firefox') worker.pop('env', None) worker.update({ 'buildername': 'OS X 10.7 {} build'.format(branch), 'sourcestamp': { 'branch': branch, 'repository': config.params['head_repository'], 'revision': config.params['head_rev'], }, 'properties': { 'product': product, 'who': config.params['owner'], 'upload_to_task_id': slugid.nice(), } })
def bb_ci_worker(config, worker): worker['properties'].update({ 'who': config.params['owner'], 'upload_to_task_id': slugid.nice(), })
def cmd(ctx, github_commit, channel, owner, pull_request, task_id, cache_urls, nix_instantiate, taskcluster_client_id, taskcluster_access_token, dry_run, ): '''A tool to be ran on each commit. ''' taskcluster_secret = 'repo:github.com/mozilla-releng/services:branch:' + channel if pull_request is not None: taskcluster_secret = 'repo:github.com/mozilla-releng/services:pull-request' taskcluster_queue = cli_common.taskcluster.get_service('queue') taskcluster_notify = cli_common.taskcluster.get_service('notify') click.echo(' => Retriving taskGroupId ... ', nl=False) with click_spinner.spinner(): task = taskcluster_queue.task(task_id) if 'taskGroupId' not in task: please_cli.utils.check_result(1, 'taskGroupId does not exists in task: {}'.format(json.dumps(task))) task_group_id = task['taskGroupId'] please_cli.utils.check_result(0, '') click.echo(' taskGroupId: ' + task_group_id) if channel in please_cli.config.DEPLOY_CHANNELS: taskcluster_notify.irc(dict(channel='#release-services', message=f'New deployment on {channel} is about to start: https://tools.taskcluster.net/groups/{task_group_id}')) message = ('release-services team is about to release a new version of mozilla/release-services ' '(*.mozilla-releng.net, *.moz.tools). Any alerts coming up soon will be best directed ' 'to #release-services IRC channel. Automated message (such as this) will be send ' 'once deployment is done. Thank you.') '''This message will only be sent when channel is production. ''' if channel is 'production': for msgChannel in ['#ci', '#moc']: taskcluster_notify.irc(dict(channel=msgChannel, message=message)) click.echo(' => Checking cache which project needs to be rebuilt') build_projects = [] project_hashes = dict() for project in sorted(PROJECTS): click.echo(' => ' + project) project_exists_in_cache, project_hash = ctx.invoke( please_cli.check_cache.cmd, project=project, cache_urls=cache_urls, nix_instantiate=nix_instantiate, channel=channel, indent=8, interactive=False, ) project_hashes[project] = project_hash if not project_exists_in_cache: build_projects.append(project) projects_to_deploy = [] if channel in please_cli.config.DEPLOY_CHANNELS: click.echo(' => Checking which project needs to be redeployed') # TODO: get status for our index branch deployed_projects = {} for project_name in sorted(PROJECTS): deployed_projects.get(project_name) # update hook for each project if please_cli.config.PROJECTS_CONFIG[project_name]['update'] is True: if channel == 'production': update_hook_nix_path_atttribute = f'updateHook.{channel}.scheduled' else: update_hook_nix_path_atttribute = f'updateHook.{channel}.notScheduled' projects_to_deploy.append(( project_name, [], 'TASKCLUSTER_HOOK', { 'enable': True, 'docker_registry': 'index.docker.io', 'docker_repo': 'mozillareleng/services', 'name-suffix': '-update-dependencies', 'nix_path_attribute': update_hook_nix_path_atttribute, }, )) if deployed_projects == project_hashes[project_name]: continue if 'deploys' not in please_cli.config.PROJECTS_CONFIG[project_name]: continue for deploy in please_cli.config.PROJECTS_CONFIG[project_name]['deploys']: for deploy_channel in deploy['options']: if channel == deploy_channel: projects_to_deploy.append(( project_name, please_cli.config.PROJECTS_CONFIG[project_name].get('requires', []), deploy['target'], deploy['options'][channel], )) click.echo(' => Creating taskcluster tasks definitions') tasks = [] # 1. build tasks build_tasks = {} for index, project in enumerate(sorted(build_projects)): project_uuid = slugid.nice().decode('utf-8') required = [] if pull_request is not None: required += [ 'CACHE_BUCKET', 'CACHE_REGION', ] secrets = cli_common.taskcluster.get_secrets( taskcluster_secret, project, required=required, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) build_tasks[project_uuid] = get_build_task( index, project, task_group_id, task_id, github_commit, owner, channel, taskcluster_secret, pull_request is None and secrets.get('CACHE_BUCKET') or None, pull_request is None and secrets.get('CACHE_REGION') or None, ) tasks.append((project_uuid, build_tasks[project_uuid])) if projects_to_deploy: # 2. maintanance on task maintanance_on_uuid = slugid.nice().decode('utf-8') if len(build_tasks.keys()) == 0: maintanance_on_dependencies = [task_id] else: maintanance_on_dependencies = [i for i in build_tasks.keys()] maintanance_on_task = get_task( task_group_id, maintanance_on_dependencies, github_commit, channel, taskcluster_secret, './please -vv tools maintanance:on ' + ' '.join(list(set([i[0] for i in projects_to_deploy]))), { 'name': '2. Maintanance ON', 'description': '', 'owner': owner, 'source': 'https://github.com/mozilla/release-services/tree/' + channel, }, ) tasks.append((maintanance_on_uuid, maintanance_on_task)) # 3. deploy tasks (if on production/staging) deploy_tasks = {} for index, (project, project_requires, deploy_target, deploy_options) in \ enumerate(sorted(projects_to_deploy, key=lambda x: x[0])): try: enable = deploy_options['enable'] except KeyError: raise click.ClickException(f'Missing {enable} in project {project} and channel {channel} deploy options') if not enable: continue project_uuid = slugid.nice().decode('utf-8') project_task = get_deploy_task( index, project, project_requires, deploy_target, deploy_options, task_group_id, maintanance_on_uuid, github_commit, owner, channel, taskcluster_secret, ) if project_task: deploy_tasks[project_uuid] = project_task tasks.append((project_uuid, deploy_tasks[project_uuid])) # 4. maintanance off task maintanance_off_uuid = slugid.nice().decode('utf-8') maintanance_off_task = get_task( task_group_id, [i for i in deploy_tasks.keys()], github_commit, channel, taskcluster_secret, './please -vv tools maintanance:off ' + ' '.join(list(set([i[0] for i in projects_to_deploy]))), { 'name': '4. Maintanance OFF', 'description': '', 'owner': owner, 'source': 'https://github.com/mozilla/release-services/tree/' + channel, }, ) maintanance_off_task['requires'] = 'all-resolved' tasks.append((maintanance_off_uuid, maintanance_off_task)) click.echo(' => Submitting taskcluster definitions to taskcluster') if dry_run: tasks2 = {task_id: task for task_id, task in tasks} for task_id, task in tasks: click.echo(' => %s [taskId: %s]' % (task['metadata']['name'], task_id)) click.echo(' dependencies:') deps = [] for dep in task['dependencies']: depName = '0. Decision task' if dep in tasks2: depName = tasks2[dep]['metadata']['name'] deps.append(' - %s [taskId: %s]' % (depName, dep)) for dep in sorted(deps): click.echo(dep) else: for task_id, task in tasks: taskcluster_queue.createTask(task_id, task)
def makeTaskId(): """Used in testing to generate task ids without talking to TaskCluster.""" return slugid.nice()
def slugId(): """ Generate a taskcluster slugid. This is a V4 UUID encoded into URL-Safe Base64 (RFC 4648, sec 5) with '=' padding removed """ return slugid.nice()
def make_decision_task(params, symbol, arguments=[], head_rev=None): """Generate a basic decision task, based on the root .taskcluster.yml""" with open('.taskcluster.yml') as f: taskcluster_yml = f.read() if not head_rev: head_rev = params['head_rev'] # do a cheap and dirty job of the template substitution that mozilla-taskcluster # does when it reads .taskcluster.yml comment = '"no push -- cron task \'{job_name}\'"'.format(**params), replacements = { '\'{{{?now}}}?\'': "{'relative-datestamp': '0 seconds'}", '{{{?owner}}}?': '*****@*****.**', '{{#shellquote}}{{{comment}}}{{/shellquote}}': comment, '{{{?source}}}?': params['head_repository'], '{{{?url}}}?': params['head_repository'], '{{{?project}}}?': params['project'], '{{{?level}}}?': params['level'], '{{{?revision}}}?': head_rev, '\'{{#from_now}}([^{]*){{/from_now}}\'': "{'relative-datestamp': '\\1'}", '{{{?pushdate}}}?': '0', # treeherder ignores pushlog_id, so set it to -1 '{{{?pushlog_id}}}?': '-1', # omitted as unnecessary # {{#as_slugid}}..{{/as_slugid}} } for pattern, replacement in replacements.iteritems(): taskcluster_yml = re.sub(pattern, replacement, taskcluster_yml) task = yaml.load(taskcluster_yml)['tasks'][0]['task'] # set some metadata task['metadata']['name'] = 'Decision task for cron job ' + params['job_name'] cron_task_id = os.environ.get('TASK_ID', '<cron task id>') descr_md = 'Created by a [cron task](https://tools.taskcluster.net/task-inspector/#{}/)' task['metadata']['description'] = descr_md.format(cron_task_id) th = task['extra']['treeherder'] th['groupSymbol'] = 'cron' th['symbol'] = symbol # add a scope based on the repository, with a cron:<job_name> suffix match = re.match(r'https://(hg.mozilla.org)/(.*?)/?$', params['head_repository']) if not match: raise Exception('Unrecognized head_repository') repo_scope = 'assume:repo:{}/{}:cron:{}'.format( match.group(1), match.group(2), params['job_name']) task.setdefault('scopes', []).append(repo_scope) # append arguments, quoted, to the decision task command shellcmd = task['payload']['command'] shellcmd[-1] = shellcmd[-1].rstrip('\n') # strip yaml artifact for arg in arguments: shellcmd[-1] += ' ' + pipes.quote(arg) task_id = slugid.nice() # set taskGroupid = taskId, as expected of decision tasks by other systems. # This creates a new taskGroup for this graph. task['taskGroupId'] = task_id # set the schedulerId based on the level task['schedulerId'] = 'gecko-level-{}'.format(params['level']) return (task_id, task)