def notifications(user): model = Notification() doc1 = model.createNotification('type', {}, user) doc2 = model.createNotification('type', {}, user) doc2['updated'] = OLD_TIME doc2['time'] = OLD_TIME model.save(doc2) yield [doc1, doc2]
def _updateProgress(self, job, total, current, message, notify, user, updates): """Helper for updating job progress information.""" state = JobStatus.toNotificationStatus(job['status']) if current is not None: current = float(current) if total is not None: total = float(total) if job['progress'] is None: if notify and job['userId']: notification = self._createProgressNotification( job, total, current, state, message) notificationId = notification['_id'] else: notificationId = None job['progress'] = { 'message': message, 'total': total, 'current': current, 'notificationId': notificationId } updates['$set']['progress'] = job['progress'] else: if total is not None: job['progress']['total'] = total updates['$set']['progress.total'] = total if current is not None: job['progress']['current'] = current updates['$set']['progress.current'] = current if message is not None: job['progress']['message'] = message updates['$set']['progress.message'] = message if notify and user: if job['progress']['notificationId'] is None: notification = self._createProgressNotification( job, total, current, state, message, user) nid = notification['_id'] job['progress']['notificationId'] = nid updates['$set']['progress.notificationId'] = nid else: notification = Notification().load( job['progress']['notificationId']) Notification().updateProgress( notification, state=state, message=job['progress']['message'], current=job['progress']['current'], total=job['progress']['total'])
def updateNotification(event): """ Update the Whole Tale task notification for a job, if present. """ job = event.info['job'] if job['progress'] and 'wt_notification_id' in job: state = JobStatus.toNotificationStatus(job['status']) notification = Notification().load(job['wt_notification_id']) state_changed = notification['data']['state'] != state message_changed = notification['data']['message'] != job['progress'][ 'message'] # Ignore duplicate events based on state and message content if not state_changed and not message_changed: return # For multi-job tasks, ignore success for intermediate events is_last = notification['data']['total'] == ( notification['data']['current']) if state == ProgressState.SUCCESS and not is_last: return # Add job IDs to the resource if 'jobs' not in notification['data']['resource']: notification['data']['resource']['jobs'] = [] if job['_id'] not in notification['data']['resource']['jobs']: notification['data']['resource']['jobs'].append(job['_id']) # If the state hasn't changed, increment. Otherwise keep previous current value. # Note, if expires parameter is not provided, updateProgress resets to 1 hour if not state_changed: Notification().updateProgress(notification, state=state, expires=notification['expires'], message=job['progress']['message'], increment=1, total=notification['data']['total']) else: Notification().updateProgress( notification, state=state, expires=notification['expires'], message=job['progress']['message'], current=notification['data']['current'], total=notification['data']['total'])
def _createUpdateStatusNotification(self, now, user, job): expires = now + datetime.timedelta(seconds=30) filtered = self.filter(job, user) filtered.pop('kwargs', None) filtered.pop('log', None) Notification().createNotification( type='job_status', data=filtered, user=user, expires=expires)
def _finish_orbital_gen(mo, id, user, orig_mo, future): resp = future.result() if resp.status_code == 200: cjson = json.loads(resp.text) cjson['generating_orbital'] = False if 'vibrations' in cjson: del cjson['vibrations'] # Add cube to cache ModelImporter.model('cubecache', 'molecules').create(id, mo, cjson) # Create notification to indicate cube can be retrieved now data = {'id': id, 'mo': orig_mo} else: data = { 'id': id, 'mo': orig_mo, 'error': 'Status code ' + str(resp.status_code) + ': Orbital could not be calculated.' } Notification().createNotification(type='cube.status', data=data, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
def __init__(self, on, interval=0.5, **kwargs): self.on = on self.interval = interval if on: self._lastSave = time.time() self.progress = Notification().initProgress(**kwargs)
def advanceWorkflow(event): """ Advance to next step in workflow. Send a notification on error. """ jobId = event.info['jobId'] stepName = event.info['stepName'] userId = event.info['userId'] user = User().load(userId, force=True, exc=True) try: DanesfieldWorkflowManager.instance().advance(jobId=jobId) except DanesfieldWorkflowException as e: logprint.warning( 'advanceWorkflow: Error advancing workflow ' 'Job={} Step={} PreviousStep={} Message=\'{}\''.format( jobId, e.step, stepName, str(e))) # Create notification for workflow error Notification().createNotification(type='danesfield_workflow_error', data={ 'step': e.step or '', 'previousStep': stepName, 'message': str(e) }, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
def _updateJob(event): """ Called when a job is saved, updated, or removed. If this is a large image job and it is ended, clean up after it. """ from girder.plugins.jobs.constants import JobStatus from girder.plugins.jobs.models.job import Job job = event.info[ 'job'] if event.name == 'jobs.job.update.after' else event.info meta = job.get('meta', {}) if (meta.get('creator') != 'large_image' or not meta.get('itemId') or meta.get('task') != 'createImageItem'): return status = job['status'] if event.name == 'model.job.remove' and status not in (JobStatus.ERROR, JobStatus.CANCELED, JobStatus.SUCCESS): status = JobStatus.CANCELED if status not in (JobStatus.ERROR, JobStatus.CANCELED, JobStatus.SUCCESS): return item = Item().load(meta['itemId'], force=True) if not item or 'largeImage' not in item: return if item.get('largeImage', {}).get('expected'): # We can get a SUCCESS message before we get the upload message, so # don't clear the expected status on success. if status != JobStatus.SUCCESS: del item['largeImage']['expected'] notify = item.get('largeImage', {}).get('notify') msg = None if notify: del item['largeImage']['notify'] if status == JobStatus.SUCCESS: msg = 'Large image created' elif status == JobStatus.CANCELED: msg = 'Large image creation canceled' else: # ERROR msg = 'FAILED: Large image creation failed' msg += ' for item %s' % item['name'] if (status in (JobStatus.ERROR, JobStatus.CANCELED) and 'largeImage' in item): del item['largeImage'] Item().save(item) if msg and event.name != 'model.job.remove': Job().updateJob(job, progressMessage=msg) if notify: Notification().createNotification( type='large_image.finished_image_item', data={ 'job_id': job['_id'], 'item_id': item['_id'], 'success': status == JobStatus.SUCCESS, 'status': status }, user={'_id': job.get('userId')}, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
def _createProgressNotification(self, job, total, current, state, message, user=None): if not user: user = User().load(job['userId'], force=True) # TODO support channel-based notifications for jobs. For # right now we'll just go through the user. return Notification().initProgress( user, job['title'], total, state=state, current=current, message=message, estimateTime=False, resource=job, resourceName=self.name)
def _updateLog(self, job, log, overwrite, now, notify, user, updates): """Helper for updating a job's log.""" if overwrite: updates['$set']['log'] = [log] else: updates['$push']['log'] = log if notify and user: expires = now + datetime.timedelta(seconds=30) Notification().createNotification( type='job_log', data={ '_id': job['_id'], 'overwrite': overwrite, 'text': log }, user=user, expires=expires)
def update(self, force=False, **kwargs): """ Update the underlying progress record. This will only actually save to the database if at least self.interval seconds have passed since the last time the record was written to the database. Accepts the same kwargs as Notification.updateProgress. :param force: Whether we should force the write to the database. Use only in cases where progress may be indeterminate for a long time. :type force: bool """ # Extend the response timeout, even if we aren't reporting the progress setResponseTimeLimit() if not self.on: return save = (time.time() - self._lastSave > self.interval) or force self.progress = Notification().updateProgress(self.progress, save, **kwargs) if save: self._lastSave = time.time()
def __exit__(self, excType, excValue, traceback): """ Once the context is exited, the progress is marked for deletion 30 seconds in the future, which should give all listeners time to poll and receive the final state of the progress record before it is deleted. """ if not self.on: return if excType is None and excValue is None: state = ProgressState.SUCCESS message = 'Done' else: state = ProgressState.ERROR message = 'Error' if isinstance(excValue, (ValidationException, RestException)): message = 'Error: '+excValue.message Notification().updateProgress( self.progress, state=state, message=message, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30) )
def run(job): Job().updateJob(job, status=JobStatus.RUNNING) print 'create bsve datasets' try: kwargs = job['kwargs'] user = kwargs['user'] bsveRoot = kwargs['bsveRoot'] extraHeaders = kwargs['extraHeaders'] Notification().createNotification(type='bsveDatasetUpdating', data=job, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30)) """ Hits the bsve urls """ # Bsve geoserver (wms get capabilities url) wms = bsveRoot + "/data/v2/sources/geotiles/meta/list" resp = requests.get(wms, headers=extraHeaders) data = json.loads(resp.text) existingLayers = getLayers(user) newLayers = set() for d in data['tiles']: typeName = d['name'] newLayers.add(typeName) # For now skip updating layers that always exist. # When we have a reliable ingestion time stamp, # we should check the creation date and update # if the ingestion date is later. if typeName in existingLayers: continue wms_params = {} wms_params['type_name'] = typeName wms_params['name'] = d.get('title') or d['styles'][0]['title'] wms_params['abstract'] = d['abstract'] wms_params['source'] = { 'layer_source': 'Reference', 'source_type': 'wms' } wms_params['geo_render'] = {'type': 'wms'} wms_params['category'] = _get_category(d) wms_params['metadata'] = _get_metadata(d) layer_type = 'raster' if 'WCS' in d['keywords'] else 'vector' createBsveDataset(user, bsveRoot, extraHeaders, wms_params, layer_type) # delete layers that no longer exist on the server for typeName in existingLayers: if typeName not in newLayers: item = existingLayers[typeName] Item().remove(item) Job().updateJob(job, status=JobStatus.SUCCESS) Notification().createNotification(type='bsveDatasetUpdated', data=job, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30)) except Exception as e: print e print traceback.print_exc() Notification().createNotification(type='bsveDatasetError', data=job, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
def createJob(self, title, type, taskName, taskEntry, modules="", args=(), kwargs=None, user=None, when=None, interval=0, public=False, handler=None, asynchronous=False, save=True, parentJob=None, otherFields=None): now = datetime.datetime.utcnow() if when is None: when = now if kwargs is None: kwargs = {} slurmOptions = self.findOne({'user': user['_id']}) if slurmOptions is None: slurmOptions = { 'user': user['_id'], 'partition': 'gpuib', 'gres': "gpu:4", 'nodes': 1, 'ntasks': 1, 'cpu_per_task': 1, 'mem_per_cpu': 16, 'time': 1, 'modules': "" } self.save(slurmOptions) otherFields = { 'otherFields': { 'slurm_info': { 'name': taskName, 'entry': taskEntry, 'partition': slurmOptions['partition'], 'nodes': slurmOptions['nodes'], 'ntasks': slurmOptions['ntasks'], 'gres': slurmOptions['gres'], 'cpu_per_task': slurmOptions['cpu_per_task'], 'mem_per_cpu': str(slurmOptions['mem_per_cpu']) + 'gb', 'time': str(slurmOptions['time']) + ':00:00', 'modules': modules } } } parentId = None if parentJob: parentId = parentJob['_id'] job = { 'title': title, 'type': type, 'args': args, 'kwargs': kwargs, 'created': now, 'updated': now, 'when': when, 'interval': interval, 'status': JobStatus.INACTIVE, 'progress': None, 'log': [], 'meta': {}, 'handler': handler, 'asynchronous': asynchronous, 'timestamps': [], 'parentId': parentId } job.update(otherFields) self.setPublic(job, public=public) if user: job['userId'] = user['_id'] self.setUserAccess(job, user=user, level=AccessType.ADMIN) else: job['userId'] = None if save: job = JobModel().save(job) if user: deserialized_kwargs = job['kwargs'] job['kwargs'] = json_util.dumps(job['kwargs']) Notification().createNotification( type='job_created', data=job, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30)) job['kwargs'] = deserialized_kwargs return job
def testNotification(self): from girder.plugins.jobs.models.job import Job from girder.plugins.jobs.constants import JobStatus from girder.models.notification import Notification, ProgressState from girder.plugins.wholetale.utils import init_progress from girder.plugins.worker.constants import PluginSettings as WorkerPluginSettings # TODO: Why do we need it here? Setting().set(WorkerPluginSettings.API_URL, 'http://localhost:8080/api/v1') total = 2 resource = {'type': 'wt_test_build_image', 'instance_id': 'instance_id'} notification = init_progress( resource, self.user, 'Test notification', 'Creating job', total ) # Job to test error path job = Job().createJob( title='Error test job', type='test', handler='my_handler', user=self.user, public=False, args=["tale_id"], kwargs={}, otherFields={'wt_notification_id': str(notification['_id'])}, ) job = Job().updateJob( job, status=JobStatus.INACTIVE, progressTotal=2, progressCurrent=0) self.assertEqual(job['status'], JobStatus.INACTIVE) time.sleep(1) notification = Notification().load(notification['_id']) self.assertEqual(notification['data']['state'], ProgressState.QUEUED) self.assertEqual(notification['data']['total'], 2) self.assertEqual(notification['data']['current'], 0) self.assertEqual(notification['data']['resource']['jobs'][0], job['_id']) # State change to ACTIVE job = Job().updateJob(job, status=JobStatus.RUNNING) self.assertEqual(job['status'], JobStatus.RUNNING) # Progress update job = Job().updateJob( job, status=JobStatus.RUNNING, progressCurrent=1, progressMessage="Error test message") time.sleep(1) notification = Notification().load(notification['_id']) self.assertEqual(notification['data']['state'], ProgressState.ACTIVE) self.assertEqual(notification['data']['total'], 2) self.assertEqual(notification['data']['current'], 1) self.assertEqual(notification['data']['message'], 'Error test message') # State change to ERROR job = Job().updateJob(job, status=JobStatus.ERROR) time.sleep(1) notification = Notification().load(notification['_id']) self.assertEqual(notification['data']['state'], ProgressState.ERROR) self.assertEqual(notification['data']['total'], 2) self.assertEqual(notification['data']['current'], 1) self.assertEqual(notification['data']['message'], 'Error test message') # New job to test success path job = Job().createJob( title='Test Job', type='test', handler='my_handler', user=self.user, public=False, args=["tale_id"], kwargs={}, otherFields={'wt_notification_id': str(notification['_id'])}, ) # State change to ACTIVE job = Job().updateJob(job, status=JobStatus.RUNNING) self.assertEqual(job['status'], JobStatus.RUNNING) # Progress update job = Job().updateJob( job, status=JobStatus.RUNNING, progressCurrent=1, progressMessage="Success test message") time.sleep(1) notification = Notification().load(notification['_id']) self.assertEqual(notification['data']['state'], ProgressState.ACTIVE) self.assertEqual(notification['data']['total'], 2) self.assertEqual(notification['data']['current'], 1) self.assertEqual(notification['data']['message'], 'Success test message') job = Job().updateJob(job, status=JobStatus.SUCCESS) time.sleep(1) notification = Notification().load(notification['_id']) self.assertEqual(notification['data']['state'], ProgressState.ACTIVE) self.assertEqual(notification['data']['total'], 2) self.assertEqual(notification['data']['current'], 1) self.assertEqual(notification['data']['message'], 'Success test message')
def testDeleteFolder(self): cbInfo = {} # Hook into model deletion with kwargs event to test it def cb(event): cbInfo['kwargs'] = event.info['kwargs'] cbInfo['doc'] = event.info['document'] with events.bound('model.folder.remove_with_kwargs', 'test', cb): # Requesting with no path should fail resp = self.request(path='/folder', method='DELETE', user=self.admin) self.assertStatus(resp, 400) # Grab one of the user's top level folders folders = Folder().childFolders( parent=self.admin, parentType='user', user=self.admin, limit=1, sort=[('name', SortDir.DESCENDING)]) folderResp = six.next(folders) # Add a subfolder and an item to that folder subfolder = Folder().createFolder( folderResp, 'sub', parentType='folder', creator=self.admin) item = Item().createItem('item', creator=self.admin, folder=subfolder) self.assertTrue('_id' in subfolder) self.assertTrue('_id' in item) # Delete the folder resp = self.request(path='/folder/%s' % folderResp['_id'], method='DELETE', user=self.admin, params={ 'progress': 'true' }) self.assertStatusOk(resp) # Make sure the folder, its subfolder, and its item were all deleted folder = Folder().load(folderResp['_id'], force=True) subfolder = Folder().load(subfolder['_id'], force=True) item = Item().load(item['_id']) self.assertEqual(folder, None) self.assertEqual(subfolder, None) self.assertEqual(item, None) # Make sure progress record exists and that it is set to expire soon notifs = list(Notification().get(self.admin)) self.assertEqual(len(notifs), 1) self.assertEqual(notifs[0]['type'], 'progress') self.assertEqual(notifs[0]['data']['state'], ProgressState.SUCCESS) self.assertEqual(notifs[0]['data']['title'], 'Deleting folder Public') self.assertEqual(notifs[0]['data']['message'], 'Done') self.assertEqual(notifs[0]['data']['total'], 3) self.assertEqual(notifs[0]['data']['current'], 3) self.assertTrue(notifs[0]['expires'] < datetime.datetime.utcnow() + datetime.timedelta(minutes=1)) # Make sure our event handler was called with expected args self.assertTrue('kwargs' in cbInfo) self.assertTrue('doc' in cbInfo) self.assertTrue('progress' in cbInfo['kwargs']) self.assertEqual(cbInfo['doc']['_id'], folderResp['_id'])
def _updateJob(event): """ Called when a job is saved, updated, or removed. If this is a histogram job and it is ended, clean up after it. """ if event.name == 'jobs.job.update.after': job = event.info['job'] else: job = event.info meta = job.get('meta', {}) if (meta.get('creator') != 'histogram' or meta.get('task') != 'createHistogram'): return status = job['status'] if event.name == 'model.job.remove' and status not in (JobStatus.ERROR, JobStatus.CANCELED, JobStatus.SUCCESS): status = JobStatus.CANCELED if status not in (JobStatus.ERROR, JobStatus.CANCELED, JobStatus.SUCCESS): return histograms = list(Histogram().find({'fakeId': meta.get('fakeId')}, limit=2)) if len(histograms) != 1: msg = 'Failed to retrieve histogram using fakeId %s.' logger.warning(msg % meta.get('fakeId')) return histogram = histograms[0] if histogram.get('expected'): # We can get a SUCCESS message before we get the upload message, so # don't clear the expected status on success. if status != JobStatus.SUCCESS: del histogram['expected'] notify = histogram.get('notify') msg = None if notify: del histogram['notify'] if status == JobStatus.SUCCESS: msg = 'Histogram created' elif status == JobStatus.CANCELED: msg = 'Histogram creation canceled' else: # ERROR msg = 'FAILED: Histogram creation failed' msg += ' for item %s' % histogram['itemId'] msg += ', file %s' % histogram['fileId'] if status == JobStatus.SUCCESS: Histogram().save(histogram) else: Histogram().remove(histogram) if msg and event.name != 'model.job.remove': Job().updateJob(job, progressMessage=msg) if notify: Notification().createNotification(type='histogram.finished_histogram', data={ 'histogram_id': histogram['_id'], 'item_id': histogram['itemId'], 'file_id': histogram['fileId'], 'fakeId': histogram['fakeId'], 'success': status == JobStatus.SUCCESS, 'status': status }, user={'_id': job.get('userId')}, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
self.setPublic(job, public=public) if user: job['userId'] = user['_id'] self.setUserAccess(job, user=user, level=AccessType.ADMIN) else: job['userId'] = None if save: job = self.save(job) if user: deserialized_kwargs = job['kwargs'] job['kwargs'] = json_util.dumps(job['kwargs']) Notification().createNotification( type='job_created', data=job, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30)) job['kwargs'] = deserialized_kwargs return job def save(self, job, *args, **kwargs): """ We extend save so that we can serialize the kwargs before sending them to the database. This will allow kwargs with $ and . characters in the keys. """ deserialized = job['kwargs'] job['kwargs'] = json_util.dumps(job['kwargs']) job = AccessControlledModel.save(self, job, *args, **kwargs)
def testDownloadResources(self): self._createFiles() resourceList = { 'collection': [str(self.collection['_id'])], 'user': [str(self.admin['_id'])] } # We should fail with bad json, an empty list, an invalid item in the # list, or a list that is an odd format. resp = self.request(path='/resource/download', method='GET', user=self.admin, params={ 'resources': 'this_is_not_json', }, isJson=False) self.assertStatus(resp, 400) resp = self.request(path='/resource/download', method='GET', user=self.admin, params={ 'resources': json.dumps('this_is_not_a_dict_of_resources') }, isJson=False) self.assertStatus(resp, 400) resp = self.request(path='/resource/download', method='GET', user=self.admin, params={ 'resources': json.dumps({'not_a_resource': ['not_an_id']}) }, isJson=False) self.assertStatus(resp, 400) resp = self.request(path='/resource/download', method='GET', user=self.admin, params={'resources': json.dumps({'item': []})}, isJson=False) self.assertStatus(resp, 400) resp = self.request(path='/resource/download', method='GET', user=self.admin, params={ 'resources': json.dumps({'item': [str(self.admin['_id'])]}) }, isJson=False) self.assertStatus(resp, 400) # Download the resources resp = self.request(path='/resource/download', method='GET', user=self.admin, params={ 'resources': json.dumps(resourceList), 'includeMetadata': True }, isJson=False) self.assertStatusOk(resp) self.assertEqual(resp.headers['Content-Type'], 'application/zip') zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') self.assertTrue(zip.testzip() is None) self.assertHasKeys(self.expectedZip, zip.namelist()) self.assertHasKeys(zip.namelist(), self.expectedZip) for name in zip.namelist(): expected = self.expectedZip[name] if isinstance(expected, dict): self.assertEqual(json.loads(zip.read(name).decode('utf8')), json.loads(json.dumps(expected, default=str))) else: if not isinstance(expected, six.binary_type): expected = expected.encode('utf8') self.assertEqual(expected, zip.read(name)) # Download the same resources again, this time triggering the large zip # file creation (artificially forced). We could do this naturally by # downloading >65536 files, but that would make the test take several # minutes. girder.utility.ziputil.Z_FILECOUNT_LIMIT = 5 resourceList = {'item': [str(item['_id']) for item in self.items]} resp = self.request(path='/resource/download', method='POST', user=self.admin, params={ 'resources': json.dumps(resourceList), 'includeMetadata': True }, isJson=False, additionalHeaders=[('X-HTTP-Method-Override', 'GET')]) self.assertStatusOk(resp) self.assertEqual(resp.headers['Content-Type'], 'application/zip') zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') self.assertTrue(zip.testzip() is None) # Test deleting resources resourceList = { 'collection': [str(self.collection['_id'])], 'folder': [str(self.adminSubFolder['_id'])], } resp = self.request(path='/resource', method='DELETE', user=self.admin, params={ 'resources': json.dumps(resourceList), 'progress': True }, isJson=False) self.assertStatusOk(resp) # Make sure progress record exists and that it is set to expire soon notifs = list(Notification().get(self.admin)) self.assertEqual(len(notifs), 1) self.assertEqual(notifs[0]['type'], 'progress') self.assertEqual(notifs[0]['data']['state'], ProgressState.SUCCESS) self.assertEqual(notifs[0]['data']['title'], 'Deleting resources') self.assertEqual(notifs[0]['data']['message'], 'Done') self.assertEqual(notifs[0]['data']['total'], 6) self.assertEqual(notifs[0]['data']['current'], 6) self.assertTrue(notifs[0]['expires'] < datetime.datetime.utcnow() + datetime.timedelta(minutes=1)) # Test deletes using a body on the request resourceList = {'item': [str(self.items[1]['_id'])]} resp = self.request(path='/resource', method='DELETE', user=self.admin, body=urllib.parse.urlencode( {'resources': json.dumps(resourceList)}), type='application/x-www-form-urlencoded', isJson=False) self.assertStatusOk(resp) # Test deletes using POST and override method resourceList = {'item': [str(self.items[0]['_id'])]} resp = self.request(path='/resource', method='POST', user=self.admin, params={'resources': json.dumps(resourceList)}, isJson=False, additionalHeaders=[('X-HTTP-Method-Override', 'DELETE')]) self.assertStatusOk(resp) # All of the items should be gone now resp = self.request(path='/item', method='GET', user=self.admin, params={'text': 'Item'}) self.assertStatusOk(resp) self.assertEqual(len(resp.json), 0) # Add a file under the admin private folder item = Item().createItem('Private Item', self.admin, self.adminPrivateFolder) _, path, contents = self._uploadFile('private_file', item) self.assertEqual(path, 'goodlogin/Private/Private Item/private_file') # Download as admin, should get private file resp = self.request(path='/resource/download', method='GET', user=self.admin, params={ 'resources': json.dumps({'user': [str(self.admin['_id'])]}) }, isJson=False) self.assertStatusOk(resp) self.assertEqual(resp.headers['Content-Type'], 'application/zip') zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') self.assertTrue(zip.testzip() is None) self.assertEqual(zip.namelist(), [path]) self.assertEqual(zip.read(path), contents) # Download as normal user, should get empty zip resp = self.request(path='/resource/download', method='GET', user=self.user, params={ 'resources': json.dumps({'user': [str(self.admin['_id'])]}) }, isJson=False) self.assertStatusOk(resp) self.assertEqual(resp.headers['Content-Type'], 'application/zip') zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r') self.assertTrue(zip.testzip() is None) self.assertEqual(zip.namelist(), [])
def createJob(self, title, type, args=(), kwargs=None, user=None, when=None, interval=0, public=False, handler=None, asynchronous=False, save=True, parentJob=None, otherFields=None): """ Create a new job record. :param title: The title of the job. :type title: str :param type: The type of the job. :type type: str :param args: Positional args of the job payload. :type args: list or tuple :param kwargs: Keyword arguments of the job payload. :type kwargs: dict :param user: The user creating the job. :type user: dict or None :param when: Minimum start time for the job (UTC). :type when: datetime :param interval: If this job should be recurring, set this to a value in seconds representing how often it should occur. Set to <= 0 for jobs that should only be run once. :type interval: int :param public: Public read access flag. :type public: bool :param handler: If this job should be handled by a specific handler, use this field to store that information. :param externalToken: If an external token was created for updating this job, pass it in and it will have the job-specific scope set. :type externalToken: token (dict) or None. :param asynchronous: Whether the job is to be run asynchronously. For now this only applies to jobs that are scheduled to run locally. :type asynchronous: bool :param save: Whether the documented should be saved to the database. :type save: bool :param parentJob: The job which will be set as a parent :type parentJob: Job :param otherFields: Any additional fields to set on the job. :type otherFields: dict """ now = datetime.datetime.utcnow() if when is None: when = now if kwargs is None: kwargs = {} otherFields = otherFields or {} parentId = None if parentJob: parentId = parentJob['_id'] job = { 'title': title, 'type': type, 'args': args, 'kwargs': kwargs, 'created': now, 'updated': now, 'when': when, 'interval': interval, 'status': JobStatus.INACTIVE, 'progress': None, 'log': [], 'meta': {}, 'handler': handler, 'asynchronous': asynchronous, 'timestamps': [], 'parentId': parentId } job.update(otherFields) self.setPublic(job, public=public) if user: job['userId'] = user['_id'] self.setUserAccess(job, user=user, level=AccessType.ADMIN) else: job['userId'] = None if save: job = self.save(job) if user: deserialized_kwargs = job['kwargs'] job['kwargs'] = json_util.dumps(job['kwargs']) Notification().createNotification( type='job_created', data=job, user=user, expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30)) job['kwargs'] = deserialized_kwargs return job