def notifications(user):
model = Notification()
doc1 = model.createNotification('type', {}, user)
doc2 = model.createNotification('type', {}, user)
doc2['updated'] = OLD_TIME
doc2['time'] = OLD_TIME
model.save(doc2)
yield [doc1, doc2]
def _updateProgress(self, job, total, current, message, notify, user,
updates):
"""Helper for updating job progress information."""
state = JobStatus.toNotificationStatus(job['status'])
if current is not None:
current = float(current)
if total is not None:
total = float(total)
if job['progress'] is None:
if notify and job['userId']:
notification = self._createProgressNotification(
job, total, current, state, message)
notificationId = notification['_id']
else:
notificationId = None
job['progress'] = {
'message': message,
'total': total,
'current': current,
'notificationId': notificationId
}
updates['$set']['progress'] = job['progress']
else:
if total is not None:
job['progress']['total'] = total
updates['$set']['progress.total'] = total
if current is not None:
job['progress']['current'] = current
updates['$set']['progress.current'] = current
if message is not None:
job['progress']['message'] = message
updates['$set']['progress.message'] = message
if notify and user:
if job['progress']['notificationId'] is None:
notification = self._createProgressNotification(
job, total, current, state, message, user)
nid = notification['_id']
job['progress']['notificationId'] = nid
updates['$set']['progress.notificationId'] = nid
else:
notification = Notification().load(
job['progress']['notificationId'])
Notification().updateProgress(
notification,
state=state,
message=job['progress']['message'],
current=job['progress']['current'],
total=job['progress']['total'])
def updateNotification(event):
"""
Update the Whole Tale task notification for a job, if present.
"""
job = event.info['job']
if job['progress'] and 'wt_notification_id' in job:
state = JobStatus.toNotificationStatus(job['status'])
notification = Notification().load(job['wt_notification_id'])
state_changed = notification['data']['state'] != state
message_changed = notification['data']['message'] != job['progress'][
'message']
# Ignore duplicate events based on state and message content
if not state_changed and not message_changed:
return
# For multi-job tasks, ignore success for intermediate events
is_last = notification['data']['total'] == (
notification['data']['current'])
if state == ProgressState.SUCCESS and not is_last:
return
# Add job IDs to the resource
if 'jobs' not in notification['data']['resource']:
notification['data']['resource']['jobs'] = []
if job['_id'] not in notification['data']['resource']['jobs']:
notification['data']['resource']['jobs'].append(job['_id'])
# If the state hasn't changed, increment. Otherwise keep previous current value.
# Note, if expires parameter is not provided, updateProgress resets to 1 hour
if not state_changed:
Notification().updateProgress(notification,
state=state,
expires=notification['expires'],
message=job['progress']['message'],
increment=1,
total=notification['data']['total'])
else:
Notification().updateProgress(
notification,
state=state,
expires=notification['expires'],
message=job['progress']['message'],
current=notification['data']['current'],
total=notification['data']['total'])
def _createUpdateStatusNotification(self, now, user, job):
expires = now + datetime.timedelta(seconds=30)
filtered = self.filter(job, user)
filtered.pop('kwargs', None)
filtered.pop('log', None)
Notification().createNotification(
type='job_status', data=filtered, user=user, expires=expires)
def _finish_orbital_gen(mo, id, user, orig_mo, future):
resp = future.result()
if resp.status_code == 200:
cjson = json.loads(resp.text)
cjson['generating_orbital'] = False
if 'vibrations' in cjson:
del cjson['vibrations']
# Add cube to cache
ModelImporter.model('cubecache', 'molecules').create(id, mo, cjson)
# Create notification to indicate cube can be retrieved now
data = {'id': id, 'mo': orig_mo}
else:
data = {
'id':
id,
'mo':
orig_mo,
'error':
'Status code ' + str(resp.status_code) +
': Orbital could not be calculated.'
}
Notification().createNotification(type='cube.status',
data=data,
user=user,
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
def __init__(self, on, interval=0.5, **kwargs):
self.on = on
self.interval = interval
if on:
self._lastSave = time.time()
self.progress = Notification().initProgress(**kwargs)
def advanceWorkflow(event):
"""
Advance to next step in workflow. Send a notification on error.
"""
jobId = event.info['jobId']
stepName = event.info['stepName']
userId = event.info['userId']
user = User().load(userId, force=True, exc=True)
try:
DanesfieldWorkflowManager.instance().advance(jobId=jobId)
except DanesfieldWorkflowException as e:
logprint.warning(
'advanceWorkflow: Error advancing workflow '
'Job={} Step={} PreviousStep={} Message=\'{}\''.format(
jobId, e.step, stepName, str(e)))
# Create notification for workflow error
Notification().createNotification(type='danesfield_workflow_error',
data={
'step': e.step or '',
'previousStep': stepName,
'message': str(e)
},
user=user,
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
def _updateJob(event):
"""
Called when a job is saved, updated, or removed. If this is a large image
job and it is ended, clean up after it.
"""
from girder.plugins.jobs.constants import JobStatus
from girder.plugins.jobs.models.job import Job
job = event.info[
'job'] if event.name == 'jobs.job.update.after' else event.info
meta = job.get('meta', {})
if (meta.get('creator') != 'large_image' or not meta.get('itemId')
or meta.get('task') != 'createImageItem'):
return
status = job['status']
if event.name == 'model.job.remove' and status not in (JobStatus.ERROR,
JobStatus.CANCELED,
JobStatus.SUCCESS):
status = JobStatus.CANCELED
if status not in (JobStatus.ERROR, JobStatus.CANCELED, JobStatus.SUCCESS):
return
item = Item().load(meta['itemId'], force=True)
if not item or 'largeImage' not in item:
return
if item.get('largeImage', {}).get('expected'):
# We can get a SUCCESS message before we get the upload message, so
# don't clear the expected status on success.
if status != JobStatus.SUCCESS:
del item['largeImage']['expected']
notify = item.get('largeImage', {}).get('notify')
msg = None
if notify:
del item['largeImage']['notify']
if status == JobStatus.SUCCESS:
msg = 'Large image created'
elif status == JobStatus.CANCELED:
msg = 'Large image creation canceled'
else: # ERROR
msg = 'FAILED: Large image creation failed'
msg += ' for item %s' % item['name']
if (status in (JobStatus.ERROR, JobStatus.CANCELED)
and 'largeImage' in item):
del item['largeImage']
Item().save(item)
if msg and event.name != 'model.job.remove':
Job().updateJob(job, progressMessage=msg)
if notify:
Notification().createNotification(
type='large_image.finished_image_item',
data={
'job_id': job['_id'],
'item_id': item['_id'],
'success': status == JobStatus.SUCCESS,
'status': status
},
user={'_id': job.get('userId')},
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
def _createProgressNotification(self, job, total, current, state, message,
user=None):
if not user:
user = User().load(job['userId'], force=True)
# TODO support channel-based notifications for jobs. For
# right now we'll just go through the user.
return Notification().initProgress(
user, job['title'], total, state=state, current=current,
message=message, estimateTime=False, resource=job, resourceName=self.name)
def _updateLog(self, job, log, overwrite, now, notify, user, updates):
"""Helper for updating a job's log."""
if overwrite:
updates['$set']['log'] = [log]
else:
updates['$push']['log'] = log
if notify and user:
expires = now + datetime.timedelta(seconds=30)
Notification().createNotification(
type='job_log', data={
'_id': job['_id'],
'overwrite': overwrite,
'text': log
}, user=user, expires=expires)
def update(self, force=False, **kwargs):
"""
Update the underlying progress record. This will only actually save
to the database if at least self.interval seconds have passed since
the last time the record was written to the database. Accepts the
same kwargs as Notification.updateProgress.
:param force: Whether we should force the write to the database. Use
only in cases where progress may be indeterminate for a long time.
:type force: bool
"""
# Extend the response timeout, even if we aren't reporting the progress
setResponseTimeLimit()
if not self.on:
return
save = (time.time() - self._lastSave > self.interval) or force
self.progress = Notification().updateProgress(self.progress, save, **kwargs)
if save:
self._lastSave = time.time()
def __exit__(self, excType, excValue, traceback):
"""
Once the context is exited, the progress is marked for deletion 30
seconds in the future, which should give all listeners time to poll and
receive the final state of the progress record before it is deleted.
"""
if not self.on:
return
if excType is None and excValue is None:
state = ProgressState.SUCCESS
message = 'Done'
else:
state = ProgressState.ERROR
message = 'Error'
if isinstance(excValue, (ValidationException, RestException)):
message = 'Error: '+excValue.message
Notification().updateProgress(
self.progress, state=state, message=message,
expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30)
)
def run(job):
Job().updateJob(job, status=JobStatus.RUNNING)
print 'create bsve datasets'
try:
kwargs = job['kwargs']
user = kwargs['user']
bsveRoot = kwargs['bsveRoot']
extraHeaders = kwargs['extraHeaders']
Notification().createNotification(type='bsveDatasetUpdating',
data=job,
user=user,
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
""" Hits the bsve urls """
# Bsve geoserver (wms get capabilities url)
wms = bsveRoot + "/data/v2/sources/geotiles/meta/list"
resp = requests.get(wms, headers=extraHeaders)
data = json.loads(resp.text)
existingLayers = getLayers(user)
newLayers = set()
for d in data['tiles']:
typeName = d['name']
newLayers.add(typeName)
# For now skip updating layers that always exist.
# When we have a reliable ingestion time stamp,
# we should check the creation date and update
# if the ingestion date is later.
if typeName in existingLayers:
continue
wms_params = {}
wms_params['type_name'] = typeName
wms_params['name'] = d.get('title') or d['styles'][0]['title']
wms_params['abstract'] = d['abstract']
wms_params['source'] = {
'layer_source': 'Reference',
'source_type': 'wms'
}
wms_params['geo_render'] = {'type': 'wms'}
wms_params['category'] = _get_category(d)
wms_params['metadata'] = _get_metadata(d)
layer_type = 'raster' if 'WCS' in d['keywords'] else 'vector'
createBsveDataset(user, bsveRoot, extraHeaders, wms_params,
layer_type)
# delete layers that no longer exist on the server
for typeName in existingLayers:
if typeName not in newLayers:
item = existingLayers[typeName]
Item().remove(item)
Job().updateJob(job, status=JobStatus.SUCCESS)
Notification().createNotification(type='bsveDatasetUpdated',
data=job,
user=user,
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
except Exception as e:
print e
print traceback.print_exc()
Notification().createNotification(type='bsveDatasetError',
data=job,
user=user,
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
def createJob(self,
title,
type,
taskName,
taskEntry,
modules="",
args=(),
kwargs=None,
user=None,
when=None,
interval=0,
public=False,
handler=None,
asynchronous=False,
save=True,
parentJob=None,
otherFields=None):
now = datetime.datetime.utcnow()
if when is None:
when = now
if kwargs is None:
kwargs = {}
slurmOptions = self.findOne({'user': user['_id']})
if slurmOptions is None:
slurmOptions = {
'user': user['_id'],
'partition': 'gpuib',
'gres': "gpu:4",
'nodes': 1,
'ntasks': 1,
'cpu_per_task': 1,
'mem_per_cpu': 16,
'time': 1,
'modules': ""
}
self.save(slurmOptions)
otherFields = {
'otherFields': {
'slurm_info': {
'name': taskName,
'entry': taskEntry,
'partition': slurmOptions['partition'],
'nodes': slurmOptions['nodes'],
'ntasks': slurmOptions['ntasks'],
'gres': slurmOptions['gres'],
'cpu_per_task': slurmOptions['cpu_per_task'],
'mem_per_cpu': str(slurmOptions['mem_per_cpu']) + 'gb',
'time': str(slurmOptions['time']) + ':00:00',
'modules': modules
}
}
}
parentId = None
if parentJob:
parentId = parentJob['_id']
job = {
'title': title,
'type': type,
'args': args,
'kwargs': kwargs,
'created': now,
'updated': now,
'when': when,
'interval': interval,
'status': JobStatus.INACTIVE,
'progress': None,
'log': [],
'meta': {},
'handler': handler,
'asynchronous': asynchronous,
'timestamps': [],
'parentId': parentId
}
job.update(otherFields)
self.setPublic(job, public=public)
if user:
job['userId'] = user['_id']
self.setUserAccess(job, user=user, level=AccessType.ADMIN)
else:
job['userId'] = None
if save:
job = JobModel().save(job)
if user:
deserialized_kwargs = job['kwargs']
job['kwargs'] = json_util.dumps(job['kwargs'])
Notification().createNotification(
type='job_created',
data=job,
user=user,
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
job['kwargs'] = deserialized_kwargs
return job
def testNotification(self):
from girder.plugins.jobs.models.job import Job
from girder.plugins.jobs.constants import JobStatus
from girder.models.notification import Notification, ProgressState
from girder.plugins.wholetale.utils import init_progress
from girder.plugins.worker.constants import PluginSettings as WorkerPluginSettings
# TODO: Why do we need it here?
Setting().set(WorkerPluginSettings.API_URL, 'http://localhost:8080/api/v1')
total = 2
resource = {'type': 'wt_test_build_image', 'instance_id': 'instance_id'}
notification = init_progress(
resource, self.user, 'Test notification', 'Creating job', total
)
# Job to test error path
job = Job().createJob(
title='Error test job',
type='test',
handler='my_handler',
user=self.user,
public=False,
args=["tale_id"],
kwargs={},
otherFields={'wt_notification_id': str(notification['_id'])},
)
job = Job().updateJob(
job, status=JobStatus.INACTIVE, progressTotal=2, progressCurrent=0)
self.assertEqual(job['status'], JobStatus.INACTIVE)
time.sleep(1)
notification = Notification().load(notification['_id'])
self.assertEqual(notification['data']['state'], ProgressState.QUEUED)
self.assertEqual(notification['data']['total'], 2)
self.assertEqual(notification['data']['current'], 0)
self.assertEqual(notification['data']['resource']['jobs'][0], job['_id'])
# State change to ACTIVE
job = Job().updateJob(job, status=JobStatus.RUNNING)
self.assertEqual(job['status'], JobStatus.RUNNING)
# Progress update
job = Job().updateJob(
job, status=JobStatus.RUNNING, progressCurrent=1,
progressMessage="Error test message")
time.sleep(1)
notification = Notification().load(notification['_id'])
self.assertEqual(notification['data']['state'], ProgressState.ACTIVE)
self.assertEqual(notification['data']['total'], 2)
self.assertEqual(notification['data']['current'], 1)
self.assertEqual(notification['data']['message'], 'Error test message')
# State change to ERROR
job = Job().updateJob(job, status=JobStatus.ERROR)
time.sleep(1)
notification = Notification().load(notification['_id'])
self.assertEqual(notification['data']['state'], ProgressState.ERROR)
self.assertEqual(notification['data']['total'], 2)
self.assertEqual(notification['data']['current'], 1)
self.assertEqual(notification['data']['message'], 'Error test message')
# New job to test success path
job = Job().createJob(
title='Test Job',
type='test',
handler='my_handler',
user=self.user,
public=False,
args=["tale_id"],
kwargs={},
otherFields={'wt_notification_id': str(notification['_id'])},
)
# State change to ACTIVE
job = Job().updateJob(job, status=JobStatus.RUNNING)
self.assertEqual(job['status'], JobStatus.RUNNING)
# Progress update
job = Job().updateJob(
job, status=JobStatus.RUNNING, progressCurrent=1,
progressMessage="Success test message")
time.sleep(1)
notification = Notification().load(notification['_id'])
self.assertEqual(notification['data']['state'], ProgressState.ACTIVE)
self.assertEqual(notification['data']['total'], 2)
self.assertEqual(notification['data']['current'], 1)
self.assertEqual(notification['data']['message'], 'Success test message')
job = Job().updateJob(job, status=JobStatus.SUCCESS)
time.sleep(1)
notification = Notification().load(notification['_id'])
self.assertEqual(notification['data']['state'], ProgressState.ACTIVE)
self.assertEqual(notification['data']['total'], 2)
self.assertEqual(notification['data']['current'], 1)
self.assertEqual(notification['data']['message'], 'Success test message')
def testDeleteFolder(self):
cbInfo = {}
# Hook into model deletion with kwargs event to test it
def cb(event):
cbInfo['kwargs'] = event.info['kwargs']
cbInfo['doc'] = event.info['document']
with events.bound('model.folder.remove_with_kwargs', 'test', cb):
# Requesting with no path should fail
resp = self.request(path='/folder', method='DELETE',
user=self.admin)
self.assertStatus(resp, 400)
# Grab one of the user's top level folders
folders = Folder().childFolders(
parent=self.admin, parentType='user', user=self.admin, limit=1,
sort=[('name', SortDir.DESCENDING)])
folderResp = six.next(folders)
# Add a subfolder and an item to that folder
subfolder = Folder().createFolder(
folderResp, 'sub', parentType='folder', creator=self.admin)
item = Item().createItem('item', creator=self.admin, folder=subfolder)
self.assertTrue('_id' in subfolder)
self.assertTrue('_id' in item)
# Delete the folder
resp = self.request(path='/folder/%s' % folderResp['_id'],
method='DELETE', user=self.admin, params={
'progress': 'true'
})
self.assertStatusOk(resp)
# Make sure the folder, its subfolder, and its item were all deleted
folder = Folder().load(folderResp['_id'], force=True)
subfolder = Folder().load(subfolder['_id'], force=True)
item = Item().load(item['_id'])
self.assertEqual(folder, None)
self.assertEqual(subfolder, None)
self.assertEqual(item, None)
# Make sure progress record exists and that it is set to expire soon
notifs = list(Notification().get(self.admin))
self.assertEqual(len(notifs), 1)
self.assertEqual(notifs[0]['type'], 'progress')
self.assertEqual(notifs[0]['data']['state'], ProgressState.SUCCESS)
self.assertEqual(notifs[0]['data']['title'],
'Deleting folder Public')
self.assertEqual(notifs[0]['data']['message'], 'Done')
self.assertEqual(notifs[0]['data']['total'], 3)
self.assertEqual(notifs[0]['data']['current'], 3)
self.assertTrue(notifs[0]['expires'] < datetime.datetime.utcnow() +
datetime.timedelta(minutes=1))
# Make sure our event handler was called with expected args
self.assertTrue('kwargs' in cbInfo)
self.assertTrue('doc' in cbInfo)
self.assertTrue('progress' in cbInfo['kwargs'])
self.assertEqual(cbInfo['doc']['_id'], folderResp['_id'])
def _updateJob(event):
"""
Called when a job is saved, updated, or removed. If this is a histogram
job and it is ended, clean up after it.
"""
if event.name == 'jobs.job.update.after':
job = event.info['job']
else:
job = event.info
meta = job.get('meta', {})
if (meta.get('creator') != 'histogram'
or meta.get('task') != 'createHistogram'):
return
status = job['status']
if event.name == 'model.job.remove' and status not in (JobStatus.ERROR,
JobStatus.CANCELED,
JobStatus.SUCCESS):
status = JobStatus.CANCELED
if status not in (JobStatus.ERROR, JobStatus.CANCELED, JobStatus.SUCCESS):
return
histograms = list(Histogram().find({'fakeId': meta.get('fakeId')},
limit=2))
if len(histograms) != 1:
msg = 'Failed to retrieve histogram using fakeId %s.'
logger.warning(msg % meta.get('fakeId'))
return
histogram = histograms[0]
if histogram.get('expected'):
# We can get a SUCCESS message before we get the upload message, so
# don't clear the expected status on success.
if status != JobStatus.SUCCESS:
del histogram['expected']
notify = histogram.get('notify')
msg = None
if notify:
del histogram['notify']
if status == JobStatus.SUCCESS:
msg = 'Histogram created'
elif status == JobStatus.CANCELED:
msg = 'Histogram creation canceled'
else: # ERROR
msg = 'FAILED: Histogram creation failed'
msg += ' for item %s' % histogram['itemId']
msg += ', file %s' % histogram['fileId']
if status == JobStatus.SUCCESS:
Histogram().save(histogram)
else:
Histogram().remove(histogram)
if msg and event.name != 'model.job.remove':
Job().updateJob(job, progressMessage=msg)
if notify:
Notification().createNotification(type='histogram.finished_histogram',
data={
'histogram_id': histogram['_id'],
'item_id': histogram['itemId'],
'file_id': histogram['fileId'],
'fakeId': histogram['fakeId'],
'success':
status == JobStatus.SUCCESS,
'status': status
},
user={'_id': job.get('userId')},
expires=datetime.datetime.utcnow() +
datetime.timedelta(seconds=30))
self.setPublic(job, public=public)
if user:
job['userId'] = user['_id']
self.setUserAccess(job, user=user, level=AccessType.ADMIN)
else:
job['userId'] = None
if save:
job = self.save(job)
if user:
deserialized_kwargs = job['kwargs']
job['kwargs'] = json_util.dumps(job['kwargs'])
Notification().createNotification(
type='job_created', data=job, user=user,
expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
job['kwargs'] = deserialized_kwargs
return job
def save(self, job, *args, **kwargs):
"""
We extend save so that we can serialize the kwargs before sending them
to the database. This will allow kwargs with $ and . characters in the
keys.
"""
deserialized = job['kwargs']
job['kwargs'] = json_util.dumps(job['kwargs'])
job = AccessControlledModel.save(self, job, *args, **kwargs)
def testDownloadResources(self):
self._createFiles()
resourceList = {
'collection': [str(self.collection['_id'])],
'user': [str(self.admin['_id'])]
}
# We should fail with bad json, an empty list, an invalid item in the
# list, or a list that is an odd format.
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={
'resources': 'this_is_not_json',
},
isJson=False)
self.assertStatus(resp, 400)
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={
'resources':
json.dumps('this_is_not_a_dict_of_resources')
},
isJson=False)
self.assertStatus(resp, 400)
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={
'resources':
json.dumps({'not_a_resource': ['not_an_id']})
},
isJson=False)
self.assertStatus(resp, 400)
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={'resources': json.dumps({'item': []})},
isJson=False)
self.assertStatus(resp, 400)
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={
'resources':
json.dumps({'item': [str(self.admin['_id'])]})
},
isJson=False)
self.assertStatus(resp, 400)
# Download the resources
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={
'resources': json.dumps(resourceList),
'includeMetadata': True
},
isJson=False)
self.assertStatusOk(resp)
self.assertEqual(resp.headers['Content-Type'], 'application/zip')
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
self.assertTrue(zip.testzip() is None)
self.assertHasKeys(self.expectedZip, zip.namelist())
self.assertHasKeys(zip.namelist(), self.expectedZip)
for name in zip.namelist():
expected = self.expectedZip[name]
if isinstance(expected, dict):
self.assertEqual(json.loads(zip.read(name).decode('utf8')),
json.loads(json.dumps(expected, default=str)))
else:
if not isinstance(expected, six.binary_type):
expected = expected.encode('utf8')
self.assertEqual(expected, zip.read(name))
# Download the same resources again, this time triggering the large zip
# file creation (artificially forced). We could do this naturally by
# downloading >65536 files, but that would make the test take several
# minutes.
girder.utility.ziputil.Z_FILECOUNT_LIMIT = 5
resourceList = {'item': [str(item['_id']) for item in self.items]}
resp = self.request(path='/resource/download',
method='POST',
user=self.admin,
params={
'resources': json.dumps(resourceList),
'includeMetadata': True
},
isJson=False,
additionalHeaders=[('X-HTTP-Method-Override',
'GET')])
self.assertStatusOk(resp)
self.assertEqual(resp.headers['Content-Type'], 'application/zip')
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
self.assertTrue(zip.testzip() is None)
# Test deleting resources
resourceList = {
'collection': [str(self.collection['_id'])],
'folder': [str(self.adminSubFolder['_id'])],
}
resp = self.request(path='/resource',
method='DELETE',
user=self.admin,
params={
'resources': json.dumps(resourceList),
'progress': True
},
isJson=False)
self.assertStatusOk(resp)
# Make sure progress record exists and that it is set to expire soon
notifs = list(Notification().get(self.admin))
self.assertEqual(len(notifs), 1)
self.assertEqual(notifs[0]['type'], 'progress')
self.assertEqual(notifs[0]['data']['state'], ProgressState.SUCCESS)
self.assertEqual(notifs[0]['data']['title'], 'Deleting resources')
self.assertEqual(notifs[0]['data']['message'], 'Done')
self.assertEqual(notifs[0]['data']['total'], 6)
self.assertEqual(notifs[0]['data']['current'], 6)
self.assertTrue(notifs[0]['expires'] < datetime.datetime.utcnow() +
datetime.timedelta(minutes=1))
# Test deletes using a body on the request
resourceList = {'item': [str(self.items[1]['_id'])]}
resp = self.request(path='/resource',
method='DELETE',
user=self.admin,
body=urllib.parse.urlencode(
{'resources': json.dumps(resourceList)}),
type='application/x-www-form-urlencoded',
isJson=False)
self.assertStatusOk(resp)
# Test deletes using POST and override method
resourceList = {'item': [str(self.items[0]['_id'])]}
resp = self.request(path='/resource',
method='POST',
user=self.admin,
params={'resources': json.dumps(resourceList)},
isJson=False,
additionalHeaders=[('X-HTTP-Method-Override',
'DELETE')])
self.assertStatusOk(resp)
# All of the items should be gone now
resp = self.request(path='/item',
method='GET',
user=self.admin,
params={'text': 'Item'})
self.assertStatusOk(resp)
self.assertEqual(len(resp.json), 0)
# Add a file under the admin private folder
item = Item().createItem('Private Item', self.admin,
self.adminPrivateFolder)
_, path, contents = self._uploadFile('private_file', item)
self.assertEqual(path, 'goodlogin/Private/Private Item/private_file')
# Download as admin, should get private file
resp = self.request(path='/resource/download',
method='GET',
user=self.admin,
params={
'resources':
json.dumps({'user': [str(self.admin['_id'])]})
},
isJson=False)
self.assertStatusOk(resp)
self.assertEqual(resp.headers['Content-Type'], 'application/zip')
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
self.assertTrue(zip.testzip() is None)
self.assertEqual(zip.namelist(), [path])
self.assertEqual(zip.read(path), contents)
# Download as normal user, should get empty zip
resp = self.request(path='/resource/download',
method='GET',
user=self.user,
params={
'resources':
json.dumps({'user': [str(self.admin['_id'])]})
},
isJson=False)
self.assertStatusOk(resp)
self.assertEqual(resp.headers['Content-Type'], 'application/zip')
zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
self.assertTrue(zip.testzip() is None)
self.assertEqual(zip.namelist(), [])
def createJob(self, title, type, args=(), kwargs=None, user=None, when=None,
interval=0, public=False, handler=None, asynchronous=False,
save=True, parentJob=None, otherFields=None):
"""
Create a new job record.
:param title: The title of the job.
:type title: str
:param type: The type of the job.
:type type: str
:param args: Positional args of the job payload.
:type args: list or tuple
:param kwargs: Keyword arguments of the job payload.
:type kwargs: dict
:param user: The user creating the job.
:type user: dict or None
:param when: Minimum start time for the job (UTC).
:type when: datetime
:param interval: If this job should be recurring, set this to a value
in seconds representing how often it should occur. Set to <= 0 for
jobs that should only be run once.
:type interval: int
:param public: Public read access flag.
:type public: bool
:param handler: If this job should be handled by a specific handler,
use this field to store that information.
:param externalToken: If an external token was created for updating this
job, pass it in and it will have the job-specific scope set.
:type externalToken: token (dict) or None.
:param asynchronous: Whether the job is to be run asynchronously. For now this
only applies to jobs that are scheduled to run locally.
:type asynchronous: bool
:param save: Whether the documented should be saved to the database.
:type save: bool
:param parentJob: The job which will be set as a parent
:type parentJob: Job
:param otherFields: Any additional fields to set on the job.
:type otherFields: dict
"""
now = datetime.datetime.utcnow()
if when is None:
when = now
if kwargs is None:
kwargs = {}
otherFields = otherFields or {}
parentId = None
if parentJob:
parentId = parentJob['_id']
job = {
'title': title,
'type': type,
'args': args,
'kwargs': kwargs,
'created': now,
'updated': now,
'when': when,
'interval': interval,
'status': JobStatus.INACTIVE,
'progress': None,
'log': [],
'meta': {},
'handler': handler,
'asynchronous': asynchronous,
'timestamps': [],
'parentId': parentId
}
job.update(otherFields)
self.setPublic(job, public=public)
if user:
job['userId'] = user['_id']
self.setUserAccess(job, user=user, level=AccessType.ADMIN)
else:
job['userId'] = None
if save:
job = self.save(job)
if user:
deserialized_kwargs = job['kwargs']
job['kwargs'] = json_util.dumps(job['kwargs'])
Notification().createNotification(
type='job_created', data=job, user=user,
expires=datetime.datetime.utcnow() + datetime.timedelta(seconds=30))
job['kwargs'] = deserialized_kwargs
return job
