def testUniqueIndex(self):
settingModel = Setting()
coll = settingModel.collection
indices = coll.index_information()
# Make sure we have just one index on key and that it specifies that it
# is unique
self.assertTrue(any(indices[index]['key'][0][0] == 'key' and
indices[index].get('unique') for index in indices))
self.assertFalse(any(indices[index]['key'][0][0] == 'key' and
not indices[index].get('unique') for index in indices))
# Delete that index, create a non-unique index, and make some duplicate
# settings so that we can test that this will be corrected.
coll.drop_index(next(index for index in indices
if indices[index]['key'][0][0] == 'key'))
coll.create_index('key')
for val in range(3, 8):
coll.insert_one({'key': 'duplicate', 'value': val})
# Check that we've broken things
indices = coll.index_information()
self.assertGreaterEqual(settingModel.get('duplicate'), 3)
self.assertEqual(settingModel.find({'key': 'duplicate'}).count(), 5)
self.assertFalse(any(indices[index]['key'][0][0] == 'key' and
indices[index].get('unique') for index in indices))
self.assertTrue(any(indices[index]['key'][0][0] == 'key' and
not indices[index].get('unique') for index in indices))
# Reconnecting the model should fix the issues we just created
settingModel.reconnect()
indices = coll.index_information()
self.assertTrue(any(indices[index]['key'][0][0] == 'key' and
indices[index].get('unique') for index in indices))
self.assertFalse(any(indices[index]['key'][0][0] == 'key' and
not indices[index].get('unique') for index in indices))
self.assertEqual(settingModel.get('duplicate'), 3)
self.assertEqual(settingModel.find({'key': 'duplicate'}).count(), 1)
def testUniqueIndex(self):
settingModel = Setting()
coll = settingModel.collection
indices = coll.index_information()
# Make sure we have just one index on key and that it specifies that it
# is unique
self.assertTrue(any(indices[index]['key'][0][0] == 'key' and
indices[index].get('unique') for index in indices))
self.assertFalse(any(indices[index]['key'][0][0] == 'key' and
not indices[index].get('unique') for index in indices))
# Delete that index, create a non-unique index, and make some duplicate
# settings so that we can test that this will be corrected.
coll.drop_index(next(index for index in indices
if indices[index]['key'][0][0] == 'key'))
coll.create_index('key')
for val in range(3, 8):
coll.insert_one({'key': 'duplicate', 'value': val})
# Check that we've broken things
indices = coll.index_information()
self.assertGreaterEqual(settingModel.get('duplicate'), 3)
self.assertEqual(settingModel.find({'key': 'duplicate'}).count(), 5)
self.assertFalse(any(indices[index]['key'][0][0] == 'key' and
indices[index].get('unique') for index in indices))
self.assertTrue(any(indices[index]['key'][0][0] == 'key' and
not indices[index].get('unique') for index in indices))
# Reconnecting the model should fix the issues we just created
settingModel.reconnect()
indices = coll.index_information()
self.assertTrue(any(indices[index]['key'][0][0] == 'key' and
indices[index].get('unique') for index in indices))
self.assertFalse(any(indices[index]['key'][0][0] == 'key' and
not indices[index].get('unique') for index in indices))
self.assertEqual(settingModel.get('duplicate'), 3)
self.assertEqual(settingModel.find({'key': 'duplicate'}).count(), 1)
def getCollectionCreationPolicyAccess(self):
cpp = Setting().get('core.collection_create_policy')
acList = {
'users': [{
'id': x
} for x in cpp.get('users', [])],
'groups': [{
'id': x
} for x in cpp.get('groups', [])]
}
for user in acList['users'][:]:
userDoc = User().load(user['id'],
force=True,
fields=['firstName', 'lastName', 'login'])
if userDoc is None:
acList['users'].remove(user)
else:
user['login'] = userDoc['login']
user['name'] = ' '.join(
(userDoc['firstName'], userDoc['lastName']))
for grp in acList['groups'][:]:
grpDoc = Group().load(grp['id'],
force=True,
fields=['name', 'description'])
if grpDoc is None:
acList['groups'].remove(grp)
else:
grp['name'] = grpDoc['name']
grp['description'] = grpDoc['description']
return acList
def testSettingsCache(db, enabledCache):
setting = Setting()
# 'foo' should be cached as the brand name
setting.set(SettingKey.BRAND_NAME, 'foo')
# change the brand name bypassing the cache via mongo
returnedSetting = setting.findOne({'key': SettingKey.BRAND_NAME})
returnedSetting['value'] = 'bar'
# verify the cache still gives us the old brand name
assert setting.get(SettingKey.BRAND_NAME) == 'foo'
# change the brand name through .set (which updates the cache)
setting.set(SettingKey.BRAND_NAME, 'bar')
# verify retrieving gives us the new value
with unittest.mock.patch.object(setting, 'findOne') as findOneMock:
assert setting.get(SettingKey.BRAND_NAME) == 'bar'
# findOne shouldn't be called since the cache is returning the setting
findOneMock.assert_not_called()
# unset the setting, invalidating the cache
setting.unset(SettingKey.BRAND_NAME)
# verify the database needs to be accessed to retrieve the setting now
with unittest.mock.patch.object(setting, 'findOne') as findOneMock:
setting.get(SettingKey.BRAND_NAME)
findOneMock.assert_called_once()
def load(info):
setDefaults()
settings = Setting()
homeDirsRoot = settings.get(PluginSettings.HOME_DIRS_ROOT)
logger.info('WT Home Dirs root: %s' % homeDirsRoot)
startDAVServer(homeDirsRoot, HomeDirectoryInitializer, HomeAuthorizer, HomePathMapper())
taleDirsRoot = settings.get(PluginSettings.TALE_DIRS_ROOT)
logger.info('WT Tale Dirs root: %s' % taleDirsRoot)
startDAVServer(taleDirsRoot, TaleDirectoryInitializer, TaleAuthorizer, TalePathMapper())
runsDirsRoot = settings.get(PluginSettings.RUNS_DIRS_ROOT)
if runsDirsRoot:
logger.info('WT Runs Dirs root: %s' % runsDirsRoot)
startDAVServer(runsDirsRoot, RunsDirectoryInitializer, RunsAuthorizer, RunsPathMapper())
events.unbind('model.user.save.created', CoreEventHandler.USER_DEFAULT_FOLDERS)
events.bind('model.user.save.created', 'wt_home_dirs', setHomeFolderMapping)
events.bind('model.tale.save.created', 'wt_home_dirs', setTaleFolderMapping)
events.bind('model.tale.remove', 'wt_home_dirs', deleteWorkspace)
hdp = Homedirpass()
info['apiRoot'].homedirpass = hdp
info['apiRoot'].homedirpass.route('GET', ('generate',), hdp.generatePassword)
info['apiRoot'].homedirpass.route('PUT', ('set',), hdp.setPassword)
Tale().exposeFields(level=AccessType.READ, fields={"workspaceId"})
def __init__(self, templatePath=None):
if not templatePath:
templatePath = os.path.join(constants.PACKAGE_DIR, 'utility',
'webroot.mako')
super(Webroot, self).__init__(templatePath)
settings = Setting()
self.vars = {
'plugins': [],
'apiRoot': '',
'staticRoot': '',
# 'title' is depreciated use brandName instead
'title': 'Girder',
'brandName': settings.get(SettingKey.BRAND_NAME),
'bannerColor': settings.get(SettingKey.BANNER_COLOR),
'contactEmail': settings.get(SettingKey.CONTACT_EMAIL_ADDRESS),
'registrationPolicy': settings.get(SettingKey.REGISTRATION_POLICY),
'enablePasswordLogin':
settings.get(SettingKey.ENABLE_PASSWORD_LOGIN)
}
events.bind('model.setting.save.after',
CoreEventHandler.WEBROOT_SETTING_CHANGE,
self._onSettingSave)
events.bind('model.setting.remove',
CoreEventHandler.WEBROOT_SETTING_CHANGE,
self._onSettingRemove)
def startMonitor(self):
settings = Setting()
self.ASSETSTORE_ID = settings.get(PluginSettings.ASSETSTORE_ID)
self.assetstore = AssetstoreModel().load(self.ASSETSTORE_ID)
self.MONITOR_PARTITION = settings.get(PluginSettings.MONITOR_PARTITION)
self.DESTINATION_TYPE = settings.get(PluginSettings.DESTINATION_TYPE)
self.DESTINATION_ID = settings.get(PluginSettings.DESTINATION_ID)
self.destination = ModelImporter.model(self.DESTINATION_TYPE).load(
self.DESTINATION_ID,
user=self.getCurrentUser(),
level=AccessType.WRITE,
exc=True)
wm = pyinotify.WatchManager() # Watch Manager
mask = pyinotify.IN_DELETE | pyinotify.IN_CREATE | pyinotify.IN_MODIFY
self.notifier = pyinotify.ThreadedNotifier(
wm,
EventHandler(self.assetstore, self.destination,
self.DESTINATION_TYPE, self.MONITOR_PARTITION))
wdd = wm.add_watch(self.MONITOR_PARTITION,
mask,
rec=True,
auto_add=True)
self.notifier.start()
SettingDefault.defaults.update({PluginSettings.MONITOR: True})
return {PluginSettings.MONITOR: settings.get(PluginSettings.MONITOR)}
def getCollectionCreationPolicyAccess(self):
cpp = Setting().get('core.collection_create_policy')
acList = {
'users': [{'id': x} for x in cpp.get('users', [])],
'groups': [{'id': x} for x in cpp.get('groups', [])]
}
for user in acList['users'][:]:
userDoc = User().load(
user['id'], force=True,
fields=['firstName', 'lastName', 'login'])
if userDoc is None:
acList['users'].remove(user)
else:
user['login'] = userDoc['login']
user['name'] = ' '.join((userDoc['firstName'], userDoc['lastName']))
for grp in acList['groups'][:]:
grpDoc = Group().load(
grp['id'], force=True, fields=['name', 'description'])
if grpDoc is None:
acList['groups'].remove(grp)
else:
grp['name'] = grpDoc['name']
grp['description'] = grpDoc['description']
return acList
def testSettingsCache(db, enabledCache):
setting = Setting()
# 'foo' should be cached as the brand name
setting.set(SettingKey.BRAND_NAME, 'foo')
# change the brand name bypassing the cache via mongo
returnedSetting = setting.findOne({'key': SettingKey.BRAND_NAME})
returnedSetting['value'] = 'bar'
# verify the cache still gives us the old brand name
assert setting.get(SettingKey.BRAND_NAME) == 'foo'
# change the brand name through .set (which updates the cache)
setting.set(SettingKey.BRAND_NAME, 'bar')
# verify retrieving gives us the new value
with mock.patch.object(setting, 'findOne') as findOneMock:
assert setting.get(SettingKey.BRAND_NAME) == 'bar'
# findOne shouldn't be called since the cache is returning the setting
findOneMock.assert_not_called()
# unset the setting, invalidating the cache
setting.unset(SettingKey.BRAND_NAME)
# verify the database needs to be accessed to retrieve the setting now
with mock.patch.object(setting, 'findOne') as findOneMock:
setting.get(SettingKey.BRAND_NAME)
findOneMock.assert_called_once()
def getSettings(self):
settings = Setting()
return {
PluginSettings.SHARED_PARTITION:
settings.get(PluginSettings.SHARED_PARTITION),
PluginSettings.CRONTAB_PARTITION:
settings.get(PluginSettings.CRONTAB_PARTITION)
}
def getSettings(self):
settings = Setting()
return {
PluginSettings.MARKDOWN: settings.get(PluginSettings.MARKDOWN),
PluginSettings.HEADER: settings.get(PluginSettings.HEADER),
PluginSettings.SUBHEADER: settings.get(PluginSettings.SUBHEADER),
PluginSettings.WELCOME_TEXT: settings.get(PluginSettings.WELCOME_TEXT),
PluginSettings.LOGO: settings.get(PluginSettings.LOGO),
}
def getSettings(self):
settings = Setting()
return {
PluginSettings.MARKDOWN: settings.get(PluginSettings.MARKDOWN),
PluginSettings.HEADER: settings.get(PluginSettings.HEADER),
PluginSettings.SUBHEADER: settings.get(PluginSettings.SUBHEADER),
PluginSettings.WELCOME_TEXT:
settings.get(PluginSettings.WELCOME_TEXT),
PluginSettings.LOGO: settings.get(PluginSettings.LOGO),
}
def _storeEndpointData(self, key, endpointId, endpointName):
settings = Setting()
endpointIds = settings.get(PluginSettings.GLOBUS_ENDPOINT_ID, None)
endpointNames = settings.get(PluginSettings.GLOBUS_ENDPOINT_NAME, None)
if endpointIds is None:
endpointIds = {}
endpointNames = {}
endpointIds[key] = endpointId
endpointNames[key] = endpointName
settings.set(PluginSettings.GLOBUS_ENDPOINT_ID, endpointIds)
settings.set(PluginSettings.GLOBUS_ENDPOINT_NAME, endpointNames)
def _getEndpointDataByKey(self, key):
settings = Setting()
endpointIds = settings.get(PluginSettings.GLOBUS_ENDPOINT_ID, None)
endpointNames = settings.get(PluginSettings.GLOBUS_ENDPOINT_NAME, None)
(endpointIds,
endpointNames) = self._maybeConvertSettings(endpointIds,
endpointNames)
if endpointIds is None:
return (None, None)
else:
if key in endpointIds:
return (endpointIds[key], endpointNames[key])
else:
return (None, None)
def getSettings(self):
settings = Setting()
return {
PluginSettings.ASSETSTORE_ID:
settings.get(PluginSettings.ASSETSTORE_ID) or '',
PluginSettings.MONITOR_PARTITION:
settings.get(PluginSettings.MONITOR_PARTITION) or '',
PluginSettings.DESTINATION_TYPE:
settings.get(PluginSettings.DESTINATION_TYPE) or '',
PluginSettings.DESTINATION_ID:
settings.get(PluginSettings.DESTINATION_ID) or '',
PluginSettings.MONITOR:
settings.get(PluginSettings.MONITOR) or ''
}
def _submitEmail(msg, recipients):
from girder.models.setting import Setting
setting = Setting()
smtp = _SMTPConnection(host=setting.get(SettingKey.SMTP_HOST),
port=setting.get(SettingKey.SMTP_PORT),
encryption=setting.get(SettingKey.SMTP_ENCRYPTION),
username=setting.get(SettingKey.SMTP_USERNAME),
password=setting.get(SettingKey.SMTP_PASSWORD))
logger.info('Sending email to %s through %s', ', '.join(recipients),
smtp.host)
with smtp:
smtp.send(msg['From'], recipients, msg.as_string())
def __init__(self):
super(Slurm, self).__init__()
self.resourceName = 'slurm'
self.user = self.getCurrentUser()
self.token = self.getCurrentToken()
self.name = 'test'
self.entry = None
self.partition = 'norm'
self.nodes = 1
self.ntasks = 2
self.gres = 'gpu:p100:1'
self.mem_per_cpu = '32gb'
settings = Setting()
self.SHARED_PARTITION = settings.get(PluginSettings.SHARED_PARTITION)
self._shared_partition_log = os.path.join(self.SHARED_PARTITION,
'logs')
self._shared_partition_work_directory = os.path.join(
self.SHARED_PARTITION, 'tmp')
self._modulesPath = os.path.join(self.SHARED_PARTITION, 'modules')
self._shellPath = os.path.join(self.SHARED_PARTITION, 'shells')
self.route('GET', ('lists', ), self.listSlurmJobs)
self.route('GET', ('slurmOption', ), self.getSlurmOption)
self.route('PUT', ('slurmOption', ), self.setSlurmOption)
self.route('GET', (), self.getSlurm)
self.route('PUT', ('cancel', ':id'), self.cancelSlurm)
self.route('POST', (), self.submitSlurmJob)
self.route('GET', ('settings', ), self.getSettings)
self.route('POST', ('update', ), self.update)
self.route('PUT', ('updatestep', ), self.updateStep)
def _startContainer(self, container):
settings = Setting()
psRoot = settings.get(PluginSettings.PRIVATE_STORAGE_PATH)
restUrl = rest.getApiUrl()
token = rest.getCurrentToken()['_id']
sessionId = str(container['sessionId'])
mountId = efs.mount(sessionId, '/tmp/' + sessionId, psRoot, restUrl, token)
container['mountId'] = mountId
container['status'] = 'Running'
self.save(container)
def _sendmail(event):
from girder.models.setting import Setting
msg = event.info['message']
recipients = event.info['recipients']
setting = Setting()
smtp = _SMTPConnection(
host=setting.get(SettingKey.SMTP_HOST, 'localhost'),
port=setting.get(SettingKey.SMTP_PORT, None),
encryption=setting.get(SettingKey.SMTP_ENCRYPTION, 'none'),
username=setting.get(SettingKey.SMTP_USERNAME, None),
password=setting.get(SettingKey.SMTP_PASSWORD, None)
)
logger.info('Sending email to %s through %s', ', '.join(recipients), smtp.host)
with smtp:
smtp.send(msg['From'], recipients, msg.as_string())
def enablePlugins(self, plugins):
# Determine what plugins have been disabled and remove their associated routes.
setting = Setting()
routeTable = setting.get(SettingKey.ROUTE_TABLE)
oldPlugins = setting.get(SettingKey.PLUGINS_ENABLED)
reservedRoutes = {GIRDER_ROUTE_ID, GIRDER_STATIC_ROUTE_ID}
routeTableChanged = False
removedRoutes = (set(oldPlugins) - set(plugins) - reservedRoutes)
for route in removedRoutes:
if route in routeTable:
del routeTable[route]
routeTableChanged = True
if routeTableChanged:
setting.set(SettingKey.ROUTE_TABLE, routeTable)
# Route cleanup is done; update list of enabled plugins.
return setting.set(SettingKey.PLUGINS_ENABLED, plugins)
def enablePlugins(self, plugins):
# Determine what plugins have been disabled and remove their associated routes.
setting = Setting()
routeTable = setting.get(SettingKey.ROUTE_TABLE)
oldPlugins = setting.get(SettingKey.PLUGINS_ENABLED)
reservedRoutes = {GIRDER_ROUTE_ID, GIRDER_STATIC_ROUTE_ID}
routeTableChanged = False
removedRoutes = (
set(oldPlugins) - set(plugins) - reservedRoutes)
for route in removedRoutes:
if route in routeTable:
del routeTable[route]
routeTableChanged = True
if routeTableChanged:
setting.set(SettingKey.ROUTE_TABLE, routeTable)
# Route cleanup is done; update list of enabled plugins.
return setting.set(SettingKey.PLUGINS_ENABLED, plugins)
def cronWatch(event):
import random
slurmJobId = str(event.info['slurmJobId'])
settings = Setting()
CRONTAB_PARTITION = settings.get(PluginSettings.CRONTAB_PARTITION)
logPath = os.path.join(CRONTAB_PARTITION, slurmJobId)
shellPath = os.path.join(os.path.dirname(__file__), 'crontab.sh')
commentId = str(random.getrandbits(128))
cron = CronTab(user=True)
job = cron.new(command=shellPath + ' ' + slurmJobId + ' ' + commentId +
' >> ' + logPath + ' 2>&1\n')
job.set_comment(commentId)
job.minute.every(1)
job.enable()
cron.write()
def tmpdir(cleanup=True):
# Make the temp dir underneath tmp_root config setting
settings = Setting()
tmp = os.path.join(settings.get(PluginSettings.SHARED_PARTITION), 'tmp')
root = os.path.abspath(tmp)
try:
os.makedirs(root)
except OSError:
if not os.path.isdir(root):
raise
path = tempfile.mkdtemp(dir=root)
try:
yield path
finally:
# Cleanup the temp dir
# if cleanup and os.path.isdir(path):
# shutil.rmtree(path)
pass
def loopWatch(slurmJobId):
settings = Setting()
SHARED_PARTITION = settings.get(PluginSettings.SHARED_PARTITION)
shared_partition_log = os.path.join(SHARED_PARTITION, 'logs')
shared_partition_work_directory = os.path.join(SHARED_PARTITION, 'tmp')
while True:
args = 'squeue -j {}'.format(slurmJobId)
output = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out_put = output.communicate()[0]
found = re.findall(slurmJobId, out_put.decode())
if len(found) == 0:
job = Job().findOne(
{'otherFields.slurm_info.slurm_id': int(slurmJobId)})
Job().updateJob(job, status=JobStatus.SUCCESS)
log_file_name = 'slurm-{}.{}.out'.format(
job['otherFields']['slurm_info']['name'], slurmJobId)
log_file_path = os.path.join(shared_partition_log, log_file_name)
f = open(log_file_path, "r")
content = f.read()
Job().updateJob(job, log=content)
f.close()
err_file_name = 'slurm-{}.{}.err'.format(
job['otherFields']['slurm_info']['name'], slurmJobId)
err_file_path = os.path.join(shared_partition_log, err_file_name)
f = open(err_file_path, "r")
content = f.read()
Job().updateJob(job, log=content)
f.close()
# Job().save(job)
# _send_to_girder
slurm_output_name = 'slurm-{}.{}'.format(
job['otherFields']['slurm_info']['name'], slurmJobId)
data = os.path.join(shared_partition_work_directory,
slurm_output_name)
girderOutput.sendOutputToGirder(job, data)
break
sleep(1)
def testLdapLogin(self):
settings = Setting()
self.assertEqual(settings.get(PluginSettings.SERVERS), [])
with self.assertRaises(ValidationException):
settings.set(PluginSettings.SERVERS, {})
settings.set(PluginSettings.SERVERS, [{
'baseDn': 'cn=Users,dc=foo,dc=bar,dc=org',
'bindName': 'cn=foo,cn=Users,dc=foo,dc=bar,dc=org',
'password': '******',
'searchField': 'mail',
'uri': 'foo.bar.org:389'
}])
with mock.patch('ldap.initialize', return_value=MockLdap()) as ldapInit:
resp = self.request('/user/authentication', basicAuth='hello:world')
self.assertEqual(len(ldapInit.mock_calls), 1)
self.assertStatusOk(resp)
# Register a new user
user = resp.json['user']
self.assertEqual(user['email'], '*****@*****.**')
self.assertEqual(user['firstName'], 'Foo')
self.assertEqual(user['lastName'], 'Bar')
self.assertEqual(user['login'], 'foobar')
# Login as an existing user
resp = self.request('/user/authentication', basicAuth='hello:world')
self.assertStatusOk(resp)
self.assertEqual(resp.json['user']['_id'], user['_id'])
with mock.patch('ldap.initialize', return_value=MockLdap(bindFail=True)):
resp = self.request('/user/authentication', basicAuth='hello:world')
self.assertStatus(resp, 401)
with mock.patch('ldap.initialize', return_value=MockLdap(searchFail=True)):
resp = self.request('/user/authentication', basicAuth='hello:world')
self.assertStatus(resp, 401)
# Test fallback to logging in with core auth
normalUser = User().createUser(
login='******', firstName='Normal', lastName='User', email='*****@*****.**',
password='******')
with mock.patch('ldap.initialize', return_value=MockLdap(searchFail=True)):
resp = self.request('/user/authentication', basicAuth='normal:normaluser')
self.assertStatusOk(resp)
self.assertEqual(str(normalUser['_id']), resp.json['user']['_id'])
# Test registering from a record that only has a cn, no sn/givenName
record = {
'cn': [b'Fizz Buzz'],
'mail': [b'*****@*****.**'],
'distinguishedName': [b'shouldbeignored']
}
with mock.patch('ldap.initialize', return_value=MockLdap(record=record)):
resp = self.request('/user/authentication', basicAuth='fizzbuzz:foo')
self.assertStatusOk(resp)
self.assertEqual(resp.json['user']['login'], 'fizz')
self.assertEqual(resp.json['user']['firstName'], 'Fizz')
self.assertEqual(resp.json['user']['lastName'], 'Buzz')
# Test falling back to other name generation behavior (first+last name)
record = {
'cn': [b'Fizz Buzz'],
'mail': [b'*****@*****.**'],
'distinguishedName': [b'shouldbeignored']
}
with mock.patch('ldap.initialize', return_value=MockLdap(record=record)):
resp = self.request('/user/authentication', basicAuth='fizzbuzz:foo')
self.assertStatusOk(resp)
self.assertEqual(resp.json['user']['login'], 'fizzbuzz')
self.assertEqual(resp.json['user']['firstName'], 'Fizz')
self.assertEqual(resp.json['user']['lastName'], 'Buzz')
def schedule(event):
"""
This is bound to the "jobs.schedule" event, and will be triggered any time
a job is scheduled. This handler will process any job that has the
handler field set to "worker_handler".
"""
job = event.info
slurm_info_new = job['otherFields']['slurm_info']
# shellScript = job['shellScript']
if job['handler'] == 'slurm_handler' and slurm_info_new[
'entry'] is not None:
# fetch_input(job)
settings = Setting()
SHARED_PARTITION = settings.get(PluginSettings.SHARED_PARTITION)
shared_partition_log = os.path.join(SHARED_PARTITION, 'logs')
shared_partition_work_directory = os.path.join(SHARED_PARTITION, 'tmp')
modulesPath = os.path.join(SHARED_PARTITION, 'modules')
pythonScriptPath = os.path.join(modulesPath, slurm_info_new['entry'])
Job().updateJob(job, status=JobStatus.QUEUED)
batchscript = """#!/bin/bash
#SBATCH --partition={partition}
#SBATCH --job-name={name}
#SBATCH --nodes={nodes}
#SBATCH --ntasks={ntasks}
#SBATCH --gres={gres}
#SBATCH --mem-per-cpu={mem_per_cpu}
#SBATCH --output={shared_partition_log}/slurm-%x.%j.out
#SBATCH --error={shared_partition_log}/slurm-%x.%j.err
source /etc/profile.d/modules.sh
module load {modules}
mkdir -p {shared_partition_work_directory}/slurm-$SLURM_JOB_NAME.$SLURM_JOB_ID
"""
execCommand = """python3.6 {pythonScriptPath} --directory {shared_partition_work_directory}/slurm-$SLURM_JOB_NAME.$SLURM_JOB_ID """
for name in job['kwargs']['inputs']:
if isinstance(job['kwargs']['inputs'][name]['data'], list):
arg = "--" + name + " " + ' '.join(
'"{0}"'.format(i)
for i in job['kwargs']['inputs'][name]['data']) + " "
else:
arg = "--" + name + " '" + str(
job['kwargs']['inputs'][name]['data']) + "' "
execCommand += arg
batchscript += execCommand
script = batchscript.format(
name=slurm_info_new['name'],
partition=slurm_info_new['partition'],
nodes=slurm_info_new['nodes'],
ntasks=slurm_info_new['ntasks'],
gres=slurm_info_new['gres'],
mem_per_cpu=slurm_info_new['mem_per_cpu'],
modules=" ".join(slurm_info_new['modules']),
shared_partition_log=shared_partition_log,
shared_partition_work_directory=shared_partition_work_directory,
pythonScriptPath=pythonScriptPath)
shellPath = os.path.join(SHARED_PARTITION, 'shells')
shellScriptPath = os.path.join(shellPath,
slurm_info_new['name'] + '.sh')
with open(shellScriptPath, "w") as sh:
sh.write(script)
try:
args = ['sbatch']
args.append(sh.name)
res = subprocess.check_output(args).strip()
if not res.startswith(b"Submitted batch"):
return None
slurmJobId = int(res.split()[-1])
# crontab method
# events.trigger('cron.watch', {'slurmJobId': slurmJobId})
# thread method
threading.Thread(target=loopWatch,
args=([str(slurmJobId)])).start()
job['otherFields']['slurm_info']['slurm_id'] = slurmJobId
Job().save(job)
Job().updateJob(job, status=JobStatus.RUNNING)
except Exception:
return 'something wrong during slurm start'
return slurmJobId
def testLdapLogin(self):
from girder.plugins.ldap.constants import PluginSettings
settings = Setting()
self.assertEqual(settings.get(PluginSettings.LDAP_SERVERS), [])
with self.assertRaises(ValidationException):
settings.set(PluginSettings.LDAP_SERVERS, {})
settings.set(PluginSettings.LDAP_SERVERS, [{
'baseDn': 'cn=Users,dc=foo,dc=bar,dc=org',
'bindName': 'cn=foo,cn=Users,dc=foo,dc=bar,dc=org',
'password': '******',
'searchField': 'mail',
'uri': 'foo.bar.org:389'
}])
with mock.patch('ldap.initialize',
return_value=MockLdap()) as ldapInit:
resp = self.request('/user/authentication',
basicAuth='hello:world')
self.assertEqual(len(ldapInit.mock_calls), 1)
self.assertStatusOk(resp)
# Register a new user
user = resp.json['user']
self.assertEqual(user['email'], '*****@*****.**')
self.assertEqual(user['firstName'], 'Foo')
self.assertEqual(user['lastName'], 'Bar')
self.assertEqual(user['login'], 'foobar')
# Login as an existing user
resp = self.request('/user/authentication',
basicAuth='hello:world')
self.assertStatusOk(resp)
self.assertEqual(resp.json['user']['_id'], user['_id'])
with mock.patch('ldap.initialize',
return_value=MockLdap(bindFail=True)):
resp = self.request('/user/authentication',
basicAuth='hello:world')
self.assertStatus(resp, 401)
with mock.patch('ldap.initialize',
return_value=MockLdap(searchFail=True)):
resp = self.request('/user/authentication',
basicAuth='hello:world')
self.assertStatus(resp, 401)
# Test fallback to logging in with core auth
normalUser = User().createUser(login='******',
firstName='Normal',
lastName='User',
email='*****@*****.**',
password='******')
with mock.patch('ldap.initialize',
return_value=MockLdap(searchFail=True)):
resp = self.request('/user/authentication',
basicAuth='normal:normaluser')
self.assertStatusOk(resp)
self.assertEqual(str(normalUser['_id']), resp.json['user']['_id'])
# Test registering from a record that only has a cn, no sn/givenName
record = {
'cn': [b'Fizz Buzz'],
'mail': [b'*****@*****.**'],
'distinguishedName': [b'shouldbeignored']
}
with mock.patch('ldap.initialize',
return_value=MockLdap(record=record)):
resp = self.request('/user/authentication',
basicAuth='fizzbuzz:foo')
self.assertStatusOk(resp)
self.assertEqual(resp.json['user']['login'], 'fizz')
self.assertEqual(resp.json['user']['firstName'], 'Fizz')
self.assertEqual(resp.json['user']['lastName'], 'Buzz')
# Test falling back to other name generation behavior (first+last name)
record = {
'cn': [b'Fizz Buzz'],
'mail': [b'*****@*****.**'],
'distinguishedName': [b'shouldbeignored']
}
with mock.patch('ldap.initialize',
return_value=MockLdap(record=record)):
resp = self.request('/user/authentication',
basicAuth='fizzbuzz:foo')
self.assertStatusOk(resp)
self.assertEqual(resp.json['user']['login'], 'fizzbuzz')
self.assertEqual(resp.json['user']['firstName'], 'Fizz')
self.assertEqual(resp.json['user']['lastName'], 'Buzz')
def stopMonitor(self):
settings = Setting()
self.notifier.stop()
SettingDefault.defaults.update({PluginSettings.MONITOR: False})
return {PluginSettings.MONITOR: settings.get(PluginSettings.MONITOR)}
def getSettings(self):
settings = Setting()
return {
PluginSettings.DEFAULT_BINS:
settings.get(PluginSettings.DEFAULT_BINS),
}
def getTaleDirPath(tale: dict, rootProp: str) -> pathlib.Path:
settings = Setting()
root = settings.get(rootProp)
taleId = str(tale['_id'])
return pathlib.Path(root) / taleId[0:2] / taleId
def getSettings(self):
settings = Setting()
return {
PluginSettings.DEFAULT_BEHAVIOR:
settings.get(PluginSettings.DEFAULT_BEHAVIOR),
}