def _main(): # pragma: no cover
"""
This is the entrypoint of the girder-sftpd program. It should not be
called from python code.
"""
import argparse
parser = argparse.ArgumentParser(
prog='girder-sftpd', description='Run the Girder SFTP service.')
parser.add_argument(
'-i', '--identity-file', required=False, help='path to identity (private key) file')
parser.add_argument('-p', '--port', required=False, default=DEFAULT_PORT, type=int)
parser.add_argument('-H', '--host', required=False, default='localhost')
args = parser.parse_args()
keyFile = args.identity_file or os.path.expanduser(os.path.join('~', '.ssh', 'id_rsa'))
try:
hostKey = paramiko.RSAKey.from_private_key_file(keyFile)
except paramiko.ssh_exception.PasswordRequiredException:
logprint.error(
'Error: encrypted key files are not supported (%s).' % keyFile, file=sys.stderr)
sys.exit(1)
server = SftpServer((args.host, args.port), hostKey)
logprint.info('Girder SFTP service listening on %s:%d.' % (args.host, args.port))
try:
server.serve_forever()
except (SystemExit, KeyboardInterrupt):
server.server_close()
def _main(): # pragma: no cover
"""
This is the entrypoint of the girder-sftpd program. It should not be
called from python code.
"""
import argparse
parser = argparse.ArgumentParser(
prog='girder-sftpd', description='Run the Girder SFTP service.')
parser.add_argument(
'-i', '--identity-file', required=False, help='path to identity (private key) file')
parser.add_argument('-p', '--port', required=False, default=DEFAULT_PORT, type=int)
parser.add_argument('-H', '--host', required=False, default='localhost')
args = parser.parse_args()
keyFile = args.identity_file or os.path.expanduser(os.path.join('~', '.ssh', 'id_rsa'))
try:
hostKey = paramiko.RSAKey.from_private_key_file(keyFile)
except paramiko.ssh_exception.PasswordRequiredException:
logprint.error(
'Error: encrypted key files are not supported (%s).' % keyFile, file=sys.stderr)
sys.exit(1)
server = SftpServer((args.host, args.port), hostKey)
logprint.info('Girder SFTP service listening on %s:%d.' % (args.host, args.port))
try:
server.serve_forever()
except (SystemExit, KeyboardInterrupt):
server.server_close()
def __init__(self, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
"""
try:
super(FUSELogError, self).__init__(operations, mountpoint, *args, **kwargs)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' % (
mountpoint, ))
def __init__(self, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
"""
try:
super(FUSELogError, self).__init__(operations, mountpoint, *args,
**kwargs)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' %
(mountpoint, ))
def __init__(self, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
"""
try:
logger.debug('Mounting %s\n' % mountpoint)
super().__init__(operations, mountpoint, *args, **kwargs)
logger.debug('Mounted %s\n' % mountpoint)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' %
(mountpoint, ))
Setting().unset(SettingKey.GIRDER_MOUNT_INFORMATION)
def __init__(self, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
"""
try:
logger.debug('Mounting %s\n' % mountpoint)
super(FUSELogError, self).__init__(operations, mountpoint, *args, **kwargs)
logger.debug('Mounted %s\n' % mountpoint)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' % (
mountpoint, ))
Setting().unset(SettingKey.GIRDER_MOUNT_INFORMATION)
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
logprint.error(message)
return
else:
raise ValidationException(message)
deps = allPlugins[plugin]['dependencies']
dag[plugin] = deps
for dep in deps:
if dep in visited:
return
visited.add(dep)
addDeps(dep)
def testLogPrint(tempLog):
tempLog = configureLogging({'log_max_info_level': 'INFO'})
infoSize1 = os.path.getsize(tempLog['info_log_file'])
errorSize1 = os.path.getsize(tempLog['error_log_file'])
logprint.info(INFO_MSG)
infoSize2 = os.path.getsize(tempLog['info_log_file'])
errorSize2 = os.path.getsize(tempLog['error_log_file'])
assert infoSize2 > infoSize1
assert errorSize2 == errorSize1
logprint.error(ERROR_MSG)
infoSize3 = os.path.getsize(tempLog['info_log_file'])
errorSize3 = os.path.getsize(tempLog['error_log_file'])
# logprint sends to stdout, which we capture except when sent via
# logprint, so we shouldn't see any additional data on the info log.
assert infoSize3 == infoSize2
assert errorSize3 > errorSize2
def testLogPrint(self):
self.configureLogging({'log_max_info_level': 'INFO'})
infoSize1 = os.path.getsize(self.infoFile)
errorSize1 = os.path.getsize(self.errorFile)
logprint.info(self.infoMessage)
infoSize2 = os.path.getsize(self.infoFile)
errorSize2 = os.path.getsize(self.errorFile)
self.assertGreater(infoSize2, infoSize1)
self.assertEqual(errorSize2, errorSize1)
logprint.error(self.errorMessage)
infoSize3 = os.path.getsize(self.infoFile)
errorSize3 = os.path.getsize(self.errorFile)
# logprint sends to stdout, which we capture except when sent via
# logprint, so we shouldn't see any additional data on the info log.
self.assertEqual(infoSize3, infoSize2)
self.assertGreater(errorSize3, errorSize2)
def testLogPrint(tempLog):
tempLog = configureLogging({'log_max_info_level': 'INFO'})
infoSize1 = os.path.getsize(tempLog['info_log_file'])
errorSize1 = os.path.getsize(tempLog['error_log_file'])
logprint.info(INFO_MSG)
infoSize2 = os.path.getsize(tempLog['info_log_file'])
errorSize2 = os.path.getsize(tempLog['error_log_file'])
assert infoSize2 > infoSize1
assert errorSize2 == errorSize1
logprint.error(ERROR_MSG)
infoSize3 = os.path.getsize(tempLog['info_log_file'])
errorSize3 = os.path.getsize(tempLog['error_log_file'])
# logprint sends to stdout, which we capture except when sent via
# logprint, so we shouldn't see any additional data on the info log.
assert infoSize3 == infoSize2
assert errorSize3 > errorSize2
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
logprint.error(message)
return
else:
raise ValidationException(message)
deps = allPlugins[plugin]['dependencies']
dag[plugin] = deps
for dep in deps:
if dep in visited:
return
visited.add(dep)
addDeps(dep)
def testLogPrint(self):
self.configureLogging({'log_max_info_level': 'INFO'})
infoSize1 = os.path.getsize(self.infoFile)
errorSize1 = os.path.getsize(self.errorFile)
logprint.info(self.infoMessage)
infoSize2 = os.path.getsize(self.infoFile)
errorSize2 = os.path.getsize(self.errorFile)
self.assertGreater(infoSize2, infoSize1)
self.assertEqual(errorSize2, errorSize1)
logprint.error(self.errorMessage)
infoSize3 = os.path.getsize(self.infoFile)
errorSize3 = os.path.getsize(self.errorFile)
# logprint sends to stdout, which we capture except when sent via
# logprint, so we shouldn't see any additional data on the info log.
self.assertEqual(infoSize3, infoSize2)
self.assertGreater(errorSize3, errorSize2)
def load(info):
"""OSUMO plugin entry point."""
# Check environment variables for anonymous user/password; bail if not set.
anonuser = os.environ.get('OSUMO_ANON_USER')
if not anonuser:
try:
f = open(os.path.join(info['pluginRootDir'],
'osumo_anonlogin.txt'))
with f:
anonuser = f.read().strip()
except IOError:
pass
if not anonuser:
error_message = ' '.join(
('Environment variable OSUMO_ANON_USER must be set, or the',
'file "osumo_anonlogin.txt" must exist.'))
logprint.error(error_message)
raise RuntimeError(error_message)
user_model = ModelImporter.model('user')
anon_user = user_model.findOne({'login': anonuser})
if anon_user is None:
anon_user = user_model.createUser(login=anonuser,
password=None,
firstName='Public',
lastName='User',
email='*****@*****.**',
admin=False,
public=False)
anon_user['status'] = 'enabled'
anon_user = user_model.save(anon_user)
Osumo._cp_config['tools.staticdir.dir'] = (os.path.join(
info['pluginRootDir'], 'web_client'))
osumo = Osumo(anonuser)
registerPluginWebroot(osumo, info['name'])
info['apiRoot'].osumo = osumo
events.bind('data.process', 'osumo', osumo.dataProcess)
def __init__(self, name, onError, operations, mountpoint, *args, **kwargs):
"""
This wraps fuse.FUSE so that errors are logged rather than raising a
RuntimeError exception.
:param name: key for the mount point.
:param onError: a function that is called with `name` if initialization
fails.
"""
try:
super(FUSELogError, self).__init__(operations, mountpoint, *args,
**kwargs)
except RuntimeError:
logprint.error(
'Failed to mount FUSE. Does the mountpoint (%r) exist and is '
'it empty? Does the user have permission to create FUSE '
'mounts? It could be another FUSE mount issue, too.' %
(mountpoint, ))
onError(name)
def main(identity_file, port, host):
"""
This is the entrypoint of the girder sftpd program. It should not be
called from python code.
"""
try:
hostKey = paramiko.RSAKey.from_private_key_file(identity_file)
except paramiko.ssh_exception.PasswordRequiredException:
logprint.error(
'Error: encrypted key files are not supported (%s).' % identity_file, file=sys.stderr)
sys.exit(1)
server = SftpServer((host, port), hostKey)
logprint.info('Girder SFTP service listening on %s:%d.' % (host, port))
try:
server.serve_forever()
except (SystemExit, KeyboardInterrupt):
server.server_close()
def main(identity_file, port, host):
"""
This is the entrypoint of the girder sftpd program. It should not be
called from python code.
"""
try:
hostKey = paramiko.RSAKey.from_private_key_file(identity_file)
except paramiko.ssh_exception.PasswordRequiredException:
logprint.error('Error: encrypted key files are not supported (%s).' %
identity_file,
file=sys.stderr)
sys.exit(1)
server = SftpServer((host, port), hostKey)
logprint.info('Girder SFTP service listening on %s:%d.' % (host, port))
try:
server.serve_forever()
except (SystemExit, KeyboardInterrupt):
server.server_close()
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
logprint.error(message)
return
else:
raise ValidationException(message)
deps = set()
for key in keys:
deps |= allPlugins[plugin][key]
dag[plugin] = deps
for dep in deps:
if dep in visited:
continue
visited.add(dep)
if dep not in plugins:
logger.info('Adding plugin %s because %s requires it' % (dep, plugin))
addDeps(dep)
def _loadPlugins(names, info):
"""Load a list of plugins with the given app info object.
This method will try to load **all** plugins in the provided list. If
an error occurs, it will be logged and the next plugin will be loaded. A
list of successfully loaded plugins will be returned.
"""
loadedPlugins = []
for name in names:
pluginObject = getPlugin(name)
if pluginObject is None:
logprint.error('Plugin %s is not installed' % name)
continue
try:
pluginObject.load(info)
except Exception:
continue
loadedPlugins.append(name)
return loadedPlugins
def addDeps(plugin):
if plugin not in allPlugins:
message = 'Required plugin %s does not exist.' % plugin
if ignoreMissing:
logprint.error(message)
return
else:
raise ValidationException(message)
deps = set()
for key in keys:
deps |= allPlugins[plugin][key]
dag[plugin] = deps
for dep in deps:
if dep in visited:
continue
visited.add(dep)
if dep not in plugins:
logger.info('Adding plugin %s because %s requires it' %
(dep, plugin))
addDeps(dep)
def advance(self, jobId):
"""
Advance the workflow.
Runs all remaining steps that have their dependencies met.
Finalizes the job if all steps are complete.
:param jobId: Identifier of the job running the workflow.
:type jobId: str
"""
with self._lock:
logprint.info(
'DanesfieldWorkflowManager.advance Job={}'.format(jobId))
jobData = self._getJobData(jobId)
incompleteSteps = [
step for step in self.workflow.steps
if (step.name not in jobData['completedSteps']
and step.name not in jobData['failedSteps'])
]
# Skip run-metrics if the AOI is unknown
# model = jobData['options'].get('classify-materials', {}).get(
# 'model')
# if model is None or model == 'STANDARD':
# try:
# incompleteSteps.remove(RunMetricsStep)
# except ValueError as e:
# pass
logprint.info(
'DanesfieldWorkflowManager.advance IncompleteSteps={}'.format(
[step.name for step in incompleteSteps]))
runningSteps = [
step for step in self.workflow.steps
if step.name in jobData['runningSteps']
]
logprint.info(
'DanesfieldWorkflowManager.advance RunningSteps={}'.format(
[step.name for step in runningSteps]))
# Finalize job if either:
# - All steps have completed, or
# - A previous step failed and no steps are running
# Note that it's possible that future steps could run
# successfully if they don't depend on the failed step;
# that's not currently handled.
if not runningSteps and \
(not incompleteSteps or jobData['failedSteps']):
self.finalizeJob(jobId)
return
readySteps = [
step for step in incompleteSteps
if step.name not in jobData['runningSteps']
and step.dependencies.issubset(jobData['completedSteps'])
]
logprint.info(
'DanesfieldWorkflowManager.advance ReadySteps={}'.format(
[step.name for step in readySteps]))
if not runningSteps and not readySteps and incompleteSteps:
logprint.error(
'DanesfieldWorkflowManager.advance StuckSteps={}'.format(
[step.name for step in incompleteSteps]))
# TODO: More error notification/handling/clean up
return
jobInfo = JobInfo(jobId=jobId,
requestInfo=jobData['requestInfo'],
workingSets=jobData['workingSets'],
standardOutput=jobData['standardOutput'],
outputFolder=jobData['outputFolder'],
options=jobData['options'])
if readySteps:
adminUser = User().getAdmins().next()
for step in readySteps:
# Create output directory for step
outputFolder = Folder().createFolder(
parent=jobInfo.outputFolder,
name=step.name,
parentType='folder',
public=False,
creator=adminUser,
reuseExisting=True)
jobData['runningSteps'].add(step.name)
step.run(jobInfo, outputFolder)
