def _getFileStats(self, pathString):
"""
Private method to get stats for a file
:Parameters:
fileId : string
A string uniquely identifying a file on this Monitor.
:return: stats
:rtype: monitors.FileStats
"""
stats = monitors.FileStats()
stats.baseName = pathModule.path(pathString).name
stats.owner = pathModule.path(pathString).owner
stats.size = pathModule.path(pathString).size
stats.mTime = pathModule.path(pathString).mtime
stats.cTime = pathModule.path(pathString).ctime
stats.aTime = pathModule.path(pathString).atime
if pathModule.path(pathString).isfile():
stats.type = monitors.FileType.File
elif pathModule.path(pathString).isdir():
stats.type = monitors.FileType.Dir
elif pathModule.path(pathString).islink():
stats.type = monitors.FileType.Link
elif pathModule.path(pathString).ismount():
stats.type = monitors.FileType.Mount
else:
stats.type = monitors.FileType.Unknown
return stats
def _get_classpath_logback(self, args):
lib_client = self.ctx.dir / "lib" / "client"
auto_download = False
if args.clientdir:
client_dir = path(args.clientdir)
elif lib_client.exists():
client_dir = lib_client
else:
auto_download = True
omero_java_dir, omero_java_txt = self._userdir_jars()
client_dir = omero_java_dir
etc_dir = old_div(self.ctx.dir, "etc")
if args.logback:
xml_file = path(args.logback)
else:
xml_file = old_div(etc_dir, "logback-cli.xml")
classpath = []
if client_dir and client_dir.exists():
classpath = [f.abspath() for f in client_dir.files("*.jar")]
if auto_download:
if classpath:
self.ctx.err('Using {}'.format(omero_java_txt.text()))
if not args.logback:
xml_file = client_dir / "logback-cli.xml"
else:
if not classpath:
self.ctx.die(103, "No JAR files found under '%s'" % client_dir)
logback = "-Dlogback.configurationFile=%s" % xml_file
return classpath, logback
def fileExists(self, fileId, current=None):
"""
Return true if fileId represents a file or directory that exists.
:Parameters:
fileId : string
A string uniquely identifying a file on this Monitor.
current
An ICE context, this parameter is required to be present
in an ICE interface method.
:return: isfile
:rtype: bolean
"""
try:
pathString = self._getPathString(fileId)
except Exception:
self.log.error('File ID ' + str(fileId) + ' not on this FSServer')
raise omero.OmeroFSError(reason='File ID ' + str(fileId) +
' not on this FSServer')
try:
pathModule.path(pathString).isfile
return True
except Exception:
return False
def addWatch(self, path, mask):
if not self.isPathWatched(path):
try:
if isbytes(path):
path_obj = pathModule.path(bytes_to_native_str(path))
else:
path_obj = pathModule.path(path)
res = pyinotify.WatchManager.add_watch(self,
path,
mask,
rec=False,
auto_add=False,
quiet=False)
self.watchPaths.update(res)
self.watchParams[path] = copy.copy(
self.watchParams[path_obj.parent])
if self.watchParams[path].getRec():
for d in path_obj.dirs():
self.addWatch(d, mask)
if self.isPathWatched(path):
self.log.info('Watch added on: %s' % path)
else:
self.log.info('Unable to add watch on: %s' % path)
except Exception as e:
self.log.error('Unable to add watch on: %s : %s' % (path, e))
def get_omero_userdir():
"""Returns the OMERO user directory"""
omero_userdir = os.environ.get('OMERO_USERDIR', None)
if omero_userdir:
return path.path(omero_userdir)
else:
return old_div(path.path(get_user_dir()), "omero")
def importer(self, args):
if args.clientdir:
client_dir = path(args.clientdir)
else:
client_dir = self.ctx.dir / "lib" / "client"
etc_dir = old_div(self.ctx.dir, "etc")
if args.logback:
xml_file = path(args.logback)
else:
xml_file = old_div(etc_dir, "logback-cli.xml")
logback = "-Dlogback.configurationFile=%s" % xml_file
try:
classpath = [file.abspath() for file in client_dir.files("*.jar")]
except OSError as e:
self.ctx.die(
102, "Cannot get JAR files from '%s' (%s)" %
(client_dir, e.strerror))
if not classpath:
self.ctx.die(103, "No JAR files found under '%s'" % client_dir)
command_args = CommandArguments(self.ctx, args)
xargs = [logback, "-Xmx1024M", "-cp", os.pathsep.join(classpath)]
xargs.append("-Domero.import.depth=%s" % args.depth)
if args.bulk and args.path:
self.ctx.die(104, "When using bulk import, omit paths")
elif args.bulk:
self.bulk_import(command_args, xargs)
else:
self.do_import(command_args, xargs)
def testRelativeTest(self):
assert (old_div(self.dir, "foo")).parpath(self.dir)
assert (self.dir / "foo" / "bar" / "baz").parpath(self.dir)
# Not relative
assert not (path("/")).parpath(self.dir)
assert not (path("/root")).parpath(self.dir)
assert not (path(".")).parpath(self.dir)
def edit_path(path_or_obj, start_text):
f = path.path(path_or_obj)
editor = os.getenv("VISUAL") or os.getenv("EDITOR")
if not editor:
if platform.system() == "Windows":
editor = "Notepad.exe"
else:
editor = "vi"
if isinstance(start_text, str):
start_text = start_text.encode("utf-8")
f.write_text(start_text)
# If absolute, then use the path
# as is (ticket:4246). Otherwise,
# use which.py to find it.
editor_parts = shlex.split(editor)
editor = editor_parts[0]
editor_obj = path.path(editor)
if editor_obj.isabs():
editor_parts[0] = editor
else:
from omero_ext.which import which
editor_parts[0] = which(editor)
editor_parts.append(f)
pid = os.spawnl(os.P_WAIT, editor_parts[0], *tuple(editor_parts))
if pid:
re = RuntimeError(
"Couldn't spawn editor: %s" % editor_parts[0])
re.pid = pid
raise re
def get_omero_user_cache_dir():
"""Returns the OMERO user cache directory"""
omero_userdir = os.environ.get('OMERO_USERDIR', None)
if omero_userdir:
return path.path(omero_userdir) / "cache"
else:
return path.path(user_cache_dir(*APPDIR_DEFAULTS))
def testOldTemplates(self):
old_templates = old_div(
path(__file__).dirname(), ".." / "old_templates.xml")
old_templates.copy(
old_div(path(self.cli.dir),
"etc" / "templates" / "grid" / "templates.xml"))
with pytest.raises(NonZeroReturnCode):
self.cli.invoke(self.args, strict=True)
def isFile(self, fileId, current=None):
"""
Return true if fileId represents a file.
:Parameters:
fileId : string
A string uniquely identifying a file on this Monitor.
current
An ICE context, this parameter is required to be present
in an ICE interface method.
:return: isfile
:rtype: float
"""
try:
pathString = self._getPathString(fileId)
except Exception as e:
self.log.error('File ID ' + str(fileId) + ' not on this FSServer')
raise omero.OmeroFSError(reason='File ID ' + str(fileId) +
' not on this FSServer')
try:
isfile = pathModule.path(pathString).isfile
except Exception as e:
self.log.error('Failed to get ' + str(fileId) + ' isfile : ' +
str(e))
raise omero.OmeroFSError(reason='Failed to get ' + str(fileId) +
' isfile : ' + str(e))
return isfile
def getTable(self, file_obj, factory, current=None):
"""
Create and/or register a table servant.
"""
# Will throw an exception if not allowed.
file_id = None
if file_obj is not None and file_obj.id is not None:
file_id = file_obj.id.val
self.logger.info("getTable: %s %s", file_id, current.ctx)
file_path = self.repo_mgr.getFilePath(file_obj)
p = path(file_path).dirname()
if not p.exists():
p.makedirs()
storage = self._storage_factory.getOrCreate(file_path, self.read_only)
table = TableI(self.ctx,
file_obj,
factory,
storage,
uuid=Ice.generateUUID(),
call_context=current.ctx,
adapter=current.adapter)
self.resources.add(table)
prx = current.adapter.add(table, table.id)
return self._table_cast(prx)
def parse(self, args):
from omero.install.config_parser import PropertyParser
pp = PropertyParser()
if args.file:
args.file.close()
pp.parse_file(str(path(args.file.name).abspath()))
else:
pp.parse_lines(self.ctx.get_config_property_lines(self.dir))
# Parse PSQL profile file
for p in pp:
if p.key == "omero.db.profile":
psql_file = self.dir / "etc" / "profiles" / p.val
pp.parse_file(str(psql_file.abspath()))
break
# Parse OMERO.web configuration properties
if not args.no_web:
pp.parse_module('omeroweb.settings')
# Display options
if args.headers:
pp.print_headers()
elif args.keys:
pp.print_keys()
elif args.rst:
pp.print_rst()
else:
pp.print_defaults()
def testDefaultSessionsDir(self):
from omero.util import get_user_dir
from omero_ext.path import path
# Default store sessions dir is under user dir
store = self.cli.controls['sessions'].store(None)
assert store.dir == path(get_user_dir()) / 'omero' / 'sessions'
def upload(self, args):
client = self.ctx.conn(args)
obj_ids = []
for local_file in args.file:
if not path(local_file).exists():
self.ctx.die(500, "File: %s does not exist" % local_file)
for local_file in args.file:
omero_format = UNKNOWN
if args.mimetype:
omero_format = args.mimetype
elif (mimetypes.guess_type(local_file) != (None, None)):
omero_format = mimetypes.guess_type(local_file)[0]
if args.data_dir:
obj = upload_ln_s(client, local_file, args.data_dir,
omero_format)
obj_id = obj.id
else:
obj = client.upload(local_file, type=omero_format)
obj_id = obj.id.val
if args.wrap:
fa = FileAnnotationI()
fa.setFile(OriginalFileI(obj_id, False))
if args.namespace:
fa.setNs(rstring(args.namespace))
fa = client.sf.getUpdateService().saveAndReturnObject(fa)
obj_ids.append(fa.id.val)
else:
obj_ids.append(obj_id)
self.ctx.set("last.upload.id", obj_id)
obj_ids = self._order_and_range_ids(obj_ids)
if args.wrap:
self.ctx.out("FileAnnotation:%s" % obj_ids)
else:
self.ctx.out("OriginalFile:%s" % obj_ids)
def store(self, args):
try:
# Read base directory from deprecated --session-dir argument
base_dir = getattr(args, "session_dir", None)
if base_dir:
warnings.warn(
"--session-dir is deprecated. Use OMERO_SESSIONDIR"
" instead.", DeprecationWarning)
# Read base directory from deprecated OMERO_SESSION_DIR envvar
base_dir = os.environ.get('OMERO_SESSION_DIR', base_dir)
if 'OMERO_SESSION_DIR' in os.environ:
warnings.warn(
"OMERO_SESSION_DIR is deprecated. Use OMERO_SESSIONDIR"
" instead.", DeprecationWarning)
# Read sessions directory from OMERO_SESSIONDIR envvar
session_dir = None
if base_dir:
from omero_ext.path import path
session_dir = path(base_dir) / "omero" / "sessions"
sessions_dir = os.environ.get('OMERO_SESSIONDIR', session_dir)
return self.FACTORY(sessions_dir)
except OSError as ose:
filename = getattr(ose, "filename", sessions_dir)
self.ctx.die(155, "Could not access session dir: %s" % filename)
def set_python_dir(self, monkeypatch):
dist_dir = path(__file__) / ".." / ".." / ".." / ".." / ".." /\
"target" # FIXME: should not be hard-coded
dist_dir = dist_dir.abspath()
monkeypatch.setattr(WebControl, '_get_python_dir',
lambda x: dist_dir / "lib" / "python")
def compare_with_reference(self, refname, generated):
reffile = path(__file__).dirname() / 'reference_templates' / refname
generated = generated.split('\n')
# reffile.write_lines(generated)
ref = reffile.lines(retain=False)
d = '\n'.join(unified_diff(ref, generated))
return d
def upload(self, args):
p = path(args.file)
if not p.exists():
self.ctx.die(502, "File does not exist: %s" % p.abspath())
import omero
c = self.ctx.conn(args)
scriptSvc = c.sf.getScriptService()
if args.official:
try:
id = scriptSvc.uploadOfficialScript(args.file, p.text())
except omero.ApiUsageException as aue:
if "editScript" in aue.message:
self.ctx.die(
502, "%s already exists; use 'replace'"
" instead" % args.file)
else:
self.ctx.die(504, "ApiUsageException: %s" % aue.message)
except omero.SecurityViolation as sv:
self.ctx.die(503, "SecurityViolation: %s" % sv.message)
else:
id = scriptSvc.uploadScript(args.file, p.text())
self.ctx.err("Uploaded %sscript" %
(args.official and "official " or ""))
self.ctx.out("OriginalFile:%s" % id)
self.ctx.set("script.file.id", id)
def check_default_xml(topdir,
prefix='',
tcp=4063,
ssl=4064,
ws=4065,
wss=4066,
transports=None,
**kwargs):
if transports is None:
transports = ['ssl', 'tcp']
routerport = ('<variable name="ROUTERPORT" value="%s%s"/>' %
(prefix, ssl))
insecure_routerport = (
'<variable name="INSECUREROUTER" value="OMERO.Glacier2'
'/router:tcp -p %s%s -h @omero.host@"/>' % (prefix, tcp))
client_endpoint_list = []
for tp in transports:
if tp == 'tcp':
client_endpoint_list.append('tcp -p %s%s' % (prefix, tcp))
if tp == 'ssl':
client_endpoint_list.append('ssl -p %s%s' % (prefix, ssl))
if tp == 'ws':
client_endpoint_list.append('ws -p %s%s' % (prefix, ws))
if tp == 'wss':
client_endpoint_list.append('wss -p %s%s' % (prefix, wss))
client_endpoints = 'client-endpoints="%s"' % ':'.join(client_endpoint_list)
for key in ['default.xml', 'windefault.xml']:
s = path(old_div(topdir, "etc" / "grid" / key)).text()
assert routerport in s
assert insecure_routerport in s
assert client_endpoints in s
def getOwner(self, fileId, current=None):
"""
Return the owner of the file.
:Parameters:
fileId : string
A string uniquely identifying a file on this Monitor.
current
An ICE context, this parameter is required to be present
in an ICE interface method.
:return: owner
:rtype: string
"""
try:
pathString = self._getPathString(fileId)
except Exception as e:
self.log.error('File ID ' + str(fileId) + ' not on this FSServer')
raise omero.OmeroFSError(reason='File ID ' + str(fileId) +
' not on this FSServer')
try:
owner = pathModule.path(pathString).owner
except Exception as e:
self.log.error('Failed to get ' + str(fileId) + ' owner : ' +
str(e))
raise omero.OmeroFSError(reason='Failed to get ' + str(fileId) +
' owner : ' + str(e))
return owner
def testBulkInclude(self):
t = path(__file__).parent / "bulk_import" / "test_include" / "inner"
b = old_div(t, "bulk.yml")
self.add_client_dir()
self.args += ["-f", "---bulk=%s" % b]
self.cli.invoke(self.args, strict=True)
def testBulkNoPaths(self):
t = path(__file__) / "bulk_import" / "test_simple"
b = old_div(t, "bulk.yml")
self.add_client_dir()
self.args += ["-f", "---bulk=%s" % b, "dne.fake"]
with pytest.raises(NonZeroReturnCode):
self.cli.invoke(self.args, strict=True)
def testImportReaders(self, tmpdir, capfd, params):
"""Test fake image import"""
fakefile = tmpdir.join("test.fake")
fakefile.write('')
flag, filename, status = params
filename = path(__file__).parent / "readers" / filename
self.add_client_dir()
self.args += ["-f", flag, filename]
self.args += [str(fakefile)]
if status:
self.cli.invoke(self.args, strict=True)
o, e = capfd.readouterr()
outputlines = str(o).split('\n')
reader = 'loci.formats.in.FakeReader'
assert outputlines[-2] == str(fakefile)
assert outputlines[-3] == \
"# Group: %s SPW: false Reader: %s" % (str(fakefile), reader)
else:
with pytest.raises(NonZeroReturnCode):
self.cli.invoke(self.args, strict=True)
o, e = capfd.readouterr()
assert "parsed into 0 group" in e
def get_omero_userdir():
"""
Returns the OMERO user directory
In 6.0.0 the default will change to use appdirs.user_data_dir.
You can enable this behaviour now by setting OMERO_USERDIR="" (empty
string) instead of unset.
Note that files in the old user directory will not be migrated.
"""
omero_userdir = os.environ.get('OMERO_USERDIR', None)
if omero_userdir:
return path.path(omero_userdir)
elif omero_userdir == "":
return path.path(user_data_dir(*APPDIR_DEFAULTS))
else:
return old_div(path.path(get_user_dir()), "omero")
def testTemplatesGeneration(self):
"""Test template files are generated by the rewrite subcommand"""
# Test non-existence of configuration files
for f in GRID_FILES:
assert not os.path.exists(
old_div(path(self.cli.dir), "etc" / "grid" / f))
for f in ETC_FILES:
assert not os.path.exists(old_div(path(self.cli.dir), "etc" / f))
# Call the jvmcf command and test file genearation
self.cli.invoke(self.args, strict=True)
for f in GRID_FILES:
assert os.path.exists(
old_div(path(self.cli.dir), "etc" / "grid" / f))
for f in ETC_FILES:
assert os.path.exists(old_div(path(self.cli.dir), "etc" / f))
def testBulkBad(self):
t = path(__file__).parent / "bulk_import" / "test_bad"
b = old_div(t, "bulk.yml")
self.add_client_dir()
self.args += ["-f", "---bulk=%s" % b]
with pytest.raises(NonZeroReturnCode):
self.cli.invoke(self.args, strict=True)
def repofile(self, db_uuid, repo_uuid=None):
if repo_uuid is None:
repo_uuid = self.repouuid()
f = self.repodir()
f = old_div(path(f), db_uuid)
f.makedirs()
f = old_div(f, "repo_uuid")
f.write_lines([repo_uuid])
def repodir(self, make=True):
self.tmp = path(self.tmpdir())
self.communicator.getProperties().setProperty("omero.repo.dir",
native_str(self.tmp))
repo = self.tmp / ".omero" / "repository"
if make:
repo.makedirs()
return str(repo)
def test_12527(self, fixture, monkeypatch):
monkeypatch.setattr(Strategy, '_system_memory_mb_java', lambda x:
(2000, 4000))
p = write_config(fixture.input)
old_templates = old_div(path(__file__).dirname(), "old_templates.xml")
xml = XML(old_templates.abspath().text())
config = ConfigXml(filename=str(p), env_config="default")
with pytest.raises(Exception):
adjust_settings(config, xml, **fixture.kwargs)
