def search_query(cls, username, q, fields=[], sanitize=True, **kwargs):
"""Search the Elasticsearch index using a query string
Use a query string to search the ES index. This method will search
on the fields **name**, and **keywords**
:param str username: username making the request
:param str q: string to query the ES index
:param list fields: list of strings
.. note:: In order to make this method scalable more fields can be
searched by passing a **fields** keyword argument. You can add
the **systemTags** field like so:
>>> Object.search('username', 'txt', fields = ['systemTags'])
"""
if isinstance(fields, basestring):
fields = fields.split(',')
search_fields = ['name', 'name._exact', 'keywords']
if fields:
search_fields += fields
sq = Q('bool',
must=query_utils.files_wildcard_query(q, search_fields),
filter=query_utils.files_access_filter(username))
s = cls.search()
s.query = sq
s = s.sort('path._exact', 'name._exact')
res, s = cls._execute_search(s)
offset, limit = cls.get_paginate_limits(res, **kwargs)
#logger.debug('limit: %s. offset: %s' % (limit, offset))
return res, s[offset:limit]
def search_query(cls, username, q, fields = [], sanitize = True, **kwargs):
"""Search the Elasticsearch index using a query string
Use a query string to search the ES index. This method will search
on the fields **name**, and **keywords**
:param str username: username making the request
:param str q: string to query the ES index
:param list fields: list of strings
.. note:: In order to make this method scalable more fields can be
searched by passing a **fields** keyword argument. You can add
the **systemTags** field like so:
>>> Object.search('username', 'txt', fields = ['systemTags'])
"""
if isinstance(fields, basestring):
fields = fields.split(',')
search_fields = ['name', 'name._exact', 'keywords']
if fields:
search_fields += fields
sq = Q('filtered',
query = query_utils.files_wildcard_query(q, search_fields),
filter = query_utils.files_access_filter(username)
)
s = cls.search()
s.query = sq
s = s.sort('path._exact', 'name._exact')
res, s = cls._execute_search(s)
offset, limit = cls.get_paginate_limits(res, **kwargs)
#logger.debug('limit: %s. offset: %s' % (limit, offset))
return res, s[offset:limit]
def from_file_path(cls, system, username, file_path):
"""Retrieves a document from the ES index based on a path.
:param str system: system id
:param str username: username making the request
:param str file_path: path of a file
:returns: instance of this class or None if the file
doesn't exists in the index
:rtype: :class:`Object`
"""
path, name = os.path.split(file_path)
path = path or '/'
q = Q('bool',
must=Q('bool',
must=[
Q({'term': {
'path._exact': path
}}),
Q({'term': {
'name._exact': name
}})
]),
filter=query_utils.files_access_filter(username, system))
s = cls.search()
s.query = q
res, s = cls._execute_search(s)
if res.hits.total:
return res[0]
else:
return None
def from_file_path(cls, system, username, file_path):
"""Retrieves a document from the ES index based on a path.
:param str system: system id
:param str username: username making the request
:param str file_path: path of a file
:returns: instance of this class or None if the file
doesn't exists in the index
:rtype: :class:`Object`
"""
path, name = os.path.split(file_path)
path = path or '/'
q = Q('filtered',
query = Q('bool',
must = [
Q({'term': {'path._exact': path}}),
Q({'term': {'name._exact': name}})
]
),
filter = query_utils.files_access_filter(username, system)
)
s = cls.search()
s.query = q
res, s = cls._execute_search(s)
if res.hits.total:
return res[0]
else:
return None
def listing_recursive(cls, system, username, file_path, **kwargs):
"""Do a listing recursevly
This method will first check if the file_path that is been listing
is not a shared file, if so, it will use :func:`_listing_recursive`.
If ``file_path`` is a shared file it will combine the listing into
common denominators. This approach is neccesary in order to avoid
showing empty shared directories or directories where there are
no files shared with the requesting user. This gives the user
a more fluent navigation.
.. example::
**Reasoning behind combining the listing into common
denomintaros**.
Say there are three folders ``a/b``, ``a/c`` and ``a/d``.
And say a user has shared a few files and directories
like this:
- ``a/b/path/to/folder``
- ``a/b/path/to/folder/file1.txt``
- ``a/c/path/another/folder``
- ``a/c/path/another/folder2``
- ``a/d/file``
Then the listing should show as this:
- ``a/b/path/to/folder``
- ``a/c/path/another``
- ``a/d/file``
Thus making a more fluent navigation.
.. note::
This function assumes than when listing the root path
(``/`` or `` ``) then we are doing a listing of shared
files.
"""
owner = file_path.split('/')[0]
#If we are listing something inside the requesting user's home dir.
if username == owner or username == 'ds_admin':
return cls._listing_recursive(system, username, file_path)
#Everything else should be something shared. If we are listing the
#root path get everything.
if file_path == '/' or file_path == '':
q = Q('filtered',
filter = query_utils.files_access_filter(username, system)
)
s = cls.search()
s = s.sort('path._path', 'name._exact')
s.query = q
logger.debug('Recursive Listing query: {}'.format(s.to_dict()))
r, s = cls._execute_search(s)
else:
#Get recursive listing for shared path.
r, s = cls._listing_recursive(system, username, file_path)
listing = merge_file_paths(system, username, file_path, s)
r.hits.total = len(listing)
offset, limit = cls.get_paginate_limits(r, **kwargs)
return r, listing[offset:limit]
def listing_recursive(cls, system, username, file_path, **kwargs):
"""Do a listing recursevly
This method will first check if the file_path that is been listing
is not a shared file, if so, it will use :func:`_listing_recursive`.
If ``file_path`` is a shared file it will combine the listing into
common denominators. This approach is neccesary in order to avoid
showing empty shared directories or directories where there are
no files shared with the requesting user. This gives the user
a more fluent navigation.
.. example::
**Reasoning behind combining the listing into common
denomintaros**.
Say there are three folders ``a/b``, ``a/c`` and ``a/d``.
And say a user has shared a few files and directories
like this:
- ``a/b/path/to/folder``
- ``a/b/path/to/folder/file1.txt``
- ``a/c/path/another/folder``
- ``a/c/path/another/folder2``
- ``a/d/file``
Then the listing should show as this:
- ``a/b/path/to/folder``
- ``a/c/path/another``
- ``a/d/file``
Thus making a more fluent navigation.
.. note::
This function assumes than when listing the root path
(``/`` or `` ``) then we are doing a listing of shared
files.
"""
owner = file_path.split('/')[0]
#If we are listing something inside the requesting user's home dir.
if username == owner or username == 'ds_admin':
return cls._listing_recursive(system, username, file_path)
#Everything else should be something shared. If we are listing the
#root path get everything.
if file_path == '/' or file_path == '':
q = Q('bool',
filter=query_utils.files_access_filter(username, system))
s = cls.search()
s = s.sort('path._exact', 'name._exact')
s.query = q
logger.debug('Recursive Listing query: {}'.format(s.to_dict()))
r, s = cls._execute_search(s)
else:
#Get recursive listing for shared path.
r, s = cls._listing_recursive(system, username, file_path)
listing = merge_file_paths(system, username, file_path, s)
r.hits.total = len(listing)
offset, limit = cls.get_paginate_limits(r, **kwargs)
return r, listing[offset:limit]
def _listing_recursive(cls, system, username, file_path):
"""Do a listing recursively
This method is an efficient way to recursively do a "listing" of a
folder. This is because of the hierarcical tokenizer we have in
`path._path`. The returning listing will be sorted by path and name
:param str system: system id
:param str username: username making the request
:param str file_path: path of the folder to list
:returns: list of :class:`Object`
:rtype: list
.. note:: the returned list does not contain the parent file
Examples:
---------
Sort listing by depth
.. code-block:: python
>>> listing = Object.listing_recursive('agave.system.id',
... 'username', 'username/path/folder')
>>> sorted(listing, key=lambda x: len(x.full_path.split('/')))
.. note:: Python sorting is stable. In theory we could sort the listing
alphabetically (default behaivour) and then sort the listing
by depth and we'll end up with a listing sorted both by depth
and alphabetically.
"""
#logger.debug('Using username: {}'.format(username))
q = Q('filtered',
query = Q('bool',
must = [
Q({'term': {'path._path': file_path}})
]
),
filter = query_utils.files_access_filter(username, system)
)
s = cls.search()
s = s.sort('path._exact', 'name._exact')
s.query = q
return cls._execute_search(s)
def _listing_recursive(cls, system, username, file_path):
"""Do a listing recursively
This method is an efficient way to recursively do a "listing" of a
folder. This is because of the hierarcical tokenizer we have in
`path._path`. The returning listing will be sorted by path and name
:param str system: system id
:param str username: username making the request
:param str file_path: path of the folder to list
:returns: list of :class:`Object`
:rtype: list
.. note:: the returned list does not contain the parent file
Examples:
---------
Sort listing by depth
.. code-block:: python
>>> listing = Object.listing_recursive('agave.system.id',
... 'username', 'username/path/folder')
>>> sorted(listing, key=lambda x: len(x.full_path.split('/')))
.. note:: Python sorting is stable. In theory we could sort the listing
alphabetically (default behaivour) and then sort the listing
by depth and we'll end up with a listing sorted both by depth
and alphabetically.
"""
#logger.debug('Using username: {}'.format(username))
q = Q('bool',
must=Q('bool', must=[Q({'term': {
'path._path': file_path
}})]),
filter=query_utils.files_access_filter(username, system))
s = cls.search()
s = s.sort('path._exact', 'name._exact')
s.query = q
return cls._execute_search(s)
def listing(cls, system, username, file_path, **kwargs):
"""Do a listing of one level.
:param str system: system id
:param str username: username making the request
:param str file_path: file path to list
:returns: list of :class:`Object`
:rtype: list
"""
q = Q('bool',
must=Q({'term': {
'path._exact': file_path
}}),
filter=query_utils.files_access_filter(username, system))
s = cls.search()
s.query = q
s = s.sort({'name._exact': 'asc'})
res, s = cls._execute_search(s)
offset, limit = cls.get_paginate_limits(res, **kwargs)
return res, s[offset:limit]
def listing(cls, system, username, file_path, **kwargs):
"""Do a listing of one level.
:param str system: system id
:param str username: username making the request
:param str file_path: file path to list
:returns: list of :class:`Object`
:rtype: list
"""
q = Q('filtered',
query = Q('bool',
must = Q({'term': {'path._exact': file_path}})
),
filter = query_utils.files_access_filter(username, system)
)
s = cls.search()
s.query = q
s = s.sort({'name._exact': 'asc'})
res, s = cls._execute_search(s)
offset, limit = cls.get_paginate_limits(res, **kwargs)
return res, s[offset:limit]
