def LQuery(*args):
if args[1] is None:
expr = Ltree('None.*')
else:
expr = Ltree("*." + str(arg[1]) + ".*") #Validation fails at this point.
return args[0].query.filter(args[0].path.lquery(expr)).all()
def test_create_category(self):
'''Ensure creating categories behave as expected.
'''
category1 = Category(name='First category', url='/category/cat-256121')
category2 = Category(name='Second category',
url='/category/cat-two-256121',
parent=category1)
self.assertEqual(category1.slug, 'cat')
self.assertEqual(category1.parent, None)
self.assertEqual(category1.path, Ltree('cat'))
self.assertEqual(category2.slug, 'cat_two')
self.assertEqual(category2.parent, category1)
self.assertEqual(category2.path, Ltree('cat.cat_two'))
async def register(self, session) -> str:
"""
Obtain and manually set object's id and ltree path.
session must be provided with tornado_sqlalchemy's ThreadPoolExecutor instance.
"""
# Prevent INSERT before instance registration
with session.no_autoflush:
# Expected to be zero if the tree is empty
node_count = await as_future(
lambda: session.query(self.__class__).count())
# Obtain the next index from the sequence
id = await as_future(
lambda: session.execute(Sequence('nodelist_id_seq')))
ltree_id = Ltree(str(id))
# TODO: validate parent's path
if type(self.parent) is not self.__class__:
if node_count > 0:
raise TreeNotEmptyError('Parentless node in non-empty tree')
else:
self.path = ltree_id
else:
self.path = self.parent.path + ltree_id
self.id = id
return self.path
def insertMateriallizedPath(last_id, start_range, end_range, parent_id,
parent_path):
for i in range(start_range, end_range):
object_created_date = datetime.today() - timedelta(hours=i)
object_updated_date = datetime.today() - timedelta(minutes=i)
desc = randomword(i + 10)
title = randomword(5)
model = MaterializedPathModel(
row_id=i,
created_at=object_created_date.strftime("%Y-%m-%d %H:%M:%S"),
updated_at=object_updated_date.strftime("%Y-%m-%d %H:%M:%S"),
parent_id=parent_id,
title=title,
description=desc,
path=Ltree(parent_path + "." + str(i)))
db.session.add(model)
if parent_id < 0:
parent_id += 2
parent_path = parent_path + "." + str(i)
if i % 3 == 0:
parent_id += 1
parent_object = MaterializedPathModel.query.filter(
MaterializedPathModel.id == parent_id).first()
if parent_object is not None:
parent_path = str(parent_object.path)
if (i % CHUNK_SIZE == 0):
logger.info('Commiting Session Rows')
db.session.commit()
db.session.commit()
def strfltee(s: str, replacements=(" ", "-")):
result = s
for repl in replacements:
result = result.replace(repl, "")
return Ltree(result)
def __init__(self, *args, **kwargs):
engine = kwargs['engine']
del kwargs['engine']
super().__init__(*args, **kwargs)
_id = engine.execute(id_seq)
self.id = _id
self.uuid = uuid.uuid4()
parent = kwargs['parent'] if 'parent' in kwargs else None
ltree_id = Ltree(str(self.id))
self.path = ltree_id if parent is None else parent.path + ltree_id
def create_root(self, db: Session, *,
obj_in: OrganizationCreateRoot) -> Organization:
parent = self.get(
db, id=obj_in.parent_id) if 'parent_id' in obj_in else None
obj_in_data = jsonable_encoder(obj_in, exclude=["owner_email"])
org = Organization(**obj_in_data, parent=None)
db.add(org)
db.commit()
db.refresh(org)
org.path = (Ltree(str(org.id)) if parent is None else
(parent.path or Ltree("_")) + Ltree(str(org.id)))
org.slug = Organization.initiate_unique_slug(name=org.name,
id=org.id,
path=org.path)
db.commit()
db.refresh(org)
return org
def __init__(self, *args, **kwargs):
if 'slug' not in kwargs:
url = kwargs.get('url')
slug = re.search(r'/category/(\D+)-\d+', url).group(1)
slug = slug.replace('-', '_')
kwargs['slug'] = slug
else:
slug = kwargs.get('slug')
ltree_slug = Ltree(slug)
parent = kwargs.get('parent')
kwargs['path'] = ltree_slug if not parent else parent.path + ltree_slug
super().__init__(*args, **kwargs)
def parent_category_term(self):
"""
Hybrid property representing the parent (CategoryTerm) of a CategoryTerm, if one exists
Performs a query based on the CategoryTerm.path Ltree property. If the path of the CategoryTerm is a single
(root) level (e.g. 'root' rather than 'root.foo.bar') then there isn't a parent CategoryTerm and this property
is made empty.
Otherwise the current path is shortened (up) by a single level and used to find the relevant parent
CategoryTerm (i.e. '1.2.3' becomes '1.2').
:rtype CategoryTerm
:return: CategoryTerm that is a parent of the current CategoryTerm as determined by the Ltree column
"""
parent_category_term_path = Ltree(self.path)
if len(parent_category_term_path) == 1:
return None
parent_category_term_path = Ltree(self.path)[:-1]
return CategoryTerm.query.filter_by(
path=parent_category_term_path).first()
def group_by_path(request: Request, path: str) -> Group:
"""Get a group specified by {group_path} in the route (or 404)."""
# If loading the specified group path into the GroupSchema changed it, do a
# 301 redirect to the resulting group path. This will happen in cases like
# the original url including capital letters in the group path, where we
# want to redirect to the proper all-lowercase path instead.
if path != request.matchdict['group_path']:
request.matchdict['group_path'] = path
proper_url = request.route_url(request.matched_route.name,
**request.matchdict)
raise HTTPMovedPermanently(location=proper_url)
query = request.query(Group).filter(Group.path == Ltree(path))
return get_resource(request, query)
def handleStorage(email, gaia_id, fs):
"""
Expect some input of the form:
{
email: "",
id: "",
storage: ...,
}
"""
User(email=email, gaia_id=gaia_id)
assert User.validate(email, gaia_id), "Invalid email and gaia_id combination"
# TODO OPTIMIZE BY SENDING IN BULK
for path, file in fs.items():
File.upsert(email, Ltree(encode(path)), file) # TODO FOR BIB
return True
def update_common_topic_tags(config_path: str) -> None:
"""Update the list of common topic tags for all groups."""
db_session = get_session_from_config(config_path)
all_groups = db_session.query(Group).all()
for group in all_groups:
# create a subquery for all tags from topics in that group - UNNEST() converts
# the arrays of tags into rows so that we can easily group and count, and
# created_time will be used to determine when a particular tag was last used
group_tags = (
db_session.query(
func.unnest(Topic._tags).label("tag"), Topic.created_time # noqa
)
.filter(Topic.group == group)
.subquery()
)
# get the list of the most common tags, based on frequency and breaking ties
# with which was used most recently
common_tags = (
db_session.query(
group_tags.columns["tag"],
func.count().label("frequency"),
func.max(group_tags.columns["created_time"]).label("last_used"),
)
.group_by("tag")
.order_by(desc("frequency"), desc("last_used"))
.limit(MAX_NUM_COMMON_TAGS)
.all()
)
group._common_topic_tags = [ # noqa
Ltree(common_tag[0]) for common_tag in common_tags
]
db_session.add(group)
db_session.commit()
def _generate_category_term_ltree_path(path_elements: Dict[str, str]) -> Ltree:
"""
Converts a series of parent Category Terms into an ltree column compatible value
Category Terms are hierarchical, with parent terms representing broader categories and children, narrower
categories. Currently this hierarchy is represented as a tree data structure, implemented in the underlying
database using an 'ltree' data type (https://www.postgresql.org/docs/current/ltree.html).
This data type stores the path from the current item to the root item, including any intermediate/ancestor items.
This path is represented as a `.` separated string of labels for each item.
For example, the path from an item to the root item could be: [Item] -> [Parent Item] -> [Root Item]. When encoded
as an ltree path this becomes: '[Root Item].[Parent Item].[Item]'. Labels are limited to alphanumeric characters
and underscores - i.e. 'root_item.parent_item.item'.
Categories used in this project are usually taken from RDF vocabularies, which typically use URIs to identify each
concept/category. The category JSON Schema used to structure imported categories consistently requires the path of
each category to be expressed as a list of identifiers, consequently these are typically lists of URIs.
As these identifiers cannot be used in an ltree path encoding, this method converts them into 'ltree safe' versions
by replacing any non-alphanumeric or underscore character with an underscore.
For example, `['http://example.com/12', 'http://example.com/1']` becomes `'http_example_com_12.http_example_com_1'`.
:type path_elements: list
:param path_elements: category scheme identifiers for the categories between a category and the root category
:rtype Ltree
:return version of the categories path encoded as a ltree data type path expression
"""
if len(path_elements.values()) == 0:
raise ValueError("Path for category cannot be empty")
path = list(
map(lambda subject: re.sub('[^0-9a-zA-Z]+', '_', subject),
path_elements.values()))
return Ltree('.'.join(path))
def test_length(self, path, length):
assert len(Ltree(path)) == length
def test_validate_with_invalid_path(self, path):
with pytest.raises(ValueError) as e:
Ltree.validate(path)
assert str(
e.value) == ("'{0}' is not a valid ltree path.".format(path))
def test_validate_with_valid_codes(self, code):
Ltree.validate(code)
def test_iadd(self, path, other, result):
ltree = Ltree(path)
ltree += other
assert ltree == result
def test_init(self):
assert Ltree('path.path') == Ltree(Ltree('path.path'))
def test_lca(self, path, others, result):
assert Ltree(path).lca(*others) == result
def test_non_equality_operator(self):
assert Ltree('path.path') != u'path.'
assert not (Ltree('path.path') != 'path.path')
def test_equality_operator(self):
assert Ltree('path.path') == 'path.path'
assert 'path.path' == Ltree('path.path')
assert Ltree('path.path') == Ltree('path.path')
def test_validate_with_valid_codes(self, code):
Ltree.validate(code)
def test_validate_with_invalid_path(self, path):
with pytest.raises(ValueError) as e:
Ltree.validate(path)
assert str(e.value) == (
"'{0}' is not a valid ltree path.".format(path)
)
def test_index(self, path, subpath, index):
assert Ltree(path).index(subpath) == index
def test_getitem(self, path, item_slice, result):
assert Ltree(path)[item_slice] == result
def test_repr(self):
return repr(Ltree('path')) == "Ltree('path')"
def test_add(self, path, other, result):
assert Ltree(path) + other == result
def test_unicode(self):
ltree = Ltree('path.path')
assert six.text_type(ltree) == 'path.path'
def test_radd(self, path, other, result):
assert other + Ltree(path) == result
def test_str(self):
ltree = Ltree('path')
assert str(ltree) == 'path'
def test_hash(self):
return hash(Ltree('path')) == hash('path')
def test_constructor_with_invalid_code(self):
with pytest.raises(ValueError) as e:
Ltree('..')
assert str(e.value) == "'..' is not a valid ltree path."
