def create_test_article(create=True, articleset=None, deduplicate=True, properties=None, **kargs):
"""Create a test article"""
from amcat.models.article import Article
# Get static properties
title = kargs.pop("title", "test title {}: {}".format(_get_next_id(), uuid4()))
date = kargs.pop("date", datetime.datetime.now())
url = kargs.pop("url", "http://example.com")
text = kargs.pop("text", "Lorum Ipsum: {}".format(_get_next_id()))
project = kargs.pop("project", articleset.project if articleset is not None else create_test_project())
parent_hash = kargs.pop("parent_hash", None)
hash = kargs.pop("hash", None)
# Caller is allowed to pas date as string
if isinstance(date, str):
date = _parse_date(date)
a = Article(title=title, date=date, url=url, text=text, project=project, parent_hash=parent_hash, hash=hash)
if properties:
for propname, value in properties.items():
if get_property_primitive_type(propname) == datetime.datetime and isinstance(value, str):
properties[propname] = _parse_date(value)
a.properties.update(properties)
if create:
Article.create_articles([a], articleset, deduplicate=deduplicate)
return a
def __setitem__(self, key: str, value: Union[str, int, float,
datetime.datetime]):
# Check for key type
if not isinstance(key, str):
raise ValueError(
"{} is not of type str, but is {} instead.".format(
key, type(key)))
if not is_valid_property_name(key):
raise ValueError("This property name is not valid: {}".format(key))
# None does not make sense in this context. The caller should use __delitem__ instead.
if value is None:
raise ValueError("Value can not be None. Delete it instead!")
# Property types are determined by their name. As a result, we expect that type.
expected_type = get_property_primitive_type(key)
# Implicitly convert ints to floats (but not the other way around)
if expected_type is float and isinstance(value, int):
value = float(value)
if not isinstance(value, expected_type):
raise ValueError(
"Expected type {} for key {}. Got {} with type {} instead.".
format(expected_type, key, value, type(value)))
super().__setitem__(key, value)
def create_test_article(create=True, articleset=None, deduplicate=True, properties=None, project=None, **kargs):
"""Create a test article"""
from amcat.models.article import Article
# Get static properties
title = kargs.pop("title", "test title {}: {}".format(_get_next_id(), uuid4()))
date = kargs.pop("date", datetime.datetime.now())
url = kargs.pop("url", "http://example.com")
text = kargs.pop("text", "Lorum Ipsum: {}".format(_get_next_id()))
if project is None:
project = articleset.project if articleset is not None else create_test_project()
parent_hash = kargs.pop("parent_hash", None)
hash = kargs.pop("hash", None)
# Caller is allowed to pas date as string
if isinstance(date, str):
date = _parse_date(date)
a = Article(title=title, date=date, url=url, text=text, project=project, parent_hash=parent_hash, hash=hash)
if properties:
for propname, value in properties.items():
if get_property_primitive_type(propname) == datetime.datetime and isinstance(value, str):
properties[propname] = _parse_date(value)
a.properties.update(properties)
if create:
Article.create_articles([a], articleset, deduplicate=deduplicate)
return a
def fromdb(cls, d):
new = PropertyMapping()
for key, value in d.items():
expected_type = get_property_primitive_type(key)
if expected_type == datetime.datetime:
new[key] = iso8601.parse_date(value)
else:
new[key] = expected_type(value)
return new
def get_property_field_cls(property):
type = get_property_primitive_type(property)
base_field = TYPE_FIELDS[type]
class PropertyField(base_field):
def get_attribute(self, instance):
return instance.article.properties.get(property)
return PropertyField
def __getattr__(self, field):
if field in self._fields:
value = None
if field in self._result:
value = self._result[field]
if get_property_primitive_type(field) == datetime.datetime:
value = iso8601.parse_date(value, default_timezone=None)
setattr(self, field, value)
return value
raise AttributeError("{!r} object has no attribute {!r}".format(self, field))
def __getattr__(self, field):
if field in self._fields:
value = None
if field in self._result:
value = self._result[field]
if get_property_primitive_type(field) == datetime.datetime:
value = iso8601.parse_date(value, default_timezone=None)
setattr(self, field, value)
return value
raise AttributeError("{!r} object has no attribute {!r}".format(self, field))
def parse_file(self, file: model_UploadedFile, data):
self.ln_query, arts = data
for data in arts:
art = {}
for field, setting in self.options['field_map'].items():
datatype = get_property_primitive_type(field)
value, typ = setting['value'], setting['type']
val = data.get(value) if typ == 'field' else value
if val:
if datatype is datetime.datetime and type(val) is str:
val = toolkit.read_date(val)
art[field] = val
yield Article(**art)
def from_fieldname(cls, fieldname):
"""Instantiate the correct type of FieldCategory for a given field."""
ptype = amcates.get_property_primitive_type(fieldname)
if ptype == datetime.datetime:
raise ValueError("Use DateFieldCategory() instead of FieldCategroy.from_fieldname for field: {}".format(fieldname))
elif ptype == int:
return IntegerFieldCategory(fieldname)
elif ptype == float:
return NumFieldCategory(fieldname)
elif ptype == str:
return StringFieldCategory(fieldname)
else:
raise ValueError("Did not recognize type {} of field {}.".format(ptype, fieldname))
def parse_file(self, file, encoding, data):
self.ln_query, arts = data
for data in arts:
art = {}
for field, setting in self.options['field_map'].items():
datatype = get_property_primitive_type(field)
value, typ = setting['value'], setting['type']
val = data.get(value) if typ == 'field' else value
if val:
if datatype is datetime.datetime and type(val) is str:
val = toolkit.read_date(val)
art[field] = val
yield Article(**art)
def parse_file(self, file: UploadedFile, _data):
for fields in _data:
data = {f["path"]: f["content"] for f in fields}
art = {}
for field, setting in self.options['field_map'].items():
datatype = get_property_primitive_type(field)
value, typ = setting['value'], setting['type']
val = data.get(value) if typ == 'field' else value
if val:
if datatype is datetime.datetime and type(val) is str:
val = toolkit.read_date(val)
art[field] = val
yield Article(**art)
def order_by(self, *fields, seed: Optional[int]=None) -> "ESQuerySet":
"""
Order by a number of fields. By default, order ascending. If a minus is given in front
of the fieldname, order in a descending manner. Random ordering can be given by passing
a single question mark. Examples:
qs.order_by("id")
qs.order_by("-id")
qs.order_by("?")
qs.order_by("date", "id")
Note that it is not possible to order a single field randomly. You can reset an ordering
by passing no fields or None explicitly:
qs.order_by()
qs.order_by(None)
If you need a deterministic ordering, but you don't care about the ordering itself use
_doc as an order field:
qs.order_by("_doc")
@param seed: a random ordering can be made deterministic by supplying a seed value.
"""
if fields == (None,):
return self._copy(ordering=())
# Check for validity
ordering = []
for field in fields:
if field == "?" or field == "_doc":
# Random ordering or deterministic ordering
ordering.append(field)
continue
order = "asc"
_field = field
if field.startswith(("+", "-")):
if field[0] == "-":
order = "desc"
_field = field[1:]
if get_property_primitive_type(_field) == str:
raise NotImplementedError("Ordering on string type not yet supported")
self._check_fields((_field,))
ordering.append((_field, order))
# Copy self and return new one
seed = seed if seed is not None else self.seed
return self._copy(ordering=tuple(ordering), seed=seed)
def order_by(self, *fields, seed: Optional[int]=None) -> "ESQuerySet":
"""
Order by a number of fields. By default, order ascending. If a minus is given in front
of the fieldname, order in a descending manner. Random ordering can be given by passing
a single question mark. Examples:
qs.order_by("id")
qs.order_by("-id")
qs.order_by("?")
qs.order_by("date", "id")
Note that it is not possible to order a single field randomly. You can reset an ordering
by passing no fields or None explicitly:
qs.order_by()
qs.order_by(None)
If you need a deterministic ordering, but you don't care about the ordering itself use
_doc as an order field:
qs.order_by("_doc")
@param seed: a random ordering can be made deterministic by supplying a seed value.
"""
if fields == (None,):
return self._copy(ordering=())
# Check for validity
ordering = []
for field in fields:
if field == "?" or field == "_doc":
# Random ordering or deterministic ordering
ordering.append(field)
continue
order = "asc"
_field = field
if field.startswith(("+", "-")):
if field[0] == "-":
order = "desc"
_field = field[1:]
if get_property_primitive_type(_field) == str:
raise NotImplementedError("Ordering on string type not yet supported")
self._check_fields((_field,))
ordering.append((_field, order))
# Copy self and return new one
seed = seed if seed is not None else self.seed
return self._copy(ordering=tuple(ordering), seed=seed)
def from_field_name(cls, field_name: str, **kwargs):
"""Construct a category object corresponding to the field_name's type. For example,
the field 'date' would map to a IntervalCategory, while author would map to
TextCategory.
@param kwargs: additional parameters passed to corresponding Category"""
is_json_field = field_name not in Article.static_fields()
field_type = get_property_primitive_type(field_name)
if field_type in (int, str, float):
return ArticleFieldCategory(is_json_field=is_json_field, field_name=field_name, **kwargs)
elif field_type == datetime.datetime:
return IntervalCategory(is_json_field=is_json_field, field_name=field_name, **kwargs)
else:
raise ValueError("Did not recognize primitive field type: {} (on {})".format(field_type, field_name))
def from_fieldname(cls, fieldname):
"""Instantiate the correct type of FieldCategory for a given field."""
ptype = amcates.get_property_primitive_type(fieldname)
if ptype == datetime.datetime:
raise ValueError(
"Use DateFieldCategory() instead of FieldCategroy.from_fieldname for field: {}"
.format(fieldname))
elif ptype == int:
return IntegerFieldCategory(fieldname)
elif ptype == float:
return NumFieldCategory(fieldname)
elif ptype == str:
return StringFieldCategory(fieldname)
else:
raise ValueError("Did not recognize type {} of field {}.".format(
ptype, fieldname))
def get_filter_clause(field: str, qualifier: Optional[str], filter_value):
"""
@param field:
@param qualifier:
@param filter_value:
@return:
"""
ptype = get_property_primitive_type(field)
if ptype == datetime.datetime:
return get_filter_clause_date(field, qualifier, filter_value)
elif ptype == int or ptype == float:
return get_filter_clause_num(field, qualifier, filter_value, ptype)
elif ptype == str:
return get_filter_clause_str(field, qualifier, filter_value)
elif ptype == set:
return get_filter_clause_sets(field, qualifier, filter_value)
else:
raise ValueError("Did not recognize primitive type {} of field {}".format(ptype, field))
def get_aggregation_choice(property: str) -> Tuple:
basename = get_property_basename(property)
ptype = get_property_primitive_type(property)
if ptype is str:
return (property + "_str", basename)
elif ptype in (int, float):
value = "{}_{}".format(basename, ptype.__name__)
label = "{} ({})".format(basename, HUMAN_READABLE_TYPES[ptype])
return (value, label)
elif ptype is datetime.datetime:
return ("{} (date)".format(basename), (
("{}_year".format(property), "Year"),
("{}_quarter".format(property), "Quarter"),
("{}_month".format(property), "Month"),
("{}_week".format(property), "Week"),
("{}_day".format(property), "Day")
))
else:
raise ValueError("Primitive type {} not recognized".format(ptype))
def get_filter_clause(field: str, qualifier: Optional[str], filter_value):
"""
@param field:
@param qualifier:
@param filter_value:
@return:
"""
ptype = get_property_primitive_type(field)
if ptype == datetime.datetime:
return get_filter_clause_date(field, qualifier, filter_value)
elif ptype == int or ptype == float:
return get_filter_clause_num(field, qualifier, filter_value, ptype)
elif ptype == str:
return get_filter_clause_str(field, qualifier, filter_value)
elif ptype == set:
return get_filter_clause_sets(field, qualifier, filter_value)
else:
raise ValueError("Did not recognize primitive type {} of field {}".format(ptype, field))
def get_aggregation_choice(property: str) -> Tuple:
basename = get_property_basename(property)
ptype = get_property_primitive_type(property)
if ptype is str:
return (property + "_str", basename)
elif ptype in (int, float):
value = "{}_{}".format(property, ptype.__name__)
label = "{} ({})".format(basename, HUMAN_READABLE_TYPES[ptype])
return (value, label)
elif ptype is datetime.datetime:
return ("{} (date)".format(basename), (
("{}_year".format(property), "Year"),
("{}_quarter".format(property), "Quarter"),
("{}_month".format(property), "Month"),
("{}_week".format(property), "Week"),
("{}_day".format(property), "Day")
))
else:
raise ValueError("Primitive type {} not recognized".format(ptype))
def from_field_name(cls, field_name: str, **kwargs):
"""Construct a category object corresponding to the field_name's type. For example,
the field 'date' would map to a IntervalCategory, while author would map to
TextCategory.
@param kwargs: additional parameters passed to corresponding Category"""
is_json_field = field_name not in Article.static_fields()
field_type = get_property_primitive_type(field_name)
if field_type in (int, str, float):
return ArticleFieldCategory(is_json_field=is_json_field,
field_name=field_name,
**kwargs)
elif field_type == datetime.datetime:
return IntervalCategory(is_json_field=is_json_field,
field_name=field_name,
**kwargs)
else:
raise ValueError(
"Did not recognize primitive field type: {} (on {})".format(
field_type, field_name))
def test_get_property_primitive_type(self):
# Special property names
self.assertEqual(get_property_primitive_type("sets"), set)
self.assertEqual(get_property_primitive_type("text"), str)
self.assertEqual(get_property_primitive_type("hash"), str)
self.assertEqual(get_property_primitive_type("date"), datetime.datetime)
self.assertEqual(get_property_primitive_type("url"), str)
self.assertEqual(get_property_primitive_type("title"), str)
self.assertEqual(get_property_primitive_type("hash"), str)
self.assertEqual(get_property_primitive_type("id"), int)
# User-defined properties
self.assertEqual(get_property_primitive_type("foo_date"), datetime.datetime)
self.assertEqual(get_property_primitive_type("foo_num"), float)
self.assertEqual(get_property_primitive_type("foo_int"), int)
self.assertEqual(get_property_primitive_type("foo_url"), str)
self.assertEqual(get_property_primitive_type("foo_id"), str)
def test_get_property_primitive_type(self):
# Special property names
self.assertEqual(get_property_primitive_type("sets"), set)
self.assertEqual(get_property_primitive_type("text"), str)
self.assertEqual(get_property_primitive_type("hash"), str)
self.assertEqual(get_property_primitive_type("date"),
datetime.datetime)
self.assertEqual(get_property_primitive_type("url"), str)
self.assertEqual(get_property_primitive_type("title"), str)
self.assertEqual(get_property_primitive_type("hash"), str)
self.assertEqual(get_property_primitive_type("id"), int)
# User-defined properties
self.assertEqual(get_property_primitive_type("foo_date"),
datetime.datetime)
self.assertEqual(get_property_primitive_type("foo_num"), float)
self.assertEqual(get_property_primitive_type("foo_int"), int)
self.assertEqual(get_property_primitive_type("foo_url"), str)
self.assertEqual(get_property_primitive_type("foo_id"), str)