def validate_fields_exist(self, field_or_fields):
"""Validates that the collection has fields with the given names.
If ``field_or_fields`` contains an embedded field name such as
``field_name.document.field``, only the root ``field_name`` is checked
for existence.
Args:
field_or_fields: a field name or iterable of field names
Raises:
ValueError: if one or more of the fields do not exist
"""
if etau.is_str(field_or_fields):
field_or_fields = [field_or_fields]
schema = self.get_field_schema()
default_fields = set(
default_sample_fields(DatasetSampleDocument,
include_private=True,
include_id=True))
for field in field_or_fields:
# We only validate that the root field exists
field_name = field.split(".", 1)[0]
if field_name not in schema and field_name not in default_fields:
raise ValueError("Field '%s' does not exist" % field_name)
def _load_dataset(name):
try:
# pylint: disable=no-member
_meta = foo.DatasetDocument.objects.get(name=name)
except DoesNotExist:
raise DoesNotExistError("Dataset '%s' not found" % name)
_sample_doc_cls = type(name, (foo.DatasetSampleDocument,), {})
num_default_fields = len(foos.default_sample_fields(include_private=True))
for sample_field in _meta.sample_fields[num_default_fields:]:
subfield = (
etau.get_class(sample_field.subfield)
if sample_field.subfield
else None
)
embedded_doc_type = (
etau.get_class(sample_field.embedded_doc_type)
if sample_field.embedded_doc_type
else None
)
_sample_doc_cls.add_field(
sample_field.name,
etau.get_class(sample_field.ftype),
subfield=subfield,
embedded_doc_type=embedded_doc_type,
save=False,
)
return _meta, _sample_doc_cls
def _validate_params(self):
default_fields = set(default_sample_fields(DatasetSampleDocument))
for field_name in self._field_names:
if field_name.startswith("_"):
raise ValueError("Cannot exclude private field '%s'" %
field_name)
if (field_name == "id") or (field_name in default_fields):
raise ValueError("Cannot exclude default field '%s'" %
field_name)
def to_mongo(self):
"""Returns the MongoDB version of the stage.
Returns:
a MongoDB aggregation pipeline (list of dicts)
"""
default_fields = default_sample_fields(DatasetSampleDocument,
include_private=True)
selected_fields = list(set(self.field_names) | set(default_fields))
return [{"$project": {fn: True for fn in selected_fields}}]
def __init__(self, field_names=None):
default_fields = default_sample_fields()
if field_names:
if etau.is_str(field_names):
field_names = [field_names]
self._field_names = list(set(field_names) | set(default_fields))
else:
self._field_names = list(default_fields)
def test_select_fields(self):
self.dataset.add_sample_field("select_fields_field", fo.IntField)
for sv in self.dataset.select_fields():
self.assertSetEqual(sv.selected_field_names,
set(default_sample_fields()))
self.assertIsNone(sv.excluded_field_names)
sv.filepath
sv.metadata
sv.tags
with self.assertRaises(NameError):
sv.select_fields_field
def get_selected_fields(self):
default_fields = default_sample_fields(DatasetSampleDocument)
return list(set(self.field_names) | set(default_fields))
def _validate(self):
invalid_fields = set(self._field_names) & set(default_sample_fields())
if invalid_fields:
raise ValueError(
"Cannot exclude default fields: %s" % list(invalid_fields)
)