def validate_fields_exist(self, field_or_fields): """Validates that the collection has fields with the given names. If ``field_or_fields`` contains an embedded field name such as ``field_name.document.field``, only the root ``field_name`` is checked for existence. Args: field_or_fields: a field name or iterable of field names Raises: ValueError: if one or more of the fields do not exist """ if etau.is_str(field_or_fields): field_or_fields = [field_or_fields] schema = self.get_field_schema() default_fields = set( default_sample_fields(DatasetSampleDocument, include_private=True, include_id=True)) for field in field_or_fields: # We only validate that the root field exists field_name = field.split(".", 1)[0] if field_name not in schema and field_name not in default_fields: raise ValueError("Field '%s' does not exist" % field_name)
def _load_dataset(name): try: # pylint: disable=no-member _meta = foo.DatasetDocument.objects.get(name=name) except DoesNotExist: raise DoesNotExistError("Dataset '%s' not found" % name) _sample_doc_cls = type(name, (foo.DatasetSampleDocument,), {}) num_default_fields = len(foos.default_sample_fields(include_private=True)) for sample_field in _meta.sample_fields[num_default_fields:]: subfield = ( etau.get_class(sample_field.subfield) if sample_field.subfield else None ) embedded_doc_type = ( etau.get_class(sample_field.embedded_doc_type) if sample_field.embedded_doc_type else None ) _sample_doc_cls.add_field( sample_field.name, etau.get_class(sample_field.ftype), subfield=subfield, embedded_doc_type=embedded_doc_type, save=False, ) return _meta, _sample_doc_cls
def _validate_params(self): default_fields = set(default_sample_fields(DatasetSampleDocument)) for field_name in self._field_names: if field_name.startswith("_"): raise ValueError("Cannot exclude private field '%s'" % field_name) if (field_name == "id") or (field_name in default_fields): raise ValueError("Cannot exclude default field '%s'" % field_name)
def to_mongo(self): """Returns the MongoDB version of the stage. Returns: a MongoDB aggregation pipeline (list of dicts) """ default_fields = default_sample_fields(DatasetSampleDocument, include_private=True) selected_fields = list(set(self.field_names) | set(default_fields)) return [{"$project": {fn: True for fn in selected_fields}}]
def __init__(self, field_names=None): default_fields = default_sample_fields() if field_names: if etau.is_str(field_names): field_names = [field_names] self._field_names = list(set(field_names) | set(default_fields)) else: self._field_names = list(default_fields)
def test_select_fields(self): self.dataset.add_sample_field("select_fields_field", fo.IntField) for sv in self.dataset.select_fields(): self.assertSetEqual(sv.selected_field_names, set(default_sample_fields())) self.assertIsNone(sv.excluded_field_names) sv.filepath sv.metadata sv.tags with self.assertRaises(NameError): sv.select_fields_field
def get_selected_fields(self): default_fields = default_sample_fields(DatasetSampleDocument) return list(set(self.field_names) | set(default_fields))
def _validate(self): invalid_fields = set(self._field_names) & set(default_sample_fields()) if invalid_fields: raise ValueError( "Cannot exclude default fields: %s" % list(invalid_fields) )