async def on_rebalance(self,
assigned: Set[TP],
revoked: Set[TP],
newly_assigned: Set[TP]) -> None:
"""Call when the cluster is rebalancing."""
T = traced_from_parent_span()
# Stop producers for revoked partitions.
revoked_tids = list(sorted(self._tps_to_transactional_ids(revoked)))
if revoked_tids:
self.log.info(
'Stopping %r transactional %s for %r revoked %s...',
len(revoked_tids),
pluralize(len(revoked_tids), 'producer'),
len(revoked),
pluralize(len(revoked), 'partition'))
await T(self._stop_transactions, tids=revoked_tids)(revoked_tids)
# Start produers for assigned partitions
assigned_tids = list(sorted(self._tps_to_transactional_ids(assigned)))
if assigned_tids:
self.log.info(
'Starting %r transactional %s for %r assigned %s...',
len(assigned_tids),
pluralize(len(assigned_tids), 'producer'),
len(assigned),
pluralize(len(assigned), 'partition'))
await T(self._start_transactions,
tids=assigned_tids)(assigned_tids)
def validate(self, value: CharacterType) -> Iterable[ValidationError]:
allow_blank = self.allow_blank
if not allow_blank and not len(value):
yield self.validation_error(f"{self.field} cannot be left blank")
max_ = self.max_length
length = len(value)
min_ = self.min_length
if min_:
if length < min_:
chars = pluralize(min_, "character")
yield self.validation_error(
f"{self.field} must have at least {min_} {chars}")
if max_:
if length > max_:
chars = pluralize(max_, "character")
yield self.validation_error(
f"{self.field} must be at least {max_} {chars}")
def _contribute_to_options(cls, options: ModelOptions) -> None:
# Find attributes and their types, and create indexes for these.
# This only happens once when the class is created, so Faust
# models are fast at runtime.
fields, defaults = annotations(
cls,
stop=Record,
skip_classvar=True,
alias_types=ALIAS_FIELD_TYPES,
localns={cls.__name__: cls},
)
options.fields = cast(Mapping, fields)
options.fieldset = frozenset(fields)
options.fieldpos = {i: k for i, k in enumerate(fields.keys())}
# extract all default values, but only for actual fields.
options.defaults = {
k: v.default if isinstance(v, FieldDescriptor) else v
for k, v in defaults.items() if k in fields
and not (isinstance(v, FieldDescriptor) and v.required)
}
# Raise error if non-defaults are mixed in with defaults
# like namedtuple/dataclasses do.
local_defaults = []
for attr_name in cls.__annotations__:
if attr_name in cls.__dict__:
default_value = cls.__dict__[attr_name]
if isinstance(default_value, FieldDescriptorT):
if not default_value.required:
local_defaults.append(attr_name)
else:
local_defaults.append(attr_name)
else:
if local_defaults:
raise TypeError(
E_NON_DEFAULT_FOLLOWS_DEFAULT.format(
cls_name=cls.__name__,
field_name=attr_name,
fields=pluralize(len(local_defaults), 'field'),
default_names=', '.join(local_defaults),
))
for field, typ in fields.items():
if is_optional(typ):
# Optional[X] also needs to be added to defaults mapping.
options.defaults.setdefault(field, None)
# Create frozenset index of default fields.
options.optionalset = frozenset(options.defaults)
def test_pluralize(n, s, suffix, expected):
assert text.pluralize(n, s, suffix=suffix) == expected
def _contribute_to_options(cls, options: ModelOptions) -> None:
# Find attributes and their types, and create indexes for these.
# This only happens once when the class is created, so Faust
# models are fast at runtime.
fields, defaults = annotations(
cls,
stop=Record,
skip_classvar=True,
alias_types=ALIAS_FIELD_TYPES,
localns={cls.__name__: cls},
)
options.fields = cast(Mapping, fields)
options.fieldset = frozenset(fields)
options.fieldpos = {i: k for i, k in enumerate(fields.keys())}
# extract all default values, but only for actual fields.
options.defaults = {
k: v.default if isinstance(v, FieldDescriptor) else v
for k, v in defaults.items() if k in fields
and not (isinstance(v, FieldDescriptor) and v.required)
}
options.models = {}
options.polyindex = {}
modelattrs = options.modelattrs = {}
def _is_concrete_type(field: str, wanted: IsInstanceArgT) -> bool:
typeinfo = options.polyindex[field]
try:
return issubclass(typeinfo.member_type, wanted)
except TypeError:
return False
# Raise error if non-defaults are mixed in with defaults
# like namedtuple/dataclasses do.
local_defaults = []
for attr_name in cls.__annotations__:
if attr_name in cls.__dict__:
default_value = cls.__dict__[attr_name]
if isinstance(default_value, FieldDescriptorT):
if not default_value.required:
local_defaults.append(attr_name)
else:
local_defaults.append(attr_name)
else:
if local_defaults:
raise TypeError(
E_NON_DEFAULT_FOLLOWS_DEFAULT.format(
cls_name=cls.__name__,
field_name=attr_name,
fields=pluralize(len(local_defaults), 'field'),
default_names=', '.join(local_defaults),
))
for field, typ in fields.items():
is_model, member_type, generic_type = _is_model(typ)
options.polyindex[field] = TypeInfo(generic_type, member_type)
if is_model:
# Extract all model fields
options.models[field] = typ
# Create mapping of model fields to polymorphic types if
# available
modelattrs[field] = generic_type
if is_optional(typ):
# Optional[X] also needs to be added to defaults mapping.
options.defaults.setdefault(field, None)
# Create frozenset index of default fields.
options.optionalset = frozenset(options.defaults)
# extract all fields that we want to coerce to a different type
# (decimals=True, isodates=True, coercions={MyClass: converter})
# Then move them to options.field_coerce, which is what the
# model.__init__ method uses to coerce any fields that need to
# be coerced.
options.field_coerce = {}
if options.isodates:
options.coercions.setdefault(DATE_TYPES, iso8601.parse)
if options.decimals:
options.coercions.setdefault(DECIMAL_TYPES, str_to_decimal)
for coerce_types, coerce_handler in options.coercions.items():
options.field_coerce.update({
field: TypeCoerce(typ, coerce_handler)
for field, typ in fields.items()
if (field not in modelattrs
and _is_concrete_type(field, coerce_types))
})
