def _create_index(self, parameters: dict) -> None:
"""
Create index with given parameters
:param parameters:
:return:
"""
left_index_schema = deepcopy(parameters)
fields = left_index_schema.pop('fields') # Key must be present
name = left_index_schema.get('name')
# Check if index with such name\parameters exists in db
found_index = self._find_index(self._run_ctx['collection'], name,
fields)
if found_index:
iname, ispec = found_index
# Exclude index desc keys which does not get to index schema
compare_keys = (ispec.keys()
| left_index_schema.keys()) - {'v', 'ns', 'key'}
if all(
ispec.get(k) == left_index_schema.get(k)
for k in compare_keys):
log.warning('Index %s already exists, ignore', fields)
return
else:
raise MigrationError(
'Index {} already exists with other parameters. Please drop it before '
'applying the migration'.format(fields))
try:
self._run_ctx['collection'].create_index(fields,
**left_index_schema)
except pymongo.errors.OperationFailure as e:
index_id = left_index_schema.get('name', fields)
raise MigrationError(
'Could not create index {}'.format(index_id)) from e
def convert_geojson(updater: DocumentUpdater, from_type: str, to_type: str):
"""Convert GeoJSON object from one type to another"""
from_ind, to_ind = None, None
if updater.migration_policy.name == 'strict':
__check_geojson_objects(updater, [from_type, to_type])
# There could be legacy coordinate pair arrays or GeoJSON objects
__check_legacy_point_coordinates(updater)
__check_value_types(updater, ['object', 'array'])
if from_type == to_type:
return
for ind, convertion in __CONVERTIONS:
if from_type in convertion:
from_ind = ind
if to_type in convertion:
to_ind = ind
if from_ind is None or to_ind is None:
raise MigrationError(
f"Unknown geo field type. Was requested: {from_type}, {to_type}")
depth = abs(from_ind - to_ind)
if from_ind <= to_ind:
__increase_geojson_nesting(updater, from_type, to_type, depth)
else:
__decrease_geojson_nesting(updater, from_type, to_type, depth)
def by_doc(ctx: ByDocContext):
# https://docs.mongodb.com/manual/reference/operator/aggregation/convert/
type_map = {
'double': float,
'string': str,
'objectId': bson.ObjectId,
'bool': bool,
'date': lambda x: dateutil_parse(str(x)),
'int': int,
'long': bson.Int64,
'decimal': float,
'binary': bson.Binary,
'object': dict
}
assert target_type in type_map
doc = ctx.document
field_name = updater.field_name
if field_name in doc:
t = type_map[target_type]
if not isinstance(doc[field_name],
t) and doc[field_name] is not None:
try:
doc[field_name] = type_map[target_type](doc[field_name])
except (TypeError, ValueError) as e:
if updater.migration_policy.name == 'strict':
raise MigrationError(
f'Cannot convert value '
f'{field_name}: {doc[field_name]} to type {t}'
) from e
def convert_type(self, updater: DocumentUpdater,
from_field_cls: Type[mongoengine.fields.BaseField],
to_field_cls: Type[mongoengine.fields.BaseField]):
"""
Convert field type from another to a current one. This method
is called only if such change was requested in a migration.
We use convertion matrix here. It contains mapping between
old and new types and appropriate converter function which
is called to perform such convertion.
Old field can be either actual field type which used before.
But in case if that field was a user defined class and does
not exist already, the BaseField will be sent.
New field will always have target mongoengine field type
:param updater:
:param from_field_cls: mongoengine field class which was used
before
:param to_field_cls: mongoengine field class which will be used
further
:return:
"""
type_converters = CONVERTION_MATRIX.get(from_field_cls) or \
CONVERTION_MATRIX.get(get_closest_parent(from_field_cls, CONVERTION_MATRIX.keys()))
if type_converters is None:
raise MigrationError(f'Type converter not found for convertion '
f'{from_field_cls!r} -> {to_field_cls!r}')
type_converter = type_converters.get(to_field_cls) or \
type_converters.get(get_closest_parent(to_field_cls, type_converters))
if type_converter is None:
raise MigrationError(f'Type converter not found for convertion '
f'{from_field_cls!r} -> {to_field_cls!r}')
type_converter(updater)
def by_doc(ctx: ByDocContext):
doc = ctx.document
if updater.field_name not in doc or doc[updater.field_name] is None:
return
f = doc[updater.field_name]
is_dict = isinstance(f, dict)
if is_dict and isinstance(f.get('_ref'), bson.DBRef): # dynamic ref
doc[updater.field_name] = str(f['_ref'].id)
elif is_dict and isinstance(f.get('_id'), bson.ObjectId): # manual ref
doc[updater.field_name] = str(f['_id'])
elif isinstance(f, bson.DBRef):
doc[updater.field_name] = str(f.id)
else:
try:
doc[updater.field_name] = str(f)
except (TypeError, ValueError) as e:
if updater.migration_policy.name == 'strict':
raise MigrationError(
f'Cannot convert value {updater.field_name}: '
f'{doc[updater.field_name]} to string') from e
def by_doc(ctx: ByDocContext):
doc = ctx.document
if updater.field_name not in doc or doc[updater.field_name] is None:
return
f = doc[updater.field_name]
if isinstance(f, (list, tuple)):
if f:
f = f[0]
if remove_cls_key and isinstance(f, dict) and '_cls' in f:
del f['_cls']
if not isinstance(
f, item_type
) and updater.migration_policy.name == 'strict':
raise InconsistencyError(
f"Field {updater.field_name} has wrong value {f!r} "
f"(should be {item_type}) in record {doc}")
else:
f = None
doc[updater.field_name] = f
elif f is not None and updater.migration_policy.name == 'strict':
raise MigrationError(
f'Could not extract item from non-list value '
f'{updater.field_name}: {doc[updater.field_name]}')
def deny(updater: DocumentUpdater):
"""Convertion is denied"""
raise MigrationError(
f"Such type_key convertion for field "
f"{updater.document_type}.{updater.field_name} is forbidden")