def handle(self, *args, **kwargs):
try:
client = services.get_mongo_client()
db = client.python_developer_db
segment_collection = db.segment_collection
self.stdout.write(
self.style.MIGRATE_HEADING(f"Serializing SQL data..."))
data = [
s.as_dict(extra_fields=True) for s in Segment.objects.all()
]
# Clear collection and migrate data
segment_collection.remove({})
self.stdout.write(
self.style.MIGRATE_HEADING(f"Inserting data into MongoDB..."))
segment_collection.insert(data)
# Output report
segment_size = len(data)
restaurant_size = sum([i["size"] for i in data])
self.stdout.write(
self.style.SUCCESS(
f"Migrated {segment_size} segments with {restaurant_size} restaurants"
))
except Exception as e:
self.stdout.write(self.style.ERROR(e))
class MongoRestaurantRepo(BaseRepo):
client = get_mongo_client()
db = client.python_developer_db
collection = db.segment_collection
def _query(self, params={}, first=False, **kwargs):
"""
Supports one condition per field
Supports None values
Duplicates need to be removed when many records are retrieved,
because a restaurant can be embedded in several segments
params: dict
{
"popularity_rate": {"gt": 5.5},
"satisfaction_rate": {"ne": None},
}
first : bool, optional
Whether you want to retrieve one or many objects
"""
params = {
field:
{f"${operator}": value
for operator, value in condition.items()}
for field, condition in params.items()
}
projection = {
"_id": 0,
"restaurants": {
"$elemMatch": {
"$and": [{
field: condition
} for field, condition in params.items()]
}
},
}
query = [
i["restaurants"][0]
for i in self.collection.find({}, projection=projection)
if i.get("restaurants")
]
if first:
return query[0]
query_without_duplicates = [
dict(i) for i in {tuple(q.items())
for q in query}
]
return query_without_duplicates
def handle(self, *args, **options):
with open(options["restaurant_json"][0], "r") as restaurant_json, open(
options["segment_json"][0], "r"
) as segment_json:
try:
client = services.get_mongo_client()
db = client.python_developer_db
self.load_segments(segment_json, db)
self.load_restaurants(restaurant_json, db)
except Exception as e:
self.stdout.write(self.style.ERROR(e))
def handle(self, *args, **kwargs):
try:
client = services.get_mongo_client()
db = client.python_developer_db
collection = db.segment_collection
self.stdout.write(
self.style.MIGRATE_HEADING(f"Cleaning SQL tables..."))
Segment.objects.all().delete()
Restaurant.objects.all().delete()
self.stdout.write(
self.style.MIGRATE_HEADING(f"Deserializing MongoDB data..."))
data = [s for s in collection.find({})]
self.stdout.write(
self.style.MIGRATE_HEADING(
f"Migrating data and creating relationships..."))
with tqdm(total=len(data)) as progress_bar:
for segment in data:
segment_data = {
k: v
for k, v in segment.items() if k not in (
"restaurants",
"size",
"_id",
)
}
# Create Segment objects
segment_object = Segment(**segment_data)
segment_object.save()
# Create Restaurant objects
restaurant_objects = set(
Restaurant(**r) for r in segment["restaurants"])
# bulk_create skips duplicates when ignore_conflicts is True
with warnings.catch_warnings():
warnings.simplefilter("ignore")
Restaurant.objects.bulk_create(restaurant_objects,
ignore_conflicts=True)
# Create relationships
segment_object.restaurants.add(*restaurant_objects)
progress_bar.update(1)
except IntegrityError as e:
return False
def handle(self, *args, **kwargs):
with open(kwargs["json"][0], "w") as f:
try:
client = services.get_mongo_client()
db = client.python_developer_db
collection = db.segment_collection
data = {
str(s["uidentifier"]): s
for s in collection.find({}, projection={"_id": False})
}
f.write(json.dumps(data))
self.stdout.write(
self.style.SUCCESS(
f"Dumped {collection.count()} segments"))
except IntegrityError as e:
self.stdout.write(self.style.ERROR(e))