def to_objects(self, error_collector: ErrorCollector):
if self.delivered == "Yes":
return []
if self.eta is None:
return []
if self.functionality.lower() in {"full", "critical care"}:
item = Item.ventilators_full_service
elif self.functionality.lower() == "limited":
item = Item.ventilators_non_full_service
else:
error_collector.report_error(
f"Unknown ventilator type: {self.functionality}")
return []
purchase = models.Purchase(
order_type=OrderType.Purchase,
item=item,
quantity=self.quantity,
received_quantity=self.quantity_delivered,
description=f"Ventilator {self.type} ({self.functionality})",
raw_data=self.raw_data,
)
deliveries = []
if self.eta is not None:
deliveries.append(
models.ScheduledDelivery(purchase=purchase,
delivery_date=self.eta,
quantity=self.quantity))
return [purchase, *deliveries]
def to_objects(self, error_collector: ErrorCollector):
if self.picked_up:
return []
if self.quantity is None:
error_collector.report_warning(
"Ignoring donation row with no quantity")
return []
purchase = Purchase(
order_type=OrderType.Donation,
item=self.item,
description=self.description,
quantity=self.quantity,
received_quantity=self.quantity if self.picked_up else 0,
vendor=self.donor,
comment=self.comments,
raw_data=self.raw_data,
donation_date=self.notification_date,
)
objs = [purchase]
delivery_date = self.guess_delivery_date(error_collector)
if delivery_date is not None:
objs.append(
ScheduledDelivery(
purchase=purchase,
delivery_date=delivery_date,
quantity=self.quantity,
))
return objs
def sanity(self, error_collector: ErrorCollector):
delivered_quantity = (self.delivery_day_1_quantity
or 0) + (self.delivery_day_2_quantity or 0)
errors = []
# lots of data doesn't have delivery dates.
if delivered_quantity > self.quantity:
error_collector.report_warning(
f"Claimed delivered quantity ({delivered_quantity}) > "
f"total quantity {self.quantity} for {self.item} from {self.vendor}"
)
# if delivered_quantity < self.quantity:
# errors.append(f'Delivery < total {delivered_quantity} < {self.quantity}')
if self.quantity is None:
errors.append("Quantity is None")
return errors
def test_unscheduled_deliveries(self):
data_import = DataImport(
status=ImportStatus.active,
data_file=DataFile.PPE_ORDERINGCHARTS_DATE_XLSX,
file_checksum="123",
)
data_import.save()
items = SourcingRow(
item=dc.Item.gown,
quantity=2000,
vendor="Gown Sellers Ltd",
description="Some gowns",
delivery_day_1=datetime.strptime("2020-04-12", "%Y-%m-%d")
- timedelta(days=5),
delivery_day_1_quantity=5,
delivery_day_2=datetime.strptime("2020-04-12", "%Y-%m-%d")
+ timedelta(days=1),
delivery_day_2_quantity=1000,
status="Completed",
received_quantity=0,
raw_data={},
).to_objects(ErrorCollector())
for item in items:
item.source = data_import
item.save()
purchase = Purchase.objects.filter(item=dc.Item.gown)
self.assertEqual(purchase.count(), 1)
self.assertEqual(purchase.first().unscheduled_quantity, 995)
def load_from_request(cls, request) -> "StandardRequestParams":
if request.GET:
params = request.GET
else:
params = request.POST
start_date = params.get("start_date")
end_date = params.get("end_date")
err_collector = ErrorCollector()
# Python defaults to Monday. Subtract one extra day to get us to Sunday
default_start = datetime.today() + timedelta(weeks=1) - timedelta(
days=datetime.today().weekday() + 1)
default_end = default_start + timedelta(days=6)
start_date = (parse_date(start_date, err_collector)
or default_start).date()
end_date = (parse_date(end_date, err_collector) or default_end).date()
if params.get("rollup") in {"mayoral", "", None}:
rollup_fn = mayoral_rollup
else:
rollup_fn = lambda x: x
if params.get("supply"):
supply_components = {
AggColumn(col)
for col in params.get("supply").split(",")
}
else:
supply_components = AggColumn.all()
print(f"Parsed request as {start_date}->{end_date}")
if len(err_collector) > 0:
err_collector.dump()
return StandardRequestParams(
start_date=start_date,
end_date=end_date,
rollup_fn=rollup_fn,
supply_components=supply_components,
)
def to_objects(self, error_collector: ErrorCollector):
errors = self.sanity()
if errors:
error_collector.report_error(
f"Refusing to generate a data model for: {self}. Errors: {errors}"
)
return []
purchase = models.Purchase(
item=self.item,
quantity=self.quantity,
vendor=self.vendor,
raw_data=self.raw_data,
order_type=OrderType.Make,
)
dates = []
if self.delivery_date:
dates.append(self.delivery_date)
elif "weekly" in self.raw_date:
import re
date_str = re.sub(r"[a-zA-Z ]+", "", self.raw_date).strip()
end_date = parse_date(date_str, error_collector)
if end_date:
dates = []
while end_date > datetime.today():
dates.append(end_date)
end_date -= timedelta(weeks=1)
delivery = [
models.ScheduledDelivery(
purchase=purchase, delivery_date=date, quantity=self.quantity,
)
for date in dates
]
return [purchase, *delivery]
def to_objects(self, error_collector: ErrorCollector):
if self.status != "Completed":
return []
errors = self.sanity(error_collector)
if errors:
error_collector.report_error(
f"Refusing to generate a data model for: {self}. Errors: {errors}"
)
return []
purchase = models.Purchase(
item=self.item,
quantity=self.quantity,
received_quantity=self.received_quantity,
vendor=self.vendor,
raw_data=self.raw_data,
order_type=OrderType.Purchase,
description=self.description,
)
deliveries = []
for day in [1, 2]:
total = 0
if getattr(self, f"delivery_day_{day}"):
quantity = getattr(self, f"delivery_day_{day}_quantity")
if quantity is None:
quantity = self.quantity - total
error_collector.report_warning(
f"Assuming that a null quantity means a full delivery for {self}"
)
deliveries.append(
models.ScheduledDelivery(
purchase=purchase,
delivery_date=getattr(self, f"delivery_day_{day}"),
quantity=quantity,
))
return [purchase, *deliveries]
def __call__(self, names):
opts = [name for name in names if re.match(self.patt, name)]
if len(opts) == 1:
return opts[0]
if len(opts) > 1 and self.take_latest:
date_strs = [
(opt, re.search(self.patt, opt).group(1)) for opt in opts
] # [(opt, ('4-23',)), ...]
parsed_dates = [
(opt, parse_date(date_str, ErrorCollector()))
for (opt, date_str) in date_strs
]
sorted_by_date = sorted(parsed_dates, key=lambda opt_date: opt_date[1])
# We want to highest date
print(sorted_by_date)
return sorted_by_date[-1][0]
return None
def import_xlsx(
path: Path,
sheet_mapping: SheetMapping,
error_collector: ErrorCollector = lambda: ErrorCollector(),
):
as_dicts = list(sheet_mapping.load_data(path))
for row in as_dicts:
mapped_row = {}
if all(row.get(col) is None for col in sheet_mapping.key_columns()):
continue
for mapping in sheet_mapping.mappings:
item = row[mapping.sheet_column_name]
if mapping.proc:
item = mapping.proc(item, error_collector)
mapped_row[mapping.obj_column_name] = item
if sheet_mapping.include_raw:
# allow serialization of datetimes
mapped_row[RAW_DATA] = json.dumps(row, cls=DjangoJSONEncoder)
if sheet_mapping.obj_constructor:
yield sheet_mapping.obj_constructor(**mapped_row)
else:
yield mapped_row
def setUp(self) -> None:
self.data_import = DataImport(
status=ImportStatus.active,
data_file=DataFile.PPE_ORDERINGCHARTS_DATE_XLSX,
file_checksum="123",
)
self.data_import.save()
items = SourcingRow(
item=dc.Item.gown,
quantity=1005,
vendor="Gown Sellers Ltd",
description="Some gowns",
delivery_day_1=datetime.strptime("2020-04-12", "%Y-%m-%d")
- timedelta(days=5),
delivery_day_1_quantity=5,
status="Completed",
received_quantity=0,
delivery_day_2=datetime.strptime("2020-04-12", "%Y-%m-%d")
+ timedelta(days=1),
delivery_day_2_quantity=1000,
raw_data={},
).to_objects(ErrorCollector())
inventory = [
Inventory(
item=dc.Item.gown,
quantity=100,
as_of=datetime(year=2020, day=11, month=4),
raw_data={},
),
# this one is old and should be superceded
Inventory(
item=dc.Item.gown,
quantity=200,
as_of=datetime(year=2020, day=10, month=4),
raw_data={},
),
]
f = Facility(name="Generic Hospital", tpe=dc.FacilityType.hospital)
items.append(f)
deliveries = [
FacilityDelivery(
date=datetime(year=2020, day=10, month=4),
quantity=1234,
facility=f,
item=dc.Item.gown,
),
FacilityDelivery(
date=datetime(year=2020, day=10, month=4),
quantity=123,
facility=f,
item=dc.Item.faceshield,
),
]
items += deliveries
items += inventory
for item in items:
item.source = self.data_import
item.save()
items = DemandRow(
item=dc.Item.gown,
demand=2457000,
week_start_date=datetime.strptime("2020-04-11", "%Y-%m-%d"),
week_end_date=datetime.strptime("2020-04-17", "%Y-%m-%d"),
raw_data={},
).to_objects(ErrorCollector())
for item in items:
item.source = self.data_import
item.save()
def import_data(
path: Path,
mappings: List[xlsx_utils.SheetMapping],
current_as_of: date,
user_provided_filename: Optional[str],
uploaded_by: Optional[str] = None,
overwrite_in_prog=False,
):
error_collector = ErrorCollector()
data_file = {mapping.data_file for mapping in mappings}
if len(data_file) != 1:
raise ImportError(
"Something is wrong, can't import from two different files..."
)
data_file = mappings[0].data_file
in_progress = import_in_progress(data_file)
if in_progress.count() > 0:
if overwrite_in_prog:
in_progress.update(status=ImportStatus.replaced)
else:
raise ImportInProgressError(in_progress.first().id)
with open(path, "rb") as f:
checksum = hashlib.sha256(f.read())
uploaded_by = uploaded_by or ""
data_import = DataImport(
status=ImportStatus.candidate,
current_as_of=current_as_of,
data_file=data_file,
uploaded_by=uploaded_by,
file_checksum=checksum,
file_name=user_provided_filename or path.name,
)
data_import.save()
for mapping in mappings:
try:
data = import_xlsx(path, mapping, error_collector)
data = list(data)
# there are a lot of deliveries, pull them out for bulk import
deliveries = []
for item in data:
try:
for obj in item.to_objects(error_collector):
obj.source = data_import
if isinstance(obj, FacilityDelivery):
deliveries.append(obj)
else:
obj.save()
except Exception as ex:
error_collector.report_error(
f"Failure importing row. This is a bug: {ex}"
)
sentry_sdk.capture_exception(ex)
FacilityDelivery.objects.bulk_create(deliveries)
except Exception:
print(f"Failure importing {path}, mapping: {mapping.sheet_name}")
raise
print(f"Errors: ")
error_collector.dump()
return data_import