def document(order_id: ObjectId, user: dict): """fetch indiviual order info or cancel it""" if request.method == "GET": with_logs = request.args.get("with_logs", default=False, type=string_to_bool) order = Orders.get_with_tasks(order_id, with_logs=with_logs) if order is None: raise errors.NotFound() return jsonify(order) if request.method == "PATCH": order = Orders.get_with_tasks(order_id) if order is None: raise errors.NotFound() request_json = request.get_json() Orders().add_shipment(order_id, request_json.get("shipment_details")) send_order_shipped_email(order_id) return jsonify(order) if request.method == "DELETE": # should prepare email message with infos from order here deleted_count = Orders().delete_one({"_id": order_id}).deleted_count if deleted_count == 0: raise errors.NotFound() # send email about deletion return jsonify({"_id": order_id})
def confirm_inserted_task(task_id: ObjectId, task_type: str): task_cls = tasks_cls_for(task_type) task = task_cls().find_one({ "_id": task_id, "status": task_cls.waiting_for_card }) if task is None: raise errors.NotFound() if request.method == "GET": order = Orders().get(task["order"]) return render_template("pub_confirm_inserted.html", order=order, task=task) elif request.method == "POST": task_cls.update_status(task_id, status=task_cls.card_inserted) task_cls.cascade_status(task_id, task_cls.card_inserted) order = Orders().get(task["order"]) return render_template("pub_thank_inserted.html", order=order, task=task) raise errors.BadRequest("?")
def add_shipment(order_id: ObjectId): order = Orders().find_one({ "_id": order_id, "status": Orders.pending_shipment }) if order is None: raise errors.NotFound() if request.method == "GET": return render_template("pub_add_shipment.html", order=order, order_id=order["_id"]) if request.method == "POST": shipment_details = request.form.get("details") if not shipment_details: raise errors.BadRequest("Missing shipment details") # store shipment details Orders().add_shipment(order_id, shipment_details) # refresh order object order = Orders.get(order_id) # send recipient an email send_order_shipped_email(order_id) return render_template("pub_thank_shipment.html", order=order, order_id=order["_id"])
def cancel(order_id: ObjectId, user: dict): order = Orders.get(order_id) if order is None: raise errors.NotFound() Orders().cancel(order_id) send_order_failed_email(order_id) return jsonify({"_id": order_id})
def get_full_context(order_id, extra={}): order = Orders().get_with_tasks(order_id, {"logs": 0}) order.update({ "id": order_id, "status": order["statuses"][-1], "min_id": order_id[:8] + order_id[-3:], }) context = {"order": order} context.update(extra) return context
def create_order_from(payload): # validate payload validate(payload, Orders().schema) payload["status"] = Orders().created payload["tasks"] = {} payload["statuses"] = [{ "status": Orders().created, "on": datetime.datetime.now(), "payload": None }] # actually create Order order_id = Orders().insert_one(payload).inserted_id # define and record fname for this order if "fname" not in payload: fname = f"{order_id}.img" else: short_id = str(order_id)[:10] if "{rand}" in payload["fname"]: fname = payload["fname"].replace("{rand}", short_id) else: stem, ext = os.path.splitext(payload["fname"]) fname = "".join((stem + f"_{short_id}", ext)) Orders.update(order_id, {"fname": fname}) # send email about new order send_order_created_email(order_id) # create creation task Orders.create_creator_task(order_id) return order_id
def get_email_for(order_id, kind, formatted=True): def _fmt(name, email): return "{name} <{email}>".format(name=name, email=email) if kind not in ("client", "recipient", "operator", "error-manager"): return None, "en" if kind == "error-manager" and FAILED_ORDER_EMAIL: return _fmt("Cardshop Error Manager", FAILED_ORDER_EMAIL), "en" order = Orders.get_with_tasks(order_id, {"logs": 0}) if kind == "client": return ( _fmt(order["client"]["name"], order["client"]["email"]), order["client"]["language"], ) if kind == "recipient": return ( _fmt(order["recipient"]["name"], order["recipient"]["email"]), order["recipient"]["language"], ) if kind == "operator": worker = Users().by_username(order["tasks"]["download"]["worker"]) return worker["email"], "en" return None, "en"
def check_autoimages(): # update images that were building logger.info("Looking for currently building images…") for image in AutoImages.all_currently_building(): logger.info(f".. {image['slug']}") # check order status order = Orders.get(image["order"]) # order is considered failed if order["status"] in Orders.FAILED_STATUSES: logger.info(f".. order failed: {order['status']}") AutoImages.update_status(image["slug"], status="failed") continue # order is considered successful if order["status"] in Orders.SUCCESS_STATUSES + [ Orders.pending_expiry ]: logger.info(f".. order succeeded: {order['status']}") torrent_url = get_public_download_torrent_url(order) AutoImages.update_status( image["slug"], status="ready", order=None, http_url=get_public_download_url(order), torrent_url=torrent_url, magnet_url=get_magnet_for_torrent(torrent_url), expire_on=get_next_month(), ) continue logger.info(f".. order still building: {order['status']}") # find images that must be recreated logger.info("Looking for images needing building…") for image in AutoImages.all_needing_rebuild(): logger.info(f".. {image['slug']} ; starting build") # create order payload = AutoImages.create_order_payload(image["slug"]) try: order_id = create_order_from(payload) except Exception as exc: logger.error(f"Error creating image `{image['slug']}`: {exc}") logger.exception(exc) AutoImages.update_status(image["slug"], status="failed") continue # update with order ID and status: building AutoImages.update_status(image["slug"], status="building", order=order_id)
def anonymize(user: dict): try: request_json = request.get_json() validate(request_json, {"order_ids": { "type": "list", "required": True }}) order_ids = [ ensure_objectid(oid) for oid in request_json.get("order_ids") ] except ValidationError as error: raise errors.BadRequest(str(error)) except Exception: raise errors.BadRequest("Orders IDs are not all valid IDs") try: Orders.anonymize(order_ids) except Exception: raise errors.NotFound() return jsonify({"_ids": order_ids})
def collection(user: dict): """ List or create orders """ if request.method == "GET": # unpack url parameters skip = request.args.get("skip", default=0, type=int) limit = request.args.get("limit", default=20, type=int) skip = 0 if skip < 0 else skip limit = 20 if limit <= 0 else limit query = {} projection = {"_id": 1} cursor = (Orders().find(query, projection).sort([ ("$natural", pymongo.DESCENDING) ]).skip(skip).limit(limit)) count = Orders().count_documents(query) orders = [Orders.get(order.get("_id")) for order in cursor] return jsonify({ "meta": { "skip": skip, "limit": limit, "count": count }, "items": orders }) if request.method == "POST": try: order_id = create_order_from(request.get_json()) except ValidationError as error: raise errors.BadRequest(str(error)) return jsonify({"_id": order_id})
def collection(user: dict): """ List or create orders """ if request.method == "GET": # unpack url parameters skip = request.args.get("skip", default=0, type=int) limit = request.args.get("limit", default=20, type=int) skip = 0 if skip < 0 else skip limit = 20 if limit <= 0 else limit orders = [ Orders.get(order.get("_id")) for order in Orders().find({}, {"_id": 1}) ] return jsonify({"meta": {"skip": skip, "limit": limit}, "items": orders}) elif request.method == "POST": # validate request json try: request_json = request.get_json() validate(request_json, Orders().schema) except ValidationError as error: raise errors.BadRequest(error.message) request_json["status"] = Orders().created request_json["tasks"] = {} request_json["statuses"] = [ {"status": Orders().created, "on": datetime.datetime.now(), "payload": None} ] # actually create Ordr order_id = Orders().insert_one(request_json).inserted_id print("ORDER_ID", order_id) # send email about new order send_order_created_email(order_id) # create creation task Orders.create_creator_task(order_id) return jsonify({"_id": order_id})
def get_email_for(order_id, kind, formatted=True): def _fmt(name, email): return "{name} <{email}>".format(name=name, email=email) if kind not in ("client", "recipient", "operator"): return [] order = Orders.get_with_tasks(order_id, {"logs": 0}) if kind == "client": return _fmt(order["client"]["name"], order["client"]["email"]) if kind == "recipient": return _fmt(order["recipient"]["name"], order["recipient"]["email"]) if kind == "operator": worker = Users().by_username(order["tasks"]["download"]["worker"]) return worker["email"] return []
def send_order_failed_email(order_id): # recipient: order failed. you'll be refunded and contacted by client send_order_email_for( order_id, "subject_order_failed", "recipient_order_failed", "recipient", "client", ) # operator: download/write failed, please check conn and SD and contact client order = Orders().get_with_tasks(order_id, {"logs": 0}) if order["tasks"].get("write") and order["tasks"]["write"]["status"] in ( WriterTasks.failed_to_download, WriterTasks.failed_to_write, ): send_order_email_for(order_id, "subject_order_failed", "operator_order_failed", "operator")
def build_shipping_document(order_id): order = Orders().get(order_id, {"logs": 0}) channel = Channels().get(order["channel"]) context = get_full_context(str(order_id), extra={"channel": channel}) context.update({"cwd": os.path.abspath(".")}) fname = "Shipping_{oid}.pdf".format(oid=context["order"]["min_id"]) fpath = os.path.join(os.getenv("TMP_DIR", "/tmp"), fname) content = jinja_env.get_template("shipping.html").render(**context) options = { "page-size": "A4", "encoding": "UTF-8", "custom-header": [("Accept-Encoding", "gzip")], "no-outline": None, "viewport-size": "1280x1024", } pdfkit.from_string(content, fpath, options=options) return fpath
def run_periodic_tasks(): logger.info("running periodic tasks !!") # manage auto-images logger.info("managing auto-images") check_autoimages() # timeout orders based on status # orders cant stay in creation for more than 12h # orders in downloading/writing cant be slower than 4Mb/s # now = datetime.datetime.now() # min_bps = int(humanfriendly.parse_size("4MiB") / 8) # for morder in Orders().find( # {"status": {"$in": [Orders().creating, Orders.downloading, Orders.writing]}}, # {"_id": 1}, # ): # order = Orders().get_with_tasks(morder["_id"]) # ls = order["statuses"][-1] # # prepare expired dt based on status # if ls["status"] == Orders().creating: # task = CreatorTasks().get(order["tasks"]["create"]) # elif ls["status"] in (Orders.downloading, Orders.writing): # size = humanfriendly.parse_size(order["config"]["size"]) # expired = now - datetime.timedelta(seconds=int(size / min_bps)) # if ls["status"] == Orders.downloading: # task_cls, task_id = DownloaderTasks, order["tasks"]["download"] # else: # task_cls, task_id = WriterTasks, order["tasks"]["write"] # else: # # last status not in-progress # continue # # compare last update with expiry datetime # if ls["on"] > expired: # continue # # timeout task # task_cls().update_status(task_id=task_id, status=task_cls.timedout) # # timeout order # task_cls().cascade_status(task_id=task_id, status=task_cls.timedout) # # notify failure # send_order_failed_email(order["_id"]) # TODO: forward to task/order mgmt logger.info("timing out expired orders") for task_cls, task in Tasks.all_inprogress(): task_id = task["_id"] ls = task["statuses"][-1] if not is_expired(ls["status"], ls["on"], task_cls.get_size(task_id)): logger.info("skipping non-expired task #{}".format(task_id)) continue logger.info("timing out task #{}".format(task_id)) order = Orders().get_with_tasks(task["order"]) # timeout task task_cls.update_status(task_id=task_id, status=Tasks.timedout) # if write, cancel peers if ls["status"] in (Tasks.wiping_sdcard, Tasks.writing): for peer in order["tasks"]["write"]: if peer["_id"] == task_id: continue task_cls.update_status(task_id=peer["_id"], status=Tasks.canceled) # cascade task_cls.cascade_status(task_id=task_id, task_status=task_cls.timedout) # notify send_order_failed_email(order["_id"]) # TODO: forward to task/order mgmt logger.info("removing expired donwload files") now = datetime.datetime.now() for order in Orders.all_pending_expiry(): ls = order["statuses"][-1] if not ls["status"] == Orders.pending_expiry: continue # wrong timing if not order["sd_card"]["expiration"] < now: continue # expiration not reached logger.info("Order #{} has reach expiration.".format(order["_id"])) Orders().update_status(order["_id"], Orders.expired)
def calculate_load(user: dict): now = datetime.datetime.now() def is_connected(worker): """whether a worker is considered connected (ping 15mn ago)""" try: return (now - worker["on"]).total_seconds() <= 900 # 15mn except Exception: raise return False def get_remaining_minutes(task): """estimated remaining duration of task based on units and when/if started""" # physical card units are 10 times download ones but creator impact is identical units = task["units"] / 10 if task["units"] > 512 else task["units"] # small images are done in about an hour # larger one increase duration with size ; especially due to upload duration = max(int(units * 1.875), 90) if units >= 32 else 60 if task["status"] in ["pending", "received"]: # task has not started return duration # already started, remove spent time from received for event in task.get("statuses", []): if event["status"] == "received": passed = datetime.datetime.now() - event["on"] return duration - (passed.total_seconds() // 60) # couldn't find received ; returning full duration return duration # get number of online workers cursor = Acknowlegments().find({}, None).sort([("$natural", pymongo.ASCENDING)]) nb_connected_workers = len([ worker for worker in cursor if worker.get("worker_type") == "creator" and is_connected(worker) ]) # get list of pending task and calculate remaining durations for those tasks = [] for task in CreatorTasks().find( { "status": { "$in": ["pending", "received", "building", "built", "uploading"] } }, { "status": 1, "statuses": 1, "order": 1, "worker": 1 }, ): task["units"] = Orders().find_one({"_id": task["order"]}, {"units": 1})["units"] task["duration"] = get_remaining_minutes(task) tasks.append(task) nb_pending_taks = len(tasks) cumulative_duration = sum([task["duration"] for task in tasks]) try: remaining_minutes = cumulative_duration // nb_connected_workers except ZeroDivisionError: remaining_minutes = None # no worker estimated_completion = (now + datetime.timedelta(seconds=remaining_minutes * 60) if remaining_minutes else now) return jsonify({ "connected_workers": nb_connected_workers, "pending_tasks": nb_pending_taks, "cumulative_duration": cumulative_duration, "remaining_minutes": remaining_minutes, "estimated_completion": estimated_completion.isoformat(), })
def update_status(task_id: ObjectId, task_type: str, user: dict): task_cls = tasks_cls_for(task_type) task = task_cls().get(task_id) if task is None: raise errors.NotFound() request_json = request.get_json() # try: # request_json = request.get_json() # validate(request_json, Orders().schema) # except ValidationError as error: # raise errors.BadRequest(error.message) # update task status status = request_json.get("status") task_cls.update_status( task_id, status=request_json.get("status"), payload=request_json.get("log"), extra_update=request_json.get("extra"), ) # update order status based on this task task_cls.cascade_status(task_id, request_json.get("status")) # send email if appropriate order_id = task["order"] # create task uploaded image if status == Tasks.uploaded_public: order = Orders().get(order_id) # set expiration date expiration = datetime.datetime.now() + datetime.timedelta( days=order["sd_card"]["duration"]) Orders().update(order_id, {"sd_card.expiration": expiration}) send_image_uploaded_public_email(order_id) elif status == Tasks.uploaded: send_image_uploaded_email(order_id) # create DownloadTask Orders().create_downloader_task( order_id, { "fname": task.get("image", {}).get("fname"), "size": task.get("image", {}).get("size"), "checksum": task.get("image", {}).get("checksum"), }, ) # download task downloaded image elif status == Tasks.downloaded: # create WriterTask(s) Orders().create_writer_tasks(order_id) # write task was registered elif status == Tasks.waiting_for_card: # send email to insert card send_insert_card_email(order_id, task_id) # write task started writing elif status == Tasks.writing: send_image_writing_email(order_id, task_id) # write task completed elif status == Tasks.written: send_image_written_email(order_id, task_id) order = Orders().get_with_tasks(order_id) # all write tasks are marked as written if not [ 1 for wt in order["tasks"]["write"] if wt["status"] != Tasks.written ]: Orders().update_status(order_id, Orders.pending_shipment) send_order_pending_shipment_email(order_id) # find matching download task and mark it for file removal DownloaderTasks().update_status( task_id=order["tasks"]["download"]["_id"], status=Tasks.pending_image_removal, ) elif status in Tasks.FAILED_STATUSES: send_order_failed_email(order_id) return jsonify({"_id": task_id})