def document(order_id: ObjectId, user: dict):
"""fetch indiviual order info or cancel it"""
if request.method == "GET":
with_logs = request.args.get("with_logs",
default=False,
type=string_to_bool)
order = Orders.get_with_tasks(order_id, with_logs=with_logs)
if order is None:
raise errors.NotFound()
return jsonify(order)
if request.method == "PATCH":
order = Orders.get_with_tasks(order_id)
if order is None:
raise errors.NotFound()
request_json = request.get_json()
Orders().add_shipment(order_id, request_json.get("shipment_details"))
send_order_shipped_email(order_id)
return jsonify(order)
if request.method == "DELETE":
# should prepare email message with infos from order here
deleted_count = Orders().delete_one({"_id": order_id}).deleted_count
if deleted_count == 0:
raise errors.NotFound()
# send email about deletion
return jsonify({"_id": order_id})
def confirm_inserted_task(task_id: ObjectId, task_type: str):
task_cls = tasks_cls_for(task_type)
task = task_cls().find_one({
"_id": task_id,
"status": task_cls.waiting_for_card
})
if task is None:
raise errors.NotFound()
if request.method == "GET":
order = Orders().get(task["order"])
return render_template("pub_confirm_inserted.html",
order=order,
task=task)
elif request.method == "POST":
task_cls.update_status(task_id, status=task_cls.card_inserted)
task_cls.cascade_status(task_id, task_cls.card_inserted)
order = Orders().get(task["order"])
return render_template("pub_thank_inserted.html",
order=order,
task=task)
raise errors.BadRequest("?")
def add_shipment(order_id: ObjectId):
order = Orders().find_one({
"_id": order_id,
"status": Orders.pending_shipment
})
if order is None:
raise errors.NotFound()
if request.method == "GET":
return render_template("pub_add_shipment.html",
order=order,
order_id=order["_id"])
if request.method == "POST":
shipment_details = request.form.get("details")
if not shipment_details:
raise errors.BadRequest("Missing shipment details")
# store shipment details
Orders().add_shipment(order_id, shipment_details)
# refresh order object
order = Orders.get(order_id)
# send recipient an email
send_order_shipped_email(order_id)
return render_template("pub_thank_shipment.html",
order=order,
order_id=order["_id"])
def cancel(order_id: ObjectId, user: dict):
order = Orders.get(order_id)
if order is None:
raise errors.NotFound()
Orders().cancel(order_id)
send_order_failed_email(order_id)
return jsonify({"_id": order_id})
def get_full_context(order_id, extra={}):
order = Orders().get_with_tasks(order_id, {"logs": 0})
order.update({
"id": order_id,
"status": order["statuses"][-1],
"min_id": order_id[:8] + order_id[-3:],
})
context = {"order": order}
context.update(extra)
return context
def create_order_from(payload):
# validate payload
validate(payload, Orders().schema)
payload["status"] = Orders().created
payload["tasks"] = {}
payload["statuses"] = [{
"status": Orders().created,
"on": datetime.datetime.now(),
"payload": None
}]
# actually create Order
order_id = Orders().insert_one(payload).inserted_id
# define and record fname for this order
if "fname" not in payload:
fname = f"{order_id}.img"
else:
short_id = str(order_id)[:10]
if "{rand}" in payload["fname"]:
fname = payload["fname"].replace("{rand}", short_id)
else:
stem, ext = os.path.splitext(payload["fname"])
fname = "".join((stem + f"_{short_id}", ext))
Orders.update(order_id, {"fname": fname})
# send email about new order
send_order_created_email(order_id)
# create creation task
Orders.create_creator_task(order_id)
return order_id
def get_email_for(order_id, kind, formatted=True):
def _fmt(name, email):
return "{name} <{email}>".format(name=name, email=email)
if kind not in ("client", "recipient", "operator", "error-manager"):
return None, "en"
if kind == "error-manager" and FAILED_ORDER_EMAIL:
return _fmt("Cardshop Error Manager", FAILED_ORDER_EMAIL), "en"
order = Orders.get_with_tasks(order_id, {"logs": 0})
if kind == "client":
return (
_fmt(order["client"]["name"], order["client"]["email"]),
order["client"]["language"],
)
if kind == "recipient":
return (
_fmt(order["recipient"]["name"], order["recipient"]["email"]),
order["recipient"]["language"],
)
if kind == "operator":
worker = Users().by_username(order["tasks"]["download"]["worker"])
return worker["email"], "en"
return None, "en"
def check_autoimages():
# update images that were building
logger.info("Looking for currently building images…")
for image in AutoImages.all_currently_building():
logger.info(f".. {image['slug']}")
# check order status
order = Orders.get(image["order"])
# order is considered failed
if order["status"] in Orders.FAILED_STATUSES:
logger.info(f".. order failed: {order['status']}")
AutoImages.update_status(image["slug"], status="failed")
continue
# order is considered successful
if order["status"] in Orders.SUCCESS_STATUSES + [
Orders.pending_expiry
]:
logger.info(f".. order succeeded: {order['status']}")
torrent_url = get_public_download_torrent_url(order)
AutoImages.update_status(
image["slug"],
status="ready",
order=None,
http_url=get_public_download_url(order),
torrent_url=torrent_url,
magnet_url=get_magnet_for_torrent(torrent_url),
expire_on=get_next_month(),
)
continue
logger.info(f".. order still building: {order['status']}")
# find images that must be recreated
logger.info("Looking for images needing building…")
for image in AutoImages.all_needing_rebuild():
logger.info(f".. {image['slug']} ; starting build")
# create order
payload = AutoImages.create_order_payload(image["slug"])
try:
order_id = create_order_from(payload)
except Exception as exc:
logger.error(f"Error creating image `{image['slug']}`: {exc}")
logger.exception(exc)
AutoImages.update_status(image["slug"], status="failed")
continue
# update with order ID and status: building
AutoImages.update_status(image["slug"],
status="building",
order=order_id)
def anonymize(user: dict):
try:
request_json = request.get_json()
validate(request_json,
{"order_ids": {
"type": "list",
"required": True
}})
order_ids = [
ensure_objectid(oid) for oid in request_json.get("order_ids")
]
except ValidationError as error:
raise errors.BadRequest(str(error))
except Exception:
raise errors.BadRequest("Orders IDs are not all valid IDs")
try:
Orders.anonymize(order_ids)
except Exception:
raise errors.NotFound()
return jsonify({"_ids": order_ids})
def collection(user: dict):
"""
List or create orders
"""
if request.method == "GET":
# unpack url parameters
skip = request.args.get("skip", default=0, type=int)
limit = request.args.get("limit", default=20, type=int)
skip = 0 if skip < 0 else skip
limit = 20 if limit <= 0 else limit
query = {}
projection = {"_id": 1}
cursor = (Orders().find(query, projection).sort([
("$natural", pymongo.DESCENDING)
]).skip(skip).limit(limit))
count = Orders().count_documents(query)
orders = [Orders.get(order.get("_id")) for order in cursor]
return jsonify({
"meta": {
"skip": skip,
"limit": limit,
"count": count
},
"items": orders
})
if request.method == "POST":
try:
order_id = create_order_from(request.get_json())
except ValidationError as error:
raise errors.BadRequest(str(error))
return jsonify({"_id": order_id})
def collection(user: dict):
"""
List or create orders
"""
if request.method == "GET":
# unpack url parameters
skip = request.args.get("skip", default=0, type=int)
limit = request.args.get("limit", default=20, type=int)
skip = 0 if skip < 0 else skip
limit = 20 if limit <= 0 else limit
orders = [
Orders.get(order.get("_id")) for order in Orders().find({}, {"_id": 1})
]
return jsonify({"meta": {"skip": skip, "limit": limit}, "items": orders})
elif request.method == "POST":
# validate request json
try:
request_json = request.get_json()
validate(request_json, Orders().schema)
except ValidationError as error:
raise errors.BadRequest(error.message)
request_json["status"] = Orders().created
request_json["tasks"] = {}
request_json["statuses"] = [
{"status": Orders().created, "on": datetime.datetime.now(), "payload": None}
]
# actually create Ordr
order_id = Orders().insert_one(request_json).inserted_id
print("ORDER_ID", order_id)
# send email about new order
send_order_created_email(order_id)
# create creation task
Orders.create_creator_task(order_id)
return jsonify({"_id": order_id})
def get_email_for(order_id, kind, formatted=True):
def _fmt(name, email):
return "{name} <{email}>".format(name=name, email=email)
if kind not in ("client", "recipient", "operator"):
return []
order = Orders.get_with_tasks(order_id, {"logs": 0})
if kind == "client":
return _fmt(order["client"]["name"], order["client"]["email"])
if kind == "recipient":
return _fmt(order["recipient"]["name"], order["recipient"]["email"])
if kind == "operator":
worker = Users().by_username(order["tasks"]["download"]["worker"])
return worker["email"]
return []
def send_order_failed_email(order_id):
# recipient: order failed. you'll be refunded and contacted by client
send_order_email_for(
order_id,
"subject_order_failed",
"recipient_order_failed",
"recipient",
"client",
)
# operator: download/write failed, please check conn and SD and contact client
order = Orders().get_with_tasks(order_id, {"logs": 0})
if order["tasks"].get("write") and order["tasks"]["write"]["status"] in (
WriterTasks.failed_to_download,
WriterTasks.failed_to_write,
):
send_order_email_for(order_id, "subject_order_failed",
"operator_order_failed", "operator")
def build_shipping_document(order_id):
order = Orders().get(order_id, {"logs": 0})
channel = Channels().get(order["channel"])
context = get_full_context(str(order_id), extra={"channel": channel})
context.update({"cwd": os.path.abspath(".")})
fname = "Shipping_{oid}.pdf".format(oid=context["order"]["min_id"])
fpath = os.path.join(os.getenv("TMP_DIR", "/tmp"), fname)
content = jinja_env.get_template("shipping.html").render(**context)
options = {
"page-size": "A4",
"encoding": "UTF-8",
"custom-header": [("Accept-Encoding", "gzip")],
"no-outline": None,
"viewport-size": "1280x1024",
}
pdfkit.from_string(content, fpath, options=options)
return fpath
def run_periodic_tasks():
logger.info("running periodic tasks !!")
# manage auto-images
logger.info("managing auto-images")
check_autoimages()
# timeout orders based on status
# orders cant stay in creation for more than 12h
# orders in downloading/writing cant be slower than 4Mb/s
# now = datetime.datetime.now()
# min_bps = int(humanfriendly.parse_size("4MiB") / 8)
# for morder in Orders().find(
# {"status": {"$in": [Orders().creating, Orders.downloading, Orders.writing]}},
# {"_id": 1},
# ):
# order = Orders().get_with_tasks(morder["_id"])
# ls = order["statuses"][-1]
# # prepare expired dt based on status
# if ls["status"] == Orders().creating:
# task = CreatorTasks().get(order["tasks"]["create"])
# elif ls["status"] in (Orders.downloading, Orders.writing):
# size = humanfriendly.parse_size(order["config"]["size"])
# expired = now - datetime.timedelta(seconds=int(size / min_bps))
# if ls["status"] == Orders.downloading:
# task_cls, task_id = DownloaderTasks, order["tasks"]["download"]
# else:
# task_cls, task_id = WriterTasks, order["tasks"]["write"]
# else:
# # last status not in-progress
# continue
# # compare last update with expiry datetime
# if ls["on"] > expired:
# continue
# # timeout task
# task_cls().update_status(task_id=task_id, status=task_cls.timedout)
# # timeout order
# task_cls().cascade_status(task_id=task_id, status=task_cls.timedout)
# # notify failure
# send_order_failed_email(order["_id"]) # TODO: forward to task/order mgmt
logger.info("timing out expired orders")
for task_cls, task in Tasks.all_inprogress():
task_id = task["_id"]
ls = task["statuses"][-1]
if not is_expired(ls["status"], ls["on"], task_cls.get_size(task_id)):
logger.info("skipping non-expired task #{}".format(task_id))
continue
logger.info("timing out task #{}".format(task_id))
order = Orders().get_with_tasks(task["order"])
# timeout task
task_cls.update_status(task_id=task_id, status=Tasks.timedout)
# if write, cancel peers
if ls["status"] in (Tasks.wiping_sdcard, Tasks.writing):
for peer in order["tasks"]["write"]:
if peer["_id"] == task_id:
continue
task_cls.update_status(task_id=peer["_id"], status=Tasks.canceled)
# cascade
task_cls.cascade_status(task_id=task_id, task_status=task_cls.timedout)
# notify
send_order_failed_email(order["_id"]) # TODO: forward to task/order mgmt
logger.info("removing expired donwload files")
now = datetime.datetime.now()
for order in Orders.all_pending_expiry():
ls = order["statuses"][-1]
if not ls["status"] == Orders.pending_expiry:
continue # wrong timing
if not order["sd_card"]["expiration"] < now:
continue # expiration not reached
logger.info("Order #{} has reach expiration.".format(order["_id"]))
Orders().update_status(order["_id"], Orders.expired)
def calculate_load(user: dict):
now = datetime.datetime.now()
def is_connected(worker):
"""whether a worker is considered connected (ping 15mn ago)"""
try:
return (now - worker["on"]).total_seconds() <= 900 # 15mn
except Exception:
raise
return False
def get_remaining_minutes(task):
"""estimated remaining duration of task based on units and when/if started"""
# physical card units are 10 times download ones but creator impact is identical
units = task["units"] / 10 if task["units"] > 512 else task["units"]
# small images are done in about an hour
# larger one increase duration with size ; especially due to upload
duration = max(int(units * 1.875), 90) if units >= 32 else 60
if task["status"] in ["pending", "received"]:
# task has not started
return duration
# already started, remove spent time from received
for event in task.get("statuses", []):
if event["status"] == "received":
passed = datetime.datetime.now() - event["on"]
return duration - (passed.total_seconds() // 60)
# couldn't find received ; returning full duration
return duration
# get number of online workers
cursor = Acknowlegments().find({}, None).sort([("$natural",
pymongo.ASCENDING)])
nb_connected_workers = len([
worker for worker in cursor
if worker.get("worker_type") == "creator" and is_connected(worker)
])
# get list of pending task and calculate remaining durations for those
tasks = []
for task in CreatorTasks().find(
{
"status": {
"$in":
["pending", "received", "building", "built", "uploading"]
}
},
{
"status": 1,
"statuses": 1,
"order": 1,
"worker": 1
},
):
task["units"] = Orders().find_one({"_id": task["order"]},
{"units": 1})["units"]
task["duration"] = get_remaining_minutes(task)
tasks.append(task)
nb_pending_taks = len(tasks)
cumulative_duration = sum([task["duration"] for task in tasks])
try:
remaining_minutes = cumulative_duration // nb_connected_workers
except ZeroDivisionError:
remaining_minutes = None # no worker
estimated_completion = (now +
datetime.timedelta(seconds=remaining_minutes * 60)
if remaining_minutes else now)
return jsonify({
"connected_workers": nb_connected_workers,
"pending_tasks": nb_pending_taks,
"cumulative_duration": cumulative_duration,
"remaining_minutes": remaining_minutes,
"estimated_completion": estimated_completion.isoformat(),
})
def update_status(task_id: ObjectId, task_type: str, user: dict):
task_cls = tasks_cls_for(task_type)
task = task_cls().get(task_id)
if task is None:
raise errors.NotFound()
request_json = request.get_json()
# try:
# request_json = request.get_json()
# validate(request_json, Orders().schema)
# except ValidationError as error:
# raise errors.BadRequest(error.message)
# update task status
status = request_json.get("status")
task_cls.update_status(
task_id,
status=request_json.get("status"),
payload=request_json.get("log"),
extra_update=request_json.get("extra"),
)
# update order status based on this task
task_cls.cascade_status(task_id, request_json.get("status"))
# send email if appropriate
order_id = task["order"]
# create task uploaded image
if status == Tasks.uploaded_public:
order = Orders().get(order_id)
# set expiration date
expiration = datetime.datetime.now() + datetime.timedelta(
days=order["sd_card"]["duration"])
Orders().update(order_id, {"sd_card.expiration": expiration})
send_image_uploaded_public_email(order_id)
elif status == Tasks.uploaded:
send_image_uploaded_email(order_id)
# create DownloadTask
Orders().create_downloader_task(
order_id,
{
"fname": task.get("image", {}).get("fname"),
"size": task.get("image", {}).get("size"),
"checksum": task.get("image", {}).get("checksum"),
},
)
# download task downloaded image
elif status == Tasks.downloaded:
# create WriterTask(s)
Orders().create_writer_tasks(order_id)
# write task was registered
elif status == Tasks.waiting_for_card:
# send email to insert card
send_insert_card_email(order_id, task_id)
# write task started writing
elif status == Tasks.writing:
send_image_writing_email(order_id, task_id)
# write task completed
elif status == Tasks.written:
send_image_written_email(order_id, task_id)
order = Orders().get_with_tasks(order_id)
# all write tasks are marked as written
if not [
1 for wt in order["tasks"]["write"]
if wt["status"] != Tasks.written
]:
Orders().update_status(order_id, Orders.pending_shipment)
send_order_pending_shipment_email(order_id)
# find matching download task and mark it for file removal
DownloaderTasks().update_status(
task_id=order["tasks"]["download"]["_id"],
status=Tasks.pending_image_removal,
)
elif status in Tasks.FAILED_STATUSES:
send_order_failed_email(order_id)
return jsonify({"_id": task_id})
