def create_file_versions(previous_node, node, current_version_node, transaction): """Create file and node_to_file version changes from diff between files of `previous_node` and `node`. `node` must be a the version node that follows `previous_node`. `current_version_node` is the newest version node following `previous_node` and `node` """ NodeToFileVersion = version_class(NodeToFile) FileVersion = version_class(File) s = db.session new_files = set(node.files) - set(previous_node.files) removed_files = set(previous_node.files) - set(node.files) for fi in new_files: fv = FileVersion(id=fi.id, path=fi.path, mimetype=fi.mimetype, filetype=fi.filetype, operation_type=Operation.INSERT, transaction=transaction) ntfv = NodeToFileVersion(nid=current_version_node.id, file_id=fi.id, operation_type=Operation.INSERT, transaction=transaction) s.add(fv) s.add(ntfv) for fi in removed_files: fv = FileVersion(id=fi.id, path=fi.path, mimetype=fi.mimetype, filetype=fi.filetype, operation_type=Operation.DELETE, transaction=transaction) # previous_file_version must be the file version with the highest transaction id previous_file_version = q(FileVersion).filter_by(id=fi.id).order_by(FileVersion.transaction_id.desc()).limit(1).scalar() if previous_file_version is not None: previous_file_version.end_transaction_id = transaction.id ntfv = NodeToFileVersion(nid=current_version_node.id, file_id=fi.id, operation_type=Operation.DELETE, transaction=transaction) # previous_file_version must be the file version with the highest transaction id previous_node_to_file_version = (q(NodeToFileVersion).filter_by(nid=current_version_node.id, file_id=fi.id) .order_by(NodeToFileVersion.transaction_id.desc()).limit(1).scalar()) if previous_node_to_file_version is not None: previous_node_to_file_version.end_transaction_id = transaction.id s.add(fv) s.add(ntfv)
def iter_attr_edits(cls, attrs, query=None): pk_cols = [k for k in inspect(cls).primary_key] cls_version = version_class(cls) pk_cols_version = [getattr(cls_version, k.name) for k in pk_cols] attrs_version = [getattr(cls_version, a) for a in attrs] cls_transaction = transaction_class(cls) if query is None: query = cls_version.query all_versions = (query.join(cls_version.transaction).with_entities( *pk_cols_version + attrs_version + [cls_transaction.issued_at]).order_by(*pk_cols_version + [cls_version.transaction_id])) def get_pk(row): return [getattr(row, k.name) for k in pk_cols_version] for pk, versions in groupby(all_versions, get_pk): # We don't yet process inserts/deletes, but should first = next(versions) attr_vals = {a: getattr(first, a) for a in attrs} attr_times = {a: [first.issued_at] for a in attrs} for version in versions: for attr in attrs: val = getattr(version, attr) if val != attr_vals[attr]: attr_times[attr].append(version.issued_at) attr_vals[attr] = val yield (pk, attr_times)
def iter_attr_edits(cls, attrs, query=None): pk_cols = [k for k in inspect(cls).primary_key] cls_version = version_class(cls) pk_cols_version = [getattr(cls_version, k.name) for k in pk_cols] attrs_version = [getattr(cls_version, a) for a in attrs] cls_transaction = transaction_class(cls) if query is None: query = cls_version.query all_versions = query.join(cls_version.transaction) \ .with_entities(*pk_cols_version + attrs_version + [cls_transaction.issued_at]) \ .order_by(*pk_cols_version + [cls_version.transaction_id]) def get_pk(row): return [getattr(row, k.name) for k in pk_cols_version] for pk, versions in groupby(all_versions, get_pk): # We don't yet process inserts/deletes, but should first = next(versions) attr_vals = {a: getattr(first, a) for a in attrs} attr_times = {a: [first.issued_at] for a in attrs} for version in versions: for attr in attrs: val = getattr(version, attr) if val != attr_vals[attr]: attr_times[attr].append(version.issued_at) attr_vals[attr] = val yield (pk, attr_times)
def get_published_version(self): Transaction = versioning_manager.transaction_cls TransactionMeta = versioning_manager.transaction_meta_cls version_cls = version_class(self.__class__) published_versions = self.versions.join(Transaction, version_cls.transaction_id == Transaction.id).\ join(Transaction.meta_relation). filter(TransactionMeta.key == u"publish") return published_versions.scalar()
def __enter__(self): self.session = s = object_session(node) if s.new or s.dirty: raise Exception("Refusing to create a new tagged node version. Session must be clean!") uow = versioning_manager.unit_of_work(s) tx = uow.create_transaction(s) if user is not None: tx.user = user if tag: if node.get_tagged_version(tag): raise ValueError("tag already exists") tx.meta[u"tag"] = tag elif publish: if node.get_published_version(): raise ValueError("publish version already exists") tx.meta[u"publish"] = publish else: NodeVersion = version_class(node.__class__) # in case you were wondering: order_by(None) resets the default order_by last_tagged_version = node.tagged_versions.order_by(None).order_by(NodeVersion.transaction_id.desc()).first() if last_tagged_version is not None: next_version = int(last_tagged_version.tag) + 1 else: node.versions[-1].tag = u"1" next_version = 2 tx.meta[u"tag"] = unicode(next_version) if comment: tx.meta[u"comment"] = comment return tx
def create_current_version(current_version_node): s = db.session Transaction = versioning_manager.transaction_cls tx = Transaction() version_id = current_version_node.system_attrs[u"version.id"] operation_type = Operation.UPDATE TransactionMeta = versioning_manager.transaction_meta_cls tx.meta_relation[u"tag"] = TransactionMeta(key=u"tag", value=unicode(version_id)) tx.meta_relation[u"mysql_migration"] = TransactionMeta(key=u"mysql_migration", value=u"migrated current node version") if u"version.comment" in current_version_node.system_attrs: tx.meta_relation[u"comment"] = TransactionMeta(key=u"comment", value=current_version_node.system_attrs[u"version.comment"]) s.add(tx) NodeVersion = version_class(Node) nv = NodeVersion(id=current_version_node.id, name=current_version_node.name, type=current_version_node.type, schema=current_version_node.schema, attrs=current_version_node.attrs, orderpos=current_version_node.orderpos, transaction=tx, operation_type=operation_type) s.add(nv) s.flush() return nv
def _send_thumbnail(thumb_type, req): try: nid = node_id_from_req_path(req) except ValueError: return 400 version_id = version_id_from_req(req) node_or_version = get_node_or_version(nid, version_id, Data) if not node_or_version.has_read_access(): return 404 FileVersion = version_class(File) if version_id: version = node_or_version files = version.files.filter_by( filetype=thumb_type, transaction_id=version.transaction_id).all() if not files: # files may be None if in this version only metadata changed # then try previous transaction_ids files = version.files.filter(FileVersion.filetype==thumb_type, FileVersion.transaction_id<=version.transaction_id). \ order_by(FileVersion.transaction_id.desc()) for f in files: if f.exists: return _request_handler.sendFile(req, f.abspath, f.mimetype) ntype, schema = version.type, version.schema else: # no version id given # XXX: better to use scalar(), but we must ensure that we have no dupes first node = node_or_version for f in node.files.filter_by(filetype=thumb_type): if f.exists: return _request_handler.sendFile(req, f.abspath, f.mimetype) try: ntype, schema = node.type, node.schema except NoResultFound: return 404 for p in _request_handler.getFileStorePaths("/img/"): for test in [ "default_thumb_%s_%s.*" % (ntype, schema), "default_thumb_%s.*" % schema, "default_thumb_%s.*" % ntype ]: fps = glob.glob(os.path.join(p, test)) if fps: thumb_mimetype, thumb_type = utils.utils.getMimeType(fps[0]) return _request_handler.sendFile(req, fps[0], thumb_mimetype, force=1) return _request_handler.sendFile(req, config.basedir + "/web/img/questionmark.png", "image/png", force=1)
def get_table_version(db: Session) -> int: version = version_class(AssociationTypePhPWDB) last_changed = db.query(version).order_by(desc(version.transaction_id)).limit(1) if last_changed.count() > 0: return last_changed[0].transaction_id else: return -1
def test_old_node_version_support(content_node_versioned_with_alias_id): """Tests the q(Node).get hack (in MtQuery)""" node = content_node_versioned_with_alias_id q = db.query version = q(Node).get(23) version_cls = version_class(node.__class__) assert isinstance(version, version_cls) assert version.orderpos == 23
def tagged_versions(self): Transaction = versioning_manager.transaction_cls TransactionMeta = versioning_manager.transaction_meta_cls version_cls = version_class(self.__class__) return (self.versions.join( Transaction, version_cls.transaction_id == Transaction.id).join( Transaction.meta_relation).filter( TransactionMeta.key == u"tag"))
def refresh(self, node): """Return a refreshed copy of `node`. Workaround for Node objects which are kept between requests. XXX: must be removed later """ from .node import Node NodeVersion = version_class(Node) if isinstance(node, NodeVersion): return self.session.query(NodeVersion).get((node.id, node.transaction.id)) else: return self.session.query(Node).get(node.id)
def _calculate_tx_id(self, obj): session = sa.orm.object_session(self) if obj: object_version = version_obj(session, obj) if object_version: return object_version.transaction_id version_cls = version_class(obj.__class__) return session.query( sa.func.max(version_cls.transaction_id)).filter( version_cls.id == obj.id).scalar()
def get_export_data(cls): if cls.__name__ == 'Payment': # Export stats for each payment type separately return {} purchase_counts = cls.query.outerjoin(cls.purchases).group_by(cls.id).with_entities(func.count(models.Ticket.id)) refund_counts = cls.query.outerjoin(cls.refunds).group_by(cls.id).with_entities(func.count(Refund.id)) cls_version = version_class(cls) cls_transaction = transaction_class(cls) changes = cls.query.join(cls.versions).group_by(cls.id) change_counts = changes.with_entities(func.count(cls_version.id)) first_changes = changes.join(cls_version.transaction) \ .with_entities(func.min(cls_transaction.issued_at).label('created')) \ .from_self() cls_ver_new = aliased(cls.versions) cls_ver_paid = aliased(cls.versions) cls_txn_new = aliased(cls_version.transaction) cls_txn_paid = aliased(cls_version.transaction) active_time = func.max(cls_txn_paid.issued_at) - func.max(cls_txn_new.issued_at) active_times = cls.query \ .join(cls_ver_new, cls_ver_new.id == cls.id) \ .join(cls_ver_paid, cls_ver_paid.id == cls.id) \ .join(cls_txn_new, cls_txn_new.id == cls_ver_new.transaction_id) \ .join(cls_txn_paid, cls_txn_paid.id == cls_ver_paid.transaction_id) \ .filter(cls_ver_new.state == 'new') \ .filter(cls_ver_paid.state == 'paid') \ .with_entities(active_time.label('active_time')) \ .group_by(cls.id) time_buckets = [timedelta(0), timedelta(minutes=1), timedelta(hours=1)] + \ [timedelta(d) for d in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 28, 60]] data = { 'public': { 'payments': { 'counts': { 'purchases': bucketise(purchase_counts, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20]), 'refunds': bucketise(refund_counts, [0, 1, 2, 3, 4]), 'changes': bucketise(change_counts, range(10)), 'created_week': export_intervals(first_changes, column('created'), 'week', 'YYYY-MM-DD'), 'active_time': bucketise([r.active_time for r in active_times], time_buckets), 'amounts': bucketise(cls.query.with_entities(cls.amount_int / 100), [0, 10, 20, 30, 40, 50, 100, 150, 200]), }, }, }, 'tables': ['payment', 'payment_version'], } count_attrs = ['state', 'reminder_sent', 'currency'] data['public']['payments']['counts'].update(export_attr_counts(cls, count_attrs)) return data
def get(self, ident): nodeclass = self._find_nodeclass() if not nodeclass: return Query.get(self, ident) active_version = Query.get(self, ident) Transaction = versioning_manager.transaction_cls if active_version is None: ver_cls = version_class(nodeclass) return (self.session.query(ver_cls).join(Transaction, ver_cls.transaction_id == Transaction.id) .join(Transaction.meta_relation) .filter_by(key=u'alias_id', value=unicode(ident)).scalar()) return active_version
def _send_thumbnail(thumb_type, req): try: nid = node_id_from_req_path(req) except ValueError: return 400 version_id = version_id_from_req(req) node_or_version = get_node_or_version(nid, version_id, Data) if not node_or_version.has_read_access(): return 404 FileVersion = version_class(File) if version_id: version = node_or_version files = version.files.filter_by(filetype=thumb_type, transaction_id=version.transaction_id).all() if not files: # files may be None if in this version only metadata changed # then try previous transaction_ids files = version.files.filter(FileVersion.filetype==thumb_type, FileVersion.transaction_id<=version.transaction_id). \ order_by(FileVersion.transaction_id.desc()) for f in files: if f.exists: return req.sendFile(f.abspath, f.mimetype) ntype, schema = version.type, version.schema else: # no version id given # XXX: better to use scalar(), but we must ensure that we have no dupes first node = node_or_version for f in node.files.filter_by(filetype=thumb_type): if f.exists: return req.sendFile(f.abspath, f.mimetype) try: ntype, schema = node.type, node.schema except NoResultFound: return 404 for p in athana.getFileStorePaths("/img/"): for test in ["default_thumb_%s_%s.*" % (ntype, schema), "default_thumb_%s.*" % schema, "default_thumb_%s.*" % ntype]: fps = glob.glob(os.path.join(p, test)) if fps: thumb_mimetype, thumb_type = utils.utils.getMimeType(fps[0]) return req.sendFile(fps[0], thumb_mimetype, force=1) return req.sendFile(config.basedir + "/web/img/questionmark.png", "image/png", force=1)
def get(self, ident): nodeclass = self._find_nodeclass() if not nodeclass: return Query.get(self, ident) else: nodeclass = nodeclass[0] active_version = Query.get(self, ident) Transaction = versioning_manager.transaction_cls if active_version is None: ver_cls = version_class(nodeclass) return (self.session.query(ver_cls).join(Transaction, ver_cls.transaction_id == Transaction.id) .join(Transaction.meta_relation) .filter_by(key=u'alias_id', value=unicode(ident)).scalar()) return active_version
def __enter__(self): self.session = s = object_session(node) if s.new or s.dirty: raise Exception( "Refusing to create a new tagged node version. Session must be clean!" ) uow = versioning_manager.unit_of_work(s) tx = uow.create_transaction(s) if user is not None: tx.user = user if tag: if node.get_tagged_version(tag): raise ValueError("tag already exists") tx.meta[u"tag"] = tag elif publish: if node.get_published_version(): raise ValueError("publish version already exists") tx.meta[u"publish"] = publish else: NodeVersion = version_class(node.__class__) # in case you were wondering: order_by(None) resets the default order_by last_tagged_version = node.tagged_versions.order_by( None).order_by( NodeVersion.transaction_id.desc()).first() if last_tagged_version is not None: next_version = int(last_tagged_version.tag) + 1 else: node.versions[-1].tag = u"1" next_version = 2 tx.meta[u"tag"] = unicode(next_version) if comment: tx.meta[u"comment"] = comment return tx
def _send_file_with_type(filetype, mimetype, req, checkonly=False): try: nid = node_id_from_req_path(req) except ValueError: return 400 version_id = version_id_from_req(req) node = get_node_or_version(nid, version_id, Content) if node is None or not node.has_data_access(): return 404 fileobj = None file_query = node.files.filter_by(filetype=filetype) # if version_id == u"published": if version_id: file_query = file_query.filter_by(transaction_id=node.transaction_id) fileobj = file_query.scalar() # fileobj may be None if in this version only metadata changed # then try previous transaction_ids if not fileobj: FileVersion = version_class(File) # this a long lasting query file_query = node.files.filter_by(filetype=filetype) fileobj = file_query.filter(FileVersion.transaction_id <= node.transaction_id).\ order_by(FileVersion.transaction_id.desc()).first() if mimetype: file_query = file_query.filter_by(mimetype=mimetype) if not fileobj: fileobj = file_query.scalar() if fileobj is not None: if checkonly: return 200 return _request_handler.sendFile(req, fileobj.abspath, fileobj.mimetype) return 404
def _send_file_with_type(filetype, mimetype, req, checkonly=False): try: nid = node_id_from_req_path(req) except ValueError: return 400 version_id = version_id_from_req(req) node = get_node_or_version(nid, version_id, Content) if node is None or not node.has_data_access(): return 404 fileobj = None file_query = node.files.filter_by(filetype=filetype) # if version_id == u"published": if version_id: file_query = file_query.filter_by(transaction_id=node.transaction_id) fileobj = file_query.scalar() # fileobj may be None if in this version only metadata changed # then try previous transaction_ids if not fileobj: FileVersion = version_class(File) # this a long lasting query file_query = node.files.filter_by(filetype=filetype) fileobj = file_query.filter(FileVersion.transaction_id <= node.transaction_id).\ order_by(FileVersion.transaction_id.desc()).first() if mimetype: file_query = file_query.filter_by(mimetype=mimetype) if not fileobj: fileobj = file_query.scalar() if fileobj is not None: if checkonly: return 200 return req.sendFile(fileobj.abspath, fileobj.mimetype) return 404
def run(self): # As we go, we check against the list of all tables, in case we forget about some # new object type (e.g. association table). # Exclude tables we know will never be exported ignore = ['alembic_version', 'transaction'] all_model_classes = {cls for cls in db.Model._decl_class_registry.values() if isinstance(cls, type) and issubclass(cls, db.Model)} all_version_classes = {version_class(c) for c in all_model_classes if is_versioned(c)} seen_model_classes = set() remaining_tables = set(db.metadata.tables) year = datetime.utcnow().year path = os.path.join('exports', str(year)) for dirname in ['public', 'private']: os.makedirs(os.path.join(path, dirname), exist_ok=True) for model_class in all_model_classes: if model_class in seen_model_classes: continue seen_model_classes.add(model_class) table = model_class.__table__.name model = model_class.__name__ if table in ignore: app.logger.debug('Ignoring %s', model) remaining_tables.remove(table) continue if not getattr(model_class, '__export_data__', True): # We don't remove the version table, as we want # to be explicit about chucking away edit stats app.logger.debug('Skipping %s', model) remaining_tables.remove(table) continue if model_class in all_version_classes: # Version tables are explicitly dumped by their parents, # as they don't make sense to be exported on their own app.logger.debug('Ignoring version model %s', model) continue if hasattr(model_class, 'get_export_data'): try: export = model_class.get_export_data() for dirname in ['public', 'private']: if dirname in export: filename = os.path.join(path, dirname, '{}.json'.format(model)) simplejson.dump(export[dirname], open(filename, 'w'), indent=4, cls=ExportEncoder) app.logger.info('Exported data from %s to %s', model, filename) except Exception as e: app.logger.error('Error exporting %s', model) raise exported_tables = export.get('tables', [table]) remaining_tables -= set(exported_tables) if remaining_tables: app.logger.warning('Remaining tables: %s', ', '.join(remaining_tables)) data = { 'timestamp': datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'), 'remaining_tables': sorted(list(remaining_tables)) } filename = os.path.join(path, 'export.json') simplejson.dump(data, open(filename, 'w'), indent=4, cls=ExportEncoder) with app.test_client() as client: for schedule in ['schedule.frab', 'schedule.json', 'schedule.ics']: resp = client.get('/{}'.format(schedule)) with open(os.path.join(path, 'public', schedule), 'wb') as f: f.write(resp.data) app.logger.info('Export complete, summary written to %s', filename)
def tagged_versions(self): Transaction = versioning_manager.transaction_cls TransactionMeta = versioning_manager.transaction_meta_cls version_cls = version_class(self.__class__) return (self.versions.join(Transaction, version_cls.transaction_id == Transaction.id).join(Transaction.meta_relation). filter(TransactionMeta.key == u"tag"))
def select_style_link(self, style): version = self._node.tag if isinstance(self._node, version_class(Node)) else None return node_url(self.id, version=version, style=style)
def filebrowser(node, req): filesize = 0 ret = list() if isinstance(node, Node): file_entity = File else: file_entity = version_class(File) paths = [ t[0] for t in node.files.with_entities(file_entity.path).filter_by( filetype=u"attachment") ] if len(paths) == 1 and os.path.isdir( config.get("paths.datadir") + paths[0]): # single file with no path path = paths[0] elif len(paths) > 0: # some single files files = [] for path in paths: file = {} if not os.path.isdir(config.get("paths.datadir") + path): # file file["mimetype"], file["type"] = getMimeType( config.get("paths.datadir") + path) icon = fileicons.get(file["mimetype"]) if not icon: icon = fileicons["other"] file["icon"] = icon file["path"] = path file["name"] = os.path.basename(path) if os.path.exists(config.get("paths.datadir") + path): size = os.path.getsize(config.get("paths.datadir") + path) else: size = 0 file["size"] = format_filesize(size) filesize += int(size) files.append(file) return files, filesize else: path = "" if path == "": # no attachment directory -> test for single file for f in node.files.filter( ~file_entity.filetype.in_(node.get_sys_filetypes())): file = {} file["mimetype"], file["type"] = getMimeType(f.getName()) file["icon"] = fileicons[file["mimetype"]] file["path"] = f.path file["name"] = f.base_name file["size"] = format_filesize(f.size) filesize += f.size ret.append(file) return ret, filesize if not path.endswith("/") and not req.params.get("path", "").startswith("/"): path += "/" path += req.params.get("path", "") if req.params.get("path", "") != "": file = {} file["type"] = "back" file["mimetype"] = "back" file["icon"] = fileicons[file["mimetype"]] file["name"] = ".." file["path"] = req.params.get("path", "") file["req_path"] = req.params.get( "path", "")[:req.params.get("path", "").rfind("/")] ret.append(file) for name in os.listdir(config.settings["paths.datadir"] + path + "/"): if name.endswith(".thumb") or name.endswith(".thumb2"): continue file = {} file_path = os.path.join(config.settings["paths.datadir"] + path, name) if os.path.isdir(file_path): # directory file["type"] = "dir" file["mimetype"] = "directory" else: # file file["mimetype"], file["type"] = getMimeType(name) file["size"] = format_filesize(os.path.getsize(file_path)) filesize += os.path.getsize(file_path) file["icon"] = fileicons[file["mimetype"]] file["path"] = os.path.join(path, name) file["name"] = name file["req_path"] = req.params.get("path", "") + "/" + file["name"] ret.append(file) return ret, format_filesize(filesize)
def export_db(): """ Export data from the DB to disk. This command is run as a last step before wiping the DB after an event, to export all the data we want to save. It saves a private and a public export to the exports directory. Model classes should implement get_export_data, which returns a dict with keys: public Public data to save in git private Private data that should be stored for a limited amount of time tables Tables this method exported, used to sanity check the export process Alternatively, add __export_data__ = False to a class to state that get_export_data shouldn't be called, and that its associated table doesn't need to be checked. """ # As we go, we check against the list of all tables, in case we forget about some # new object type (e.g. association table). # Exclude tables we know will never be exported ignore = ["alembic_version", "transaction"] all_model_classes = { cls for cls in db.Model._decl_class_registry.values() if isinstance(cls, type) and issubclass(cls, db.Model) } all_version_classes = { version_class(c) for c in all_model_classes if is_versioned(c) } seen_model_classes = set() remaining_tables = set(db.metadata.tables) year = datetime.utcnow().year path = os.path.join("exports", str(year)) for dirname in ["public", "private"]: os.makedirs(os.path.join(path, dirname), exist_ok=True) for model_class in all_model_classes: if model_class in seen_model_classes: continue seen_model_classes.add(model_class) table = model_class.__table__.name model = model_class.__name__ if table in ignore: app.logger.debug("Ignoring %s", model) remaining_tables.remove(table) continue if not getattr(model_class, "__export_data__", True): # We don't remove the version table, as we want # to be explicit about chucking away edit stats app.logger.debug("Skipping %s", model) remaining_tables.remove(table) continue if model_class in all_version_classes: # Version tables are explicitly dumped by their parents, # as they don't make sense to be exported on their own app.logger.debug("Ignoring version model %s", model) continue if hasattr(model_class, "get_export_data"): try: export = model_class.get_export_data() for dirname in ["public", "private"]: if dirname in export: filename = os.path.join(path, dirname, "{}.json".format(model)) simplejson.dump( export[dirname], open(filename, "w"), indent=4, cls=ExportEncoder, ) app.logger.info("Exported data from %s to %s", model, filename) except Exception as e: app.logger.error("Error exporting %s", model) raise exported_tables = export.get("tables", [table]) remaining_tables -= set(exported_tables) if remaining_tables: app.logger.warning("Remaining tables: %s", ", ".join(remaining_tables)) data = { "timestamp": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), "remaining_tables": sorted(list(remaining_tables)), } filename = os.path.join(path, "export.json") simplejson.dump(data, open(filename, "w"), indent=4, cls=ExportEncoder) with app.test_client() as client: for schedule in ["schedule.frab", "schedule.json", "schedule.ics"]: resp = client.get("/{}".format(schedule)) with open(os.path.join(path, "public", schedule), "wb") as f: f.write(resp.data) app.logger.info("Export complete, summary written to %s", filename)
def get_export_data(cls): if cls.__name__ == "Payment": # Export stats for each payment type separately return {} purchase_counts = ( cls.query.outerjoin(cls.purchases) .group_by(cls.id) .with_entities(func.count(Ticket.id)) ) refund_counts = ( cls.query.outerjoin(cls.refunds) .group_by(cls.id) .with_entities(func.count(Refund.id)) ) cls_version = version_class(cls) cls_transaction = transaction_class(cls) changes = cls.query.join(cls.versions).group_by(cls.id) change_counts = changes.with_entities(func.count(cls_version.id)) first_changes = ( changes.join(cls_version.transaction) .with_entities(func.min(cls_transaction.issued_at).label("created")) .from_self() ) cls_ver_new = aliased(cls.versions) cls_ver_paid = aliased(cls.versions) cls_txn_new = aliased(cls_version.transaction) cls_txn_paid = aliased(cls_version.transaction) active_time = func.max(cls_txn_paid.issued_at) - func.max(cls_txn_new.issued_at) active_times = ( cls.query.join(cls_ver_new, cls_ver_new.id == cls.id) .join(cls_ver_paid, cls_ver_paid.id == cls.id) .join(cls_txn_new, cls_txn_new.id == cls_ver_new.transaction_id) .join(cls_txn_paid, cls_txn_paid.id == cls_ver_paid.transaction_id) .filter(cls_ver_new.state == "new") .filter(cls_ver_paid.state == "paid") .with_entities(active_time.label("active_time")) .group_by(cls.id) ) time_buckets = [timedelta(0), timedelta(minutes=1), timedelta(hours=1)] + [ timedelta(d) for d in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 28, 60] ] data = { "public": { "payments": { "counts": { "purchases": bucketise( purchase_counts, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20] ), "refunds": bucketise(refund_counts, [0, 1, 2, 3, 4]), "changes": bucketise(change_counts, range(10)), "created_week": export_intervals( first_changes, column("created"), "week", "YYYY-MM-DD" ), "active_time": bucketise( [r.active_time for r in active_times], time_buckets ), "amounts": bucketise( cls.query.with_entities(cls.amount_int / 100), [0, 10, 20, 30, 40, 50, 100, 150, 200], ), } } }, "tables": ["payment", "payment_version"], } count_attrs = ["state", "reminder_sent", "currency"] data["public"]["payments"]["counts"].update( export_attr_counts(cls, count_attrs) ) return data