class TwitterArchive(db.Model, TimestampMixin): __tablename__ = 'twitter_archives' id = db.Column(db.Integer, primary_key=True) account_id = db.Column(db.String, db.ForeignKey('accounts.id', onupdate='CASCADE', ondelete='CASCADE'), nullable=False) account = db.relationship(Account, backref=db.backref( 'twitter_archives', order_by=lambda: db.desc(TwitterArchive.id))) body = db.deferred(db.Column(db.LargeBinary, nullable=False)) chunks = db.Column(db.Integer) chunks_successful = db.Column(db.Integer, server_default='0', nullable=False) chunks_failed = db.Column(db.Integer, server_default='0', nullable=False) def status(self): if self.chunks is None or self.chunks_failed > 0: return 'failed' if self.chunks_successful == self.chunks: return 'successful' return 'pending'
class Visitors(db.Model): __tablename__ = 'visitors' id_number = db.Column(db.String(64), primary_key=True) id_type = db.Column(db.String(64)) name = db.Column(db.String(64)) gender = db.deferred(db.Column(db.String(128))) phone_number = db.Column(db.String(128)) address = db.Column(db.String(32), nullable=True) avatar = db.Column(db.String(128)) def __init__(self, id_number, id_type, name, gender, phone_number, address): self.id_number = id_number self.id_type = id_type self.name = name self.gender = gender self.phone_number = phone_number self.address = address def avatar_url(self, _external=False): if self.avatar: avatar_json = json.loads(self.avatar) if avatar_json['use_out_url']: return avatar_json['url'] else: return url_for('_uploads.uploaded_file', setname=avatars.name, filename=avatar_json['url'], _external=_external) else: return url_for('static', filename='img/avatar.png', _external=_external)
class SiteMeta(db.Model): """ Site table """ __tablename__ = "meta" __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} id = db.Column(db.Integer, primary_key=True, nullable=False) name = db.Column(db.String(255), nullable=False, index=True) value = db.deferred(db.Column(LONGTEXT, default="", nullable=False)) @staticmethod def add(data): for name, value in data.items(): meta = SiteMeta.query.filter_by(name=name).first() if meta is not None: continue meta = SiteMeta(name=name, value=value) db.session.add(meta) db.session.commit() @staticmethod def setting(data): for name, value in data.items(): meta = SiteMeta.query.filter_by(name=name).first() if not meta: meta = SiteMeta(name=name, value=value) db.session.add(meta) return meta.value = value db.session.commit() @staticmethod def all(): return SiteMeta.query.all()
class FlicketSubscription(db.Model): __tablename__ = 'flicket_ticket_subscription' id = db.Column(db.Integer, primary_key=True) ticket_id = db.Column(db.Integer, db.ForeignKey(FlicketTicket.id)) ticket = db.relationship(FlicketTicket) user_id = db.Column(db.Integer, db.ForeignKey(User.id)) user = db.relationship(User) user_def = db.deferred(db.select([User.email]).where(User.id == user_id)) def __repr__(self): return '<Class FlicketSubscription: ticket_id={}, user_id={}>'
class Book(db.Model): __tablename__ = 'books' id = db.Column(db.Integer, primary_key=True) isbn = db.Column(db.String(16), unique=True) title = db.Column(db.String(128)) origin_title = db.Column(db.String(128)) subtitle = db.Column(db.String(128)) author = db.Column(db.String(128)) translator = db.Column(db.String(64)) publisher = db.Column(db.String(64)) image = db.Column(db.String(128)) pubdate = db.Column(db.String(32)) pages = db.Column(db.Integer) price = db.Column(db.String(16)) binding = db.Column(db.String(16)) numbers = db.Column(db.Integer, default=5) summary = db.deferred(db.Column(db.Text, default="")) summary_html = db.deferred(db.Column(db.Text)) catalog = db.deferred(db.Column(db.Text, default="")) catalog_html = db.deferred(db.Column(db.Text)) hidden = db.Column(db.Boolean, default=0) transactions = db.relationship('Transaction', backref=db.backref('book', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') logs = db.relationship('Log', backref=db.backref('book', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') comments = db.relationship('Comment', backref='book', lazy='dynamic', cascade='all, delete-orphan') @property def tags_string(self): return ",".join([tag.name for tag in self.tags.all()]) @tags_string.setter def tags_string(self, value): self.tags = [] tags_list = value.split(u',') for str in tags_list: tag = Tag.query.filter(Tag.name.ilike(str)).first() if tag is None: tag = Tag(name=str) self.tags.append(tag) db.session.add(self) db.session.commit() def can_borrow(self): return (not self.hidden) and self.can_borrow_number() > 0 def can_borrow_number(self): return self.numbers - Log.query.filter_by(book_id=self.id, returned=0).count() @staticmethod def on_changed_summary(target, value, oldvalue, initiaor): allowed_tags = [ 'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p' ] target.summary_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True)) @staticmethod def on_changed_catalog(target, value, oldvalue, initiaor): allowed_tags = [ 'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p' ] target.catalog_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True)) def __repr__(self): return u'<Book %r>' % self.title
class User(UserMixin, db.Model): __tablename__ = 'users' id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(64), unique=True) name = db.Column(db.String(64)) password_hash = db.deferred(db.Column(db.String(128))) major = db.Column(db.String(128)) role_id = db.Column(db.Integer, db.ForeignKey('roles.id')) headline = db.Column(db.String(32), nullable=True) about_me = db.deferred(db.Column(db.Text, nullable=True)) about_me_html = db.deferred(db.Column(db.Text, nullable=True)) avatar = db.Column(db.String(128)) member_since = db.Column(db.DateTime(), default=datetime.utcnow) @property def password(self): raise AttributeError('password is not readable attribute') @password.setter def password(self, password): self.password_hash = generate_password_hash(password) def verify_password(self, password): return check_password_hash(self.password_hash, password) def __init__(self, **kwargs): super(User, self).__init__(**kwargs) if self.role is None: if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower( ): self.role = Role.query.filter_by(permissions=0x1ff).first() if self.role is None: self.role = Role.query.filter_by(default=True).first() self.member_since = datetime.now() def can(self, permissions): return self.role is not None and \ (self.role.permissions & permissions) == permissions def is_administrator(self): return self.can(Permission.ADMINISTER) investments = db.relationship('Investment', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') def __repr__(self): return '<User %r>' % self.email def get_id(self): return self.id def avatar_url(self, _external=False): if self.avatar: avatar_json = json.loads(self.avatar) if avatar_json['use_out_url']: return avatar_json['url'] else: return url_for('_uploads.uploaded_file', setname=avatars.name, filename=avatar_json['url'], _external=_external) else: return url_for('static', filename='img/avatar.png', _external=_external) @staticmethod def on_changed_about_me(target, value, oldvalue, initiaor): allowed_tags = [ 'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p' ] target.about_me_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True))
class User(UserMixin, db.Model): __tablename__ = 'users' id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(64), unique=True) name = db.Column(db.String(64)) password_hash = db.deferred(db.Column(db.String(128))) major = db.Column(db.String(128)) role_id = db.Column(db.Integer, db.ForeignKey('roles.id')) headline = db.Column(db.String(32), nullable=True) about_me = db.deferred(db.Column(db.Text, nullable=True)) about_me_html = db.deferred(db.Column(db.Text, nullable=True)) avatar = db.Column(db.String(128)) member_since = db.Column(db.DateTime(), default=datetime.utcnow) @property def password(self): raise AttributeError('password is not readable attribute') @password.setter def password(self, password): self.password_hash = generate_password_hash(password) def verify_password(self, password): return check_password_hash(self.password_hash, password) def __init__(self, **kwargs): super(User, self).__init__(**kwargs) if self.role is None: if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower( ): self.role = Role.query.filter_by(permissions=0x1ff).first() if self.role is None: self.role = Role.query.filter_by(default=True).first() self.member_since = datetime.now() def can(self, permissions): return self.role is not None and \ (self.role.permissions & permissions) == permissions def is_administrator(self): return self.can(Permission.ADMINISTER) logs = db.relationship('Log', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') comments = db.relationship('Comment', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') def __repr__(self): return '<User %r>' % self.email def borrowing(self, book): return self.logs.filter_by(book_id=book.id, returned=0).first() def can_borrow_book(self): return self.logs.filter( Log.returned == 0, Log.return_timestamp < datetime.now()).count() == 0 def borrow_book(self, book): if self.logs.filter(Log.returned == 0, Log.return_timestamp < datetime.now()).count() > 0: return False, u"Unable to borrow, you have expired books not returned" if self.borrowing(book): return False, u'Looks like you have borrowed this book!!' if not book.can_borrow(): return False, u'This book is too hot, we have no copies, please wait for others to return and borrow again.' db.session.add(Log(self, book)) return True, u'You successfully get to a copy %s' % book.title def return_book(self, log): if log.returned == 1 or log.user_id != self.id: return False, u'Did not find this record' log.returned = 1 log.return_timestamp = datetime.now() db.session.add(log) return True, u'You returned a copy %s' % log.book.title def avatar_url(self, _external=False): if self.avatar: avatar_json = json.loads(self.avatar) if avatar_json['use_out_url']: return avatar_json['url'] else: return url_for('_uploads.uploaded_file', setname=avatars.name, filename=avatar_json['url'], _external=_external) else: return url_for('static', filename='img/avatar.png', _external=_external) @staticmethod def on_changed_about_me(target, value, oldvalue, initiaor): allowed_tags = [ 'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p' ] target.about_me_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True))
class User(UserMixin, db.Model): __tablename__ = 'users' id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(64), unique=True) name = db.Column(db.String(64)) password_hash = db.deferred(db.Column(db.String(128))) major = db.Column(db.String(128)) role_id = db.Column(db.Integer, db.ForeignKey('roles.id')) headline = db.Column(db.String(32), nullable=True) about_me = db.deferred(db.Column(db.Text, nullable=True)) about_me_html = db.deferred(db.Column(db.Text, nullable=True)) avatar = db.Column(db.String(128)) category = db.Column(db.Integer) member_since = db.Column(db.DateTime(), default=datetime.utcnow) balance = db.Column(db.Float()) transactions = db.relationship('Transaction', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') @property def password(self): raise AttributeError('password is not readable attribute') @password.setter def password(self, password): self.password_hash = generate_password_hash(password) def verify_password(self, password): return check_password_hash(self.password_hash, password) def __init__(self, **kwargs): super(User, self).__init__(**kwargs) if self.role is None: if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower( ): self.role = Role.query.filter_by(permissions=0x1ff).first() if self.role is None: self.role = Role.query.filter_by(default=True).first() self.member_since = datetime.now() def has_discount(self): if self.category != 0: return True else: return False def calculate_discount(self, sum): if self.category == Category.STUDENT: return sum * 0.1 elif self.category == Category.LARGE_FAMILIES: return sum * 0.5 def can(self, permissions): return self.role is not None and \ (self.role.permissions & permissions) == permissions def is_administrator(self): return self.can(Permission.ADMINISTER) logs = db.relationship('Log', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') comments = db.relationship('Comment', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') def __repr__(self): return '<User %r>' % self.email def borrowing(self, book): return self.logs.filter_by(book_id=book.id, returned=0).first() def can_borrow_book(self): return self.logs.filter( Log.return_request == 0, Log.return_timestamp < datetime.now()).count() == 0 def borrow_book(self, book): if self.logs.filter(Log.return_request == 0, Log.return_timestamp < datetime.now()).count() > 0: return False, u"Unable to borrow, you have not returned the expired books" if self.borrowing(book): return False, u'It looks like you have borrowed this book!' if not book.can_borrow(): return False, u'This book is too popular, please wait for someone to return it later' if self.balance < book.price: return False, u'Not enough money' self.balance -= book.price transaction = Transaction(self.id, book.id, book.price, Operations.BORROW_BOOK) db.session.add(transaction) db.session.add(self) db.session.add(Log(self, book)) return True, u'You successfully got a copy %s' % book.title def approve_return_book(self, log, with_fine): if log.returned == 1 or not self.can( Permission.APPROVE_REQUEST_RETURN): return False, u'Did not find this record' log.returned = 1 user = User.query.get(log.user_id) book = Book.query.get(log.book_id) delata_time = log.return_timestamp - log.borrow_timestamp if delata_time.days == 0: delata_time = timedelta(days=1) total_sum = book.price - (book.price / 100) * delata_time.days transaction = Transaction(user.id, book.id, total_sum, Operations.RETURN_BOOK) user.balance += total_sum db.session.add(transaction) if with_fine == '1': user.balance -= book.price - total_sum fine_transaction = Transaction(user.id, book.id, total_sum, Operations.FINE) db.session.add(fine_transaction) else: if (user.has_discount()): discount = user.calculate_discount(total_sum) discount_transaction = Transaction(user.id, book.id, discount, Operations.DISCOUNT) user.balance += discount db.session.add(discount_transaction) db.session.add(user) db.session.add(log) return True, u'You returned a copy %s' % log.book.title def create_return_request(self, log): if log.returned == 1 or log.user_id != self.id or log.return_request == 1: return False, u'Did not find this record' log.return_request = 1 log.return_timestamp = datetime.now() db.session.add(log) return True, u'You returned a copy %s' % log.book.title def avatar_url(self, _external=False): if self.avatar: avatar_json = json.loads(self.avatar) if avatar_json['use_out_url']: return avatar_json['url'] else: return url_for('_uploads.uploaded_file', setname=avatars.name, filename=avatar_json['url'], _external=_external) else: return url_for('static', filename='img/avatar.png', _external=_external) @staticmethod def on_changed_about_me(target, value, oldvalue, initiaor): allowed_tags = [ 'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p' ] target.about_me_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True))
class Topic(db.Model): __tablename__ = DB_PREFIX + "topic" id = db.Column(db.Integer, primary_key=True, nullable=False) name = db.Column(db.String(255), default="", index=True, nullable=False) category_id = db.Column(db.Integer, default=0) # 专题所属分类 brief = db.deferred(db.Column(db.Text, default="", nullable=False)) avatar = db.Column(db.String(255), default="", nullable=False) updatetime = db.Column(db.DateTime, default=datetime.now, nullable=False) timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False) articles = db.relationship("ArticleTopic", backref="topic", lazy="dynamic") @staticmethod def prefix_autosearch(name, page, per_page): """ 标签名前缀查询 """ Article = app.article.models.Article ArticleTopic = app.article.models.ArticleTopic topics = db.session.query(Topic).outerjoin( ArticleTopic, Article).filter(Topic.name.ilike("{0}%".format(name))).filter( Article.access == "public").group_by(Topic).order_by( func.count(ArticleTopic.topic_id).desc()) return paginate(topics, page, per_page=per_page, error_out=False) @staticmethod def get_user_article_topics(user_id, page): per_page = current_app.config["USER_TOPIC_PAGE"] ArticleTopic = app.article.models.ArticleTopic Article = app.article.models.Article topics = db.session.query(Topic, func.count(ArticleTopic.topic_id).label("count"))\ .outerjoin(ArticleTopic, Article).group_by(Topic) if not current_user.is_authenticated or current_user.id != user_id: topics = topics.filter( db.and_(Article.user_id == user_id, Article.access == "public")) else: topics = topics.filter(Article.user_id == user_id) topics = topics.order_by(func.count(ArticleTopic.topic_id).desc()) topics = paginate(topics, page, per_page=per_page, error_out=False) return topics @staticmethod def get_article_topics(num): """ 所有文章的专题列表 @param num: 返回专题个数 """ ArticleTopic = app.article.models.ArticleTopic Article = app.article.models.Article topics = db.session.query(Topic) topics = topics.outerjoin(ArticleTopic).filter( ArticleTopic.article.has( Article.access == "public")).group_by(Topic).order_by( func.count(ArticleTopic.topic_id).desc()) return topics.limit(num).all(), topics.count() def get_articles(self, page, per_page): Article = app.article.models.Article ArticleTopic = app.article.models.ArticleTopic articles = db.session.query(Article).filter_by( access="public").outerjoin(ArticleTopic).filter( ArticleTopic.topic_id == self.id) return paginate(articles, page, per_page=per_page, error_out=False) def count_article(self): Article = app.article.models.Article return self.articles.outerjoin(Article).filter( Article.access == "public").count()
class Article(db.Model): __tablename__ = DB_PREFIX + PREFIX + "article" id = db.Column(db.Integer, primary_key=True, nullable=False) user_id = db.Column(db.Integer, db.ForeignKey(User.__tablename__ + ".id", ondelete="CASCADE", onupdate="CASCADE"), nullable=False) title = db.Column(db.String(255), default="", nullable=False) url = db.Column(db.String(255), default="", nullable=False) access = db.Column(db.String(255), default="public", nullable=False) # 访问权限 public=> 公开 private=>私人 markdown = db.deferred(db.Column(db.Text, default="", nullable=False)) html = db.deferred(db.Column(db.Text, default="", nullable=False)) updatetime = db.Column(db.DateTime, default=datetime.now, nullable=False) timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False) abstract = db.deferred(db.Column(db.Text, default="", nullable=False)) abstract_timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False) cover = db.Column(db.String(255), default="") cover_timestamp = db.Column(db.DateTime, default=datetime.now) pathname = db.Column(db.String(255), default=uuid.uuid1().hex, nullable=False) # 文章的url路径名 images = db.relationship("ArticleImage", backref="article", lazy="select", passive_deletes=True) topics = db.relationship("ArticleTopic", backref="article", lazy="select", passive_deletes=True) @staticmethod def new(title, access, topics, user_id): """ 新建文章 """ article = Article() article.title = title article.access = access article.user_id = user_id db.session.add(article) db.session.commit() article.pathname = article.create_pathname() # 保存文章标签 article.update_topics(topics[:5]) return article @staticmethod def search(keyword, page, per_page): query = "%{0}%".format(keyword) articles = Article.query.filter( db.and_(Article.access == "public", Article.title.ilike(query))).order_by( Article.updatetime.desc()) return articles.paginate(page, per_page=per_page, error_out=False) @staticmethod def get_articles(page, per_page): """ @param page: 当前页 @param per_page: 每页个数 """ articles = Article.query.filter_by(access="public") return articles.paginate(page, per_page=per_page, error_out=False) @staticmethod def newest_articles(num): return Article.query.filter_by(access="public").order_by( Article.timestamp.desc()).limit(num).all() def create_pathname(self): """ 创建文章路径名 """ code = generate_code(4) return "".join([code, str(self.id)]) def setting(self, title, access, topics): """ 文章设置 """ self.title = title self.access = access self.update_topics(topics) def get_cover(self): """ 取文章内容第一个图片作为文章的封面 """ if self.updatetime > self.cover_timestamp: soup = BeautifulSoup(self.html, "lxml") img = soup.find("img") if img is not None: self.cover = img['src'] else: self.cover = "" self.cover_timestamp = self.updatetime return self.cover def update_topics(self, topics): """ 更新文章标签 """ new_topics = set([topic.strip().lower() for topic in topics]) old_topics = set([topic.topic.name for topic in self.topics]) delete_topics = old_topics - new_topics add_topics = new_topics - old_topics for name in delete_topics: self.delete_topic(name) for name in add_topics: self.add_topic(name) def add_topic(self, name): """ 添加标签 """ if not name: return False topic = Topic.query.filter_by(name=name).first() if not topic: topic = Topic() topic.name = name db.session.add(topic) db.session.commit() for t in self.topics: if t.topic.name == name.lower(): return False article_topic = ArticleTopic() article_topic.article_id = self.id article_topic.topic_id = topic.id db.session.add(article_topic) return True def delete_topic(self, name): """ 删除标签 """ topic = Topic.query.filter_by(name=name).first() if not topic: return False ArticleTopic.query.filter_by(topic_id=topic.id).delete() return True def get_abstract(self): if not self.html: return "" if self.updatetime > self.abstract_timestamp: soup = BeautifulSoup(self.html, "lxml") self.abstract = soup.get_text()[0:400] self.abstract_timestamp = self.updatetime return self.abstract def save_content(self, markdown, html): """ 保存文章内容 """ self.markdown = markdown self.html = html self.updatetime = datetime.now() def delete(self): """ 删除文章 """ # 删除文章图片 for image in self.images: image.delete() # 删除标签 for topic in self.topics: topic.delete() # 删除文章自己 db.session.delete(self)
class Person(db.Model): __tablename__ = 'person' id = db.Column(db.Integer, primary_key=True) email = db.Column(db.Text) name = db.Column(db.Text) other_names = db.Column(JSONB) github_login = db.Column(db.Text) github_about = db.deferred(db.Column(JSONB)) parsed_name = db.Column(JSONB) impact = db.Column(db.Float) impact_percentile = db.Column(db.Float) num_downloads = db.Column(db.Integer) num_downloads_percentile = db.Column(db.Float) num_citations = db.Column(db.Integer) num_citations_percentile = db.Column(db.Float) pagerank = db.Column(db.Float) pagerank_percentile = db.Column(db.Float) is_organization = db.Column(db.Boolean) main_language = db.Column(db.Text) def __repr__(self): return u'<Person "{name}" ({id})>'.format(name=self.name, id=self.id) contributions = db.relationship( 'Contribution', # lazy='select', cascade="all, delete-orphan", backref="person") def set_is_organization(self): # set this using the sql in the sql folder! set_person_is_organization.sql # this is just a placeholder to remind us to run it :) pass def refresh(self, refsets_dict): self.set_scores() self.set_subscore_percentiles(refsets_dict) self.set_impact() self.set_impact_percentiles(refsets_dict) @property def display_pagerank(self): return self.pagerank / 100.0 @property def subscores(self): ret = [] if not self.impact: # no academic impact, so don't return subscores return ret ret.append({ "display_name": "Downloads", "icon": "fa-download", "name": "num_downloads", "percentile": self.num_downloads_percentile, "val": self.num_downloads }) ret.append({ "display_name": "Dependency PageRank", "icon": "fa-recycle", "name": "pagerank", "percentile": self.pagerank_percentile, "val": self.display_pagerank }) ret.append({ "display_name": "Citations", "icon": "fa-file-text-o", "name": "num_mentions", "percentile": self.num_citations_percentile, "val": self.num_citations }) return ret def to_dict(self, max_person_packages=None, include_top_collabs=True): ret = self.as_package_snippet person_packages = self.get_person_packages() ret["num_packages"] = len(person_packages) ret["num_packages_r"] = len( [pp for pp in person_packages if pp.package.language == 'r']) ret["num_packages_python"] = len( [pp for pp in person_packages if pp.package.language == 'python']) # tags tags_dict = defaultdict(int) for pp in person_packages: for tag in pp.package.tags: tags_dict[tag] += 1 tags_to_return = min(5, len(tags_dict)) sorted_tags_to_return = sorted(tags_dict.items(), key=lambda x: x[1], reverse=True)[0:tags_to_return] ret["top_person_tags"] = [{ "name": name, "count": count } for name, count in sorted_tags_to_return] # co-collaborators if include_top_collabs: my_collabs = defaultdict(float) for pp in person_packages: for collab_person_id, collab_credit in pp.package.credit.iteritems( ): if int(collab_person_id ) != self.id: #don't measure my own collab strength collab_strength = collab_credit * pp.person_package_credit my_collabs[collab_person_id] += collab_strength sorted_collabs_to_return = sorted(my_collabs.items(), key=lambda x: x[1], reverse=True) ret["top_collabs"] = [] num_collabs_to_return = min(5, len(sorted_collabs_to_return)) for person_id, collab_score in sorted_collabs_to_return[ 0:num_collabs_to_return]: person = Person.query.get(int(person_id)) if person: person_dict = person.as_package_snippet person_dict[ "collab_score"] = collab_score * 4 # to make a 0.25*0.25 connection strength of 1 ret["top_collabs"].append(person_dict) else: print u"ERROR: person {} not found; maybe deduped? skipping.".format( person_id) # person packages if max_person_packages: person_packages_to_return = min(max_person_packages, len(person_packages)) ret["person_packages"] = [ p.as_person_snippet for p in person_packages[0:person_packages_to_return] ] else: ret["person_packages"] = [p.to_dict() for p in person_packages] ret["subscores"] = self.subscores return ret @property def as_snippet(self): return self.to_dict(max_person_packages=3, include_top_collabs=False) @property def as_package_snippet(self): ret = { "id": self.id, "name": self.display_name, "single_name": self.single_name, "person_name": self.name, # helps with namespacing in UI "github_login": self.github_login, "icon": self.icon, "icon_small": self.icon_small, "is_organization": self.is_organization, "main_language": self.main_language, "impact": self.impact, "impact_percentile": self.impact_percentile, "id": self.id, "subscores": self.subscores } return ret def set_main_language(self): person_package_summary_dict = self.as_snippet if person_package_summary_dict[ "num_packages_r"] > person_package_summary_dict[ "num_packages_python"]: self.main_language = "r" else: self.main_language = "python" @classmethod def shortcut_percentile_refsets(cls): print "getting the percentile refsets...." ref_list = defaultdict(dict) q = db.session.query(cls.num_downloads, cls.pagerank, cls.num_citations, cls.impact) q = q.filter( cls.num_downloads != None ) # only academic contributions, so would have some fractional downloads q = q.filter( or_(cls.pagerank > 0, cls.num_citations > 0, cls.num_downloads > 0)) # only academic contributions rows = q.all() ref_list["num_downloads"] = sorted( [row[0] for row in rows if row[0] != None]) ref_list["pagerank"] = sorted( [row[1] for row in rows if row[1] != None]) ref_list["num_citations"] = sorted( [row[2] for row in rows if row[2] != None]) ref_list["impact"] = sorted([row[3] for row in rows if row[3] != None]) # only compare impacts against other things with impacts ref_list["impact"] = [val for val in ref_list["impact"] if val > 0] return ref_list def _calc_percentile(self, refset, value): if value is None: # distinguish between that and zero return None try: matching_index = refset.index(value) percentile = float(matching_index) / len(refset) except ValueError: # not in index. maybe has no impact because no academic contributions print u"not setting percentile for {}; looks like not academic".format( self.name) percentile = None return percentile def set_num_downloads_percentile(self, refset): self.num_downloads_percentile = self._calc_percentile( refset, self.num_downloads) def set_pagerank_percentile(self, refset): self.pagerank_percentile = self._calc_percentile(refset, self.pagerank) def set_num_citations_percentile(self, refset): self.num_citations_percentile = self._calc_percentile( refset, self.num_citations) def set_impact_percentile(self, refset): self.impact_percentile = self._calc_percentile(refset, self.impact) def set_subscore_percentiles(self, refsets_dict): self.set_num_downloads_percentile(refsets_dict["num_downloads"]) self.set_pagerank_percentile(refsets_dict["pagerank"]) self.set_num_citations_percentile(refsets_dict["num_citations"]) def set_impact_percentiles(self, refsets_dict): self.set_impact_percentile(refsets_dict["impact"]) def set_github_about(self): if self.github_login is None: return None self.github_about = get_profile(self.github_login) try: if not self.name: self.name = self.github_about["name"] if not self.email: self.email = self.github_about["email"] except KeyError: # our github_about is an error object, # it's got no info about the person in it. return False def set_impact(self): try: self.impact = (self.pagerank_percentile + self.num_downloads_percentile + self.num_citations_percentile) / 3.0 except TypeError: #something was null self.impact = 0 def set_scores(self): self.pagerank = 0 self.num_downloads = 0 self.num_citations = 0 for pp in self.get_person_packages(): # only count up academic packages if pp.package.is_academic: # only count up impact for packages in our main language if pp.package.language == self.main_language: if pp.person_package_pagerank: self.pagerank += pp.person_package_pagerank if pp.person_package_num_downloads: self.num_downloads += pp.person_package_num_downloads if pp.person_package_num_citations: self.num_citations += pp.person_package_num_citations safe_commit(db) @property def name_normalized_for_maximal_deduping(self): if not self.name: return None if self.is_organization: return self.name.lower() try: first_initial = self.parsed_name["first"][0].lower() except (KeyError, AttributeError, IndexError): first_initial = "?" try: last_orig = self.parsed_name["last"] last = unicodedata.normalize('NFKD', last_orig).encode('ascii', 'ignore') last = last.lower() except (KeyError, AttributeError): last = "?" normalized_name = u"{} {}".format(first_initial, last) # print "normalized_name", normalized_name return normalized_name def set_parsed_name(self): if not self.name: self.parsed_name = None return name = HumanName(self.name) self.parsed_name = name.as_dict() def _make_gravatar_url(self, size): try: if self.email is not None: hash = hashlib.md5(self.email).hexdigest() else: hash = hashlib.md5(str(self.id)).hexdigest() except UnicodeEncodeError: print "UnicodeEncodeError making gravatar url from email" hash = 42 url = "http://www.gravatar.com/avatar/{hash}.jpg?s={size}&d=retro".format( hash=hash, size=size) return url @property def icon(self): return self._make_gravatar_url(160) @property def icon_small(self): return self._make_gravatar_url(30) def has_role_on_project(self, role, package_id): for c in self.contributions: if c.role == role and c.package_id == package_id: return True return False def num_commits_on_project(self, package_id): for c in self.contributions: if c.role == "github_contributor" and c.package_id == package_id: return c.quantity return False @property def single_name(self): if self.is_organization: return self.display_name elif self.parsed_name and self.parsed_name["last"]: return self.parsed_name["last"] return self.display_name @property def display_name(self): if self.name: return self.name elif self.github_login: return self.github_login elif self.email: return self.email.split("@")[0] else: return "name unknown" # could be a property, but kinda slow, so better as explicity method methinks def get_person_packages(self): person_packages = defaultdict(PersonPackage) for contrib in self.contributions: person_packages[contrib.package.id].set_role(contrib) person_packages_list = person_packages.values() person_packages_list.sort(key=lambda x: x.person_package_impact, reverse=True) return person_packages_list def num_commits_on_project(self, package_id): for c in self.contributions: if c.role == "github_contributor" and c.package_id == package_id: return c.quantity return False @classmethod def decide_who_to_dedup(cls, people): if len(people) <= 1: # this name has no dups return None people_with_github = [p for p in people if p.github_login] people_with_no_github = [p for p in people if not p.github_login] # don't merge people with github together # so we only care about merging if there are people with no github if not people_with_no_github: return None if people_with_github: # merge people with no github into first person with github dedup_target = people_with_github[0] people_to_merge = people_with_no_github else: # pick first person with no github as target, rest as mergees dedup_target = people_with_no_github[0] people_to_merge = people_with_no_github[1:] return { "dedup_target": dedup_target, "people_to_merge": people_to_merge } @classmethod def dedup(cls, dedup_target, people_to_merge): print u"person we will merge into: {}".format(dedup_target.id) print u"people to merge: {}".format([p.id for p in people_to_merge]) for person_to_delete in people_to_merge: contributions_to_change = person_to_delete.contributions for contrib in contributions_to_change: contrib.person = dedup_target db.session.add(contrib) print u"now going to delete {}".format(person_to_delete) db.session.delete(person_to_delete)
class Person(db.Model): __tablename__ = 'person' id = db.Column(db.Integer, primary_key=True) email = db.Column(db.Text) name = db.Column(db.Text) other_names = db.Column(JSONB) github_login = db.Column(db.Text) github_about = db.deferred(db.Column(JSONB)) bucket = db.Column(JSONB) impact = db.Column(db.Float) impact_rank = db.Column(db.Integer) pagerank_score = db.Column(db.Float) num_downloads_score = db.Column(db.Float) num_citations_score = db.Column(db.Float) parsed_name = db.Column(JSONB) is_academic = db.Column(db.Boolean) is_organization = db.Column(db.Boolean) main_language = db.Column(db.Text) type = db.Column(db.Text) def __repr__(self): return u'<Person "{name}" ({id})>'.format(name=self.name, id=self.id) contributions = db.relationship( 'Contribution', # lazy='select', cascade="all, delete-orphan", backref="person") def set_is_academic(self): # set this using the sql in the sql folder! set_person_is_academic.sql # this is just a placeholder to remind us to run it :) pass def set_is_organization(self): # set this using the sql in the sql folder! set_person_is_organization.sql # this is just a placeholder to remind us to run it :) pass @property def subscores(self): ret = [] ret.append({ "name": "num_citations", "score": self.num_citations_score, "val": None }) ret.append({ "name": "pagerank", "score": self.pagerank_score, "val": None }) ret.append({ "name": "num_downloads", "score": self.num_downloads_score, "val": None }) # select id, name, email, num_citations_score from person where is_organization=false and main_lanugage='python' order by num_downloads_score desc limit 5 maxes = { "python": { "num_citations": 1312.07783912007676, "pagerank": 6828.41972274044019, "num_downloads": 16419.5532038200363 }, "r": { "num_citations": 1866, "pagerank": 9270, "num_downloads": 18213 } } for my_dict in ret: if my_dict["score"]: my_dict["score"] = 1000.00 * my_dict["score"] / maxes[ self.main_language][my_dict["name"]] return ret def to_dict(self, max_person_packages=None, include_top_collabs=True): ret = self.as_package_snippet person_packages = self.get_person_packages() ret["num_packages"] = len(person_packages) ret["num_packages_r"] = len( [pp for pp in person_packages if pp.package.language == 'r']) ret["num_packages_python"] = len( [pp for pp in person_packages if pp.package.language == 'python']) # tags tags_dict = defaultdict(int) for pp in person_packages: for tag in pp.package.tags: tags_dict[tag] += 1 tags_to_return = min(5, len(tags_dict)) sorted_tags_to_return = sorted(tags_dict.items(), key=lambda x: x[1], reverse=True)[0:tags_to_return] ret["top_person_tags"] = [{ "name": name, "count": count } for name, count in sorted_tags_to_return] # co-collaborators if include_top_collabs: my_collabs = defaultdict(float) for pp in person_packages: for collab_person_id, collab_credit in pp.package.credit.iteritems( ): if int(collab_person_id ) != self.id: #don't measure my own collab strength collab_strength = collab_credit * pp.person_package_credit my_collabs[collab_person_id] += collab_strength sorted_collabs_to_return = sorted(my_collabs.items(), key=lambda x: x[1], reverse=True) ret["top_collabs"] = [] num_collabs_to_return = min(5, len(sorted_collabs_to_return)) for person_id, collab_score in sorted_collabs_to_return[ 0:num_collabs_to_return]: person = Person.query.get(int(person_id)) person_dict = person.as_package_snippet person_dict[ "collab_score"] = collab_score * 4 # to make a 0.25*0.25 connection strength of 1 ret["top_collabs"].append(person_dict) # person packages if max_person_packages: person_packages_to_return = min(max_person_packages, len(person_packages)) ret["person_packages"] = [ p.as_person_snippet for p in person_packages[0:person_packages_to_return] ] else: ret["person_packages"] = [p.to_dict() for p in person_packages] ret["subscores"] = self.subscores return ret @property def as_snippet(self): return self.to_dict(max_person_packages=3, include_top_collabs=False) @property def as_package_snippet(self): ret = { "id": self.id, "name": self.display_name, "single_name": self.single_name, "github_login": self.github_login, "icon": self.icon, "icon_small": self.icon_small, "is_academic": self.is_academic, "is_organization": self.is_organization, "main_language": self.main_language, "impact": self.impact, "impact_rank": self.impact_rank, "impact_rank_max": self.impact_rank_max, "pagerank_score": self.pagerank_score, "num_downloads_score": self.num_downloads_score, "num_citations_score": self.num_citations_score, "id": self.id } return ret def set_main_language(self): person_package_summary_dict = self.as_snippet if person_package_summary_dict[ "num_packages_r"] > person_package_summary_dict[ "num_packages_python"]: self.main_language = "r" else: self.main_language = "python" @classmethod def shortcut_impact_rank(cls): print "getting the lookup for ranking impact...." impact_rank_lookup = defaultdict(dict) for main_language in ["python", "r"]: q = db.session.query(cls.id) q = add_person_leaderboard_filters(q) q = q.filter(Person.main_language == main_language) q = q.order_by(cls.impact.desc()) # the important part :) rows = q.all() ids_sorted_by_impact = [row[0] for row in rows] for my_id in ids_sorted_by_impact: zero_based_rank = ids_sorted_by_impact.index(my_id) impact_rank_lookup[main_language][my_id] = zero_based_rank + 1 return impact_rank_lookup def set_impact_rank(self, impact_rank_lookup): try: self.impact_rank = impact_rank_lookup[self.main_language][self.id] except KeyError: # maybe because organization, or name=="UNKNOWN" print "couldn't find my id" self.impact_rank = None print "self.impact_rank", self.impact_rank @property def impact_rank_max(self): # select count(id), main_language # from person # where is_organization=false # and (name is null or name!='UKNOWN') # and (email is null or email!='UNKNOWN') # group by main_language if self.main_language == "python": return 62951 elif self.main_language == "r": return 10447 def set_github_about(self): if self.github_login is None: return None self.github_about = get_profile(self.github_login) try: if not self.name: self.name = self.github_about["name"] if not self.email: self.email = self.github_about["email"] except KeyError: # our github_about is an error object, # it's got no info about the person in it. return False def set_impact(self): self.impact = sum( [pp.person_package_impact for pp in self.get_person_packages()]) return self.impact def set_scores(self): person_packages = self.get_person_packages() self.pagerank_score = sum([ pp.person_package_pagerank_score for pp in person_packages if pp.person_package_pagerank_score ]) self.num_downloads_score = sum( [pp.person_package_num_downloads_score for pp in person_packages]) self.num_citations_score = sum( [pp.person_package_num_citations_score for pp in person_packages]) def set_parsed_name(self): if not self.name: self.parsed_name = None return name = HumanName(self.name) self.parsed_name = name.as_dict() def _make_gravatar_url(self, size): try: if self.email is not None: hash = hashlib.md5(self.email).hexdigest() else: hash = hashlib.md5("*****@*****.**").hexdigest() except UnicodeEncodeError: print "UnicodeEncodeError making gravatar url from email" hash = 42 url = "http://www.gravatar.com/avatar/{hash}.jpg?s={size}&d=mm".format( hash=hash, size=size) return url @property def icon(self): return self._make_gravatar_url(160) @property def icon_small(self): return self._make_gravatar_url(30) def has_role_on_project(self, role, package_id): for c in self.contributions: if c.role == role and c.package_id == package_id: return True return False def num_commits_on_project(self, package_id): for c in self.contributions: if c.role == "github_contributor" and c.package_id == package_id: return c.quantity return False @property def single_name(self): if self.is_organization: return self.display_name elif self.parsed_name and self.parsed_name["last"]: return self.parsed_name["last"] return self.display_name @property def display_name(self): if self.name: return self.name elif self.github_login: return self.github_login elif self.email: return self.email.split("@")[0] else: return "name unknown" # could be a property, but kinda slow, so better as explicity method methinks def get_person_packages(self): person_packages = defaultdict(PersonPackage) for contrib in self.contributions: person_packages[contrib.package.id].set_role(contrib) person_packages_list = person_packages.values() person_packages_list.sort(key=lambda x: x.person_package_impact, reverse=True) return person_packages_list def num_commits_on_project(self, package_id): for c in self.contributions: if c.role == "github_contributor" and c.package_id == package_id: return c.quantity return False
class Package(db.Model): id = db.Column(db.Text, primary_key=True) host = db.Column(db.Text) project_name = db.Column(db.Text) import_name = db.Column(db.Text) unique_import_name = db.Column(db.Boolean) setup_py_import_name = db.Column(db.Text) github_owner = db.Column(db.Text) github_repo_name = db.Column(db.Text) github_api_raw = db.deferred(db.Column(JSONB)) github_contributors = db.deferred(db.Column(JSONB)) api_raw = db.deferred(db.Column(JSONB)) downloads = db.deferred(db.Column(MutableDict.as_mutable(JSONB))) all_r_reverse_deps = db.deferred(db.Column(JSONB)) tags = db.Column(JSONB) proxy_papers = db.deferred(db.Column(db.Text)) is_academic = db.Column(db.Boolean) num_citations_by_source = db.Column(JSONB) is_distinctive_name = db.Column(db.Boolean) host_reverse_deps = db.deferred(db.Column(JSONB)) github_reverse_deps = db.deferred(db.Column(JSONB)) dependencies = db.deferred(db.Column(JSONB)) bucket = db.deferred(db.Column(MutableDict.as_mutable(JSONB))) bucket2 = db.deferred(db.Column(MutableDict.as_mutable(JSONB))) requires_files = db.deferred(db.Column(MutableDict.as_mutable(JSONB))) setup_py = db.deferred(db.Column(db.Text)) setup_py_hash = db.deferred(db.Column(db.Text)) impact = db.Column(db.Float) impact_rank = db.Column(db.Integer) num_downloads = db.Column(db.Integer) num_downloads_percentile = db.Column(db.Float) num_downloads_score = db.Column(db.Float) pagerank = db.Column(db.Float) pagerank_percentile = db.Column(db.Float) pagerank_score = db.Column(db.Float) num_citations = db.Column(db.Integer) num_citations_percentile = db.Column(db.Float) num_citations_score = db.Column(db.Float) neighborhood_size = db.Column(db.Float) indegree = db.Column(db.Float) eccentricity = db.Column(db.Float) closeness = db.Column(db.Float) betweenness = db.Column(db.Float) eigenvector_centrality = db.Column(db.Float) max_path_length = db.Column(db.Integer) avg_path_length = db.Column(db.Float) longest_path = db.Column(JSONB) avg_outdegree_of_indegrees = db.Column(db.Float) avg_pagerank_of_indegrees = db.Column(db.Float) num_stars = db.Column(db.Integer) summary = db.Column(db.Text) num_committers = db.Column(db.Integer) num_commits = db.Column(db.Integer) num_authors = db.Column(db.Integer) rev_deps_tree = db.Column(JSONB) credit = db.Column(JSONB) contributions = db.relationship( 'Contribution', # lazy='subquery', cascade="all, delete-orphan", backref="package" ) __mapper_args__ = { 'polymorphic_on': host, 'with_polymorphic': '*' } def __repr__(self): return u'<Package {name}>'.format( name=self.id) @property def language(self): return "unknown" def to_dict(self, full=True): ret = { "name": self.project_name, "as_snippet": self.as_snippet, "github_owner": self.github_owner, "github_repo_name": self.github_repo_name, "host": self.host, "indegree": self.indegree, "neighborhood_size": self.neighborhood_size, "num_authors": self.num_authors, "num_commits": self.num_commits, "num_committers": self.num_committers, "num_citations": self.num_citations, "num_citations_score": self.num_citations_score, "num_citations_percentile": self.num_citations_percentile, "pagerank": self.pagerank, "pagerank_score": self.pagerank_score, "pagerank_percentile": self.pagerank_percentile, "num_downloads": self.num_downloads, "num_downloads_score": self.num_downloads_score, "num_downloads_percentile": self.num_downloads_percentile, "num_stars": self.num_stars, "impact": self.impact, "impact_rank": self.impact_rank, "rev_deps_tree": self.tree, "is_academic": self.is_academic, # current implementation requires api_raw, so slows down db because deferred # "source_url": self.source_url, "summary": self.summary, "tags": self.tags } return ret @property def tree(self): return self.rev_deps_tree @property def as_snippet_without_people(self): ret = { "_host_url": self.host_url, "name": self.project_name, "language": self.language, "is_academic": self.is_academic, "impact": self.impact, "impact_rank": self.impact_rank, "impact_rank_max": self.impact_rank_max, "pagerank_score": self.pagerank_score, "num_downloads_score": self.num_downloads_score, "num_citations_score": self.num_citations_score, "pagerank_percentile": self.pagerank_percentile, "num_downloads_percentile": self.num_downloads_percentile, "num_citations_percentile": self.num_citations_percentile, "pagerank": self.pagerank, "num_downloads": self.num_downloads, "num_citations": self.num_citations, "subscores": self.subscores, "is_academic": self.is_academic, "summary": prep_summary(self.summary), "tags": self.tags } return ret @property def subscores(self): names = [ "num_downloads", "pagerank", "num_citations" ] ret = [] for name in names: if name == "pagerank": score = self.pagerank_score else: score = getattr(self, name) ret.append({ "name": name, "score": getattr(self, name + "_score"), "val": score }) return ret @property def as_snippet(self): ret = self.as_snippet_without_people # ret["contributions"] = defaultdict(list) # for c in self.contributions: # ret["contributions"][c.role].append(u"{}: {}".format( # c.percent, c.person.display_name)) # for role in ret["contributions"]: # ret["contributions"][role].sort(reverse=True) ret["contribs"] = [] if self.credit: ret["num_contributors"] = len(self.credit) top_five_people = sorted(self.credit, key=self.credit.get, reverse=True)[0:5] for person_id in top_five_people: person_id = int(person_id) person = Person.query.get(person_id) person_snippet = person.as_package_snippet person_snippet["credit"] = self.credit[str(person_id)] person_snippet["roles"] = [] for contrib in self.contributions: if contrib.person_id == person_id: person_snippet["roles"].append(contrib.role) ret["contribs"].append(person_snippet) return ret @classmethod def valid_package_names(cls, module_names): """ this will normally be called by subclasses, to filter by specific package hosts """ lowercase_module_names = [n.lower() for n in module_names] q = db.session.query(cls.project_name) q = q.filter(func.lower(cls.project_name).in_(lowercase_module_names)) response = [row[0] for row in q.all()] return response def test(self): print "{}: I'm a test!".format(self) def save_all_people(self): self.save_github_owners_and_contributors() self.save_host_contributors() def save_github_owners_and_contributors(self): self.save_github_contribs_to_db() self.save_github_owner_to_db() def save_host_contributors(self): # this needs to be overridden, because it depends on whether we've # got a pypi or cran package...they have diff metadata formats. raise NotImplementedError @property def host_url(self): # this needs to be overridden, because it depends on whether we've # got a pypi or cran package raise NotImplementedError @property def all_people(self): people = list(set([c.person for c in self.contributions])) return people @property def all_authors(self): people = list(set([c.person for c in self.contributions if c.role=="author"])) return people @property def all_github_owners(self): people = list(set([c.person for c in self.contributions if c.role=="github_owner"])) return people def set_credit(self): people = self.contributors_with_credit() credit_dict = {} for person in people: credit_dict[int(person.id)] = person.credit self.credit = credit_dict def get_credit_for_person(self, person_id): return self.credit[str(person_id)] @property def has_github_commits(self): return self.max_github_commits > 0 @property def max_github_commits(self): if len(self.contributions) == 0: return 0 all_commits = [c.quantity for c in self.contributions if c.quantity and c.role=="github_contributor"] if not all_commits: return None return max(all_commits) def contributors_with_credit(self): people_for_contributions = self.all_people author_committers = [] non_author_committers = [] # if no github contributors, split cred equally among all authors # if github contributors, give contributions tied with maximum github contribution # by making them a virtual committer with that many commits for person in people_for_contributions: person.credit = 0 # initialize if person.has_role_on_project("author", self.id): if self.has_github_commits: # print u"{} is an author committer".format(person.name) author_committers += [person] else: # print u"{} is an author and there are no committers".format(person.name) equal_author_share = float(1)/len(self.all_authors) person.credit += equal_author_share elif person.has_role_on_project("github_contributor", self.id): # print u"{} is a non-author committer".format(person.name) non_author_committers += [person] # give all non-author committers their number of real commits for person in non_author_committers: person.num_counting_commits = person.num_commits_on_project(self.id) # give all virtual committers the max number of commits for person in author_committers: person.num_counting_commits = self.max_github_commits # calc how many commits were handed out, real + virtual total_author_and_nonauthor_commits = 0 author_and_nonauthor_commmiters = non_author_committers + author_committers for person in author_and_nonauthor_commmiters: total_author_and_nonauthor_commits += person.num_counting_commits # assign credit to be the fraction of commits they have out of total # print "total_author_and_nonauthor_commits", total_author_and_nonauthor_commits for person in author_and_nonauthor_commmiters: person.credit = float(person.num_counting_commits) / total_author_and_nonauthor_commits # print u"{} has with {} commits, {} credit".format( # person.name, person.num_commits, person.credit) # finally, handle github owners by giving them a little boost for person in people_for_contributions: if person.has_role_on_project("github_owner", self.id): person.credit += 0.01 people_for_contributions.sort(key=lambda x: x.credit, reverse=True) # for person in people_for_contributions: # print u"{credit}: {name} has contribution".format( # name = person.name, # credit = round(person.credit, 3) # ) return people_for_contributions def save_github_contribs_to_db(self): if isinstance(self.github_contributors, dict): # it's an error resp from the API, doh. return None if self.github_contributors is None: return None total_contributions_count = sum([c['contributions'] for c in self.github_contributors]) for github_contrib in self.github_contributors: person = get_or_make_person(github_login=github_contrib["login"]) percent_total_contribs = round( github_contrib["contributions"] / float(total_contributions_count) * 100, 3 ) self._save_contribution( person, "github_contributor", quantity=github_contrib["contributions"], percent=percent_total_contribs ) self.num_github_committers = len(self.github_contributors) def save_github_owner_to_db(self): if not self.github_owner: return False person = get_or_make_person(github_login=self.github_owner) self._save_contribution(person, "github_owner") def set_num_committers_and_commits(self): if not self.set_github_contributors: return None try: self.num_committers = len(self.github_contributors) self.num_commits = sum([contrib["contributions"] for contrib in self.github_contributors]) except TypeError: self.num_committers = 0 self.num_commits = 0 def _save_contribution(self, person, role, quantity=None, percent=None): print u"saving contribution {} for {}".format(role, person) extant_contrib = self.get_contribution(person.id, role) if extant_contrib is None: # make the new contrib. # there's got to be a better way to make this args thing... kwargs_dict = { "role": role } if quantity is not None: kwargs_dict["quantity"] = quantity if percent is not None: kwargs_dict["percent"] = percent new_contrib = Contribution(**kwargs_dict) # set the contrib in its various places. self.contributions.append(new_contrib) person.contributions.append(new_contrib) db.session.merge(person) def get_contribution(self, person_id, role): for contrib in self.contributions: if contrib.person.id == person_id and contrib.role == role: return contrib return None def set_github_contributors(self): self.github_contributors = github_api.get_repo_contributors( self.github_owner, self.github_repo_name ) print "found github contributors", self.github_contributors self.set_num_committers_and_commits() def set_github_repo_id(self): # override in subclass raise NotImplementedError def set_tags(self): # override in subclass raise NotImplementedError def set_is_distinctive_name(self): self.is_distinctive_name = False if not self.bucket: return if not 'num_hits_raw' in self.bucket: return if self.bucket["num_hits_raw"] <= 0: return ratio = float(self.bucket["num_hits_with_language"])/self.bucket["num_hits_raw"] print "distinctiveness ratio for {} is {}".format( self.project_name, ratio) if self.host == "cran": if ratio > 0.20: self.is_distinctive_name = True print "IS DISTINCTIVE! going for it" else: if ratio > 0.27: self.is_distinctive_name = True print "IS DISTINCTIVE! going for it" return def get_sources_to_query(self): # i bet there is a better way to do this!! :) sources_to_query = [ full_text_source.Pmc # , # full_text_source.Arxiv, # full_text_source.Citeseer, ] return sources_to_query @property def citations_dict(self): citations_dict = self.set_num_citations_by_source() response_dict = defaultdict(dict) for source_class in self.get_sources_to_query(): source = source_class() response_dict[source.name] = { "count": citations_dict[source.name], "url": source.query_url(self.full_text_query) } return response_dict @property def full_text_query(self): queries = [] # add the cran or pypi url to start with host_url = self.host_url.replace("https://", "").replace("http://", "") queries.append('"{}"'.format(host_url)) # then github url if we know it if self.github_owner: github_url = '"github.com/{}/{}"'.format(self.github_owner, self.github_repo_name) queries.append(github_url) # also look up its project name if is unique # this line here now because haven't run them all previously self.set_is_distinctive_name() if self.is_distinctive_name: queries.append('"{}"'.format(self.project_name)) else: print "{} isn't a rare package name, so not looking up by name".format(self.project_name) query = " OR ".join(queries) query += ' NOT AUTH:"{}"'.format(self.project_name) print "query", query return query def set_num_citations_by_source(self): if not self.num_citations_by_source: self.num_citations_by_source = {} self.num_citations = 0 for source_class in self.get_sources_to_query(): if source_class.__name__ == "Pmc" and ("-" in self.project_name): # pmc can't do a query if a hyphen in the name, so just bail self.num_citations = 0 return {} source = source_class() num_found = source.run_query(self.full_text_query) self.num_citations_by_source[source.name] = num_found self.num_citations += num_found print "num citations found", num_found return self.num_citations_by_source def set_distinctiveness(self): source = full_text_source.Citeseer() self.bucket2 = {} raw_query = '"{name}"'.format( name=self.project_name) # raw_query = '"{name}" NOT AUTH:"{name}"'.format( # name=self.project_name) num_hits_raw = source.run_query(raw_query) self.bucket2["num_hits_raw"] = num_hits_raw num_hits_with_language = source.run_query(self.distinctiveness_query) self.bucket2["num_hits_with_language"] = num_hits_with_language if self.bucket2["num_hits_raw"] > 0: ratio = float(self.bucket2["num_hits_with_language"])/self.bucket2["num_hits_raw"] else: ratio = None print "{}: solo search finds {}, ratio is {}".format( self.project_name, self.bucket2["num_hits_raw"], ratio ) def set_igraph_data(self, our_igraph_data): try: self.pagerank = our_igraph_data[self.project_name]["pagerank"] self.neighborhood_size = our_igraph_data[self.project_name]["neighborhood_size"] self.indegree = our_igraph_data[self.project_name]["indegree"] self.eccentricity = our_igraph_data[self.project_name]["eccentricity"] self.closeness = our_igraph_data[self.project_name]["closeness"] self.betweenness = our_igraph_data[self.project_name]["betweenness"] self.eigenvector_centrality = our_igraph_data[self.project_name]["eigenvector_centrality"] self.longest_path = our_igraph_data[self.project_name]["longest_path"] self.max_path_length = our_igraph_data[self.project_name]["max_path_length"] self.avg_path_length = our_igraph_data[self.project_name]["avg_path_length"] self.avg_outdegree_of_indegrees = our_igraph_data[self.project_name]["avg_outdegree_of_indegrees"] self.avg_pagerank_of_indegrees = our_igraph_data[self.project_name]["avg_pagerank_of_indegrees"] print "pagerank of {} is {}".format(self.project_name, self.pagerank) except KeyError: print "network params for {} were not calculated".format(self.project_name) self.pagerank = None self.neighborhood_size = None self.indegree = None self.eccentricity = None self.closeness = None self.betweenness = None self.eigenvector_centrality = None self.longest_path = None self.max_path_length = None self.avg_path_length = None self.avg_outdegree_of_indegrees = None self.avg_pagerank_of_indegrees = None def refresh_github_ids(self): if not self.github_owner: return None self.github_api_raw = github_api.get_repo_data(self.github_owner, self.github_repo_name) try: (self.github_owner, self.github_repo_name) = self.github_api_raw["full_name"].split("/") except KeyError: self.github_owner = None self.github_repo_name = None @classmethod def shortcut_percentile_refsets(cls): print "getting the percentile refsets...." ref_list = defaultdict(dict) q = db.session.query( cls.num_downloads, cls.pagerank, cls.num_citations ) rows = q.all() ref_list["num_downloads"] = sorted([row[0] for row in rows if row[0] != None]) ref_list["pagerank"] = sorted([row[1] for row in rows if row[1] != None]) ref_list["num_citations"] = sorted([row[2] for row in rows if row[2] != None]) return ref_list def _calc_percentile(self, refset, value): if value is None: # distinguish between that and zero return None matching_index = refset.index(value) percentile = float(matching_index) / len(refset) return percentile def set_num_downloads_percentile(self, refset): self.num_downloads_percentile = self._calc_percentile(refset, self.num_downloads) def set_pagerank_percentile(self, refset): self.pagerank_percentile = self._calc_percentile(refset, self.pagerank) def set_num_citations_percentile(self, refset): self.num_citations_percentile = self._calc_percentile(refset, self.num_citations) def set_all_percentiles(self, refsets_dict): self.set_num_downloads_percentile(refsets_dict["num_downloads"]) self.set_pagerank_percentile(refsets_dict["pagerank"]) self.set_num_citations_percentile(refsets_dict["num_citations"]) @classmethod def _shortcut_rank(cls, name_to_rank_by): print "getting the lookup for ranking impact...." property_to_rank_by = getattr(cls, name_to_rank_by) q = db.session.query(cls.id) q = q.order_by(property_to_rank_by.desc()) # the important part :) rows = q.all() impact_rank_lookup = {} ids_sorted = [row[0] for row in rows] for my_id in ids_sorted: zero_based_rank = ids_sorted.index(my_id) impact_rank_lookup[my_id] = zero_based_rank + 1 return impact_rank_lookup @classmethod def shortcut_impact_rank(cls): return cls._shortcut_rank("impact") def set_impact_rank(self, impact_rank_lookup): self.impact_rank = impact_rank_lookup[self.id] print "self.impact_rank", self.impact_rank @classmethod def shortcut_num_citations_rank(cls): print "getting the lookup for num_citations ranking shortcut...." q = db.session.query(cls.id) q = q.order_by(cls.num_citations.desc()) # the important part :) rows = q.all() impact_rank_lookup = {} ids_sorted_by_impact = [row[0] for row in rows] for my_id in ids_sorted_by_impact: zero_based_rank = ids_sorted_by_impact.index(my_id) impact_rank_lookup[my_id] = zero_based_rank + 1 return impact_rank_lookup def set_impact_rank(self, impact_rank_lookup): self.impact_rank = impact_rank_lookup[self.id] print "self.impact_rank", self.impact_rank @property def pagerank_score_multiplier(self): return self.pagerank_score @property def num_downloads_score_multiplier(self): return 1000.0/self.num_downloads_offset_to_recenter_scores # makes it out of 1000 @property def num_citations_score_multiplier(self): return 1000.0/self.num_citations_offset_to_recenter_scores # makes it out of 1000 def set_pagerank_score(self): if not self.pagerank: self.pagerank_score = None return self.pagerank_score try: raw = math.log10(float(self.pagerank)/self.pagerank_max) adjusted = (raw + self.pagerank_offset_to_recenter_scores) * self.pagerank_score_multiplier except ValueError: adjusted = None self.pagerank_score = adjusted return self.pagerank_score def set_num_downloads_score(self): if not self.num_downloads: self.num_downloads_score = None return self.num_downloads_score try: raw = math.log10(float(self.num_downloads)/self.num_downloads_max) adjusted = (raw + self.num_downloads_offset_to_recenter_scores) * self.num_downloads_score_multiplier except ValueError: adjusted = None self.num_downloads_score = adjusted return self.num_downloads_score def set_num_citations_score(self): if not self.num_citations: self.num_citations_score = 0 return self.num_citations_score try: raw = math.log10(float(self.num_citations)/self.num_citations_max) adjusted = (raw + self.num_citations_offset_to_recenter_scores) * self.num_citations_score_multiplier except ValueError: adjusted = None self.num_citations_score = adjusted return self.num_citations_score def set_impact(self): score_components = [] pagerank_score = self.set_pagerank_score() if pagerank_score != None: score_components.append(pagerank_score) num_downloads_score = self.set_num_downloads_score() if num_downloads_score != None: score_components.append(num_downloads_score) if score_components: pagerank_and_downloads_mean = numpy.mean(score_components) else: pagerank_and_downloads_mean = None num_citations_score = self.set_num_citations_score() # twice the mean and twenty percent of the scaled citations combo = (pagerank_and_downloads_mean*2 + num_citations_score*0.2) / 2.2 self.impact = combo print u"self.impact for {} is {}".format(self.id, self.impact) @classmethod def shortcut_rev_deps_pairs(cls): NUM_TOP_NODES = 1000 command = """select package, used_by, pagerank, (coalesce((github_repo.api_raw->>'stargazers_count')::int, 0) + coalesce(package.num_stars, 0)) as num_stars from dep_nodes_ncol_{host}_reverse left outer join github_repo on dep_nodes_ncol_{host}_reverse.used_by = 'github:' || github_repo.id left outer join package on dep_nodes_ncol_{host}_reverse.used_by = package.project_name""".format( host=cls.class_host) rev_deps_by_package = defaultdict(list) res = db.session.connection().execute(sql.text(command)) rows = res.fetchall() non_zero_pageranks = [row[2] for row in rows if row[2]] min_pagerank = min(non_zero_pageranks) for row in rows: my_name = row[0] child_name = row[1] child_pagerank = row[2] child_stars = row[3] if not child_pagerank: child_pagerank = min_pagerank rev_deps_by_package[my_name].append([ child_name, child_pagerank, child_stars ]) return rev_deps_by_package def set_rev_deps_tree(self, rev_deps_lookup): node = RevDepNode( parent=None, name=self.project_name, pagerank=self.pagerank, generation=0, stars=None, root_pagerank=self.pagerank ) node.is_root = True node.set_children(rev_deps_lookup) self.rev_deps_tree = node.to_dict()
class BookCatalog(db.Model): __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} __tablename__ = "catalog" id = db.Column(db.Integer, primary_key=True, nullable=False) title = db.Column(db.String(255), default="", nullable=False, index=True) markdown = db.deferred(db.Column(LONGTEXT, default="", nullable=False)) html = db.deferred(db.Column(LONGTEXT, default="", nullable=False)) # 有些字段内容比较多而且不经常使用,比如是一个新闻的内容字段, # 在列表页或者封面页我们完全不必读取这个大的字段到内存里,这时候我们就可以设定这个列为 deferred publish_markdown = db.deferred( db.Column(LONGTEXT, default='', nullable=False)) publish_html = db.deferred(db.Column(LONGTEXT, default='', nullable=False)) status = db.Column(db.Integer, default=0, nullable=True, index=True) abstract = db.deferred(db.Column(db.String(255), default="")) publish_order = db.Column(db.Integer, default=0, nullable=True, index=True) pos = db.Column(db.Integer, default=0, nullable=False, index=True) parent_id = db.Column(db.Integer, default=0, nullable=False, index=True) is_dir = db.Column(db.Boolean, default=False, nullable=False, index=True) publish_timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) first_publish = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) updatetime = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) book_id = db.Column(db.Integer, db.ForeignKey(Book.__tablename__ + ".id", ondelete="CASCADE", onupdate="CASCADE"), nullable=False) @staticmethod def add(book_id, title, parent_id=0, is_dir=False): parent_id = 0 if parent_id is None else int(parent_id) if parent_id != 0: if not BookCatalog.get(parent_id): return catalog = BookCatalog( title=title, is_dir=is_dir, book_id=book_id, ) if parent_id: catalog.parent_id = parent_id catalog.pos = BookCatalog.max_pos(book_id, parent_id) + 1 db.session.add(catalog) db.session.commit() catalog.book.updatetime = datetime.now() return catalog @staticmethod def get(id): return BookCatalog.query.filter_by(id=id).first() @staticmethod def get_deep(id=None): """ 返回目录深度 """ if not id: return 1 else: catalog = BookCatalog.query.filter_by(id=id).first() if not catalog: return 0 return BookCatalog.get_deep(catalog.parent_id) + 1 @staticmethod def reader(book_id, id=None): catalog = BookCatalog.query.filter_by(book_id=book_id) \ .options(db.undefer("publish_html")) if id: catalog = catalog.filter_by(id=id) else: catalog = catalog.filter_by(parent_id=0) \ .order_by(BookCatalog.pos) return catalog.first() @staticmethod def page(page, per_page): last_catalog = Book.create_subquery("last_catalog") catalogs = db.session.query(Book, BookCatalog) \ .outerjoin(last_catalog, last_catalog.c.id == Book.id) \ .outerjoin(BookCatalog, db.and_( BookCatalog.book_id == Book.id, BookCatalog.publish_order == last_catalog.c.order)) \ .options( db.Load(Book).undefer("brief"), db.Load(BookCatalog).undefer("abstract")) \ .filter(db.and_( Book.access == 1, last_catalog.c.order.isnot(None))) \ .order_by(BookCatalog.first_publish.desc()) return paginate(catalogs, page, per_page=per_page, error_out=False) @staticmethod def max_pos(book_id, id=0): if id: catalogs = BookCatalog.query.filter_by(parent_id=id).filter_by( book_id=book_id).all() else: catalogs = BookCatalog.query.filter( db.and_(BookCatalog.book_id == book_id, BookCatalog.parent_id == 0)).all() if catalogs: return max([catalog.pos for catalog in catalogs]) return 0 @staticmethod def max_order(book_id): catalog = BookCatalog.query.filter(BookCatalog.book_id == book_id) \ .order_by(BookCatalog.publish_order.desc()).first() return 0 if catalog is None else catalog.publish_order @staticmethod def prev(catalog): prev_catalog = BookCatalog.query.with_entities( BookCatalog.id, BookCatalog.parent_id, BookCatalog.is_dir) \ .filter(db.and_( BookCatalog.book_id == catalog.book_id, BookCatalog.pos < catalog.pos, BookCatalog.parent_id == catalog.parent_id)) \ .order_by(BookCatalog.pos.desc()).first() if prev_catalog and prev_catalog.is_dir: child = BookCatalog.last_child(prev_catalog) if child: return child if prev_catalog or not catalog.parent_id: return prev_catalog return BookCatalog.query.with_entities(BookCatalog.id) \ .filter_by(id=catalog.parent_id).first() @staticmethod def next(catalog, is_first=True): if catalog: if catalog.is_dir and is_first: next_catalog = BookCatalog.first_child(catalog) if next_catalog: return next_catalog next_catalog = BookCatalog.query.with_entities(BookCatalog.id, BookCatalog.parent_id, BookCatalog.is_dir).filter(db.and_( BookCatalog.book_id == catalog.book_id, BookCatalog.pos > catalog.pos, BookCatalog.parent_id == catalog.parent_id)) \ .order_by(BookCatalog.pos).first() if next_catalog or not catalog.parent_id: return next_catalog catalog = BookCatalog.query.with_entities( BookCatalog.id, BookCatalog.parent_id, BookCatalog.is_dir, BookCatalog.pos, BookCatalog.book_id) \ .filter_by(id=catalog.parent_id).first() return BookCatalog.next(catalog, False) else: next_catalog = BookCatalog.query.with_entities(BookCatalog.id) \ .filter(db.and_( BookCatalog.book_id == catalog.book_id, BookCatalog.parent_id == 0)) \ .order_by(BookCatalog.pos).first() return next_catalog @staticmethod def first_child(catalog): return BookCatalog.query.with_entities(BookCatalog.id, BookCatalog.parent_id, BookCatalog.title, BookCatalog.is_dir) \ .filter_by(parent_id=catalog.book_id) \ .order_by(BookCatalog.pos).first() @staticmethod def last_child(catalog): return BookCatalog.query.with_entities(BookCatalog.id, BookCatalog.parent_id, BookCatalog.title, BookCatalog.id) \ .filter_by(parent_id=catalog.id) \ .order_by(BookCatalog.pos.desc()).first() def set_abstract(self): soup = BeautifulSoup(self.publish_html, "html.parser") self.abstract = soup.text[:110] + u"..." def sort(self, next_id=None): if not next_id: self.pos = BookCatalog.max_pos(self.book_id) + 1 self.parent_id = 0 else: next = BookCatalog.get(next_id) if not next: return False self.pos = next.pos self.parent_id = next.parent_id nexts = BookCatalog.query.filter( db.and_(BookCatalog.book_id == self.book_id, BookCatalog.parent_id == next.parent_id, BookCatalog.pos > next.pos)).all() for _next in nexts: _next.pos += 1 next.pos += 1 self.updatetime = datetime.now() self.book.updatetime = self.updatetime return True def rename(self, title): self.title = title self.updatetime = datetime.now() self.book.updatetime = self.updatetime def save(self, markdown, html): self.markdown = markdown self.html = html self.updatetime = datetime.now() self.book.updatetime = self.updatetime def delete(self): catalogs = BookCatalog.query.filter_by(parent_id=self.id).all() for catalog in catalogs: catalog.delete() self.book.updatetime = datetime.now() db.session.delete(self)
class Book(db.Model): __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} __tablename__ = "book" id = db.Column(db.Integer, primary_key=True, nullable=False) name = db.Column(db.String(255), default="", nullable=False, index=True) access = db.Column(db.Integer, default=1, nullable=False, index=True) status = db.Column(db.Integer, default=0, nullable=False, index=True) # publish status brief = db.deferred(db.Column(db.Text, default="", nullable=False)) #简介 select_catalog = db.Column(db.Integer, default=0, nullable=False) publish_timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) updatetime = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False, index=True) cover = db.Column(db.String(255), default="", nullable=False) user_id = db.Column(db.Integer, db.ForeignKey(User.__tablename__ + ".id", ondelete="CASCADE", onupdate="CASCADE"), nullable=False) catalogs = db.relationship("BookCatalog", backref="book", lazy="dynamic", passive_deletes=True) images = db.relationship("BookImage", backref="book", lazy="select", passive_deletes=True) @staticmethod def add(name, brief, access, user_id): book = Book(name=name, brief=brief, access=access, user_id=user_id) db.session.add(book) db.session.commit() return book @staticmethod def get(id): return Book.query.filter_by(id=id).first() @staticmethod def create_subquery(sub_type, user_id=None): if sub_type == "last_catalog": return db.session.query( Book.id, func.max(BookCatalog.publish_order).label("order")).outerjoin( BookCatalog, db.and_(BookCatalog.book_id == Book.id, BookCatalog.status == 1)).group_by( Book.id).subquery("order") @staticmethod def page(page, per_page): books = Book.query.options(db.Load(Book).undefer("brief")) \ .order_by(Book.timestamp.desc()) \ .paginate(page, per_page=per_page, error_out=False) return books @staticmethod def info(id): return Book.query.filter_by(id=id) \ .options(db.undefer("brief")).first() def setting(self, name, brief, access): self.name = name self.brief = brief self.access = access def _deep_catalogs(self, catalogs, catalog_dict): for catalog in catalogs: catalog.catalogs = self._deep_catalogs( self._sort_catalogs(catalog_dict.get(catalog.id, [])), catalog_dict) return catalogs def _sort_catalogs(self, catalogs): return sorted(catalogs, key=lambda x: x.pos) def tree_catalogs(self): catalogs = self.catalogs.options( load_only("title", "parent_id", "pos", "is_dir", "book_id")).all() catalog_dict = {} for catalog in catalogs: if catalog.parent_id not in catalog_dict: catalog_dict[catalog.parent_id] = [] catalog_dict[catalog.parent_id].append(catalog) return self._deep_catalogs( self._sort_catalogs(catalog_dict.get(0, [])), catalog_dict) def origin_cover(self): image_path = current_app.config["BOOK_COVER_PATH"] return "/".join([image_path, self.cover]) def thumbnail_cover(self): image_path = current_app.config["BOOK_COVER_PATH"] return "/".join([image_path, "thumbnail_{}".format(self.cover)]) def publish(self): now = datetime.now() self.catalog_publish(now) self.status = 1 self.publish_timestamp = now def catalog_publish(self, date): catalogs = BookCatalog.query.filter(BookCatalog.book_id == self.id) \ .filter(BookCatalog.updatetime > self.publish_timestamp).options( db.undefer("markdown").undefer("html") \ .undefer("publish_markdown").undefer("publish_html") ).all() max_order = BookCatalog.max_order(self.id) max_catalog = None for catalog in catalogs: catalog.publish_markdown = catalog.markdown catalog.publish_html = catalog.html catalog.publish_timestamp = date if len(catalog.publish_markdown) > 10: catalog.set_abstract() if catalog.first_publish == catalog.timestamp: catalog.first_publish = date if not catalog.publish_order: max_order += 1 catalog.publish_order = max_order catalog.status = 1 else: catalog.status = 0 def delete_cover(self): if not self.cover: return file.delete_cover(self.cover) def delete(self): for image in self.images: image.delete() for catalog in self.catalogs: catalog.delete() self.delete_cover() db.session.delete(self)
class Item(db.Model): __tablename__ = 'items' id = db.Column(db.Integer, primary_key=True) itemtype = db.Column(db.String(128)) platform = db.Column(db.String(128)) title = db.Column(db.String(128)) author = db.Column(db.String(128)) publisher = db.Column(db.String(64)) image = db.Column(db.String(128)) pubdate = db.Column(db.String(32)) price = db.Column(db.String(16)) summary = db.deferred(db.Column(db.Text, default="")) summary_html = db.deferred(db.Column(db.Text)) hidden = db.Column(db.Integer, default=0) amount = db.Column(db.Integer) logs = db.relationship('Log', backref=db.backref('item', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') comments = db.relationship('Comment', backref='item', lazy='dynamic', cascade='all, delete-orphan') @property def tags_string(self): return ",".join([tag.name for tag in self.tags.all()]) @tags_string.setter def tags_string(self, value): self.tags = [] tags_list = value.split(u',') for str in tags_list: tag = Tag.query.filter(Tag.name.ilike(str)).first() if tag is None: tag = Tag(name=str) self.tags.append(tag) db.session.add(self) db.session.commit() def can_borrow(self,user_id,item_id): if not self.hidden and self.can_borrow_number() > 0 and Cart.query.filter_by(user_id=user_id,item_id=item_id).first() == None and Log.query.filter_by(user_id=user_id,item_id=item_id,returned=0).first() == None: return True else: return False #return ((not self.hidden) and self.can_borrow_number() > 0) or Cart.query.filter_by(user_id=user_id,item_id=item_id) != None def can_borrow_number(self): return self.amount - Log.query.filter_by(item_id=self.id, returned=0).count() @staticmethod def on_changed_summary(target, value, oldvalue, initiaor): allowed_tags = ['a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'] target.summary_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True)) def __repr__(self): return u'<Item %r>' % self.title
class User(UserMixin, db.Model): __tablename__ = 'users' id = db.Column(db.Integer, primary_key=True) email = db.Column(db.String(64), unique=True) name = db.Column(db.String(64)) password_hash = db.deferred(db.Column(db.String(128))) major = db.Column(db.String(128)) role_id = db.Column(db.Integer, db.ForeignKey('roles.id')) headline = db.Column(db.String(32), nullable=True) about_me = db.deferred(db.Column(db.Text, nullable=True)) about_me_html = db.deferred(db.Column(db.Text, nullable=True)) avatar = db.Column(db.String(128)) member_since = db.Column(db.DateTime(), default=datetime.utcnow) @property def password(self): raise AttributeError('password is not readable attribute') @password.setter def password(self, password): self.password_hash = generate_password_hash(password) def verify_password(self, password): return check_password_hash(self.password_hash, password) def __init__(self, **kwargs): super(User, self).__init__(**kwargs) if self.role is None: if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower(): self.role = Role.query.filter_by(permissions=0x1ff).first() if self.role is None: self.role = Role.query.filter_by(rdefault=True).first() self.member_since = datetime.now() def can(self, permissions): return self.role is not None and \ (self.role.permissions & permissions) == permissions def is_administrator(self): return self.can(Permission.ADMINISTER) logs = db.relationship('Log', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') comments = db.relationship('Comment', backref=db.backref('user', lazy='joined'), lazy='dynamic', cascade='all, delete-orphan') def __repr__(self): return '<User %r>' % self.email def borrowing(self, item): return self.logs.filter_by(item_id=item.id, returned=0).first() def can_borrow_item(self): return self.logs.filter(Log.returned == 0, Log.return_timestamp < datetime.now()).count() == 0 def borrow_item(self, item): if self.logs.filter(Log.returned == 0, Log.return_timestamp < datetime.now()).count() > 0: return False, u"Nie można wypożyczyć,zaległe książki nie zostały zwrócone!" if self.borrowing(item): return False, u'Wygląda na to, że już wypożyczyłeś tą książkę!!' if not item.can_borrow(id,item.id): return False, u'Brak egzemplarzy do wypożyczenia!' db.session.add(Log(self, item)) return True, u'Wypożyczono pomyślnie %s' % item.title def return_item(self, log): if log.returned == 1 or (self.id != 1 and log.user_id != self.id): print(log.returned) print(log.user_id) print("user"+str(self.id)) return False, u'Nie znaleziono takiego wypożyczenia!' log.returned = 1 log.return_timestamp = datetime.now() db.session.add(log) db.session.commit() return True, u'Zwrot zakończony pomyślnie %s' % log.item.title def avatar_url(self, _external=False): if self.avatar: avatar_json = json.loads(self.avatar) if avatar_json['use_out_url']: return avatar_json['url'] else: return url_for('_uploads.uploaded_file', setname=avatars.name, filename=avatar_json['url'], _external=_external) else: return url_for('static', filename='img/toad.png', _external=_external) @staticmethod def on_changed_about_me(target, value, oldvalue, initiaor): allowed_tags = ['a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'] target.about_me_html = bleach.linkify( bleach.clean(markdown(value, output_format='html'), tags=allowed_tags, strip=True))