class TwitterArchive(db.Model, TimestampMixin):
__tablename__ = 'twitter_archives'
id = db.Column(db.Integer, primary_key=True)
account_id = db.Column(db.String,
db.ForeignKey('accounts.id',
onupdate='CASCADE',
ondelete='CASCADE'),
nullable=False)
account = db.relationship(Account,
backref=db.backref(
'twitter_archives',
order_by=lambda: db.desc(TwitterArchive.id)))
body = db.deferred(db.Column(db.LargeBinary, nullable=False))
chunks = db.Column(db.Integer)
chunks_successful = db.Column(db.Integer,
server_default='0',
nullable=False)
chunks_failed = db.Column(db.Integer, server_default='0', nullable=False)
def status(self):
if self.chunks is None or self.chunks_failed > 0:
return 'failed'
if self.chunks_successful == self.chunks:
return 'successful'
return 'pending'
class Visitors(db.Model):
__tablename__ = 'visitors'
id_number = db.Column(db.String(64), primary_key=True)
id_type = db.Column(db.String(64))
name = db.Column(db.String(64))
gender = db.deferred(db.Column(db.String(128)))
phone_number = db.Column(db.String(128))
address = db.Column(db.String(32), nullable=True)
avatar = db.Column(db.String(128))
def __init__(self, id_number, id_type, name, gender, phone_number,
address):
self.id_number = id_number
self.id_type = id_type
self.name = name
self.gender = gender
self.phone_number = phone_number
self.address = address
def avatar_url(self, _external=False):
if self.avatar:
avatar_json = json.loads(self.avatar)
if avatar_json['use_out_url']:
return avatar_json['url']
else:
return url_for('_uploads.uploaded_file',
setname=avatars.name,
filename=avatar_json['url'],
_external=_external)
else:
return url_for('static',
filename='img/avatar.png',
_external=_external)
class SiteMeta(db.Model):
""" Site table """
__tablename__ = "meta"
__table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
id = db.Column(db.Integer, primary_key=True, nullable=False)
name = db.Column(db.String(255), nullable=False, index=True)
value = db.deferred(db.Column(LONGTEXT, default="", nullable=False))
@staticmethod
def add(data):
for name, value in data.items():
meta = SiteMeta.query.filter_by(name=name).first()
if meta is not None:
continue
meta = SiteMeta(name=name, value=value)
db.session.add(meta)
db.session.commit()
@staticmethod
def setting(data):
for name, value in data.items():
meta = SiteMeta.query.filter_by(name=name).first()
if not meta:
meta = SiteMeta(name=name, value=value)
db.session.add(meta)
return
meta.value = value
db.session.commit()
@staticmethod
def all():
return SiteMeta.query.all()
class FlicketSubscription(db.Model):
__tablename__ = 'flicket_ticket_subscription'
id = db.Column(db.Integer, primary_key=True)
ticket_id = db.Column(db.Integer, db.ForeignKey(FlicketTicket.id))
ticket = db.relationship(FlicketTicket)
user_id = db.Column(db.Integer, db.ForeignKey(User.id))
user = db.relationship(User)
user_def = db.deferred(db.select([User.email]).where(User.id == user_id))
def __repr__(self):
return '<Class FlicketSubscription: ticket_id={}, user_id={}>'
class Book(db.Model):
__tablename__ = 'books'
id = db.Column(db.Integer, primary_key=True)
isbn = db.Column(db.String(16), unique=True)
title = db.Column(db.String(128))
origin_title = db.Column(db.String(128))
subtitle = db.Column(db.String(128))
author = db.Column(db.String(128))
translator = db.Column(db.String(64))
publisher = db.Column(db.String(64))
image = db.Column(db.String(128))
pubdate = db.Column(db.String(32))
pages = db.Column(db.Integer)
price = db.Column(db.String(16))
binding = db.Column(db.String(16))
numbers = db.Column(db.Integer, default=5)
summary = db.deferred(db.Column(db.Text, default=""))
summary_html = db.deferred(db.Column(db.Text))
catalog = db.deferred(db.Column(db.Text, default=""))
catalog_html = db.deferred(db.Column(db.Text))
hidden = db.Column(db.Boolean, default=0)
transactions = db.relationship('Transaction',
backref=db.backref('book', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
logs = db.relationship('Log',
backref=db.backref('book', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
comments = db.relationship('Comment',
backref='book',
lazy='dynamic',
cascade='all, delete-orphan')
@property
def tags_string(self):
return ",".join([tag.name for tag in self.tags.all()])
@tags_string.setter
def tags_string(self, value):
self.tags = []
tags_list = value.split(u',')
for str in tags_list:
tag = Tag.query.filter(Tag.name.ilike(str)).first()
if tag is None:
tag = Tag(name=str)
self.tags.append(tag)
db.session.add(self)
db.session.commit()
def can_borrow(self):
return (not self.hidden) and self.can_borrow_number() > 0
def can_borrow_number(self):
return self.numbers - Log.query.filter_by(book_id=self.id,
returned=0).count()
@staticmethod
def on_changed_summary(target, value, oldvalue, initiaor):
allowed_tags = [
'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li',
'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'
]
target.summary_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags,
strip=True))
@staticmethod
def on_changed_catalog(target, value, oldvalue, initiaor):
allowed_tags = [
'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li',
'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'
]
target.catalog_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags,
strip=True))
def __repr__(self):
return u'<Book %r>' % self.title
class User(UserMixin, db.Model):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(64), unique=True)
name = db.Column(db.String(64))
password_hash = db.deferred(db.Column(db.String(128)))
major = db.Column(db.String(128))
role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
headline = db.Column(db.String(32), nullable=True)
about_me = db.deferred(db.Column(db.Text, nullable=True))
about_me_html = db.deferred(db.Column(db.Text, nullable=True))
avatar = db.Column(db.String(128))
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
@property
def password(self):
raise AttributeError('password is not readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower(
):
self.role = Role.query.filter_by(permissions=0x1ff).first()
if self.role is None:
self.role = Role.query.filter_by(default=True).first()
self.member_since = datetime.now()
def can(self, permissions):
return self.role is not None and \
(self.role.permissions & permissions) == permissions
def is_administrator(self):
return self.can(Permission.ADMINISTER)
investments = db.relationship('Investment',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
def __repr__(self):
return '<User %r>' % self.email
def get_id(self):
return self.id
def avatar_url(self, _external=False):
if self.avatar:
avatar_json = json.loads(self.avatar)
if avatar_json['use_out_url']:
return avatar_json['url']
else:
return url_for('_uploads.uploaded_file',
setname=avatars.name,
filename=avatar_json['url'],
_external=_external)
else:
return url_for('static',
filename='img/avatar.png',
_external=_external)
@staticmethod
def on_changed_about_me(target, value, oldvalue, initiaor):
allowed_tags = [
'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li',
'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'
]
target.about_me_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags,
strip=True))
class User(UserMixin, db.Model):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(64), unique=True)
name = db.Column(db.String(64))
password_hash = db.deferred(db.Column(db.String(128)))
major = db.Column(db.String(128))
role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
headline = db.Column(db.String(32), nullable=True)
about_me = db.deferred(db.Column(db.Text, nullable=True))
about_me_html = db.deferred(db.Column(db.Text, nullable=True))
avatar = db.Column(db.String(128))
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
@property
def password(self):
raise AttributeError('password is not readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower(
):
self.role = Role.query.filter_by(permissions=0x1ff).first()
if self.role is None:
self.role = Role.query.filter_by(default=True).first()
self.member_since = datetime.now()
def can(self, permissions):
return self.role is not None and \
(self.role.permissions & permissions) == permissions
def is_administrator(self):
return self.can(Permission.ADMINISTER)
logs = db.relationship('Log',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
comments = db.relationship('Comment',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
def __repr__(self):
return '<User %r>' % self.email
def borrowing(self, book):
return self.logs.filter_by(book_id=book.id, returned=0).first()
def can_borrow_book(self):
return self.logs.filter(
Log.returned == 0,
Log.return_timestamp < datetime.now()).count() == 0
def borrow_book(self, book):
if self.logs.filter(Log.returned == 0,
Log.return_timestamp < datetime.now()).count() > 0:
return False, u"Unable to borrow, you have expired books not returned"
if self.borrowing(book):
return False, u'Looks like you have borrowed this book!!'
if not book.can_borrow():
return False, u'This book is too hot, we have no copies, please wait for others to return and borrow again.'
db.session.add(Log(self, book))
return True, u'You successfully get to a copy %s' % book.title
def return_book(self, log):
if log.returned == 1 or log.user_id != self.id:
return False, u'Did not find this record'
log.returned = 1
log.return_timestamp = datetime.now()
db.session.add(log)
return True, u'You returned a copy %s' % log.book.title
def avatar_url(self, _external=False):
if self.avatar:
avatar_json = json.loads(self.avatar)
if avatar_json['use_out_url']:
return avatar_json['url']
else:
return url_for('_uploads.uploaded_file',
setname=avatars.name,
filename=avatar_json['url'],
_external=_external)
else:
return url_for('static',
filename='img/avatar.png',
_external=_external)
@staticmethod
def on_changed_about_me(target, value, oldvalue, initiaor):
allowed_tags = [
'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li',
'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'
]
target.about_me_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags,
strip=True))
class User(UserMixin, db.Model):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(64), unique=True)
name = db.Column(db.String(64))
password_hash = db.deferred(db.Column(db.String(128)))
major = db.Column(db.String(128))
role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
headline = db.Column(db.String(32), nullable=True)
about_me = db.deferred(db.Column(db.Text, nullable=True))
about_me_html = db.deferred(db.Column(db.Text, nullable=True))
avatar = db.Column(db.String(128))
category = db.Column(db.Integer)
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
balance = db.Column(db.Float())
transactions = db.relationship('Transaction',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
@property
def password(self):
raise AttributeError('password is not readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower(
):
self.role = Role.query.filter_by(permissions=0x1ff).first()
if self.role is None:
self.role = Role.query.filter_by(default=True).first()
self.member_since = datetime.now()
def has_discount(self):
if self.category != 0:
return True
else:
return False
def calculate_discount(self, sum):
if self.category == Category.STUDENT:
return sum * 0.1
elif self.category == Category.LARGE_FAMILIES:
return sum * 0.5
def can(self, permissions):
return self.role is not None and \
(self.role.permissions & permissions) == permissions
def is_administrator(self):
return self.can(Permission.ADMINISTER)
logs = db.relationship('Log',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
comments = db.relationship('Comment',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
def __repr__(self):
return '<User %r>' % self.email
def borrowing(self, book):
return self.logs.filter_by(book_id=book.id, returned=0).first()
def can_borrow_book(self):
return self.logs.filter(
Log.return_request == 0,
Log.return_timestamp < datetime.now()).count() == 0
def borrow_book(self, book):
if self.logs.filter(Log.return_request == 0,
Log.return_timestamp < datetime.now()).count() > 0:
return False, u"Unable to borrow, you have not returned the expired books"
if self.borrowing(book):
return False, u'It looks like you have borrowed this book!'
if not book.can_borrow():
return False, u'This book is too popular, please wait for someone to return it later'
if self.balance < book.price:
return False, u'Not enough money'
self.balance -= book.price
transaction = Transaction(self.id, book.id, book.price,
Operations.BORROW_BOOK)
db.session.add(transaction)
db.session.add(self)
db.session.add(Log(self, book))
return True, u'You successfully got a copy %s' % book.title
def approve_return_book(self, log, with_fine):
if log.returned == 1 or not self.can(
Permission.APPROVE_REQUEST_RETURN):
return False, u'Did not find this record'
log.returned = 1
user = User.query.get(log.user_id)
book = Book.query.get(log.book_id)
delata_time = log.return_timestamp - log.borrow_timestamp
if delata_time.days == 0:
delata_time = timedelta(days=1)
total_sum = book.price - (book.price / 100) * delata_time.days
transaction = Transaction(user.id, book.id, total_sum,
Operations.RETURN_BOOK)
user.balance += total_sum
db.session.add(transaction)
if with_fine == '1':
user.balance -= book.price - total_sum
fine_transaction = Transaction(user.id, book.id, total_sum,
Operations.FINE)
db.session.add(fine_transaction)
else:
if (user.has_discount()):
discount = user.calculate_discount(total_sum)
discount_transaction = Transaction(user.id, book.id, discount,
Operations.DISCOUNT)
user.balance += discount
db.session.add(discount_transaction)
db.session.add(user)
db.session.add(log)
return True, u'You returned a copy %s' % log.book.title
def create_return_request(self, log):
if log.returned == 1 or log.user_id != self.id or log.return_request == 1:
return False, u'Did not find this record'
log.return_request = 1
log.return_timestamp = datetime.now()
db.session.add(log)
return True, u'You returned a copy %s' % log.book.title
def avatar_url(self, _external=False):
if self.avatar:
avatar_json = json.loads(self.avatar)
if avatar_json['use_out_url']:
return avatar_json['url']
else:
return url_for('_uploads.uploaded_file',
setname=avatars.name,
filename=avatar_json['url'],
_external=_external)
else:
return url_for('static',
filename='img/avatar.png',
_external=_external)
@staticmethod
def on_changed_about_me(target, value, oldvalue, initiaor):
allowed_tags = [
'a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i', 'li',
'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p'
]
target.about_me_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags,
strip=True))
class Topic(db.Model):
__tablename__ = DB_PREFIX + "topic"
id = db.Column(db.Integer, primary_key=True, nullable=False)
name = db.Column(db.String(255), default="", index=True, nullable=False)
category_id = db.Column(db.Integer, default=0) # 专题所属分类
brief = db.deferred(db.Column(db.Text, default="", nullable=False))
avatar = db.Column(db.String(255), default="", nullable=False)
updatetime = db.Column(db.DateTime, default=datetime.now, nullable=False)
timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False)
articles = db.relationship("ArticleTopic", backref="topic", lazy="dynamic")
@staticmethod
def prefix_autosearch(name, page, per_page):
""" 标签名前缀查询 """
Article = app.article.models.Article
ArticleTopic = app.article.models.ArticleTopic
topics = db.session.query(Topic).outerjoin(
ArticleTopic,
Article).filter(Topic.name.ilike("{0}%".format(name))).filter(
Article.access == "public").group_by(Topic).order_by(
func.count(ArticleTopic.topic_id).desc())
return paginate(topics, page, per_page=per_page, error_out=False)
@staticmethod
def get_user_article_topics(user_id, page):
per_page = current_app.config["USER_TOPIC_PAGE"]
ArticleTopic = app.article.models.ArticleTopic
Article = app.article.models.Article
topics = db.session.query(Topic,
func.count(ArticleTopic.topic_id).label("count"))\
.outerjoin(ArticleTopic, Article).group_by(Topic)
if not current_user.is_authenticated or current_user.id != user_id:
topics = topics.filter(
db.and_(Article.user_id == user_id,
Article.access == "public"))
else:
topics = topics.filter(Article.user_id == user_id)
topics = topics.order_by(func.count(ArticleTopic.topic_id).desc())
topics = paginate(topics, page, per_page=per_page, error_out=False)
return topics
@staticmethod
def get_article_topics(num):
""" 所有文章的专题列表
@param num: 返回专题个数
"""
ArticleTopic = app.article.models.ArticleTopic
Article = app.article.models.Article
topics = db.session.query(Topic)
topics = topics.outerjoin(ArticleTopic).filter(
ArticleTopic.article.has(
Article.access == "public")).group_by(Topic).order_by(
func.count(ArticleTopic.topic_id).desc())
return topics.limit(num).all(), topics.count()
def get_articles(self, page, per_page):
Article = app.article.models.Article
ArticleTopic = app.article.models.ArticleTopic
articles = db.session.query(Article).filter_by(
access="public").outerjoin(ArticleTopic).filter(
ArticleTopic.topic_id == self.id)
return paginate(articles, page, per_page=per_page, error_out=False)
def count_article(self):
Article = app.article.models.Article
return self.articles.outerjoin(Article).filter(
Article.access == "public").count()
class Article(db.Model):
__tablename__ = DB_PREFIX + PREFIX + "article"
id = db.Column(db.Integer, primary_key=True, nullable=False)
user_id = db.Column(db.Integer,
db.ForeignKey(User.__tablename__ + ".id",
ondelete="CASCADE",
onupdate="CASCADE"),
nullable=False)
title = db.Column(db.String(255), default="", nullable=False)
url = db.Column(db.String(255), default="", nullable=False)
access = db.Column(db.String(255), default="public",
nullable=False) # 访问权限 public=> 公开 private=>私人
markdown = db.deferred(db.Column(db.Text, default="", nullable=False))
html = db.deferred(db.Column(db.Text, default="", nullable=False))
updatetime = db.Column(db.DateTime, default=datetime.now, nullable=False)
timestamp = db.Column(db.DateTime, default=datetime.now, nullable=False)
abstract = db.deferred(db.Column(db.Text, default="", nullable=False))
abstract_timestamp = db.Column(db.DateTime,
default=datetime.now,
nullable=False)
cover = db.Column(db.String(255), default="")
cover_timestamp = db.Column(db.DateTime, default=datetime.now)
pathname = db.Column(db.String(255),
default=uuid.uuid1().hex,
nullable=False) # 文章的url路径名
images = db.relationship("ArticleImage",
backref="article",
lazy="select",
passive_deletes=True)
topics = db.relationship("ArticleTopic",
backref="article",
lazy="select",
passive_deletes=True)
@staticmethod
def new(title, access, topics, user_id):
""" 新建文章 """
article = Article()
article.title = title
article.access = access
article.user_id = user_id
db.session.add(article)
db.session.commit()
article.pathname = article.create_pathname()
# 保存文章标签
article.update_topics(topics[:5])
return article
@staticmethod
def search(keyword, page, per_page):
query = "%{0}%".format(keyword)
articles = Article.query.filter(
db.and_(Article.access == "public",
Article.title.ilike(query))).order_by(
Article.updatetime.desc())
return articles.paginate(page, per_page=per_page, error_out=False)
@staticmethod
def get_articles(page, per_page):
"""
@param page: 当前页
@param per_page: 每页个数
"""
articles = Article.query.filter_by(access="public")
return articles.paginate(page, per_page=per_page, error_out=False)
@staticmethod
def newest_articles(num):
return Article.query.filter_by(access="public").order_by(
Article.timestamp.desc()).limit(num).all()
def create_pathname(self):
""" 创建文章路径名 """
code = generate_code(4)
return "".join([code, str(self.id)])
def setting(self, title, access, topics):
""" 文章设置 """
self.title = title
self.access = access
self.update_topics(topics)
def get_cover(self):
""" 取文章内容第一个图片作为文章的封面 """
if self.updatetime > self.cover_timestamp:
soup = BeautifulSoup(self.html, "lxml")
img = soup.find("img")
if img is not None:
self.cover = img['src']
else:
self.cover = ""
self.cover_timestamp = self.updatetime
return self.cover
def update_topics(self, topics):
""" 更新文章标签 """
new_topics = set([topic.strip().lower() for topic in topics])
old_topics = set([topic.topic.name for topic in self.topics])
delete_topics = old_topics - new_topics
add_topics = new_topics - old_topics
for name in delete_topics:
self.delete_topic(name)
for name in add_topics:
self.add_topic(name)
def add_topic(self, name):
""" 添加标签 """
if not name:
return False
topic = Topic.query.filter_by(name=name).first()
if not topic:
topic = Topic()
topic.name = name
db.session.add(topic)
db.session.commit()
for t in self.topics:
if t.topic.name == name.lower():
return False
article_topic = ArticleTopic()
article_topic.article_id = self.id
article_topic.topic_id = topic.id
db.session.add(article_topic)
return True
def delete_topic(self, name):
""" 删除标签 """
topic = Topic.query.filter_by(name=name).first()
if not topic:
return False
ArticleTopic.query.filter_by(topic_id=topic.id).delete()
return True
def get_abstract(self):
if not self.html:
return ""
if self.updatetime > self.abstract_timestamp:
soup = BeautifulSoup(self.html, "lxml")
self.abstract = soup.get_text()[0:400]
self.abstract_timestamp = self.updatetime
return self.abstract
def save_content(self, markdown, html):
""" 保存文章内容 """
self.markdown = markdown
self.html = html
self.updatetime = datetime.now()
def delete(self):
""" 删除文章 """
# 删除文章图片
for image in self.images:
image.delete()
# 删除标签
for topic in self.topics:
topic.delete()
# 删除文章自己
db.session.delete(self)
class Person(db.Model):
__tablename__ = 'person'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.Text)
name = db.Column(db.Text)
other_names = db.Column(JSONB)
github_login = db.Column(db.Text)
github_about = db.deferred(db.Column(JSONB))
parsed_name = db.Column(JSONB)
impact = db.Column(db.Float)
impact_percentile = db.Column(db.Float)
num_downloads = db.Column(db.Integer)
num_downloads_percentile = db.Column(db.Float)
num_citations = db.Column(db.Integer)
num_citations_percentile = db.Column(db.Float)
pagerank = db.Column(db.Float)
pagerank_percentile = db.Column(db.Float)
is_organization = db.Column(db.Boolean)
main_language = db.Column(db.Text)
def __repr__(self):
return u'<Person "{name}" ({id})>'.format(name=self.name, id=self.id)
contributions = db.relationship(
'Contribution',
# lazy='select',
cascade="all, delete-orphan",
backref="person")
def set_is_organization(self):
# set this using the sql in the sql folder! set_person_is_organization.sql
# this is just a placeholder to remind us to run it :)
pass
def refresh(self, refsets_dict):
self.set_scores()
self.set_subscore_percentiles(refsets_dict)
self.set_impact()
self.set_impact_percentiles(refsets_dict)
@property
def display_pagerank(self):
return self.pagerank / 100.0
@property
def subscores(self):
ret = []
if not self.impact:
# no academic impact, so don't return subscores
return ret
ret.append({
"display_name": "Downloads",
"icon": "fa-download",
"name": "num_downloads",
"percentile": self.num_downloads_percentile,
"val": self.num_downloads
})
ret.append({
"display_name": "Dependency PageRank",
"icon": "fa-recycle",
"name": "pagerank",
"percentile": self.pagerank_percentile,
"val": self.display_pagerank
})
ret.append({
"display_name": "Citations",
"icon": "fa-file-text-o",
"name": "num_mentions",
"percentile": self.num_citations_percentile,
"val": self.num_citations
})
return ret
def to_dict(self, max_person_packages=None, include_top_collabs=True):
ret = self.as_package_snippet
person_packages = self.get_person_packages()
ret["num_packages"] = len(person_packages)
ret["num_packages_r"] = len(
[pp for pp in person_packages if pp.package.language == 'r'])
ret["num_packages_python"] = len(
[pp for pp in person_packages if pp.package.language == 'python'])
# tags
tags_dict = defaultdict(int)
for pp in person_packages:
for tag in pp.package.tags:
tags_dict[tag] += 1
tags_to_return = min(5, len(tags_dict))
sorted_tags_to_return = sorted(tags_dict.items(),
key=lambda x: x[1],
reverse=True)[0:tags_to_return]
ret["top_person_tags"] = [{
"name": name,
"count": count
} for name, count in sorted_tags_to_return]
# co-collaborators
if include_top_collabs:
my_collabs = defaultdict(float)
for pp in person_packages:
for collab_person_id, collab_credit in pp.package.credit.iteritems(
):
if int(collab_person_id
) != self.id: #don't measure my own collab strength
collab_strength = collab_credit * pp.person_package_credit
my_collabs[collab_person_id] += collab_strength
sorted_collabs_to_return = sorted(my_collabs.items(),
key=lambda x: x[1],
reverse=True)
ret["top_collabs"] = []
num_collabs_to_return = min(5, len(sorted_collabs_to_return))
for person_id, collab_score in sorted_collabs_to_return[
0:num_collabs_to_return]:
person = Person.query.get(int(person_id))
if person:
person_dict = person.as_package_snippet
person_dict[
"collab_score"] = collab_score * 4 # to make a 0.25*0.25 connection strength of 1
ret["top_collabs"].append(person_dict)
else:
print u"ERROR: person {} not found; maybe deduped? skipping.".format(
person_id)
# person packages
if max_person_packages:
person_packages_to_return = min(max_person_packages,
len(person_packages))
ret["person_packages"] = [
p.as_person_snippet
for p in person_packages[0:person_packages_to_return]
]
else:
ret["person_packages"] = [p.to_dict() for p in person_packages]
ret["subscores"] = self.subscores
return ret
@property
def as_snippet(self):
return self.to_dict(max_person_packages=3, include_top_collabs=False)
@property
def as_package_snippet(self):
ret = {
"id": self.id,
"name": self.display_name,
"single_name": self.single_name,
"person_name": self.name, # helps with namespacing in UI
"github_login": self.github_login,
"icon": self.icon,
"icon_small": self.icon_small,
"is_organization": self.is_organization,
"main_language": self.main_language,
"impact": self.impact,
"impact_percentile": self.impact_percentile,
"id": self.id,
"subscores": self.subscores
}
return ret
def set_main_language(self):
person_package_summary_dict = self.as_snippet
if person_package_summary_dict[
"num_packages_r"] > person_package_summary_dict[
"num_packages_python"]:
self.main_language = "r"
else:
self.main_language = "python"
@classmethod
def shortcut_percentile_refsets(cls):
print "getting the percentile refsets...."
ref_list = defaultdict(dict)
q = db.session.query(cls.num_downloads, cls.pagerank,
cls.num_citations, cls.impact)
q = q.filter(
cls.num_downloads != None
) # only academic contributions, so would have some fractional downloads
q = q.filter(
or_(cls.pagerank > 0, cls.num_citations > 0,
cls.num_downloads > 0)) # only academic contributions
rows = q.all()
ref_list["num_downloads"] = sorted(
[row[0] for row in rows if row[0] != None])
ref_list["pagerank"] = sorted(
[row[1] for row in rows if row[1] != None])
ref_list["num_citations"] = sorted(
[row[2] for row in rows if row[2] != None])
ref_list["impact"] = sorted([row[3] for row in rows if row[3] != None])
# only compare impacts against other things with impacts
ref_list["impact"] = [val for val in ref_list["impact"] if val > 0]
return ref_list
def _calc_percentile(self, refset, value):
if value is None: # distinguish between that and zero
return None
try:
matching_index = refset.index(value)
percentile = float(matching_index) / len(refset)
except ValueError:
# not in index. maybe has no impact because no academic contributions
print u"not setting percentile for {}; looks like not academic".format(
self.name)
percentile = None
return percentile
def set_num_downloads_percentile(self, refset):
self.num_downloads_percentile = self._calc_percentile(
refset, self.num_downloads)
def set_pagerank_percentile(self, refset):
self.pagerank_percentile = self._calc_percentile(refset, self.pagerank)
def set_num_citations_percentile(self, refset):
self.num_citations_percentile = self._calc_percentile(
refset, self.num_citations)
def set_impact_percentile(self, refset):
self.impact_percentile = self._calc_percentile(refset, self.impact)
def set_subscore_percentiles(self, refsets_dict):
self.set_num_downloads_percentile(refsets_dict["num_downloads"])
self.set_pagerank_percentile(refsets_dict["pagerank"])
self.set_num_citations_percentile(refsets_dict["num_citations"])
def set_impact_percentiles(self, refsets_dict):
self.set_impact_percentile(refsets_dict["impact"])
def set_github_about(self):
if self.github_login is None:
return None
self.github_about = get_profile(self.github_login)
try:
if not self.name:
self.name = self.github_about["name"]
if not self.email:
self.email = self.github_about["email"]
except KeyError:
# our github_about is an error object,
# it's got no info about the person in it.
return False
def set_impact(self):
try:
self.impact = (self.pagerank_percentile +
self.num_downloads_percentile +
self.num_citations_percentile) / 3.0
except TypeError: #something was null
self.impact = 0
def set_scores(self):
self.pagerank = 0
self.num_downloads = 0
self.num_citations = 0
for pp in self.get_person_packages():
# only count up academic packages
if pp.package.is_academic:
# only count up impact for packages in our main language
if pp.package.language == self.main_language:
if pp.person_package_pagerank:
self.pagerank += pp.person_package_pagerank
if pp.person_package_num_downloads:
self.num_downloads += pp.person_package_num_downloads
if pp.person_package_num_citations:
self.num_citations += pp.person_package_num_citations
safe_commit(db)
@property
def name_normalized_for_maximal_deduping(self):
if not self.name:
return None
if self.is_organization:
return self.name.lower()
try:
first_initial = self.parsed_name["first"][0].lower()
except (KeyError, AttributeError, IndexError):
first_initial = "?"
try:
last_orig = self.parsed_name["last"]
last = unicodedata.normalize('NFKD',
last_orig).encode('ascii', 'ignore')
last = last.lower()
except (KeyError, AttributeError):
last = "?"
normalized_name = u"{} {}".format(first_initial, last)
# print "normalized_name", normalized_name
return normalized_name
def set_parsed_name(self):
if not self.name:
self.parsed_name = None
return
name = HumanName(self.name)
self.parsed_name = name.as_dict()
def _make_gravatar_url(self, size):
try:
if self.email is not None:
hash = hashlib.md5(self.email).hexdigest()
else:
hash = hashlib.md5(str(self.id)).hexdigest()
except UnicodeEncodeError:
print "UnicodeEncodeError making gravatar url from email"
hash = 42
url = "http://www.gravatar.com/avatar/{hash}.jpg?s={size}&d=retro".format(
hash=hash, size=size)
return url
@property
def icon(self):
return self._make_gravatar_url(160)
@property
def icon_small(self):
return self._make_gravatar_url(30)
def has_role_on_project(self, role, package_id):
for c in self.contributions:
if c.role == role and c.package_id == package_id:
return True
return False
def num_commits_on_project(self, package_id):
for c in self.contributions:
if c.role == "github_contributor" and c.package_id == package_id:
return c.quantity
return False
@property
def single_name(self):
if self.is_organization:
return self.display_name
elif self.parsed_name and self.parsed_name["last"]:
return self.parsed_name["last"]
return self.display_name
@property
def display_name(self):
if self.name:
return self.name
elif self.github_login:
return self.github_login
elif self.email:
return self.email.split("@")[0]
else:
return "name unknown"
# could be a property, but kinda slow, so better as explicity method methinks
def get_person_packages(self):
person_packages = defaultdict(PersonPackage)
for contrib in self.contributions:
person_packages[contrib.package.id].set_role(contrib)
person_packages_list = person_packages.values()
person_packages_list.sort(key=lambda x: x.person_package_impact,
reverse=True)
return person_packages_list
def num_commits_on_project(self, package_id):
for c in self.contributions:
if c.role == "github_contributor" and c.package_id == package_id:
return c.quantity
return False
@classmethod
def decide_who_to_dedup(cls, people):
if len(people) <= 1:
# this name has no dups
return None
people_with_github = [p for p in people if p.github_login]
people_with_no_github = [p for p in people if not p.github_login]
# don't merge people with github together
# so we only care about merging if there are people with no github
if not people_with_no_github:
return None
if people_with_github:
# merge people with no github into first person with github
dedup_target = people_with_github[0]
people_to_merge = people_with_no_github
else:
# pick first person with no github as target, rest as mergees
dedup_target = people_with_no_github[0]
people_to_merge = people_with_no_github[1:]
return {
"dedup_target": dedup_target,
"people_to_merge": people_to_merge
}
@classmethod
def dedup(cls, dedup_target, people_to_merge):
print u"person we will merge into: {}".format(dedup_target.id)
print u"people to merge: {}".format([p.id for p in people_to_merge])
for person_to_delete in people_to_merge:
contributions_to_change = person_to_delete.contributions
for contrib in contributions_to_change:
contrib.person = dedup_target
db.session.add(contrib)
print u"now going to delete {}".format(person_to_delete)
db.session.delete(person_to_delete)
class Person(db.Model):
__tablename__ = 'person'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.Text)
name = db.Column(db.Text)
other_names = db.Column(JSONB)
github_login = db.Column(db.Text)
github_about = db.deferred(db.Column(JSONB))
bucket = db.Column(JSONB)
impact = db.Column(db.Float)
impact_rank = db.Column(db.Integer)
pagerank_score = db.Column(db.Float)
num_downloads_score = db.Column(db.Float)
num_citations_score = db.Column(db.Float)
parsed_name = db.Column(JSONB)
is_academic = db.Column(db.Boolean)
is_organization = db.Column(db.Boolean)
main_language = db.Column(db.Text)
type = db.Column(db.Text)
def __repr__(self):
return u'<Person "{name}" ({id})>'.format(name=self.name, id=self.id)
contributions = db.relationship(
'Contribution',
# lazy='select',
cascade="all, delete-orphan",
backref="person")
def set_is_academic(self):
# set this using the sql in the sql folder! set_person_is_academic.sql
# this is just a placeholder to remind us to run it :)
pass
def set_is_organization(self):
# set this using the sql in the sql folder! set_person_is_organization.sql
# this is just a placeholder to remind us to run it :)
pass
@property
def subscores(self):
ret = []
ret.append({
"name": "num_citations",
"score": self.num_citations_score,
"val": None
})
ret.append({
"name": "pagerank",
"score": self.pagerank_score,
"val": None
})
ret.append({
"name": "num_downloads",
"score": self.num_downloads_score,
"val": None
})
# select id, name, email, num_citations_score from person where is_organization=false and main_lanugage='python' order by num_downloads_score desc limit 5
maxes = {
"python": {
"num_citations": 1312.07783912007676,
"pagerank": 6828.41972274044019,
"num_downloads": 16419.5532038200363
},
"r": {
"num_citations": 1866,
"pagerank": 9270,
"num_downloads": 18213
}
}
for my_dict in ret:
if my_dict["score"]:
my_dict["score"] = 1000.00 * my_dict["score"] / maxes[
self.main_language][my_dict["name"]]
return ret
def to_dict(self, max_person_packages=None, include_top_collabs=True):
ret = self.as_package_snippet
person_packages = self.get_person_packages()
ret["num_packages"] = len(person_packages)
ret["num_packages_r"] = len(
[pp for pp in person_packages if pp.package.language == 'r'])
ret["num_packages_python"] = len(
[pp for pp in person_packages if pp.package.language == 'python'])
# tags
tags_dict = defaultdict(int)
for pp in person_packages:
for tag in pp.package.tags:
tags_dict[tag] += 1
tags_to_return = min(5, len(tags_dict))
sorted_tags_to_return = sorted(tags_dict.items(),
key=lambda x: x[1],
reverse=True)[0:tags_to_return]
ret["top_person_tags"] = [{
"name": name,
"count": count
} for name, count in sorted_tags_to_return]
# co-collaborators
if include_top_collabs:
my_collabs = defaultdict(float)
for pp in person_packages:
for collab_person_id, collab_credit in pp.package.credit.iteritems(
):
if int(collab_person_id
) != self.id: #don't measure my own collab strength
collab_strength = collab_credit * pp.person_package_credit
my_collabs[collab_person_id] += collab_strength
sorted_collabs_to_return = sorted(my_collabs.items(),
key=lambda x: x[1],
reverse=True)
ret["top_collabs"] = []
num_collabs_to_return = min(5, len(sorted_collabs_to_return))
for person_id, collab_score in sorted_collabs_to_return[
0:num_collabs_to_return]:
person = Person.query.get(int(person_id))
person_dict = person.as_package_snippet
person_dict[
"collab_score"] = collab_score * 4 # to make a 0.25*0.25 connection strength of 1
ret["top_collabs"].append(person_dict)
# person packages
if max_person_packages:
person_packages_to_return = min(max_person_packages,
len(person_packages))
ret["person_packages"] = [
p.as_person_snippet
for p in person_packages[0:person_packages_to_return]
]
else:
ret["person_packages"] = [p.to_dict() for p in person_packages]
ret["subscores"] = self.subscores
return ret
@property
def as_snippet(self):
return self.to_dict(max_person_packages=3, include_top_collabs=False)
@property
def as_package_snippet(self):
ret = {
"id": self.id,
"name": self.display_name,
"single_name": self.single_name,
"github_login": self.github_login,
"icon": self.icon,
"icon_small": self.icon_small,
"is_academic": self.is_academic,
"is_organization": self.is_organization,
"main_language": self.main_language,
"impact": self.impact,
"impact_rank": self.impact_rank,
"impact_rank_max": self.impact_rank_max,
"pagerank_score": self.pagerank_score,
"num_downloads_score": self.num_downloads_score,
"num_citations_score": self.num_citations_score,
"id": self.id
}
return ret
def set_main_language(self):
person_package_summary_dict = self.as_snippet
if person_package_summary_dict[
"num_packages_r"] > person_package_summary_dict[
"num_packages_python"]:
self.main_language = "r"
else:
self.main_language = "python"
@classmethod
def shortcut_impact_rank(cls):
print "getting the lookup for ranking impact...."
impact_rank_lookup = defaultdict(dict)
for main_language in ["python", "r"]:
q = db.session.query(cls.id)
q = add_person_leaderboard_filters(q)
q = q.filter(Person.main_language == main_language)
q = q.order_by(cls.impact.desc()) # the important part :)
rows = q.all()
ids_sorted_by_impact = [row[0] for row in rows]
for my_id in ids_sorted_by_impact:
zero_based_rank = ids_sorted_by_impact.index(my_id)
impact_rank_lookup[main_language][my_id] = zero_based_rank + 1
return impact_rank_lookup
def set_impact_rank(self, impact_rank_lookup):
try:
self.impact_rank = impact_rank_lookup[self.main_language][self.id]
except KeyError: # maybe because organization, or name=="UNKNOWN"
print "couldn't find my id"
self.impact_rank = None
print "self.impact_rank", self.impact_rank
@property
def impact_rank_max(self):
# select count(id), main_language
# from person
# where is_organization=false
# and (name is null or name!='UKNOWN')
# and (email is null or email!='UNKNOWN')
# group by main_language
if self.main_language == "python":
return 62951
elif self.main_language == "r":
return 10447
def set_github_about(self):
if self.github_login is None:
return None
self.github_about = get_profile(self.github_login)
try:
if not self.name:
self.name = self.github_about["name"]
if not self.email:
self.email = self.github_about["email"]
except KeyError:
# our github_about is an error object,
# it's got no info about the person in it.
return False
def set_impact(self):
self.impact = sum(
[pp.person_package_impact for pp in self.get_person_packages()])
return self.impact
def set_scores(self):
person_packages = self.get_person_packages()
self.pagerank_score = sum([
pp.person_package_pagerank_score for pp in person_packages
if pp.person_package_pagerank_score
])
self.num_downloads_score = sum(
[pp.person_package_num_downloads_score for pp in person_packages])
self.num_citations_score = sum(
[pp.person_package_num_citations_score for pp in person_packages])
def set_parsed_name(self):
if not self.name:
self.parsed_name = None
return
name = HumanName(self.name)
self.parsed_name = name.as_dict()
def _make_gravatar_url(self, size):
try:
if self.email is not None:
hash = hashlib.md5(self.email).hexdigest()
else:
hash = hashlib.md5("*****@*****.**").hexdigest()
except UnicodeEncodeError:
print "UnicodeEncodeError making gravatar url from email"
hash = 42
url = "http://www.gravatar.com/avatar/{hash}.jpg?s={size}&d=mm".format(
hash=hash, size=size)
return url
@property
def icon(self):
return self._make_gravatar_url(160)
@property
def icon_small(self):
return self._make_gravatar_url(30)
def has_role_on_project(self, role, package_id):
for c in self.contributions:
if c.role == role and c.package_id == package_id:
return True
return False
def num_commits_on_project(self, package_id):
for c in self.contributions:
if c.role == "github_contributor" and c.package_id == package_id:
return c.quantity
return False
@property
def single_name(self):
if self.is_organization:
return self.display_name
elif self.parsed_name and self.parsed_name["last"]:
return self.parsed_name["last"]
return self.display_name
@property
def display_name(self):
if self.name:
return self.name
elif self.github_login:
return self.github_login
elif self.email:
return self.email.split("@")[0]
else:
return "name unknown"
# could be a property, but kinda slow, so better as explicity method methinks
def get_person_packages(self):
person_packages = defaultdict(PersonPackage)
for contrib in self.contributions:
person_packages[contrib.package.id].set_role(contrib)
person_packages_list = person_packages.values()
person_packages_list.sort(key=lambda x: x.person_package_impact,
reverse=True)
return person_packages_list
def num_commits_on_project(self, package_id):
for c in self.contributions:
if c.role == "github_contributor" and c.package_id == package_id:
return c.quantity
return False
class Package(db.Model):
id = db.Column(db.Text, primary_key=True)
host = db.Column(db.Text)
project_name = db.Column(db.Text)
import_name = db.Column(db.Text)
unique_import_name = db.Column(db.Boolean)
setup_py_import_name = db.Column(db.Text)
github_owner = db.Column(db.Text)
github_repo_name = db.Column(db.Text)
github_api_raw = db.deferred(db.Column(JSONB))
github_contributors = db.deferred(db.Column(JSONB))
api_raw = db.deferred(db.Column(JSONB))
downloads = db.deferred(db.Column(MutableDict.as_mutable(JSONB)))
all_r_reverse_deps = db.deferred(db.Column(JSONB))
tags = db.Column(JSONB)
proxy_papers = db.deferred(db.Column(db.Text))
is_academic = db.Column(db.Boolean)
num_citations_by_source = db.Column(JSONB)
is_distinctive_name = db.Column(db.Boolean)
host_reverse_deps = db.deferred(db.Column(JSONB))
github_reverse_deps = db.deferred(db.Column(JSONB))
dependencies = db.deferred(db.Column(JSONB))
bucket = db.deferred(db.Column(MutableDict.as_mutable(JSONB)))
bucket2 = db.deferred(db.Column(MutableDict.as_mutable(JSONB)))
requires_files = db.deferred(db.Column(MutableDict.as_mutable(JSONB)))
setup_py = db.deferred(db.Column(db.Text))
setup_py_hash = db.deferred(db.Column(db.Text))
impact = db.Column(db.Float)
impact_rank = db.Column(db.Integer)
num_downloads = db.Column(db.Integer)
num_downloads_percentile = db.Column(db.Float)
num_downloads_score = db.Column(db.Float)
pagerank = db.Column(db.Float)
pagerank_percentile = db.Column(db.Float)
pagerank_score = db.Column(db.Float)
num_citations = db.Column(db.Integer)
num_citations_percentile = db.Column(db.Float)
num_citations_score = db.Column(db.Float)
neighborhood_size = db.Column(db.Float)
indegree = db.Column(db.Float)
eccentricity = db.Column(db.Float)
closeness = db.Column(db.Float)
betweenness = db.Column(db.Float)
eigenvector_centrality = db.Column(db.Float)
max_path_length = db.Column(db.Integer)
avg_path_length = db.Column(db.Float)
longest_path = db.Column(JSONB)
avg_outdegree_of_indegrees = db.Column(db.Float)
avg_pagerank_of_indegrees = db.Column(db.Float)
num_stars = db.Column(db.Integer)
summary = db.Column(db.Text)
num_committers = db.Column(db.Integer)
num_commits = db.Column(db.Integer)
num_authors = db.Column(db.Integer)
rev_deps_tree = db.Column(JSONB)
credit = db.Column(JSONB)
contributions = db.relationship(
'Contribution',
# lazy='subquery',
cascade="all, delete-orphan",
backref="package"
)
__mapper_args__ = {
'polymorphic_on': host,
'with_polymorphic': '*'
}
def __repr__(self):
return u'<Package {name}>'.format(
name=self.id)
@property
def language(self):
return "unknown"
def to_dict(self, full=True):
ret = {
"name": self.project_name,
"as_snippet": self.as_snippet,
"github_owner": self.github_owner,
"github_repo_name": self.github_repo_name,
"host": self.host,
"indegree": self.indegree,
"neighborhood_size": self.neighborhood_size,
"num_authors": self.num_authors,
"num_commits": self.num_commits,
"num_committers": self.num_committers,
"num_citations": self.num_citations,
"num_citations_score": self.num_citations_score,
"num_citations_percentile": self.num_citations_percentile,
"pagerank": self.pagerank,
"pagerank_score": self.pagerank_score,
"pagerank_percentile": self.pagerank_percentile,
"num_downloads": self.num_downloads,
"num_downloads_score": self.num_downloads_score,
"num_downloads_percentile": self.num_downloads_percentile,
"num_stars": self.num_stars,
"impact": self.impact,
"impact_rank": self.impact_rank,
"rev_deps_tree": self.tree,
"is_academic": self.is_academic,
# current implementation requires api_raw, so slows down db because deferred
# "source_url": self.source_url,
"summary": self.summary,
"tags": self.tags
}
return ret
@property
def tree(self):
return self.rev_deps_tree
@property
def as_snippet_without_people(self):
ret = {
"_host_url": self.host_url,
"name": self.project_name,
"language": self.language,
"is_academic": self.is_academic,
"impact": self.impact,
"impact_rank": self.impact_rank,
"impact_rank_max": self.impact_rank_max,
"pagerank_score": self.pagerank_score,
"num_downloads_score": self.num_downloads_score,
"num_citations_score": self.num_citations_score,
"pagerank_percentile": self.pagerank_percentile,
"num_downloads_percentile": self.num_downloads_percentile,
"num_citations_percentile": self.num_citations_percentile,
"pagerank": self.pagerank,
"num_downloads": self.num_downloads,
"num_citations": self.num_citations,
"subscores": self.subscores,
"is_academic": self.is_academic,
"summary": prep_summary(self.summary),
"tags": self.tags
}
return ret
@property
def subscores(self):
names = [
"num_downloads",
"pagerank",
"num_citations"
]
ret = []
for name in names:
if name == "pagerank":
score = self.pagerank_score
else:
score = getattr(self, name)
ret.append({
"name": name,
"score": getattr(self, name + "_score"),
"val": score
})
return ret
@property
def as_snippet(self):
ret = self.as_snippet_without_people
# ret["contributions"] = defaultdict(list)
# for c in self.contributions:
# ret["contributions"][c.role].append(u"{}: {}".format(
# c.percent, c.person.display_name))
# for role in ret["contributions"]:
# ret["contributions"][role].sort(reverse=True)
ret["contribs"] = []
if self.credit:
ret["num_contributors"] = len(self.credit)
top_five_people = sorted(self.credit, key=self.credit.get, reverse=True)[0:5]
for person_id in top_five_people:
person_id = int(person_id)
person = Person.query.get(person_id)
person_snippet = person.as_package_snippet
person_snippet["credit"] = self.credit[str(person_id)]
person_snippet["roles"] = []
for contrib in self.contributions:
if contrib.person_id == person_id:
person_snippet["roles"].append(contrib.role)
ret["contribs"].append(person_snippet)
return ret
@classmethod
def valid_package_names(cls, module_names):
"""
this will normally be called by subclasses, to filter by specific package hosts
"""
lowercase_module_names = [n.lower() for n in module_names]
q = db.session.query(cls.project_name)
q = q.filter(func.lower(cls.project_name).in_(lowercase_module_names))
response = [row[0] for row in q.all()]
return response
def test(self):
print "{}: I'm a test!".format(self)
def save_all_people(self):
self.save_github_owners_and_contributors()
self.save_host_contributors()
def save_github_owners_and_contributors(self):
self.save_github_contribs_to_db()
self.save_github_owner_to_db()
def save_host_contributors(self):
# this needs to be overridden, because it depends on whether we've
# got a pypi or cran package...they have diff metadata formats.
raise NotImplementedError
@property
def host_url(self):
# this needs to be overridden, because it depends on whether we've
# got a pypi or cran package
raise NotImplementedError
@property
def all_people(self):
people = list(set([c.person for c in self.contributions]))
return people
@property
def all_authors(self):
people = list(set([c.person for c in self.contributions if c.role=="author"]))
return people
@property
def all_github_owners(self):
people = list(set([c.person for c in self.contributions if c.role=="github_owner"]))
return people
def set_credit(self):
people = self.contributors_with_credit()
credit_dict = {}
for person in people:
credit_dict[int(person.id)] = person.credit
self.credit = credit_dict
def get_credit_for_person(self, person_id):
return self.credit[str(person_id)]
@property
def has_github_commits(self):
return self.max_github_commits > 0
@property
def max_github_commits(self):
if len(self.contributions) == 0:
return 0
all_commits = [c.quantity for c in self.contributions
if c.quantity and c.role=="github_contributor"]
if not all_commits:
return None
return max(all_commits)
def contributors_with_credit(self):
people_for_contributions = self.all_people
author_committers = []
non_author_committers = []
# if no github contributors, split cred equally among all authors
# if github contributors, give contributions tied with maximum github contribution
# by making them a virtual committer with that many commits
for person in people_for_contributions:
person.credit = 0 # initialize
if person.has_role_on_project("author", self.id):
if self.has_github_commits:
# print u"{} is an author committer".format(person.name)
author_committers += [person]
else:
# print u"{} is an author and there are no committers".format(person.name)
equal_author_share = float(1)/len(self.all_authors)
person.credit += equal_author_share
elif person.has_role_on_project("github_contributor", self.id):
# print u"{} is a non-author committer".format(person.name)
non_author_committers += [person]
# give all non-author committers their number of real commits
for person in non_author_committers:
person.num_counting_commits = person.num_commits_on_project(self.id)
# give all virtual committers the max number of commits
for person in author_committers:
person.num_counting_commits = self.max_github_commits
# calc how many commits were handed out, real + virtual
total_author_and_nonauthor_commits = 0
author_and_nonauthor_commmiters = non_author_committers + author_committers
for person in author_and_nonauthor_commmiters:
total_author_and_nonauthor_commits += person.num_counting_commits
# assign credit to be the fraction of commits they have out of total
# print "total_author_and_nonauthor_commits", total_author_and_nonauthor_commits
for person in author_and_nonauthor_commmiters:
person.credit = float(person.num_counting_commits) / total_author_and_nonauthor_commits
# print u"{} has with {} commits, {} credit".format(
# person.name, person.num_commits, person.credit)
# finally, handle github owners by giving them a little boost
for person in people_for_contributions:
if person.has_role_on_project("github_owner", self.id):
person.credit += 0.01
people_for_contributions.sort(key=lambda x: x.credit, reverse=True)
# for person in people_for_contributions:
# print u"{credit}: {name} has contribution".format(
# name = person.name,
# credit = round(person.credit, 3)
# )
return people_for_contributions
def save_github_contribs_to_db(self):
if isinstance(self.github_contributors, dict):
# it's an error resp from the API, doh.
return None
if self.github_contributors is None:
return None
total_contributions_count = sum([c['contributions'] for c in self.github_contributors])
for github_contrib in self.github_contributors:
person = get_or_make_person(github_login=github_contrib["login"])
percent_total_contribs = round(
github_contrib["contributions"] / float(total_contributions_count) * 100,
3
)
self._save_contribution(
person,
"github_contributor",
quantity=github_contrib["contributions"],
percent=percent_total_contribs
)
self.num_github_committers = len(self.github_contributors)
def save_github_owner_to_db(self):
if not self.github_owner:
return False
person = get_or_make_person(github_login=self.github_owner)
self._save_contribution(person, "github_owner")
def set_num_committers_and_commits(self):
if not self.set_github_contributors:
return None
try:
self.num_committers = len(self.github_contributors)
self.num_commits = sum([contrib["contributions"] for contrib in self.github_contributors])
except TypeError:
self.num_committers = 0
self.num_commits = 0
def _save_contribution(self, person, role, quantity=None, percent=None):
print u"saving contribution {} for {}".format(role, person)
extant_contrib = self.get_contribution(person.id, role)
if extant_contrib is None:
# make the new contrib.
# there's got to be a better way to make this args thing...
kwargs_dict = {
"role": role
}
if quantity is not None:
kwargs_dict["quantity"] = quantity
if percent is not None:
kwargs_dict["percent"] = percent
new_contrib = Contribution(**kwargs_dict)
# set the contrib in its various places.
self.contributions.append(new_contrib)
person.contributions.append(new_contrib)
db.session.merge(person)
def get_contribution(self, person_id, role):
for contrib in self.contributions:
if contrib.person.id == person_id and contrib.role == role:
return contrib
return None
def set_github_contributors(self):
self.github_contributors = github_api.get_repo_contributors(
self.github_owner,
self.github_repo_name
)
print "found github contributors", self.github_contributors
self.set_num_committers_and_commits()
def set_github_repo_id(self):
# override in subclass
raise NotImplementedError
def set_tags(self):
# override in subclass
raise NotImplementedError
def set_is_distinctive_name(self):
self.is_distinctive_name = False
if not self.bucket:
return
if not 'num_hits_raw' in self.bucket:
return
if self.bucket["num_hits_raw"] <= 0:
return
ratio = float(self.bucket["num_hits_with_language"])/self.bucket["num_hits_raw"]
print "distinctiveness ratio for {} is {}".format(
self.project_name, ratio)
if self.host == "cran":
if ratio > 0.20:
self.is_distinctive_name = True
print "IS DISTINCTIVE! going for it"
else:
if ratio > 0.27:
self.is_distinctive_name = True
print "IS DISTINCTIVE! going for it"
return
def get_sources_to_query(self):
# i bet there is a better way to do this!! :)
sources_to_query = [
full_text_source.Pmc
# ,
# full_text_source.Arxiv,
# full_text_source.Citeseer,
]
return sources_to_query
@property
def citations_dict(self):
citations_dict = self.set_num_citations_by_source()
response_dict = defaultdict(dict)
for source_class in self.get_sources_to_query():
source = source_class()
response_dict[source.name] = {
"count": citations_dict[source.name],
"url": source.query_url(self.full_text_query)
}
return response_dict
@property
def full_text_query(self):
queries = []
# add the cran or pypi url to start with
host_url = self.host_url.replace("https://", "").replace("http://", "")
queries.append('"{}"'.format(host_url))
# then github url if we know it
if self.github_owner:
github_url = '"github.com/{}/{}"'.format(self.github_owner, self.github_repo_name)
queries.append(github_url)
# also look up its project name if is unique
# this line here now because haven't run them all previously
self.set_is_distinctive_name()
if self.is_distinctive_name:
queries.append('"{}"'.format(self.project_name))
else:
print "{} isn't a rare package name, so not looking up by name".format(self.project_name)
query = " OR ".join(queries)
query += ' NOT AUTH:"{}"'.format(self.project_name)
print "query", query
return query
def set_num_citations_by_source(self):
if not self.num_citations_by_source:
self.num_citations_by_source = {}
self.num_citations = 0
for source_class in self.get_sources_to_query():
if source_class.__name__ == "Pmc" and ("-" in self.project_name):
# pmc can't do a query if a hyphen in the name, so just bail
self.num_citations = 0
return {}
source = source_class()
num_found = source.run_query(self.full_text_query)
self.num_citations_by_source[source.name] = num_found
self.num_citations += num_found
print "num citations found", num_found
return self.num_citations_by_source
def set_distinctiveness(self):
source = full_text_source.Citeseer()
self.bucket2 = {}
raw_query = '"{name}"'.format(
name=self.project_name)
# raw_query = '"{name}" NOT AUTH:"{name}"'.format(
# name=self.project_name)
num_hits_raw = source.run_query(raw_query)
self.bucket2["num_hits_raw"] = num_hits_raw
num_hits_with_language = source.run_query(self.distinctiveness_query)
self.bucket2["num_hits_with_language"] = num_hits_with_language
if self.bucket2["num_hits_raw"] > 0:
ratio = float(self.bucket2["num_hits_with_language"])/self.bucket2["num_hits_raw"]
else:
ratio = None
print "{}: solo search finds {}, ratio is {}".format(
self.project_name,
self.bucket2["num_hits_raw"],
ratio
)
def set_igraph_data(self, our_igraph_data):
try:
self.pagerank = our_igraph_data[self.project_name]["pagerank"]
self.neighborhood_size = our_igraph_data[self.project_name]["neighborhood_size"]
self.indegree = our_igraph_data[self.project_name]["indegree"]
self.eccentricity = our_igraph_data[self.project_name]["eccentricity"]
self.closeness = our_igraph_data[self.project_name]["closeness"]
self.betweenness = our_igraph_data[self.project_name]["betweenness"]
self.eigenvector_centrality = our_igraph_data[self.project_name]["eigenvector_centrality"]
self.longest_path = our_igraph_data[self.project_name]["longest_path"]
self.max_path_length = our_igraph_data[self.project_name]["max_path_length"]
self.avg_path_length = our_igraph_data[self.project_name]["avg_path_length"]
self.avg_outdegree_of_indegrees = our_igraph_data[self.project_name]["avg_outdegree_of_indegrees"]
self.avg_pagerank_of_indegrees = our_igraph_data[self.project_name]["avg_pagerank_of_indegrees"]
print "pagerank of {} is {}".format(self.project_name, self.pagerank)
except KeyError:
print "network params for {} were not calculated".format(self.project_name)
self.pagerank = None
self.neighborhood_size = None
self.indegree = None
self.eccentricity = None
self.closeness = None
self.betweenness = None
self.eigenvector_centrality = None
self.longest_path = None
self.max_path_length = None
self.avg_path_length = None
self.avg_outdegree_of_indegrees = None
self.avg_pagerank_of_indegrees = None
def refresh_github_ids(self):
if not self.github_owner:
return None
self.github_api_raw = github_api.get_repo_data(self.github_owner, self.github_repo_name)
try:
(self.github_owner, self.github_repo_name) = self.github_api_raw["full_name"].split("/")
except KeyError:
self.github_owner = None
self.github_repo_name = None
@classmethod
def shortcut_percentile_refsets(cls):
print "getting the percentile refsets...."
ref_list = defaultdict(dict)
q = db.session.query(
cls.num_downloads,
cls.pagerank,
cls.num_citations
)
rows = q.all()
ref_list["num_downloads"] = sorted([row[0] for row in rows if row[0] != None])
ref_list["pagerank"] = sorted([row[1] for row in rows if row[1] != None])
ref_list["num_citations"] = sorted([row[2] for row in rows if row[2] != None])
return ref_list
def _calc_percentile(self, refset, value):
if value is None: # distinguish between that and zero
return None
matching_index = refset.index(value)
percentile = float(matching_index) / len(refset)
return percentile
def set_num_downloads_percentile(self, refset):
self.num_downloads_percentile = self._calc_percentile(refset, self.num_downloads)
def set_pagerank_percentile(self, refset):
self.pagerank_percentile = self._calc_percentile(refset, self.pagerank)
def set_num_citations_percentile(self, refset):
self.num_citations_percentile = self._calc_percentile(refset, self.num_citations)
def set_all_percentiles(self, refsets_dict):
self.set_num_downloads_percentile(refsets_dict["num_downloads"])
self.set_pagerank_percentile(refsets_dict["pagerank"])
self.set_num_citations_percentile(refsets_dict["num_citations"])
@classmethod
def _shortcut_rank(cls, name_to_rank_by):
print "getting the lookup for ranking impact...."
property_to_rank_by = getattr(cls, name_to_rank_by)
q = db.session.query(cls.id)
q = q.order_by(property_to_rank_by.desc()) # the important part :)
rows = q.all()
impact_rank_lookup = {}
ids_sorted = [row[0] for row in rows]
for my_id in ids_sorted:
zero_based_rank = ids_sorted.index(my_id)
impact_rank_lookup[my_id] = zero_based_rank + 1
return impact_rank_lookup
@classmethod
def shortcut_impact_rank(cls):
return cls._shortcut_rank("impact")
def set_impact_rank(self, impact_rank_lookup):
self.impact_rank = impact_rank_lookup[self.id]
print "self.impact_rank", self.impact_rank
@classmethod
def shortcut_num_citations_rank(cls):
print "getting the lookup for num_citations ranking shortcut...."
q = db.session.query(cls.id)
q = q.order_by(cls.num_citations.desc()) # the important part :)
rows = q.all()
impact_rank_lookup = {}
ids_sorted_by_impact = [row[0] for row in rows]
for my_id in ids_sorted_by_impact:
zero_based_rank = ids_sorted_by_impact.index(my_id)
impact_rank_lookup[my_id] = zero_based_rank + 1
return impact_rank_lookup
def set_impact_rank(self, impact_rank_lookup):
self.impact_rank = impact_rank_lookup[self.id]
print "self.impact_rank", self.impact_rank
@property
def pagerank_score_multiplier(self):
return self.pagerank_score
@property
def num_downloads_score_multiplier(self):
return 1000.0/self.num_downloads_offset_to_recenter_scores # makes it out of 1000
@property
def num_citations_score_multiplier(self):
return 1000.0/self.num_citations_offset_to_recenter_scores # makes it out of 1000
def set_pagerank_score(self):
if not self.pagerank:
self.pagerank_score = None
return self.pagerank_score
try:
raw = math.log10(float(self.pagerank)/self.pagerank_max)
adjusted = (raw + self.pagerank_offset_to_recenter_scores) * self.pagerank_score_multiplier
except ValueError:
adjusted = None
self.pagerank_score = adjusted
return self.pagerank_score
def set_num_downloads_score(self):
if not self.num_downloads:
self.num_downloads_score = None
return self.num_downloads_score
try:
raw = math.log10(float(self.num_downloads)/self.num_downloads_max)
adjusted = (raw + self.num_downloads_offset_to_recenter_scores) * self.num_downloads_score_multiplier
except ValueError:
adjusted = None
self.num_downloads_score = adjusted
return self.num_downloads_score
def set_num_citations_score(self):
if not self.num_citations:
self.num_citations_score = 0
return self.num_citations_score
try:
raw = math.log10(float(self.num_citations)/self.num_citations_max)
adjusted = (raw + self.num_citations_offset_to_recenter_scores) * self.num_citations_score_multiplier
except ValueError:
adjusted = None
self.num_citations_score = adjusted
return self.num_citations_score
def set_impact(self):
score_components = []
pagerank_score = self.set_pagerank_score()
if pagerank_score != None:
score_components.append(pagerank_score)
num_downloads_score = self.set_num_downloads_score()
if num_downloads_score != None:
score_components.append(num_downloads_score)
if score_components:
pagerank_and_downloads_mean = numpy.mean(score_components)
else:
pagerank_and_downloads_mean = None
num_citations_score = self.set_num_citations_score()
# twice the mean and twenty percent of the scaled citations
combo = (pagerank_and_downloads_mean*2 + num_citations_score*0.2) / 2.2
self.impact = combo
print u"self.impact for {} is {}".format(self.id, self.impact)
@classmethod
def shortcut_rev_deps_pairs(cls):
NUM_TOP_NODES = 1000
command = """select package,
used_by,
pagerank,
(coalesce((github_repo.api_raw->>'stargazers_count')::int, 0)
+ coalesce(package.num_stars, 0)) as num_stars
from dep_nodes_ncol_{host}_reverse
left outer join github_repo
on dep_nodes_ncol_{host}_reverse.used_by = 'github:' || github_repo.id
left outer join package
on dep_nodes_ncol_{host}_reverse.used_by = package.project_name""".format(
host=cls.class_host)
rev_deps_by_package = defaultdict(list)
res = db.session.connection().execute(sql.text(command))
rows = res.fetchall()
non_zero_pageranks = [row[2] for row in rows if row[2]]
min_pagerank = min(non_zero_pageranks)
for row in rows:
my_name = row[0]
child_name = row[1]
child_pagerank = row[2]
child_stars = row[3]
if not child_pagerank:
child_pagerank = min_pagerank
rev_deps_by_package[my_name].append([
child_name,
child_pagerank,
child_stars
])
return rev_deps_by_package
def set_rev_deps_tree(self, rev_deps_lookup):
node = RevDepNode(
parent=None,
name=self.project_name,
pagerank=self.pagerank,
generation=0,
stars=None,
root_pagerank=self.pagerank
)
node.is_root = True
node.set_children(rev_deps_lookup)
self.rev_deps_tree = node.to_dict()
class BookCatalog(db.Model):
__table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
__tablename__ = "catalog"
id = db.Column(db.Integer, primary_key=True, nullable=False)
title = db.Column(db.String(255), default="", nullable=False, index=True)
markdown = db.deferred(db.Column(LONGTEXT, default="", nullable=False))
html = db.deferred(db.Column(LONGTEXT, default="", nullable=False))
# 有些字段内容比较多而且不经常使用，比如是一个新闻的内容字段，
# 在列表页或者封面页我们完全不必读取这个大的字段到内存里，这时候我们就可以设定这个列为 deferred
publish_markdown = db.deferred(
db.Column(LONGTEXT, default='', nullable=False))
publish_html = db.deferred(db.Column(LONGTEXT, default='', nullable=False))
status = db.Column(db.Integer, default=0, nullable=True, index=True)
abstract = db.deferred(db.Column(db.String(255), default=""))
publish_order = db.Column(db.Integer, default=0, nullable=True, index=True)
pos = db.Column(db.Integer, default=0, nullable=False, index=True)
parent_id = db.Column(db.Integer, default=0, nullable=False, index=True)
is_dir = db.Column(db.Boolean, default=False, nullable=False, index=True)
publish_timestamp = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
first_publish = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
updatetime = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
timestamp = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
book_id = db.Column(db.Integer,
db.ForeignKey(Book.__tablename__ + ".id",
ondelete="CASCADE",
onupdate="CASCADE"),
nullable=False)
@staticmethod
def add(book_id, title, parent_id=0, is_dir=False):
parent_id = 0 if parent_id is None else int(parent_id)
if parent_id != 0:
if not BookCatalog.get(parent_id):
return
catalog = BookCatalog(
title=title,
is_dir=is_dir,
book_id=book_id,
)
if parent_id:
catalog.parent_id = parent_id
catalog.pos = BookCatalog.max_pos(book_id, parent_id) + 1
db.session.add(catalog)
db.session.commit()
catalog.book.updatetime = datetime.now()
return catalog
@staticmethod
def get(id):
return BookCatalog.query.filter_by(id=id).first()
@staticmethod
def get_deep(id=None):
""" 返回目录深度 """
if not id:
return 1
else:
catalog = BookCatalog.query.filter_by(id=id).first()
if not catalog:
return 0
return BookCatalog.get_deep(catalog.parent_id) + 1
@staticmethod
def reader(book_id, id=None):
catalog = BookCatalog.query.filter_by(book_id=book_id) \
.options(db.undefer("publish_html"))
if id:
catalog = catalog.filter_by(id=id)
else:
catalog = catalog.filter_by(parent_id=0) \
.order_by(BookCatalog.pos)
return catalog.first()
@staticmethod
def page(page, per_page):
last_catalog = Book.create_subquery("last_catalog")
catalogs = db.session.query(Book, BookCatalog) \
.outerjoin(last_catalog, last_catalog.c.id == Book.id) \
.outerjoin(BookCatalog, db.and_(
BookCatalog.book_id == Book.id,
BookCatalog.publish_order == last_catalog.c.order)) \
.options(
db.Load(Book).undefer("brief"),
db.Load(BookCatalog).undefer("abstract")) \
.filter(db.and_(
Book.access == 1,
last_catalog.c.order.isnot(None))) \
.order_by(BookCatalog.first_publish.desc())
return paginate(catalogs, page, per_page=per_page, error_out=False)
@staticmethod
def max_pos(book_id, id=0):
if id:
catalogs = BookCatalog.query.filter_by(parent_id=id).filter_by(
book_id=book_id).all()
else:
catalogs = BookCatalog.query.filter(
db.and_(BookCatalog.book_id == book_id,
BookCatalog.parent_id == 0)).all()
if catalogs:
return max([catalog.pos for catalog in catalogs])
return 0
@staticmethod
def max_order(book_id):
catalog = BookCatalog.query.filter(BookCatalog.book_id == book_id) \
.order_by(BookCatalog.publish_order.desc()).first()
return 0 if catalog is None else catalog.publish_order
@staticmethod
def prev(catalog):
prev_catalog = BookCatalog.query.with_entities(
BookCatalog.id,
BookCatalog.parent_id,
BookCatalog.is_dir) \
.filter(db.and_(
BookCatalog.book_id == catalog.book_id,
BookCatalog.pos < catalog.pos,
BookCatalog.parent_id == catalog.parent_id)) \
.order_by(BookCatalog.pos.desc()).first()
if prev_catalog and prev_catalog.is_dir:
child = BookCatalog.last_child(prev_catalog)
if child:
return child
if prev_catalog or not catalog.parent_id:
return prev_catalog
return BookCatalog.query.with_entities(BookCatalog.id) \
.filter_by(id=catalog.parent_id).first()
@staticmethod
def next(catalog, is_first=True):
if catalog:
if catalog.is_dir and is_first:
next_catalog = BookCatalog.first_child(catalog)
if next_catalog:
return next_catalog
next_catalog = BookCatalog.query.with_entities(BookCatalog.id,
BookCatalog.parent_id, BookCatalog.is_dir).filter(db.and_(
BookCatalog.book_id == catalog.book_id,
BookCatalog.pos > catalog.pos,
BookCatalog.parent_id == catalog.parent_id)) \
.order_by(BookCatalog.pos).first()
if next_catalog or not catalog.parent_id:
return next_catalog
catalog = BookCatalog.query.with_entities(
BookCatalog.id,
BookCatalog.parent_id,
BookCatalog.is_dir,
BookCatalog.pos,
BookCatalog.book_id) \
.filter_by(id=catalog.parent_id).first()
return BookCatalog.next(catalog, False)
else:
next_catalog = BookCatalog.query.with_entities(BookCatalog.id) \
.filter(db.and_(
BookCatalog.book_id == catalog.book_id,
BookCatalog.parent_id == 0)) \
.order_by(BookCatalog.pos).first()
return next_catalog
@staticmethod
def first_child(catalog):
return BookCatalog.query.with_entities(BookCatalog.id,
BookCatalog.parent_id, BookCatalog.title, BookCatalog.is_dir) \
.filter_by(parent_id=catalog.book_id) \
.order_by(BookCatalog.pos).first()
@staticmethod
def last_child(catalog):
return BookCatalog.query.with_entities(BookCatalog.id,
BookCatalog.parent_id, BookCatalog.title, BookCatalog.id) \
.filter_by(parent_id=catalog.id) \
.order_by(BookCatalog.pos.desc()).first()
def set_abstract(self):
soup = BeautifulSoup(self.publish_html, "html.parser")
self.abstract = soup.text[:110] + u"..."
def sort(self, next_id=None):
if not next_id:
self.pos = BookCatalog.max_pos(self.book_id) + 1
self.parent_id = 0
else:
next = BookCatalog.get(next_id)
if not next:
return False
self.pos = next.pos
self.parent_id = next.parent_id
nexts = BookCatalog.query.filter(
db.and_(BookCatalog.book_id == self.book_id,
BookCatalog.parent_id == next.parent_id,
BookCatalog.pos > next.pos)).all()
for _next in nexts:
_next.pos += 1
next.pos += 1
self.updatetime = datetime.now()
self.book.updatetime = self.updatetime
return True
def rename(self, title):
self.title = title
self.updatetime = datetime.now()
self.book.updatetime = self.updatetime
def save(self, markdown, html):
self.markdown = markdown
self.html = html
self.updatetime = datetime.now()
self.book.updatetime = self.updatetime
def delete(self):
catalogs = BookCatalog.query.filter_by(parent_id=self.id).all()
for catalog in catalogs:
catalog.delete()
self.book.updatetime = datetime.now()
db.session.delete(self)
class Book(db.Model):
__table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
__tablename__ = "book"
id = db.Column(db.Integer, primary_key=True, nullable=False)
name = db.Column(db.String(255), default="", nullable=False, index=True)
access = db.Column(db.Integer, default=1, nullable=False, index=True)
status = db.Column(db.Integer, default=0, nullable=False,
index=True) # publish status
brief = db.deferred(db.Column(db.Text, default="", nullable=False)) #简介
select_catalog = db.Column(db.Integer, default=0, nullable=False)
publish_timestamp = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
updatetime = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
timestamp = db.Column(db.DateTime,
default=datetime.now,
nullable=False,
index=True)
cover = db.Column(db.String(255), default="", nullable=False)
user_id = db.Column(db.Integer,
db.ForeignKey(User.__tablename__ + ".id",
ondelete="CASCADE",
onupdate="CASCADE"),
nullable=False)
catalogs = db.relationship("BookCatalog",
backref="book",
lazy="dynamic",
passive_deletes=True)
images = db.relationship("BookImage",
backref="book",
lazy="select",
passive_deletes=True)
@staticmethod
def add(name, brief, access, user_id):
book = Book(name=name, brief=brief, access=access, user_id=user_id)
db.session.add(book)
db.session.commit()
return book
@staticmethod
def get(id):
return Book.query.filter_by(id=id).first()
@staticmethod
def create_subquery(sub_type, user_id=None):
if sub_type == "last_catalog":
return db.session.query(
Book.id,
func.max(BookCatalog.publish_order).label("order")).outerjoin(
BookCatalog,
db.and_(BookCatalog.book_id == Book.id,
BookCatalog.status == 1)).group_by(
Book.id).subquery("order")
@staticmethod
def page(page, per_page):
books = Book.query.options(db.Load(Book).undefer("brief")) \
.order_by(Book.timestamp.desc()) \
.paginate(page, per_page=per_page, error_out=False)
return books
@staticmethod
def info(id):
return Book.query.filter_by(id=id) \
.options(db.undefer("brief")).first()
def setting(self, name, brief, access):
self.name = name
self.brief = brief
self.access = access
def _deep_catalogs(self, catalogs, catalog_dict):
for catalog in catalogs:
catalog.catalogs = self._deep_catalogs(
self._sort_catalogs(catalog_dict.get(catalog.id, [])),
catalog_dict)
return catalogs
def _sort_catalogs(self, catalogs):
return sorted(catalogs, key=lambda x: x.pos)
def tree_catalogs(self):
catalogs = self.catalogs.options(
load_only("title", "parent_id", "pos", "is_dir", "book_id")).all()
catalog_dict = {}
for catalog in catalogs:
if catalog.parent_id not in catalog_dict:
catalog_dict[catalog.parent_id] = []
catalog_dict[catalog.parent_id].append(catalog)
return self._deep_catalogs(
self._sort_catalogs(catalog_dict.get(0, [])), catalog_dict)
def origin_cover(self):
image_path = current_app.config["BOOK_COVER_PATH"]
return "/".join([image_path, self.cover])
def thumbnail_cover(self):
image_path = current_app.config["BOOK_COVER_PATH"]
return "/".join([image_path, "thumbnail_{}".format(self.cover)])
def publish(self):
now = datetime.now()
self.catalog_publish(now)
self.status = 1
self.publish_timestamp = now
def catalog_publish(self, date):
catalogs = BookCatalog.query.filter(BookCatalog.book_id == self.id) \
.filter(BookCatalog.updatetime > self.publish_timestamp).options(
db.undefer("markdown").undefer("html") \
.undefer("publish_markdown").undefer("publish_html")
).all()
max_order = BookCatalog.max_order(self.id)
max_catalog = None
for catalog in catalogs:
catalog.publish_markdown = catalog.markdown
catalog.publish_html = catalog.html
catalog.publish_timestamp = date
if len(catalog.publish_markdown) > 10:
catalog.set_abstract()
if catalog.first_publish == catalog.timestamp:
catalog.first_publish = date
if not catalog.publish_order:
max_order += 1
catalog.publish_order = max_order
catalog.status = 1
else:
catalog.status = 0
def delete_cover(self):
if not self.cover:
return
file.delete_cover(self.cover)
def delete(self):
for image in self.images:
image.delete()
for catalog in self.catalogs:
catalog.delete()
self.delete_cover()
db.session.delete(self)
class Item(db.Model):
__tablename__ = 'items'
id = db.Column(db.Integer, primary_key=True)
itemtype = db.Column(db.String(128))
platform = db.Column(db.String(128))
title = db.Column(db.String(128))
author = db.Column(db.String(128))
publisher = db.Column(db.String(64))
image = db.Column(db.String(128))
pubdate = db.Column(db.String(32))
price = db.Column(db.String(16))
summary = db.deferred(db.Column(db.Text, default=""))
summary_html = db.deferred(db.Column(db.Text))
hidden = db.Column(db.Integer, default=0)
amount = db.Column(db.Integer)
logs = db.relationship('Log',
backref=db.backref('item', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
comments = db.relationship('Comment', backref='item',
lazy='dynamic',
cascade='all, delete-orphan')
@property
def tags_string(self):
return ",".join([tag.name for tag in self.tags.all()])
@tags_string.setter
def tags_string(self, value):
self.tags = []
tags_list = value.split(u',')
for str in tags_list:
tag = Tag.query.filter(Tag.name.ilike(str)).first()
if tag is None:
tag = Tag(name=str)
self.tags.append(tag)
db.session.add(self)
db.session.commit()
def can_borrow(self,user_id,item_id):
if not self.hidden and self.can_borrow_number() > 0 and Cart.query.filter_by(user_id=user_id,item_id=item_id).first() == None and Log.query.filter_by(user_id=user_id,item_id=item_id,returned=0).first() == None:
return True
else:
return False
#return ((not self.hidden) and self.can_borrow_number() > 0) or Cart.query.filter_by(user_id=user_id,item_id=item_id) != None
def can_borrow_number(self):
return self.amount - Log.query.filter_by(item_id=self.id, returned=0).count()
@staticmethod
def on_changed_summary(target, value, oldvalue, initiaor):
allowed_tags = ['a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i',
'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p']
target.summary_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags, strip=True))
def __repr__(self):
return u'<Item %r>' % self.title
class User(UserMixin, db.Model):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(64), unique=True)
name = db.Column(db.String(64))
password_hash = db.deferred(db.Column(db.String(128)))
major = db.Column(db.String(128))
role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))
headline = db.Column(db.String(32), nullable=True)
about_me = db.deferred(db.Column(db.Text, nullable=True))
about_me_html = db.deferred(db.Column(db.Text, nullable=True))
avatar = db.Column(db.String(128))
member_since = db.Column(db.DateTime(), default=datetime.utcnow)
@property
def password(self):
raise AttributeError('password is not readable attribute')
@password.setter
def password(self, password):
self.password_hash = generate_password_hash(password)
def verify_password(self, password):
return check_password_hash(self.password_hash, password)
def __init__(self, **kwargs):
super(User, self).__init__(**kwargs)
if self.role is None:
if self.email.lower() == current_app.config['FLASKY_ADMIN'].lower():
self.role = Role.query.filter_by(permissions=0x1ff).first()
if self.role is None:
self.role = Role.query.filter_by(rdefault=True).first()
self.member_since = datetime.now()
def can(self, permissions):
return self.role is not None and \
(self.role.permissions & permissions) == permissions
def is_administrator(self):
return self.can(Permission.ADMINISTER)
logs = db.relationship('Log',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
comments = db.relationship('Comment',
backref=db.backref('user', lazy='joined'),
lazy='dynamic',
cascade='all, delete-orphan')
def __repr__(self):
return '<User %r>' % self.email
def borrowing(self, item):
return self.logs.filter_by(item_id=item.id, returned=0).first()
def can_borrow_item(self):
return self.logs.filter(Log.returned == 0, Log.return_timestamp < datetime.now()).count() == 0
def borrow_item(self, item):
if self.logs.filter(Log.returned == 0, Log.return_timestamp < datetime.now()).count() > 0:
return False, u"Nie można wypożyczyć,zaległe książki nie zostały zwrócone!"
if self.borrowing(item):
return False, u'Wygląda na to, że już wypożyczyłeś tą książkę!!'
if not item.can_borrow(id,item.id):
return False, u'Brak egzemplarzy do wypożyczenia!'
db.session.add(Log(self, item))
return True, u'Wypożyczono pomyślnie %s' % item.title
def return_item(self, log):
if log.returned == 1 or (self.id != 1 and log.user_id != self.id):
print(log.returned)
print(log.user_id)
print("user"+str(self.id))
return False, u'Nie znaleziono takiego wypożyczenia!'
log.returned = 1
log.return_timestamp = datetime.now()
db.session.add(log)
db.session.commit()
return True, u'Zwrot zakończony pomyślnie %s' % log.item.title
def avatar_url(self, _external=False):
if self.avatar:
avatar_json = json.loads(self.avatar)
if avatar_json['use_out_url']:
return avatar_json['url']
else:
return url_for('_uploads.uploaded_file', setname=avatars.name, filename=avatar_json['url'],
_external=_external)
else:
return url_for('static', filename='img/toad.png', _external=_external)
@staticmethod
def on_changed_about_me(target, value, oldvalue, initiaor):
allowed_tags = ['a', 'abbr', 'acronym', 'b', 'blockquate', 'code', 'em', 'i',
'li', 'ol', 'pre', 'strong', 'ul', 'h1', 'h2', 'h3', 'p']
target.about_me_html = bleach.linkify(
bleach.clean(markdown(value, output_format='html'),
tags=allowed_tags, strip=True))
