class Package(db.Model, ModelMixin):
__tablename__ = 'packages'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(200), unique=True)
latest_version = db.Column(db.String(20))
updated_at = db.Column(db.DateTime)
last_check = db.Column(db.DateTime)
repos = association_proxy('requirements', 'repo')
pypi = xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
def __init__(self, name):
self.name = name.lower()
def __repr__(self):
return "<Package %s>" % self.name
@property
def url(self):
return "https://pypi.python.org/pypi/%s" % self.original_name
@classmethod
def update_all_packages(cls):
packages = cls.query.filter(
or_(cls.last_check <= datetime.utcnow() - timedelta(days=1),
cls.last_check == None)).all()
for package in packages:
with ignored(Exception):
package.update_from_pypi()
db.session.commit()
@classmethod
@cache.cached(timeout=3600, key_prefix='all_packages')
def get_all_names(cls):
packages = cls.pypi.list_packages()
packages = filter(None, packages)
return {name.lower(): name for name in packages}
@property
def original_name(self):
return self.get_all_names()[self.name.lower()]
def find_latest_version(self):
version = self.pypi.package_releases(self.original_name)[0]
logger.info("Latest version of %s is %s", self.original_name, version)
return version
def update_from_pypi(self):
"""
Updates the latest version of the package by asking PyPI.
"""
latest = self.find_latest_version()
self.last_check = datetime.utcnow()
if self.latest_version != latest:
self.latest_version = latest
self.updated_at = datetime.utcnow()
class Requirement(db.Model, ModelMixin):
__tablename__ = 'requirements'
repo_id = db.Column(db.Integer, db.ForeignKey(Repo.id), primary_key=True)
package_id = db.Column(db.Integer,
db.ForeignKey(Package.id),
primary_key=True)
specs = db.Column(JSONType())
package = db.relationship(Package,
backref=db.backref('requirements',
cascade='all, delete-orphan'))
repo = db.relationship(Repo,
backref=db.backref('requirements',
cascade="all, delete-orphan"))
def __init__(self, repo, package, specs=None):
self.repo = repo
self.package = package
self.specs = specs
def __repr__(self):
return "<Requirement: %s requires %s with %s>" % (
self.repo.name, self.package.name, self.specs)
@property
def required_version(self):
logger.debug("Finding version of %s", self)
for specifier, version in self.specs:
logger.debug("specifier: %s, version: %s", specifier, version)
if specifier == '==':
return version
@property
def up_to_date(self):
latest_version = self.package.latest_version
if not latest_version:
raise Exception("Latest version of the package is unknown.")
try:
return Version(self.required_version) == Version(latest_version)
except IrrationalVersionError:
return poor_mans_version_compare(self.required_version,
latest_version)
class User(db.Model, ModelMixin):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(200))
email = db.Column(db.String(200), nullable=False)
github_id = db.Column(db.Integer, unique=True)
github_token = db.Column(db.Integer, unique=True)
email_sent_at = db.Column(db.DateTime)
repos = db.relationship('Repo',
backref='user',
cascade="all, delete-orphan")
def __init__(self, github_token):
self.github_token = github_token
def __repr__(self):
return "<User %s>" % self.name
def get_outdated_requirements(self):
outdateds = []
for repo in self.repos:
for req in repo.requirements:
if not req.up_to_date:
logger.debug("%s is outdated", req)
outdateds.append(req)
return outdateds
@classmethod
def send_emails(cls):
users = cls.query.filter(
or_(cls.email_sent_at <= datetime.utcnow() - timedelta(days=7),
cls.email_sent_at == None)).all()
for user in users:
with ignored(Exception):
logger.info(user)
user.send_email()
user.email_sent_at = datetime.utcnow()
db.session.commit()
def send_email(self):
outdateds = self.get_outdated_requirements()
if outdateds:
html = render_template('email.html', reqs=outdateds)
message = pystmark.Message(
sender='*****@*****.**',
to=self.email,
subject="There are updated packages in PyPI",
html=html)
response = pystmark.send(message,
current_app.config['POSTMARK_APIKEY'])
response.raise_for_status()
else:
logger.info("No outdated requirement.")
def get_emails_from_github(self):
params = {'access_token': self.github_token}
headers = {'Accept': 'application/vnd.github.v3'}
emails = github.get('user/emails', params=params, headers=headers)
return [e for e in emails if e['verified']]
class Repo(db.Model, ModelMixin):
__tablename__ = 'repos'
__table_args__ = (UniqueConstraint('user_id', 'github_id'), )
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey(User.id))
github_id = db.Column(db.Integer)
name = db.Column(db.String(200))
last_check = db.Column(db.DateTime)
last_modified = db.Column(db.String(40))
packages = association_proxy('requirements', 'package')
def __init__(self, github_id, user):
self.github_id = github_id
self.user = user
def __repr__(self):
return "<Repo %s>" % self.name
@property
def url(self):
return "https://github.com/%s" % self.name
@classmethod
def update_all_repos(cls):
repos = cls.query.all()
for repo in repos:
with ignored(Exception):
repo.update_requirements()
db.session.commit()
def update_requirements(self):
"""
Fetches the content of the requirements.txt files from GitHub,
parses the file and adds each requirement to the repo.
"""
for project_name, specs in self.parse_requirements_file():
# specs may be empty list if no version is specified in file
# No need to add to table since we can't check updates.
if specs:
# There must be '==' operator in specs.
operators = [s[0] for s in specs]
if '==' in operators:
# If the project is not registered on PyPI,
# we are not adding it.
if project_name.lower() in Package.get_all_names():
self.add_new_requirement(project_name, specs)
self.last_check = datetime.utcnow()
def add_new_requirement(self, name, specs):
from pypi_notifier.models.requirement import Requirement
package = Package.get_or_create(name=name)
requirement = Requirement.get_or_create(repo=self, package=package)
requirement.specs = specs
self.requirements.append = requirement
def parse_requirements_file(self):
contents = self.fetch_requirements()
if contents:
contents = strip_requirements(contents)
if contents:
for req in parse_requirements(contents):
yield req.project_name.lower(), req.specs
def fetch_requirements(self):
logger.info("Fetching requirements of repo: %s", self)
path = 'repos/%s/contents/requirements.txt' % self.name
headers = None
if self.last_modified:
headers = {'If-Modified-Since': self.last_modified}
params = {'access_token': self.user.github_token}
response = github.raw_request('GET',
path,
headers=headers,
params=params)
logger.debug("Response: %s", response)
if response.status_code == 200:
self.last_modified = response.headers['Last-Modified']
response = response.json()
if response['encoding'] == 'base64':
return base64.b64decode(response['content'])
else:
raise Exception("Unknown encoding: %s" % response['encoding'])
elif response.status_code == 304: # Not modified
return None
elif response.status_code == 401:
# User's token is not valid. Let's delete the user.
db.session.delete(self.user)
elif response.status_code == 404:
# requirements.txt file is not found.
# Remove the repo so we won't check it again.
db.session.delete(self)
else:
raise Exception("Unknown status code: %s" % response.status_code)