class Job(sqlobject.SQLObject):
status = sqlobject.IntCol(default=JobStatus.UNKNOWN)
package = sqlobject.ForeignKey('Package', cascade=True)
dist = sqlobject.StringCol(default='unreleased')
arch = sqlobject.StringCol(default='any')
creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now)
build_host = sqlobject.StringCol(default=None)
status_changed = sqlobject.DateTimeCol(default=None)
build_start = sqlobject.DateTimeCol(default=None)
build_end = sqlobject.DateTimeCol(default=None)
def __init__(self, *args, **kwargs):
sqlobject.SQLObject.__init__(self, *args, **kwargs)
self.status_lock = status_lock
def __setattr__(self, name, value):
if name == "status":
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
def dict(self):
result = {
"id": self.id,
"task": "%s/%s" % (self.dist, self.arch),
"build_host": self.build_host,
"status": JobStatus.whatis(self.status),
"dist": self.dist,
"arch": self.arch,
"creation_date": try_strftime(self.creation_date),
"build_start": try_strftime(self.build_start),
"build_end": try_strftime(self.build_end),
"status_changed": try_strftime(self.status_changed)
}
return result
def start(self, slave, builder):
if self.status != JobStatus.WAIT_LOCKED:
raise ValueError("JobStatus is not WAIT_LOCKED")
else:
self.status = JobStatus.BUILDING
self.build_start = sqlobject.DateTimeCol.now()
self.build_host = slave.name
kwargs = self.dict()
kwargs.update(self.package.dict())
return slave.build(self.id, builder, kwargs)
class Record(Base):
"""
Template table for other record tables to derive from.
"""
# Add a foreign key to id in User table.
# This is user_id in SQL or userID in Python.
user = so.ForeignKey('User')
# The time the entry was is recorded for. Defaults to current time
# if not set or set to None.
timestamp = so.DateTimeCol(default=so.DateTimeCol.now)
def _set_userID(self, userID):
"""
Override default set method to enforce that the user ID exists in User
table when a record entry is created.
There may be a better way to do this with a contrainst or changing
the foreign key creation.
"""
try:
User.get(userID)
except so.SQLObjectNotFound:
raise so.SQLObjectNotFound(
"Unable to create entry since user ID "
"{0} is not in User table.".format(userID))
else:
self._SO_set_userID(userID)
class comment(so.SQLObject):
post = so.ForeignKey('post', notNone=1)
ctime = so.DateTimeCol(notNone=1,default=so.DateTimeCol.now)
name = so.StringCol(length=128, notNone=1,default='')
email = so.StringCol(length=128, notNone=1,default='')
title = so.StringCol(length=128, notNone=1,default='')
body = so.StringCol(notNone=1,default='')
class Srv4FileInCatalog(sqlobject.SQLObject):
"""Assignment of a particular srv4 file to a specific catalog.
There could be one more layer, to which arch and osrel could be moved.
But for now, it's going to be a not-normalized structure.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
osrel = sqlobject.ForeignKey('OsRelease', notNone=True)
catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True)
srv4file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
created_on = sqlobject.DateTimeCol(
notNone=True,
default=sqlobject.DateTimeCol.now)
created_by = sqlobject.UnicodeCol(length=50, notNone=True)
uniqueness_idx = sqlobject.DatabaseIndex(
'arch', 'osrel', 'catrel', 'srv4file',
unique=True)
# http://turbogears.org/1.0/docs/SQLObject/Caching.html#what-does-cache-false-do
class sqlmeta:
cacheValues = False
def __unicode__(self):
return (u"%s is in catalog %s %s %s"
% (self.srv4file,
self.arch.name, self.osrel.full_name, self.catrel.name))
class User(Base):
"""
Table of app users.
"""
# A GUID used to uniquely identify this item across systems and on the
# internet. Does alternateID also enforce unique?
#guid = so.StringCol(notNull=True, default=UUID, unique=True, alternateID=True)
# Username defaults to None/Null, but if set it must be unique.
username = so.UnicodeCol(default=None, unique=True, length=31)
# The time the user profile was created. Defaults to current time.
created = so.DateTimeCol(default=so.DateTimeCol.now)
class PythonScore(sqlobject.SQLObject):
"""
OO mapping of the score for a Python file
"""
username = sqlobject.StringCol()
pathname = sqlobject.StringCol()
revision = sqlobject.StringCol()
score = sqlobject.FloatCol()
old_score = sqlobject.FloatCol()
credit = sqlobject.FloatCol()
date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now)
class URL(sqlobject.SQLObject):
class sqlmeta:
table = 'url'
url = sqlobject.StringCol()
clean = sqlobject.StringCol()
author = sqlobject.ForeignKey('Author')
channel = sqlobject.ForeignKey('Channel')
citations = sqlobject.IntCol(default=0)
posted = sqlobject.DateTimeCol(default=datetime.now)
comments = sqlobject.MultipleJoin('Comments')
class ZeropointEntry(sqlobject.inheritance.InheritableSQLObject):
user = sqlobject.StringCol()
time = sqlobject.DateTimeCol()
cluster = sqlobject.StringCol()
filter = sqlobject.StringCol()
zp = sqlobject.FloatCol()
mangledSpecification = sqlobject.StringCol()
def _get_specification(self):
return unmangleSpecification(self.mangledSpecification)
def _set_specification(self, **specification):
self.mangledSpecification = mangleSpecification(specification)
class Table_Archive(sqlobject.SQLObject):
date = sqlobject.DateTimeCol()
In_Temp = sqlobject.FloatCol()
Out_Temp = sqlobject.FloatCol()
Attic_Temp = sqlobject.FloatCol()
In_Humid = sqlobject.FloatCol()
Out_Humid = sqlobject.FloatCol()
Attic_Humid = sqlobject.FloatCol()
Out_Wind_Avg = sqlobject.FloatCol()
Out_Wind_Max = sqlobject.FloatCol()
Out_Rain_Minute = sqlobject.FloatCol()
System_CPU = sqlobject.FloatCol()
System_RAM = sqlobject.FloatCol()
Now_Feel = sqlobject.FloatCol()
class Campaign(so.SQLObject):
"""
Model a Campaign, which can be assigned to a Tweet as a label.
Used to group Tweets which are added to the DB because they matched
the same campaign, such as a search topic. See docs/models.md document.
"""
class sqlmeta:
defaultOrder = "name"
# Campaign name can be any case and may have spaces.
name = so.StringCol(alternateID=True, length=50)
# Query string to use on Twitter API search, whether manually or on
# schedule. This is optional, to allow campaigns which are not searches.
searchQuery = so.StringCol(default=None)
createdAt = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Link to Tweet objects assigned to the Campaign.
tweets = so.SQLRelatedJoin("Tweet",
intermediateTable="tweet_campaign",
createRelatedTable=False)
@classmethod
def getOrCreate(cls, campaignName, query=None):
"""
Get a campaign otherwise create and return one.
Query may be empty as in some cases like a utility's campaign label
the campaign is a label for grouping rather than searching.
"""
try:
return cls.byName(campaignName)
except SQLObjectNotFound:
return cls(name=campaignName, searchQuery=query)
@classmethod
def getOrRaise(cls, campaignName):
"""
Get campaign by name otherwise raise an error, with instructions.
"""
try:
return cls.byName(campaignName)
except SQLObjectNotFound as e:
raise type(e)(
"Use the campaign manager to create the Campaign"
" as name and search query. Name not found: {!r}".format(
campaignName))
class Page(so.SQLObject):
"""
Model a URI for a webpage on the internet.
Do not worry about duplicate pairs of domain and path, since we want
to allow those to occur on an import and to clean them up later.
A page may have a null Folder (as unsorted), though a folder must always
have a parent folder, even if the top folder is "root".
"""
# The host website for the page.
# TODO: Ensure this is always converted lowercase rather than raising
# an error.
domain = so.ForeignKey('Domain', notNull=True)
# The location of the webpage relative to the domain.
# TODO: Should this start with forwardslash? Check what happens when
# splitting and joining.
# TODO: Create custom validator?
path = so.UnicodeCol(notNull=True)
# Webpage title, usually taken from the metadata of the HTML head section.
title = so.UnicodeCol(default=None)
# The date and time when the record was created. Defaults to the
# current time.
created_at = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Optional preview image for the link, scraped from the metadata.
image_url = so.UnicodeCol(default=None)
description = so.UnicodeCol(default=None)
# The folder this link is placed into. If null then the link must still
# be sorted. Domain and path pairs must be unique in a folder.
folder = so.ForeignKey('Folder')
unique_idx = so.DatabaseIndex(domain, path, folder, unique=True)
source = so.ForeignKey('Source', notNull=True)
# Link to labels which this page is assigned to.
labels = so.SQLRelatedJoin('Labels',
intermediateTable='page_label',
createRelatedTable=False)
def get_url(self):
return "".join((self.domain.value, self.path))
class Srv4FileStatsBlob(sqlobject.SQLObject):
"""Holds serialized data structures in JSON.
This table holds potentially large amounts of data (>1MB per row),
and is separated to make Srv4FileStats lighter. Sometimes, we don't
need to retrieve the heavy pickled data if we want to read just a few
text fields.
"""
class sqlmeta:
cacheValues = False
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
json = sqlobject.BLOBCol(notNone=True, length=(2**24))
content_md5_sum = sqlobject.UnicodeCol(notNone=True, unique=False, length=32)
mime_type = sqlobject.UnicodeCol(notNone=True, length=250)
created_on = sqlobject.DateTimeCol(
notNone=True,
default=sqlobject.DateTimeCol.now)
class Category(so.SQLObject):
"""
Model a Category, which can be assigned to Profiles.
Group similar profiles in a category. See docs/models.md document.
"""
class sqlmeta:
defaultOrder = "name"
# Category name can be any case and may have spaces.
name = so.StringCol(alternateID=True, length=50)
createdAt = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Get Profile objects assigned to the Category.
profiles = so.SQLRelatedJoin("Profile",
intermediateTable="profile_category",
createRelatedTable=False)
class Log(sqlobject.SQLObject):
timestamp = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now())
status = sqlobject.BoolCol(default=True)
section = sqlobject.StringCol()
message = sqlobject.StringCol()
def __init__(self, *args, **kwargs):
sqlobject.SQLObject.__init__(self, *args, **kwargs)
def __setattr__(self, name, value):
sqlobject.SQLObject.__setattr__(self, name, value)
def dict(self):
result = {
"id": self.id,
"timestamp": try_strftime(self.timestamp, default='never'),
"section": self.section,
"status": self.status,
"message": self.message
}
return result
class Domain(so.SQLObject):
"""
Model a website domain.
TODO: Add columns for metadata including title, keywords, image and icon.
"""
class sqlmeta:
defaultOrder = 'value'
# Full hostname or domain of the website.
value = so.UnicodeCol(alternateID=True, validator=URL)
# The date and time when the record was created. Defaults to the
# current time.
datetime_created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Link to Page objects which have paths relative to the Domain name.
pages = so.SQLMultipleJoin('Page')
# Get Page objects for the domain as a list. This is less efficient
# for filtering, but slightly more convenient for development.
pages_list = so.MultipleJoin('Page')
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True)
catalogname = sqlobject.UnicodeCol(notNone=True)
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
registered = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True)
in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile', joinColumn='id')
def DeleteAllDependentObjects(self):
data_obj = self.data_obj
self.data_obj = None
if data_obj:
# It could be already missing
data_obj.destroySelf()
self.RemoveAllCswFiles()
self.RemoveAllCheckpkgResults()
self.RemoveOverrides()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CswFile.sqlmeta.table,
CswFile.q.srv4_file == self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel,
arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))))
def RemoveAllCheckpkgResults(self):
logging.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file == self)))
def RemoveOverrides(self):
logging.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file == self)))
def __unicode__(self):
return (u"Package: %s-%s, %s" %
(self.catalogname, self.version_string, self.arch.name))
def GetStatsStruct(self):
return cPickle.loads(str(self.data_obj.pickle))
def GetRestRepr(self):
mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail"
data = {
'arch': self.arch.name,
'basename': self.basename,
# For compatibility with the catalog parser from catalog.py
'file_basename': self.basename,
'catalogname': self.catalogname,
'filename_arch': self.filename_arch.name,
'maintainer_email': self.maintainer.email,
'maintainer_full_name': self.maintainer.full_name,
'md5_sum': self.md5_sum,
'mtime': unicode(self.mtime),
'osrel': self.os_rel.short_name,
'pkgname': self.pkginst.pkgname,
'rev': self.rev,
'size': self.size,
'version_string': self.version_string,
# For compatibility with the catalog parser from catalog.py
'version': self.version_string,
# 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]),
}
return mimetype, data
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True, length=250)
catalogname = sqlobject.UnicodeCol(notNone=True, length=250)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
osrel_str = sqlobject.UnicodeCol(notNone=True, length=9) # "SunOS5.10"
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
pkginst_str = sqlobject.UnicodeCol(notNone=True, length=255)
registered_level_one = sqlobject.BoolCol(notNone=True)
registered_level_two = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False, length=250)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True, length=250)
bundle = sqlobject.UnicodeCol(length=250)
in_catalogs = sqlobject.MultipleJoin(
'Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile',
joinColumn='id')
catalog_idx = sqlobject.DatabaseIndex('catalogname')
basename_idx = sqlobject.DatabaseIndex('basename')
pkginst_idx = sqlobject.DatabaseIndex('pkginst')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
def __unicode__(self):
return u'%s/%s, %s' % (self.pkginst.pkgname, self.catalogname, self.md5_sum)
def __str__(self):
return str(unicode(self))
def DeleteAllDependentObjects(self):
"""Prepares the object to be deleted.
Use this function with caution.
"""
self.DeleteDependentObjectsPopulatedFromPackageItself()
logger.debug('Removing all dependent objects from %s; it will cause the '
'package to be removed from all catalogs.', self)
self.RemoveCatalogAssignments()
self.RemoveAllCheckpkgResults()
def DeleteDependentObjectsPopulatedFromPackageItself(self):
"""Removing all the objects that only depend on the package contents.
It doesn't touch rows that are created for other reasons, e.g. assignments
of packages to catalogs.
"""
logger.debug('%s - Deleting objects that only depend on the package '
'contents', self)
self.RemoveAllCswFiles()
self.RemoveOverrides()
self.RemoveDepends()
self.RemoveIncompatibles()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CswFile.sqlmeta.table,
CswFile.q.srv4_file==self)))
def RemoveDepends(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4DependsOn.sqlmeta.table,
Srv4DependsOn.q.srv4_file==self)))
def RemoveIncompatibles(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4IncompatibleWith.sqlmeta.table,
Srv4IncompatibleWith.q.srv4_file==self)))
def RemoveCatalogAssignments(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4FileInCatalog.sqlmeta.table,
Srv4FileInCatalog.q.srv4file==self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file==self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logger.debug("%s: RemoveCheckpkgResults(%s, %s, %s)",
self, os_rel, arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))))
def RemoveAllCheckpkgResults(self):
logger.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file==self)))
def RemoveOverrides(self):
logger.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file==self)))
def GetUnicodeOrNone(self, s):
"""Tries to decode UTF-8.
If the object does not decode as UTF-8, it's forced to do so, while
ignoring any potential errors.
Returns: a unicode object or a None type.
"""
if s is None:
return None
if type(s) != unicode:
try:
s = unicode(s, 'utf-8')
except UnicodeDecodeError, e:
s = s.decode("utf-8", "ignore")
s = s + u" (bad unicode detected)"
return s
class Package(threading.Thread, sqlobject.SQLObject):
"""
check package with pkgname, pkgver, reponame, action, hashsum
action will be ['commit', 'release', 'candidate', 'rebuild', ...]
"""
pkgname = sqlobject.StringCol()
pkgver = sqlobject.StringCol()
reponame = sqlobject.StringCol(default="default")
action = sqlobject.StringCol(default="commit")
build_args = sqlobject.StringCol(default=None)
hashsum = sqlobject.StringCol(default=None)
expired = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now)
priority = sqlobject.StringCol(default=None)
jobs = sqlobject.MultipleJoin('Job', joinColumn='package_id')
triggered = sqlobject.IntCol(default=1)
upload_status = sqlobject.IntCol(default=UploadStatus.UNKNOWN)
status_changed = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now)
deps = sqlobject.RelatedJoin('Package',
joinColumn='pkga',
otherColumn='pkgb')
notify = None
def __init__(self, *args, **kwargs):
sqlobject.SQLObject.__init__(self, *args, **kwargs)
threading.Thread.__init__(self)
self.do_quit = threading.Event()
self.status_lock = status_lock
def __setattr__(self, name, value):
if name == "upload_status":
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
def dict(self):
result = {
"id": self.id,
"pkgname": self.pkgname,
"pkgver": self.pkgver,
"reponame": self.reponame,
"action": self.action,
"hashsum": self.hashsum,
"expired": try_strftime(self.expired, default='never'),
"priority": self.priority,
"triggered": self.triggered,
"build_args":
self.build_args.split('|') if self.build_args else [],
"upload_status": UploadStatus.whatis(self.upload_status),
"status_changed": try_strftime(self.status_changed)
}
return result
@staticmethod
def version_compare(a, b):
if version.parse(a.pkgver) >= version.parse(b.pkgver):
return True
return False
def giveup(self):
self.upload_status = UploadStatus.UPLOAD_GIVEUP
for job in self.jobs:
if job.status == JobStatus.WAIT:
job.status = JobStatus.GIVEUP
def is_allowed_to_build(self):
for dep in Package.selectBy(id=self)[0].deps:
if Package.selectBy(
id=dep)[0].upload_status != UploadStatus.UPLOAD_OK:
if self.is_maybe_giveup():
self.giveup()
return False
return True
def is_maybe_giveup(self):
for dep in Package.selectBy(id=self)[0].deps:
for job in Package.selectBy(id=dep)[0].jobs:
if Job.selectBy(id=job)[0].status in JobFailedStatus:
return True
return False
def add_dep(self, dep):
for exist_dep in self.deps:
if exist_dep.id == dep.id:
return
self.addPackage(dep)
def add_deps(self, deps):
for dep in deps:
self.add_dep(dep)
class Job(threading.Thread, sqlobject.SQLObject):
"""Class implementing a build job"""
status = sqlobject.IntCol(default=JobStatus.UNKNOWN)
mailto = sqlobject.StringCol(default=None)
package = sqlobject.ForeignKey('Package', cascade=True)
dist = sqlobject.StringCol(default='sid')
arch = sqlobject.StringCol(default='any')
creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now)
status_changed = sqlobject.DateTimeCol(default=None)
build_start = sqlobject.DateTimeCol(default=None)
build_end = sqlobject.DateTimeCol(default=None)
host = sqlobject.StringCol(default=None)
deps = sqlobject.RelatedJoin('Job', joinColumn='joba', otherColumn='jobb')
log = sqlobject.SingleJoin('Log')
notify = None
def __init__(self, *args, **kwargs):
"""Init job"""
threading.Thread.__init__(self)
sqlobject.SQLObject.__init__(self, *args, **kwargs)
self.do_quit = threading.Event()
self.status_lock = threading.Lock()
def __setattr__(self, name, value):
"""Override setattr to log build status changes"""
if name == "status":
RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\
% (self.id, self.package.name, self.package.version,
self.dist, self.arch,
JobStatus.whatis(self.status),
JobStatus.whatis(value)))
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
@property
def logfile(self):
"""Compute and return logfile name"""
build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'),
self.package.name, self.package.version,
self.dist, self.arch,
self.creation_date.strftime("%Y%m%d-%H%M%S"),
self.id)
return build_log_file
def preexec_child(self):
"""Start a new group process before executing child"""
os.setsid()
def get_source_cmd(self):
"""Return command used for grabing source for this distribution
Substitutions are done in the command strings:
$d => The distro's name
$a => the target architecture
$p => the package's name
$v => the package's version
$j => rebuildd job id
"""
try:
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version, 'p': self.package.name, \
'j': str(self.id) }
t = Template(RebuilddConfig().get('build', 'source_cmd'))
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("get_source_cmd has invalid format: %s" % error)
return None
class Trend(so.SQLObject):
"""
A trending topic on Twitter, meaning a lot of Twitter accounts were talking
about that topic.
A topic exists at a point in time and maps to a specific place. It's term
can either be a hashtag (starts with '#' and has no spaces) or a keyword
phrase (no '#' and can have multiple words).
Note that the topic is not unique, since the topic can be repeated in
many locations and it can be repeated in one location across time.
The topic has a trending volume figure, which is how many tweets there are
about the topic in the past 24 hours (according to Twitter API docs).
Adding up trends for a Place taken at the SAME time each day should give
an accurate total of tweets for the period, since there should not be any
overlap in tweets across two consecutive 24-hour periods. One might use
the earliest record available for the day, assuming the cron job
runs soon after midnight, so that any ad hoc data will not skew the
results. However, the value will for the 24 hours of the PREVIOUS day.
Note that the topic volume shown is always GLOBAL total volume i.e.
independent of the location used to look up the topic. Volume usually
ranges from around 10,000 to 1 million and smaller values are returned
as null by Twitter API.
However, it is still useful to count the number of places which a tppic
is trending in as an indication of how widespread it is.
"""
class sqlmeta:
defaultOrder = "-timestamp"
_connection = conn
# The topic which trended.
topic = so.StringCol(length=64)
topicIdx = so.DatabaseIndex(topic)
# Whether the topic is a hashtag i.e. starts with '#'.
hashtag = so.BoolCol(default=False)
# Number of global tweets about topic in past 24 hours. Required since
# there no default set here. Null values are allowed.
volume = so.IntCol(notNull=False)
# The place associated with this trend record. See `setPlace` for why
# this is an optional field.
place = so.ForeignKey("Place", notNull=False, default=None)
placeIdx = so.DatabaseIndex(place)
# Date and time when record was created.
timestamp = so.DateTimeCol(default=so.DateTimeCol.now)
timestampIdx = so.DatabaseIndex(timestamp)
def setPlace(self, woeid):
"""
Link an existing Trend and Place records, given a Place WOEID.
Expects a WOEID int, gets ID for the Place, then stores it as the
foreign key for the Trend.
This doesn't work to be placed in __init__ since then its called
on a select and doen't work for modelCreate because the input kwargs
are validated before the method is called.
:param woeid: integer value for WOEID of the Place to link to.
:return self: returns object instance.
"""
assert isinstance(
woeid,
int), "Expected WOEID as an `int`, but " "got type `{0}`.".format(
type(woeid).__name__)
try:
self.placeID = Place.byWoeid(woeid).id
except so.SQLObjectNotFound as e:
raise type(e)(
"Place with WOEID {0} could not be found in the db.".format(
woeid))
return self
def _set_topic(self, value):
"""
Override the topic setting method, so that hashtag boolean is updated
automatically whenever topic is set.
:param value: string value to set as the topic.
"""
self._SO_set_topic(value)
if value.startswith("#"):
self._SO_set_hashtag(True)
else:
self._SO_set_hashtag(False)
@classmethod
def getColumnNames(cls):
"""
Return a list of column names for the class, as strings. This is
created from a dictionary, so the order is not guaranteed.
"""
return list(cls.sqlmeta.columns.keys())
def getData(self, quiet=True):
"""
Output the current record with key:value pairs for column name
and value. Note that this is not suitable to converted to JSON
because of the data types of values.
"""
data = {col: getattr(self, col) for col in self.getColumnNames()}
if not quiet:
for k, v in data.items():
# Align key to the right.
print("{0:>15} : {1}".format(k, v))
class Profile(so.SQLObject):
"""
Models a user profile on Twitter.
Note that URL columns are named as 'Url', since SQLOlbject converts
'imageURL' to db column named 'image_ur_l'.
Twitter screen name and username rules:
https://help.twitter.com/en/managing-your-account/twitter-username-rules
We use slightly higher values than the ones there, to be safe.
Notes on screen name:
- This should not have unique restriction as users can edit their
screen name so others can take an older screen name. Or someone
could delete and recreate their account.
- Twitter itself enforces uniqueness across case.
"""
# Profile's ID (integer), as assigned by Twitter when the Profile was
# created. This is a global ID, rather than an ID specific to our local db.
guid = so.IntCol(alternateID=True)
# Profile screen name.
screenName = so.StringCol(notNull=True, length=30)
# Profile display Name.
name = so.StringCol(notNull=True, length=60)
# Description, as set in profile's bio.
description = so.StringCol(default=None)
# Location, as set in profile's bio.
location = so.StringCol(default=None)
# Link to the profile's image online. This will only be thumbnail size.
imageUrl = so.StringCol(default=None, validator=URL)
# Count of profile's followers.
followersCount = so.IntCol(notNull=True)
# Count of profile's statuses (tweets) posted by this profile.
statusesCount = so.IntCol(notNull=True)
# Profile's verified status.
verified = so.BoolCol(notNull=True, default=False)
# Join the Profile with its created tweets in the Tweet table.
tweets = so.MultipleJoin("Tweet")
# Date and time when follower and status counts were last updated.
modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
modifiedIdx = so.DatabaseIndex(modified)
# Get Category objects which this Profile has been assigned to, if any.
categories = so.SQLRelatedJoin("Category",
intermediateTable="profile_category",
createRelatedTable=False)
def set(self, **kwargs):
"""
Override the update hook to update the modified field if necessary.
"""
if ("followersCount" in kwargs
or "statusesCount" in kwargs) and "modified" not in kwargs:
kwargs["modified"] = so.DateTimeCol.now()
super(Profile, self).set(**kwargs)
def getFlatDescription(self):
"""
Return the description with newline characters replaced with spaces.
"""
if self.description is not None:
return lib.text_handling.flattenText(self.description)
return None
def getProfileUrl(self):
"""
Get link to the profile's page online.
:return: Twitter profile's URL, as a string.
"""
return "https://twitter.com/{0}".format(self.screenName)
def getLargeImageUrl(self):
"""
Get link to a large version profile's image, based on thumbnail URL.
The image URL comes from the API as '..._normal.jpeg', but
from API calls on loading a twitter.com page, it is possible to
see that the image media server allows variations of the last part,
to return a large image. Such as
- '..._bigger.jpeg' (which is not much bigger than the normal
thumbnail)
- '..._400x400.jpeg' (which is much bigger).
:return: image URL using 400x400 size parameter, or None if value
was not set.
"""
if self.imageUrl:
return self.imageUrl.replace("_normal", "_400x400")
return None
def prettyPrint(self):
"""
Method to print the attributes of the Profile instance neatly.
:return: dictionary of data which was printed.
"""
output = """\
Screen name : @{screenName}
Name : {name}
Verified : {verified}
Followers : {followers:,d}
Statuses : {statuses:,d}
DB tweets : {tweetCount}
Description : {description}
Profile URL : {url}
Image URL : {imageUrl}
Stats modified : {statsModified}
"""
data = dict(
screenName=self.screenName,
name=self.name,
verified=self.verified,
followers=self.followersCount,
statuses=self.statusesCount,
tweetCount=len(self.tweets),
description=self.getFlatDescription(),
url=self.getProfileUrl(),
imageUrl=self.getLargeImageUrl(),
statsModified=self.modified,
)
print(output.format(**data))
return data
class policy(sqlobject.SQLObject):
_connection = conn
url = sqlobject.StringCol(length=14, unique=True)
name = sqlobject.StringCol(length=255)
date = sqlobject.DateTimeCol(default=None)
text = sqlobject.StringCol()
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True)
catalogname = sqlobject.UnicodeCol(notNone=True)
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
registered = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True)
in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile', joinColumn='id')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
self._cached_pkgstats = None
def DeleteAllDependentObjects(self):
data_obj = self.data_obj
self.data_obj = None
if data_obj:
# It could be already missing
data_obj.destroySelf()
self.RemoveAllCswFiles()
self.RemoveAllCheckpkgResults()
self.RemoveOverrides()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CswFile.sqlmeta.table,
CswFile.q.srv4_file == self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel,
arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))))
def RemoveAllCheckpkgResults(self):
logging.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file == self)))
def RemoveOverrides(self):
logging.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file == self)))
def __unicode__(self):
return (u"Package: %s-%s, %s" %
(self.catalogname, self.version_string, self.arch.name))
def GetStatsStruct(self):
if not self._cached_pkgstats:
self._cached_pkgstats = cPickle.loads(str(self.data_obj.pickle))
return self._cached_pkgstats
def _GetBuildSource(self):
data = self.GetStatsStruct()
build_src = None
if "OPENCSW_REPOSITORY" in data["pkginfo"]:
build_src = data["pkginfo"]["OPENCSW_REPOSITORY"]
return build_src
def GetSvnUrl(self):
build_src = self._GetBuildSource()
svn_url = None
if build_src:
svn_url = re.sub(r'([^@]*).*', r'\1/Makefile', build_src)
return svn_url
def GetTracUrl(self):
build_src = self._GetBuildSource()
trac_url = None
if build_src:
trac_url = re.sub(
r'https://gar.svn.(sf|sourceforge).net/svnroot/gar/([^@]+)@(.*)',
r'http://sourceforge.net/apps/trac/gar/browser/\2/Makefile?rev=\3',
build_src)
return trac_url
def GetVendorUrl(self):
data = self.GetStatsStruct()
vendor_url = None
if "VENDOR" in data["pkginfo"]:
vendor_url = re.split(r"\s+", data["pkginfo"]["VENDOR"])[0]
return vendor_url
def GetRestRepr(self):
mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail"
data = {
'arch': self.arch.name,
'basename': self.basename,
# For compatibility with the catalog parser from catalog.py
'file_basename': self.basename,
'catalogname': self.catalogname,
'filename_arch': self.filename_arch.name,
'maintainer_email': self.maintainer.email,
'maintainer_full_name': self.maintainer.full_name,
'md5_sum': self.md5_sum,
'mtime': unicode(self.mtime),
'osrel': self.os_rel.short_name,
'pkgname': self.pkginst.pkgname,
'rev': self.rev,
'size': self.size,
'version_string': self.version_string,
# For compatibility with the catalog parser from catalog.py
'version': self.version_string,
# 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]),
'vendor_url': self.GetVendorUrl(),
'repository_url': self.GetSvnUrl(),
}
return mimetype, data
class Tweet(so.SQLObject):
"""
Models a tweet on Twitter.
If we are inserting the Tweet in our db, we expect to always have the
author's profile in the Profile table. If the tweet is a reply, we will
have references to the target Profile and original Tweet as GUID integers.
But we are unlikely to have those object stored in our db. Use
the `.getInReplyToTweet` and `.getInReplyToProfile` methods to see if those
exist in the db, otherwise use the GUIDs to look up data from the
Twitter API and then store them locally as db records.
For relating a Tweet to its author Profile with a foreign key, a
`setProfileByGuid` method could be implemented to set the profile
foreign key using a given GUID, but that would require doing a search
each time. So, when creating a Tweet object or multiple objects for one
Profile, it is preferable to get the Profile object's ID once
and then repeately pass that in as an argument for each Tweet object
that is created for that Profile.
For ordering, the '-guid' syntax is here is preferred, since
'guid DESC' results in an error when getting tweets of a Profile object,
even though doing a query on Tweet class itself is fine.
`AttributeError: 'Tweet' object has no attribute 'guid DESC'`
The error is also raised for multiple names e.g. '-guid, message'.
"""
class sqlmeta:
# Show recent Tweets (with higher GUID values) first.
defaultOrder = "-guid"
# Tweet ID (integer), as assigned by Twitter when the Tweet was posted.
# This is a global ID, rather than specific to our local db.
guid = so.IntCol(alternateID=True)
# Link to Tweet's author in the Profile table. Delete Tweet if
# the Profile is deleted.
profile = so.ForeignKey("Profile", notNull=True, cascade=True)
profileIdx = so.DatabaseIndex(profile)
# Date and time the tweet was posted.
createdAt = so.DateTimeCol(notNull=True)
createdAtIdx = so.DatabaseIndex(createdAt)
# Tweet message text. Length is not validated since expanded tweets can
# be longer than the standard 280 (previously 140) characters.
message = so.StringCol(notNull=True)
# Count of favorites on this Tweet.
favoriteCount = so.IntCol(notNull=True)
# Count of retweets of this Tweet.
retweetCount = so.IntCol(notNull=True)
# If the tweet is a reply, the GUID of the Tweet which the reply is
# directed at (from reply_to_status_id field). This does not require
# the Tweet to be in the local db.
inReplyToTweetGuid = so.IntCol(default=None)
# If the tweet is a reply, the GUID of the Profile which the reply is
# directed at (from reply_to_user_id field). This does not require
# the Tweet to be in the local db.
inReplyToProfileGuid = so.IntCol(default=None)
# Date and time when favorite and retweet counts where last updated.
modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
modifiedIdx = so.DatabaseIndex(modified)
# Get Campaign objects which this Profile has been assigned to, if any.
campaigns = so.SQLRelatedJoin("Campaign",
intermediateTable="tweet_campaign",
createRelatedTable=False)
def set(self, **kwargs):
"""
Override the update hook to update the modified field if necessary.
"""
if ("favoriteCount" in kwargs
or "retweetCount" in kwargs) and "modified" not in kwargs:
kwargs["modified"] = so.DateTimeCol.now()
super(Tweet, self).set(**kwargs)
def isRT(self):
return self.message.startswith("RT ")
def isReply(self):
return self.inReplyToProfileGuid is not None
def getFlatMessage(self):
"""
Return the message with newline characters replaced with spaces.
"""
return lib.text_handling.flattenText(self.message)
def getInReplyToTweet(self):
"""
If this Tweet is a reply, get the original Tweet it was directed at.
:return: single Tweet object. Return None if this is not a reply. Raise
an error if the Tweet is not in the local db.
"""
if self.inReplyToTweetGuid:
try:
return Tweet.byGuid(self.inReplyToTweetGuid)
except SQLObjectNotFound as e:
raise type(e)(
"Could not find Tweet in db with GUID: {0}".format(
self.inReplyToTweetGuid))
return None
def getInReplyToProfile(self):
"""
If this Tweet is a reply, get the Profile which it was directed at.
:return: single Profile object. Return None if this is not a reply.
Raise an error if the Tweet is not in the local db.
"""
if self.inReplyToProfileGuid:
try:
return Profile.byGuid(self.inReplyToProfileGuid)
except SQLObjectNotFound as e:
raise type(e)(
"Could not find Profile in db with GUID: {0}".format(
self.inReplyToProfileGuid))
return None
def getTweetURL(self):
"""
Return URL for the tweet as a string, using tweet author's screen name
and the tweet's GUID.
"""
return "https://twitter.com/{screenName}/status/{tweetID}".format(
screenName=self.profile.screenName, tweetID=self.guid)
def prettyPrint(self):
"""
Method to print the attributes of the Tweet instance neatly.
:return: dictionary of data which was printed.
"""
output = """\
Author : @{screenName} - {name} - {followers:,d} followers
Created at : {createdAt}
Message : {message}
Favorites : {favoriteCount:,d}
Retweets : {retweetCount:,d}
Reply To User ID : {replyProf}
Reply To Tweet ID : {replyTweet}
URL : {url}
Stats modified : {statsModified}
"""
author = self.profile
data = dict(
screenName=author.screenName,
createdAt=self.createdAt,
name=author.name,
followers=author.followersCount,
message=self.getFlatMessage(),
favoriteCount=self.favoriteCount,
retweetCount=self.retweetCount,
replyProf=self.inReplyToProfileGuid,
replyTweet=self.inReplyToTweetGuid,
url=self.getTweetURL(),
statsModified=self.modified,
)
print(output.format(**data))
return data
def report(self):
"""
Return Tweet and Profile data as dict for writing a CSV report.
"""
author = self.profile
return {
"Screen name": author.screenName,
"Followers": author.followersCount,
"Tweet URL": self.getTweetURL(),
"Tweet ID": self.guid,
"Tweeted at": str(self.createdAt),
"Is reply": "Y" if self.isReply() else "N",
"Is RT": "Y" if self.isRT() else "N",
"Message": self.message,
"Favs": self.favoriteCount,
"RTs": self.retweetCount,
}
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True)
catalogname = sqlobject.UnicodeCol(notNone=True)
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
registered = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True)
in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile', joinColumn='id')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
self._cached_pkgstats = None
def DeleteAllDependentObjects(self):
data_obj = self.data_obj
self.data_obj = None
if data_obj:
# It could be already missing
data_obj.destroySelf()
self.RemoveAllCswFiles()
self.RemoveAllCheckpkgResults()
self.RemoveOverrides()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CswFile.sqlmeta.table,
CswFile.q.srv4_file == self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel,
arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))))
def RemoveAllCheckpkgResults(self):
logging.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file == self)))
def RemoveOverrides(self):
logging.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file == self)))
def __unicode__(self):
return (u"Package: %s-%s, %s" %
(self.catalogname, self.version_string, self.arch.name))
def GetUnicodeOrNone(self, s):
"""Tries to decode UTF-8.
If the object does not decode as UTF-8, it's forced to do so, while
ignoring any potential errors.
Returns: a unicode object or a None type.
"""
if s is None:
return None
if type(s) != unicode:
try:
s = unicode(s, 'utf-8')
except UnicodeDecodeError, e:
s = s.decode("utf-8", "ignore")
s = s + u" (bad unicode detected)"
return s
class Table_Rain(sqlobject.SQLObject):
time = sqlobject.DateTimeCol()
quantity = sqlobject.FloatCol()
class post(so.SQLObject):
ctime = so.DateTimeCol(notNone=1,default=so.DateTimeCol.now)
title = so.StringCol(length=128, notNone=1,default='')
body = so.StringCol(notNone=1,default='')
comments = so.MultipleJoin('comment')
class PlaceJob(so.SQLObject):
"""
Listing of places which we want to regularly get trend data for.
The WOEID of a place is used to look up trends at the location,
for records which have status set to enabled. An item can be marked
as disabled, so that it will be skipped by a procedure but kept in the
table so it can be enabled again easily.
A place can appear only once in the table. Only towns or countries can be
added to the job list, due to how Twitter API works - this is enforced
when a record is created.
The table starts off empty and desired places can be added or removed
depending on admin user's preferences.
"""
class sqlmeta:
# Order as enabled (True) items first, then by jobs with oldest
# (or null) last completed timestamps first and then by oldest
# (or null) last attempted timestamps. Therefore when running jobs, the
# ones which have not been completed before or for the longest time
# are given priority over ones which were recently completed.
# And any disabled jobs will be at the bottom when viewing a report.
defaultOrder = "enabled DESC, last_completed ASC, last_attempted ASC"
_connection = conn
# Create a reference to Place table. Place IDs cannot be repeated in
# this job table.
place = so.ForeignKey("Place", unique=True)
# Create an index on place.
placeIdx = so.DatabaseIndex(place)
# Date and time when record was created.
created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# When the job was last attempted regardless of outcome.
lastAttempted = so.DateTimeCol(notNull=False, default=None)
# When the job item was last completed successfully. Defaults to null.
lastCompleted = so.DateTimeCol(notNull=False, default=None)
# Create an index on last completed.
lastCompletedIdx = so.DatabaseIndex(lastCompleted)
# Boolean flag for whether the job item is enabled or should be skipped.
enabled = so.BoolCol(notNull=True, default=True)
def start(self):
"""
Use this function to update the last attempted time.
"""
self._set_lastAttempted(so.DateTimeCol.now())
def end(self):
"""
Use this function to update the last run time, if successful.
"""
self._set_lastCompleted(so.DateTimeCol.now())
def setEnabled(self):
"""
Set the job to enabled.
"""
self.enabled = True
def setDisabled(self):
"""
Set the job to disabled.
"""
self.enabled = False
def getStatus(self, asText=False):
"""
Get the status from when the job was last run.
If last attempted time is None, then return None since we cannot
confirm success or failure.
Return True for success, if last completed time is not None and
is after last attempted time. Otherwise return False for failure.
:param asText: Default False. If True, return status as human-readable
string.
:return status: job status as OK (True) or failed (False) or not
run (None). Returned as human-readable string if asText is True.
"""
if self.lastAttempted:
if self.lastCompleted and self.lastCompleted > self.lastAttempted:
status = "OK" if asText else True
else:
status = "failed" if asText else False
else:
status = "not run" if asText else None
return status
class Place(InheritableSQLObject):
"""
A place in the world. This is created from the Yahoo Where On Earth
locations as returned by Twitter API.
This table has childName to indicate which table the object is in and
therefore the parent Place's location type.
Name is *not* an alternateID, since place names can be duplicated around
the world e.g. Barcelona in Venezuela and Spain.
Therefore `.byName` is not available, but we can do a `.selectBy` with
both town name and the country's ID set in the where clause, to ensure
we get one result.
Default order by ID is omitted as it causes ambiguity issues on some
selects. And timestamp is not recognised as a column on the subclasses
so cannot be used either.
"""
_connection = conn
# WOEID integer value from Yahoo system.
# Note that `.byWoeid` can be used on Place class, but cannot be used
# on any subclasses. This is because of the `id` in the order by statement
# having ambiguous meaning in `where (Place.id = {subclass}.id)`.
woeid = so.IntCol(alternateID=True)
# Name of the place.
name = so.StringCol(length=64, default=None)
nameIdx = so.DatabaseIndex(name)
# Date and time when record was created.
timestamp = so.DateTimeCol(default=so.DateTimeCol.now)
timestampIdx = so.DatabaseIndex(timestamp)
# Get all the trend records relating to this Place.
hasTrends = so.MultipleJoin("Trend")
@classmethod
def getColumnNames(cls):
"""
Return a list of column names for the class, as strings. This is
created from a dictionary, so the order is not guaranteed.
"""
return list(cls.sqlmeta.columns.keys())
def getData(self, quiet=True):
"""
Output the current record with key:value pairs for column name
and value. Note that this is not suitable to be converted directly to
JSON because of the data types of some values.
TODO: Ensure that attributes of the parent and child are all accessed,
to get more use out of this.
"""
data = {col: getattr(self, col) for col in self.getColumnNames()}
if not quiet:
for k, v in data.items():
print(f"{k:>15} : {v}")
return data
