class Event(DictSQLObject):
"""
Collects information about Events
Referenced to by the Guest and ChatMsg Tables
"""
name = so.UnicodeCol()
description = so.UnicodeCol()
class CswFile(sqlobject.SQLObject):
"""Represents a file in a catalog.
There can be multiple files with the same basename and the same path,
belonging to different packages.
This class needs to also contain files from the operating system,
coming from SUNW packages, for which we don't have the original srv4
files. (Even if we could, they are generally not accessible.) They
need to be specific to Solaris release and architecture, so we need
a way to link CswFile with a specific catalog.
Fake registered Srv4FileStats object would do, but we would have to
ensure that they can't be associated with a catalog. Also, we'd have
to generate fake md5 sums for them.
"""
basename = sqlobject.UnicodeCol(length=255, notNone=True)
path = sqlobject.UnicodeCol(notNone=True, length=900)
line = sqlobject.UnicodeCol(notNone=True, length=900)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
srv4_file = sqlobject.ForeignKey('Srv4FileStats')
basename_idx = sqlobject.DatabaseIndex('basename')
def __unicode__(self):
return u"File: %s" % os.path.join(self.path, self.basename)
class CheckpkgOverride(sqlobject.SQLObject):
# Overrides don't need to contain catalog parameters.
srv4_file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
pkgname = sqlobject.UnicodeCol(default=None, length=250)
tag_name = sqlobject.UnicodeCol(notNone=True, length=250)
tag_info = sqlobject.UnicodeCol(default=None, length=250)
def __unicode__(self):
return (u"Override: %s: %s %s" %
(self.pkgname,
self.tag_name,
self.tag_info or ""))
def DoesApply(self, tag):
"""Figures out if this override applies to the given tag."""
basket_a = {}
basket_b = {}
if self.pkgname:
basket_a["pkgname"] = self.pkgname
basket_b["pkgname"] = tag.pkgname
if self.tag_info:
basket_a["tag_info"] = self.tag_info
basket_b["tag_info"] = tag.tag_info
basket_a["tag_name"] = self.tag_name
basket_b["tag_name"] = tag.tag_name
return basket_a == basket_b
class OsRelease(sqlobject.SQLObject):
"Short name: SunOS5.9, long name: Solaris 9"
short_name = sqlobject.UnicodeCol(length=40, unique=True, notNone=True)
full_name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
def __unicode__(self):
return u"OS release: %s" % self.full_name
class Maintainer(sqlobject.SQLObject):
"""The maintainer of the package, identified by the e-mail address."""
email = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
full_name = sqlobject.UnicodeCol(length=255, default=None)
# TODO: Add more fields: status (active/retired/unregistered)
# There are some emails that are just errors, it would be good to
# distinguish emails that once were velid @opencsw.org addresses from pure
# bugs, e.g. "*****@*****.**".
def ObfuscatedEmail(self):
if self.email:
email = self.email.split("@")
else:
email = ["unknown"]
if len(email) == 2:
username, domain = email
else:
username, domain = email[0], "no domain"
username = username[:-3] + "..."
return "@".join((username, domain))
def __unicode__(self):
return u"%s <%s>" % (self.full_name or "Maintainer full name unknown",
self.ObfuscatedEmail())
def GetRestRepr(self):
return {
'maintainer_email': self.email,
'maintainer_full_name': self.full_name,
'maintainer_id': self.id,
}
class CatalogGenData(sqlobject.SQLObject): """Fields required to generate the catalog. Having this smaller table lets us avoid fetching the main big data structure. """ deps = sqlobject.BLOBCol(notNone=True, length=(2 ** 14 - 1)) i_deps = sqlobject.BLOBCol(notNone=True, length=(2 ** 14 - 1)) pkginfo_name = sqlobject.BLOBCol(notNone=True, length=(2 ** 14 - 1)) pkgname = sqlobject.UnicodeCol(default=None, length=250) md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
class ElfdumpInfoBlob(sqlobject.SQLObject):
"""Holds JSON with elfdump information for a package.
Indexed by the md5 sum of the binary.
"""
class sqlmeta:
cacheValues = False
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
json = sqlobject.BLOBCol(notNone=True, length=(2**24 - 1))
content_md5_sum = sqlobject.UnicodeCol(notNone=True, unique=False, length=32)
mime_type = sqlobject.UnicodeCol(notNone=True, length=250)
class Lyric(sqlobject.SQLObject):
_connection = conn
sid = sqlobject.IntCol(length=14, unique=True)
song = sqlobject.UnicodeCol(length=255)
lrc = sqlobject.UnicodeCol(length=255)
artist_id = sqlobject.ForeignKey('Artist')
aid = sqlobject.IntCol(length=14)
#new add field
local_path = sqlobject.UnicodeCol(length=255, default=None)
sizes = sqlobject.IntCol(default=None)
source = sqlobject.UnicodeCol(default=None)
class Maintainer(sqlobject.SQLObject):
"""The maintainer of the package, identified by the e-mail address."""
email = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
full_name = sqlobject.UnicodeCol(length=255, default=None)
def ObfuscatedEmail(self):
username, domain = self.email.split("@")
username = username[:-3] + "..."
return "@".join((username, domain))
def __unicode__(self):
return u"%s <%s>" % (self.full_name or "Maintainer full name unknown",
self.ObfuscatedEmail())
class Page(so.SQLObject):
"""
Model a URI for a webpage on the internet.
Do not worry about duplicate pairs of domain and path, since we want
to allow those to occur on an import and to clean them up later.
A page may have a null Folder (as unsorted), though a folder must always
have a parent folder, even if the top folder is "root".
"""
# The host website for the page.
# TODO: Ensure this is always converted lowercase rather than raising
# an error.
domain = so.ForeignKey('Domain', notNull=True)
# The location of the webpage relative to the domain.
# TODO: Should this start with forwardslash? Check what happens when
# splitting and joining.
# TODO: Create custom validator?
path = so.UnicodeCol(notNull=True)
# Webpage title, usually taken from the metadata of the HTML head section.
title = so.UnicodeCol(default=None)
# The date and time when the record was created. Defaults to the
# current time.
created_at = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Optional preview image for the link, scraped from the metadata.
image_url = so.UnicodeCol(default=None)
description = so.UnicodeCol(default=None)
# The folder this link is placed into. If null then the link must still
# be sorted. Domain and path pairs must be unique in a folder.
folder = so.ForeignKey('Folder')
unique_idx = so.DatabaseIndex(domain, path, folder, unique=True)
source = so.ForeignKey('Source', notNull=True)
# Link to labels which this page is assigned to.
labels = so.SQLRelatedJoin('Labels',
intermediateTable='page_label',
createRelatedTable=False)
def get_url(self):
return "".join((self.domain.value, self.path))
class Srv4FileInCatalog(sqlobject.SQLObject):
"""Assignment of a particular srv4 file to a specific catalog.
There could be one more layer, to which arch and osrel could be moved.
But for now, it's going to be a not-normalized structure.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
osrel = sqlobject.ForeignKey('OsRelease', notNone=True)
catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True)
srv4file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
created_on = sqlobject.DateTimeCol(
notNone=True,
default=sqlobject.DateTimeCol.now)
created_by = sqlobject.UnicodeCol(length=50, notNone=True)
uniqueness_idx = sqlobject.DatabaseIndex(
'arch', 'osrel', 'catrel', 'srv4file',
unique=True)
# http://turbogears.org/1.0/docs/SQLObject/Caching.html#what-does-cache-false-do
class sqlmeta:
cacheValues = False
def __unicode__(self):
return (u"%s is in catalog %s %s %s"
% (self.srv4file,
self.arch.name, self.osrel.full_name, self.catrel.name))
class CatalogRelease(sqlobject.SQLObject):
"Release names: potato, etc."
name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
type = sqlobject.ForeignKey('CatalogReleaseType', notNone=True)
def __unicode__(self):
return u"Catalog release: %s" % self.name
class DataSource(sqlobject.SQLObject):
"""Represents: a /var/sadm/install/contents file, or CSW catalog.
- "filesystem"
- "catalog"
"""
name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
class Folder(so.SQLObject):
"""
Model a folder, which can contain Page objects.
A folder can be a tree structure with one parent and many children. A
folder may contain zero or more Page objects.
Folder name has a unique constrainst. If the same folder name needs
to be used in different parts of the folder tree, it should probably
by a label instead.
"""
name = so.UnicodeCol(alternateID=True)
# Self-join to an optional parent folder. If NULL, then it is at the top
# level.
# TODO: Unique constraint to prevent multiple top level folders?
# Or use "root" instead and the res are unsorted?
parent = so.ForeignKey('Folder', default=None)
# Link to the child Folder records of a Folder.
children = so.SQLMultipleJoin('Folder')
# Link to Page records within a Folder.
pages = so.SQLMultipleJoin('Page')
class Srv4FileStatsBlob(sqlobject.SQLObject):
"""Holds serialized data structures in JSON.
This table holds potentially large amounts of data (>1MB per row),
and is separated to make Srv4FileStats lighter. Sometimes, we don't
need to retrieve the heavy pickled data if we want to read just a few
text fields.
"""
class sqlmeta:
cacheValues = False
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
json = sqlobject.BLOBCol(notNone=True, length=(2**24))
content_md5_sum = sqlobject.UnicodeCol(notNone=True, unique=False, length=32)
mime_type = sqlobject.UnicodeCol(notNone=True, length=250)
created_on = sqlobject.DateTimeCol(
notNone=True,
default=sqlobject.DateTimeCol.now)
class CheckpkgErrorTag(CheckpkgErrorTagMixin, sqlobject.SQLObject):
srv4_file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
pkgname = sqlobject.UnicodeCol(default=None, length=250)
tag_name = sqlobject.UnicodeCol(notNone=True, length=250)
tag_info = sqlobject.UnicodeCol(default=None, length=250)
msg = sqlobject.UnicodeCol(default=None, length=250)
# To cache results from checkpkg
overridden = sqlobject.BoolCol(default=False)
# The same package might have different sets of errors for different
# catalogs or Solaris releases.
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
arch = sqlobject.ForeignKey('Architecture', notNone=True)
catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True)
def __unicode__(self):
return (u"CheckpkgErrorTag: %s %s %s"
% (self.pkgname, self.tag_name, self.tag_info))
class Format(so.SQLObject):
"""
Model a datasource format.
For example, a manual entry, user-created CSV, bookmark export,
history export, or OneTab export.
"""
name = so.UnicodeCol(alternateID=True, validator=validators.LowerCaseStr)
class Guest(DictSQLObject):
"""
Collects information about Guests
Refrenced to by the ChatMsg Table
"""
name = so.UnicodeCol()
admin = so.BoolCol(default=False)
parprop = so.IntCol(default=-1)
token = so.StringCol(length=8)
event = so.ForeignKey('Event')
class CswFile(sqlobject.SQLObject):
"""Represents a file in a catalog.
There can be multiple files with the same basename and the same path,
belonging to different packages.
This class needs to also contain files from the operating system,
coming from SUNW packages, for which we don't have the original srv4
files. (Even if we could, they are generally not accessible.) They
need to be specific to Solaris release and architecture, so we need
a way to link CswFile with a specific catalog.
Fake registered Srv4FileStats object would do, but we would have to
ensure that they can't be associated with a catalog. Also, we'd have
to generate fake md5 sums for them.
"""
class sqlmeta:
# MySQL uses case-insensitive collation by default, which doesn't make sense
# for file names. If the utf8_bin (or other case sensitive) collation is not
# used, it reports e.g. Zcat and zcat as a file collision, while it really
# isn't one.
createSQL = {
'mysql' : [
'ALTER TABLE csw_file CONVERT TO CHARACTER SET utf8 '
'COLLATE utf8_bin']
}
basename = sqlobject.UnicodeCol(length=255, notNone=True)
path = sqlobject.UnicodeCol(notNone=True, length=900)
line = sqlobject.UnicodeCol(notNone=True, length=900)
# Symlinks don't have permissions on their own
perm_user = sqlobject.UnicodeCol(notNone=False, length=255)
perm_group = sqlobject.UnicodeCol(notNone=False, length=255)
perm_mode = sqlobject.UnicodeCol(notNone=False, length=5)
target = sqlobject.UnicodeCol(notNone=False, length=900)
mimetype = sqlobject.UnicodeCol(notNone=False, length=255)
machine = sqlobject.UnicodeCol(notNone=False, length=255)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
srv4_file = sqlobject.ForeignKey('Srv4FileStats')
basename_idx = sqlobject.DatabaseIndex('basename')
path_idx = sqlobject.DatabaseIndex({'column': 'path', 'length': 255})
def FullPath(self):
return os.path.join(self.path, self.basename)
def __unicode__(self):
return u"%s %s %s %s %s" % (self.perm_user, self.perm_group, self.perm_mode,
self.FullPath(), self.mimetype)
class JSONData(Record):
"""
Table to hold any form of data in JSON format.
Todo: extract date from data if available and overwrite value in
inherited timestamp column. Consider conversion of datatypes.
"""
# Optional label to describe the area or topic or page the data was
# submitted on.
area = so.UnicodeCol(default='', length=31)
data = so.JSONCol(default=None)
class User(Base):
"""
Table of app users.
"""
# A GUID used to uniquely identify this item across systems and on the
# internet. Does alternateID also enforce unique?
#guid = so.StringCol(notNull=True, default=UUID, unique=True, alternateID=True)
# Username defaults to None/Null, but if set it must be unique.
username = so.UnicodeCol(default=None, unique=True, length=31)
# The time the user profile was created. Defaults to current time.
created = so.DateTimeCol(default=so.DateTimeCol.now)
class UwatchPkgVersion(sqlobject.SQLObject):
id_pkg = sqlobject.IntCol(dbName="ID_PKG")
gar_path = sqlobject.UnicodeCol(length=255, dbName="PKG_GAR_PATH")
pkgname = sqlobject.UnicodeCol(length=64, dbName="PKG_NAME")
catalogname = sqlobject.UnicodeCol(length=64, dbName="PKG_CATALOGNAME")
gar_version = sqlobject.UnicodeCol(length=255, dbName="PKG_GAR_VERSION")
upstream_version = sqlobject.UnicodeCol(length=255, dbName="PKG_UPSTREAM_VERSION")
master_sites = sqlobject.UnicodeCol(length=255, dbName="PKG_UPSTREAM_MASTER_SITES")
distfiles = sqlobject.UnicodeCol(length=255, dbName="PKG_GAR_DISTFILES")
regex = sqlobject.UnicodeCol(length=255, dbName="PKG_UFILES_REGEXP")
class sqlmeta(object):
table = "UWATCH_PKG_VERSION"
idName = "id_pkg"
class Label(so.SQLObject):
"""
Model a label.
Unlike a folder, a label does not fit into a tree hierarchy and its name
must be unique in order to look it up properly.
"""
# Descriptive name of the label, which must be unique.
name = so.UnicodeCol(alternateID=True)
# Link to pages assigned to the label.
pages = so.SQLRelatedJoin('Page',
intermediateTable='page_label',
createRelatedTable=False)
class Domain(so.SQLObject):
"""
Model a website domain.
TODO: Add columns for metadata including title, keywords, image and icon.
"""
class sqlmeta:
defaultOrder = 'value'
# Full hostname or domain of the website.
value = so.UnicodeCol(alternateID=True, validator=URL)
# The date and time when the record was created. Defaults to the
# current time.
datetime_created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Link to Page objects which have paths relative to the Domain name.
pages = so.SQLMultipleJoin('Page')
# Get Page objects for the domain as a list. This is less efficient
# for filtering, but slightly more convenient for development.
pages_list = so.MultipleJoin('Page')
class Host(sqlobject.SQLObject):
"Hostname, as returned by socket.getfqdn()"
fqdn = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
arch = sqlobject.ForeignKey('Architecture', notNone=True)
class CswConfig(sqlobject.SQLObject): option_key = sqlobject.UnicodeCol(length=255, unique=True) float_value = sqlobject.FloatCol(default=None) int_value = sqlobject.IntCol(default=None) str_value = sqlobject.UnicodeCol(default=None, length=250)
class Architecture(sqlobject.SQLObject):
"One of: 'sparc', 'x86'."
name = sqlobject.UnicodeCol(length=40, unique=True, notNone=True)
def __unicode__(self):
return u"Architecture: %s" % self.name
class CatalogReleaseType(sqlobject.SQLObject):
"Unstable, testing, stable."
name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)
class Pkginst(sqlobject.SQLObject):
pkgname = sqlobject.UnicodeCol(length=250, unique=True, notNone=True)
catalogname = sqlobject.UnicodeCol(default=None, length=250)
pkg_desc = sqlobject.UnicodeCol(default=None, length=250)
srv4_files = sqlobject.MultipleJoin('Srv4FileStats')
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True, length=250)
catalogname = sqlobject.UnicodeCol(notNone=True, length=250)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
osrel_str = sqlobject.UnicodeCol(notNone=True, length=9) # "SunOS5.10"
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
pkginst_str = sqlobject.UnicodeCol(notNone=True, length=255)
registered_level_one = sqlobject.BoolCol(notNone=True)
registered_level_two = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False, length=250)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True, length=250)
bundle = sqlobject.UnicodeCol(length=250)
in_catalogs = sqlobject.MultipleJoin(
'Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile',
joinColumn='id')
catalog_idx = sqlobject.DatabaseIndex('catalogname')
basename_idx = sqlobject.DatabaseIndex('basename')
pkginst_idx = sqlobject.DatabaseIndex('pkginst')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
def __unicode__(self):
return u'%s/%s, %s' % (self.pkginst.pkgname, self.catalogname, self.md5_sum)
def __str__(self):
return str(unicode(self))
def DeleteAllDependentObjects(self):
"""Prepares the object to be deleted.
Use this function with caution.
"""
self.DeleteDependentObjectsPopulatedFromPackageItself()
logger.debug('Removing all dependent objects from %s; it will cause the '
'package to be removed from all catalogs.', self)
self.RemoveCatalogAssignments()
self.RemoveAllCheckpkgResults()
def DeleteDependentObjectsPopulatedFromPackageItself(self):
"""Removing all the objects that only depend on the package contents.
It doesn't touch rows that are created for other reasons, e.g. assignments
of packages to catalogs.
"""
logger.debug('%s - Deleting objects that only depend on the package '
'contents', self)
self.RemoveAllCswFiles()
self.RemoveOverrides()
self.RemoveDepends()
self.RemoveIncompatibles()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CswFile.sqlmeta.table,
CswFile.q.srv4_file==self)))
def RemoveDepends(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4DependsOn.sqlmeta.table,
Srv4DependsOn.q.srv4_file==self)))
def RemoveIncompatibles(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4IncompatibleWith.sqlmeta.table,
Srv4IncompatibleWith.q.srv4_file==self)))
def RemoveCatalogAssignments(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4FileInCatalog.sqlmeta.table,
Srv4FileInCatalog.q.srv4file==self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file==self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logger.debug("%s: RemoveCheckpkgResults(%s, %s, %s)",
self, os_rel, arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))))
def RemoveAllCheckpkgResults(self):
logger.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file==self)))
def RemoveOverrides(self):
logger.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file==self)))
def GetUnicodeOrNone(self, s):
"""Tries to decode UTF-8.
If the object does not decode as UTF-8, it's forced to do so, while
ignoring any potential errors.
Returns: a unicode object or a None type.
"""
if s is None:
return None
if type(s) != unicode:
try:
s = unicode(s, 'utf-8')
except UnicodeDecodeError, e:
s = s.decode("utf-8", "ignore")
s = s + u" (bad unicode detected)"
return s
