class Message(orm.SQLObject):
"""The class generates a Message table containing information
about user messages.
Args:
text (str, optional):
time (int, optional):
filePicture (byte, optional):
fileVideo (byte, optional):
fileAudio (byte, optional):
fileDocument (byte, optional):
emoji (str, optional):
editedTime (int, optional):
editedStatus (bool, optional):
Returns:
None
"""
text = orm.StringCol(default=None)
time = orm.IntCol(default=None)
filePicture = orm.BLOBCol(default=None)
fileVideo = orm.BLOBCol(default=None)
fileAudio = orm.BLOBCol(default=None)
fileDocument = orm.BLOBCol(default=None)
emoji = orm.BLOBCol(default=None)
editedTime = orm.IntCol(default=None)
editedStatus = orm.BoolCol(default=False)
userConfig = orm.ForeignKey('UserConfig', refColumn="uuid")
flow = orm.ForeignKey('Flow', refColumn="flowId")
class PhotometricCalibration(ZeropointEntry):
zperr = sqlobject.FloatCol()
colorterm = sqlobject.FloatCol()
colorerr = sqlobject.FloatCol(default=0.)
fixedcolor = sqlobject.BoolCol(default=True)
table_name = 'photometric_calibration'
class Job(sqlobject.SQLObject):
job_id = sqlobject.StringCol()
vault_name = sqlobject.StringCol(length=20, default=None)
action = sqlobject.StringCol(length=20, default="")
creation_date = sqlobject.DateCol(default=None)
has_completed = sqlobject.BoolCol(default=False)
completion_date = sqlobject.DateCol(default=None)
status_code = sqlobject.StringCol(length=10, default="")
response_text = sqlobject.StringCol()
class Guest(DictSQLObject):
"""
Collects information about Guests
Refrenced to by the ChatMsg Table
"""
name = so.UnicodeCol()
admin = so.BoolCol(default=False)
parprop = so.IntCol(default=-1)
token = so.StringCol(length=8)
event = so.ForeignKey('Event')
class CheckpkgErrorTag(CheckpkgErrorTagMixin, sqlobject.SQLObject):
srv4_file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
pkgname = sqlobject.UnicodeCol(default=None, length=250)
tag_name = sqlobject.UnicodeCol(notNone=True, length=250)
tag_info = sqlobject.UnicodeCol(default=None, length=250)
msg = sqlobject.UnicodeCol(default=None, length=250)
# To cache results from checkpkg
overridden = sqlobject.BoolCol(default=False)
# The same package might have different sets of errors for different
# catalogs or Solaris releases.
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
arch = sqlobject.ForeignKey('Architecture', notNone=True)
catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True)
def __unicode__(self):
return (u"CheckpkgErrorTag: %s %s %s"
% (self.pkgname, self.tag_name, self.tag_info))
class Log(sqlobject.SQLObject):
timestamp = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now())
status = sqlobject.BoolCol(default=True)
section = sqlobject.StringCol()
message = sqlobject.StringCol()
def __init__(self, *args, **kwargs):
sqlobject.SQLObject.__init__(self, *args, **kwargs)
def __setattr__(self, name, value):
sqlobject.SQLObject.__setattr__(self, name, value)
def dict(self):
result = {
"id": self.id,
"timestamp": try_strftime(self.timestamp, default='never'),
"section": self.section,
"status": self.status,
"message": self.message
}
return result
class Source(so.SQLObject):
"""
Model a datasource of exported or manually created webpage data.
"""
# Creation date of source. This could be the last day that a source
# was used and that could be related to moving away from using a device
# or exporting data from a company computer before leaving a job.
date_created = so.DateCol(notNull=True)
date_created_idx = so.DatabaseIndex(date_created)
# The format of the datasource.
format_ = so.ForeignKey('Format', notNull=True)
# The web browser where the data originated.
browser = so.ForeignKey('Browser')
# The location where one lived and worked when creating the datasource.
location = so.ForeignKey('Location')
# True if it was work related, false if it was personal.
is_work = so.BoolCol(notNull=True)
class UserConfig(orm.SQLObject):
"""The class generates a table containing data
about the user and his settings.
Args:
uuid (int, required):
login (str, required):
password (str, required):
hash_password (str, optional)
username (str, optional):
isBot (bool, optional): default False
authId (str, optional):
email (str, optional):
avatar (str, optional):
bio (str, optional):
salt (str, optional):
key (str, optional):
Returns:
None
"""
# added alternateID for added class method @byUUID
# which will return that object
uuid = orm.IntCol(alternateID=True, unique=True, notNone=True)
login = orm.StringCol()
password = orm.StringCol()
hashPassword = orm.StringCol(default=None)
username = orm.StringCol(default=None)
isBot = orm.BoolCol(default=False)
authId = orm.StringCol(default=None)
email = orm.StringCol(default=None)
avatar = orm.BLOBCol(default=None)
bio = orm.StringCol(default=None)
salt = orm.BLOBCol(default=None)
key = orm.BLOBCol(default=None)
# Connection to the Message table
message = orm.MultipleJoin('Message')
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True, length=250)
catalogname = sqlobject.UnicodeCol(notNone=True, length=250)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
osrel_str = sqlobject.UnicodeCol(notNone=True, length=9) # "SunOS5.10"
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
pkginst_str = sqlobject.UnicodeCol(notNone=True, length=255)
registered_level_one = sqlobject.BoolCol(notNone=True)
registered_level_two = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False, length=250)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True, length=250)
bundle = sqlobject.UnicodeCol(length=250)
in_catalogs = sqlobject.MultipleJoin(
'Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile',
joinColumn='id')
catalog_idx = sqlobject.DatabaseIndex('catalogname')
basename_idx = sqlobject.DatabaseIndex('basename')
pkginst_idx = sqlobject.DatabaseIndex('pkginst')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
def __unicode__(self):
return u'%s/%s, %s' % (self.pkginst.pkgname, self.catalogname, self.md5_sum)
def __str__(self):
return str(unicode(self))
def DeleteAllDependentObjects(self):
"""Prepares the object to be deleted.
Use this function with caution.
"""
self.DeleteDependentObjectsPopulatedFromPackageItself()
logger.debug('Removing all dependent objects from %s; it will cause the '
'package to be removed from all catalogs.', self)
self.RemoveCatalogAssignments()
self.RemoveAllCheckpkgResults()
def DeleteDependentObjectsPopulatedFromPackageItself(self):
"""Removing all the objects that only depend on the package contents.
It doesn't touch rows that are created for other reasons, e.g. assignments
of packages to catalogs.
"""
logger.debug('%s - Deleting objects that only depend on the package '
'contents', self)
self.RemoveAllCswFiles()
self.RemoveOverrides()
self.RemoveDepends()
self.RemoveIncompatibles()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CswFile.sqlmeta.table,
CswFile.q.srv4_file==self)))
def RemoveDepends(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4DependsOn.sqlmeta.table,
Srv4DependsOn.q.srv4_file==self)))
def RemoveIncompatibles(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4IncompatibleWith.sqlmeta.table,
Srv4IncompatibleWith.q.srv4_file==self)))
def RemoveCatalogAssignments(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4FileInCatalog.sqlmeta.table,
Srv4FileInCatalog.q.srv4file==self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file==self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logger.debug("%s: RemoveCheckpkgResults(%s, %s, %s)",
self, os_rel, arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))))
def RemoveAllCheckpkgResults(self):
logger.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file==self)))
def RemoveOverrides(self):
logger.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file==self)))
def GetUnicodeOrNone(self, s):
"""Tries to decode UTF-8.
If the object does not decode as UTF-8, it's forced to do so, while
ignoring any potential errors.
Returns: a unicode object or a None type.
"""
if s is None:
return None
if type(s) != unicode:
try:
s = unicode(s, 'utf-8')
except UnicodeDecodeError, e:
s = s.decode("utf-8", "ignore")
s = s + u" (bad unicode detected)"
return s
class Profile(so.SQLObject):
"""
Models a user profile on Twitter.
Note that URL columns are named as 'Url', since SQLOlbject converts
'imageURL' to db column named 'image_ur_l'.
Twitter screen name and username rules:
https://help.twitter.com/en/managing-your-account/twitter-username-rules
We use slightly higher values than the ones there, to be safe.
Notes on screen name:
- This should not have unique restriction as users can edit their
screen name so others can take an older screen name. Or someone
could delete and recreate their account.
- Twitter itself enforces uniqueness across case.
"""
# Profile's ID (integer), as assigned by Twitter when the Profile was
# created. This is a global ID, rather than an ID specific to our local db.
guid = so.IntCol(alternateID=True)
# Profile screen name.
screenName = so.StringCol(notNull=True, length=30)
# Profile display Name.
name = so.StringCol(notNull=True, length=60)
# Description, as set in profile's bio.
description = so.StringCol(default=None)
# Location, as set in profile's bio.
location = so.StringCol(default=None)
# Link to the profile's image online. This will only be thumbnail size.
imageUrl = so.StringCol(default=None, validator=URL)
# Count of profile's followers.
followersCount = so.IntCol(notNull=True)
# Count of profile's statuses (tweets) posted by this profile.
statusesCount = so.IntCol(notNull=True)
# Profile's verified status.
verified = so.BoolCol(notNull=True, default=False)
# Join the Profile with its created tweets in the Tweet table.
tweets = so.MultipleJoin("Tweet")
# Date and time when follower and status counts were last updated.
modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
modifiedIdx = so.DatabaseIndex(modified)
# Get Category objects which this Profile has been assigned to, if any.
categories = so.SQLRelatedJoin("Category",
intermediateTable="profile_category",
createRelatedTable=False)
def set(self, **kwargs):
"""
Override the update hook to update the modified field if necessary.
"""
if ("followersCount" in kwargs
or "statusesCount" in kwargs) and "modified" not in kwargs:
kwargs["modified"] = so.DateTimeCol.now()
super(Profile, self).set(**kwargs)
def getFlatDescription(self):
"""
Return the description with newline characters replaced with spaces.
"""
if self.description is not None:
return lib.text_handling.flattenText(self.description)
return None
def getProfileUrl(self):
"""
Get link to the profile's page online.
:return: Twitter profile's URL, as a string.
"""
return "https://twitter.com/{0}".format(self.screenName)
def getLargeImageUrl(self):
"""
Get link to a large version profile's image, based on thumbnail URL.
The image URL comes from the API as '..._normal.jpeg', but
from API calls on loading a twitter.com page, it is possible to
see that the image media server allows variations of the last part,
to return a large image. Such as
- '..._bigger.jpeg' (which is not much bigger than the normal
thumbnail)
- '..._400x400.jpeg' (which is much bigger).
:return: image URL using 400x400 size parameter, or None if value
was not set.
"""
if self.imageUrl:
return self.imageUrl.replace("_normal", "_400x400")
return None
def prettyPrint(self):
"""
Method to print the attributes of the Profile instance neatly.
:return: dictionary of data which was printed.
"""
output = """\
Screen name : @{screenName}
Name : {name}
Verified : {verified}
Followers : {followers:,d}
Statuses : {statuses:,d}
DB tweets : {tweetCount}
Description : {description}
Profile URL : {url}
Image URL : {imageUrl}
Stats modified : {statsModified}
"""
data = dict(
screenName=self.screenName,
name=self.name,
verified=self.verified,
followers=self.followersCount,
statuses=self.statusesCount,
tweetCount=len(self.tweets),
description=self.getFlatDescription(),
url=self.getProfileUrl(),
imageUrl=self.getLargeImageUrl(),
statsModified=self.modified,
)
print(output.format(**data))
return data
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True)
catalogname = sqlobject.UnicodeCol(notNone=True)
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
registered = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True)
in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile', joinColumn='id')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
self._cached_pkgstats = None
def DeleteAllDependentObjects(self):
data_obj = self.data_obj
self.data_obj = None
if data_obj:
# It could be already missing
data_obj.destroySelf()
self.RemoveAllCswFiles()
self.RemoveAllCheckpkgResults()
self.RemoveOverrides()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CswFile.sqlmeta.table,
CswFile.q.srv4_file == self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel,
arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))))
def RemoveAllCheckpkgResults(self):
logging.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file == self)))
def RemoveOverrides(self):
logging.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file == self)))
def __unicode__(self):
return (u"Package: %s-%s, %s" %
(self.catalogname, self.version_string, self.arch.name))
def GetStatsStruct(self):
if not self._cached_pkgstats:
self._cached_pkgstats = cPickle.loads(str(self.data_obj.pickle))
return self._cached_pkgstats
def _GetBuildSource(self):
data = self.GetStatsStruct()
build_src = None
if "OPENCSW_REPOSITORY" in data["pkginfo"]:
build_src = data["pkginfo"]["OPENCSW_REPOSITORY"]
return build_src
def GetSvnUrl(self):
build_src = self._GetBuildSource()
svn_url = None
if build_src:
svn_url = re.sub(r'([^@]*).*', r'\1/Makefile', build_src)
return svn_url
def GetTracUrl(self):
build_src = self._GetBuildSource()
trac_url = None
if build_src:
trac_url = re.sub(
r'https://gar.svn.(sf|sourceforge).net/svnroot/gar/([^@]+)@(.*)',
r'http://sourceforge.net/apps/trac/gar/browser/\2/Makefile?rev=\3',
build_src)
return trac_url
def GetVendorUrl(self):
data = self.GetStatsStruct()
vendor_url = None
if "VENDOR" in data["pkginfo"]:
vendor_url = re.split(r"\s+", data["pkginfo"]["VENDOR"])[0]
return vendor_url
def GetRestRepr(self):
mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail"
data = {
'arch': self.arch.name,
'basename': self.basename,
# For compatibility with the catalog parser from catalog.py
'file_basename': self.basename,
'catalogname': self.catalogname,
'filename_arch': self.filename_arch.name,
'maintainer_email': self.maintainer.email,
'maintainer_full_name': self.maintainer.full_name,
'md5_sum': self.md5_sum,
'mtime': unicode(self.mtime),
'osrel': self.os_rel.short_name,
'pkgname': self.pkginst.pkgname,
'rev': self.rev,
'size': self.size,
'version_string': self.version_string,
# For compatibility with the catalog parser from catalog.py
'version': self.version_string,
# 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]),
'vendor_url': self.GetVendorUrl(),
'repository_url': self.GetSvnUrl(),
}
return mimetype, data
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True)
catalogname = sqlobject.UnicodeCol(notNone=True)
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
registered = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True)
in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile', joinColumn='id')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
self._cached_pkgstats = None
def DeleteAllDependentObjects(self):
data_obj = self.data_obj
self.data_obj = None
if data_obj:
# It could be already missing
data_obj.destroySelf()
self.RemoveAllCswFiles()
self.RemoveAllCheckpkgResults()
self.RemoveOverrides()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CswFile.sqlmeta.table,
CswFile.q.srv4_file == self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel,
arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))))
def RemoveAllCheckpkgResults(self):
logging.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file == self)))
def RemoveOverrides(self):
logging.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file == self)))
def __unicode__(self):
return (u"Package: %s-%s, %s" %
(self.catalogname, self.version_string, self.arch.name))
def GetUnicodeOrNone(self, s):
"""Tries to decode UTF-8.
If the object does not decode as UTF-8, it's forced to do so, while
ignoring any potential errors.
Returns: a unicode object or a None type.
"""
if s is None:
return None
if type(s) != unicode:
try:
s = unicode(s, 'utf-8')
except UnicodeDecodeError, e:
s = s.decode("utf-8", "ignore")
s = s + u" (bad unicode detected)"
return s
class PlaceJob(so.SQLObject):
"""
Listing of places which we want to regularly get trend data for.
The WOEID of a place is used to look up trends at the location,
for records which have status set to enabled. An item can be marked
as disabled, so that it will be skipped by a procedure but kept in the
table so it can be enabled again easily.
A place can appear only once in the table. Only towns or countries can be
added to the job list, due to how Twitter API works - this is enforced
when a record is created.
The table starts off empty and desired places can be added or removed
depending on admin user's preferences.
"""
class sqlmeta:
# Order as enabled (True) items first, then by jobs with oldest
# (or null) last completed timestamps first and then by oldest
# (or null) last attempted timestamps. Therefore when running jobs, the
# ones which have not been completed before or for the longest time
# are given priority over ones which were recently completed.
# And any disabled jobs will be at the bottom when viewing a report.
defaultOrder = "enabled DESC, last_completed ASC, last_attempted ASC"
_connection = conn
# Create a reference to Place table. Place IDs cannot be repeated in
# this job table.
place = so.ForeignKey("Place", unique=True)
# Create an index on place.
placeIdx = so.DatabaseIndex(place)
# Date and time when record was created.
created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# When the job was last attempted regardless of outcome.
lastAttempted = so.DateTimeCol(notNull=False, default=None)
# When the job item was last completed successfully. Defaults to null.
lastCompleted = so.DateTimeCol(notNull=False, default=None)
# Create an index on last completed.
lastCompletedIdx = so.DatabaseIndex(lastCompleted)
# Boolean flag for whether the job item is enabled or should be skipped.
enabled = so.BoolCol(notNull=True, default=True)
def start(self):
"""
Use this function to update the last attempted time.
"""
self._set_lastAttempted(so.DateTimeCol.now())
def end(self):
"""
Use this function to update the last run time, if successful.
"""
self._set_lastCompleted(so.DateTimeCol.now())
def setEnabled(self):
"""
Set the job to enabled.
"""
self.enabled = True
def setDisabled(self):
"""
Set the job to disabled.
"""
self.enabled = False
def getStatus(self, asText=False):
"""
Get the status from when the job was last run.
If last attempted time is None, then return None since we cannot
confirm success or failure.
Return True for success, if last completed time is not None and
is after last attempted time. Otherwise return False for failure.
:param asText: Default False. If True, return status as human-readable
string.
:return status: job status as OK (True) or failed (False) or not
run (None). Returned as human-readable string if asText is True.
"""
if self.lastAttempted:
if self.lastCompleted and self.lastCompleted > self.lastAttempted:
status = "OK" if asText else True
else:
status = "failed" if asText else False
else:
status = "not run" if asText else None
return status
class Trend(so.SQLObject):
"""
A trending topic on Twitter, meaning a lot of Twitter accounts were talking
about that topic.
A topic exists at a point in time and maps to a specific place. It's term
can either be a hashtag (starts with '#' and has no spaces) or a keyword
phrase (no '#' and can have multiple words).
Note that the topic is not unique, since the topic can be repeated in
many locations and it can be repeated in one location across time.
The topic has a trending volume figure, which is how many tweets there are
about the topic in the past 24 hours (according to Twitter API docs).
Adding up trends for a Place taken at the SAME time each day should give
an accurate total of tweets for the period, since there should not be any
overlap in tweets across two consecutive 24-hour periods. One might use
the earliest record available for the day, assuming the cron job
runs soon after midnight, so that any ad hoc data will not skew the
results. However, the value will for the 24 hours of the PREVIOUS day.
Note that the topic volume shown is always GLOBAL total volume i.e.
independent of the location used to look up the topic. Volume usually
ranges from around 10,000 to 1 million and smaller values are returned
as null by Twitter API.
However, it is still useful to count the number of places which a tppic
is trending in as an indication of how widespread it is.
"""
class sqlmeta:
defaultOrder = "-timestamp"
_connection = conn
# The topic which trended.
topic = so.StringCol(length=64)
topicIdx = so.DatabaseIndex(topic)
# Whether the topic is a hashtag i.e. starts with '#'.
hashtag = so.BoolCol(default=False)
# Number of global tweets about topic in past 24 hours. Required since
# there no default set here. Null values are allowed.
volume = so.IntCol(notNull=False)
# The place associated with this trend record. See `setPlace` for why
# this is an optional field.
place = so.ForeignKey("Place", notNull=False, default=None)
placeIdx = so.DatabaseIndex(place)
# Date and time when record was created.
timestamp = so.DateTimeCol(default=so.DateTimeCol.now)
timestampIdx = so.DatabaseIndex(timestamp)
def setPlace(self, woeid):
"""
Link an existing Trend and Place records, given a Place WOEID.
Expects a WOEID int, gets ID for the Place, then stores it as the
foreign key for the Trend.
This doesn't work to be placed in __init__ since then its called
on a select and doen't work for modelCreate because the input kwargs
are validated before the method is called.
:param woeid: integer value for WOEID of the Place to link to.
:return self: returns object instance.
"""
assert isinstance(
woeid,
int), "Expected WOEID as an `int`, but " "got type `{0}`.".format(
type(woeid).__name__)
try:
self.placeID = Place.byWoeid(woeid).id
except so.SQLObjectNotFound as e:
raise type(e)(
"Place with WOEID {0} could not be found in the db.".format(
woeid))
return self
def _set_topic(self, value):
"""
Override the topic setting method, so that hashtag boolean is updated
automatically whenever topic is set.
:param value: string value to set as the topic.
"""
self._SO_set_topic(value)
if value.startswith("#"):
self._SO_set_hashtag(True)
else:
self._SO_set_hashtag(False)
@classmethod
def getColumnNames(cls):
"""
Return a list of column names for the class, as strings. This is
created from a dictionary, so the order is not guaranteed.
"""
return list(cls.sqlmeta.columns.keys())
def getData(self, quiet=True):
"""
Output the current record with key:value pairs for column name
and value. Note that this is not suitable to converted to JSON
because of the data types of values.
"""
data = {col: getattr(self, col) for col in self.getColumnNames()}
if not quiet:
for k, v in data.items():
# Align key to the right.
print("{0:>15} : {1}".format(k, v))
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True)
catalogname = sqlobject.UnicodeCol(notNone=True)
# The data structure can be missing - necessary for fake SUNW
# packages.
data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
latest = sqlobject.BoolCol(notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
registered = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True)
in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile', joinColumn='id')
def DeleteAllDependentObjects(self):
data_obj = self.data_obj
self.data_obj = None
if data_obj:
# It could be already missing
data_obj.destroySelf()
self.RemoveAllCswFiles()
self.RemoveAllCheckpkgResults()
self.RemoveOverrides()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CswFile.sqlmeta.table,
CswFile.q.srv4_file == self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel,
arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self,
CheckpkgErrorTag.q.os_rel == os_rel,
CheckpkgErrorTag.q.arch == arch,
CheckpkgErrorTag.q.catrel == catrel))))
def RemoveAllCheckpkgResults(self):
logging.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file == self)))
def RemoveOverrides(self):
logging.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(
sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file == self)))
def __unicode__(self):
return (u"Package: %s-%s, %s" %
(self.catalogname, self.version_string, self.arch.name))
def GetStatsStruct(self):
return cPickle.loads(str(self.data_obj.pickle))
def GetRestRepr(self):
mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail"
data = {
'arch': self.arch.name,
'basename': self.basename,
# For compatibility with the catalog parser from catalog.py
'file_basename': self.basename,
'catalogname': self.catalogname,
'filename_arch': self.filename_arch.name,
'maintainer_email': self.maintainer.email,
'maintainer_full_name': self.maintainer.full_name,
'md5_sum': self.md5_sum,
'mtime': unicode(self.mtime),
'osrel': self.os_rel.short_name,
'pkgname': self.pkginst.pkgname,
'rev': self.rev,
'size': self.size,
'version_string': self.version_string,
# For compatibility with the catalog parser from catalog.py
'version': self.version_string,
# 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]),
}
return mimetype, data