class CswFile(sqlobject.SQLObject):
"""Represents a file in a catalog.
There can be multiple files with the same basename and the same path,
belonging to different packages.
This class needs to also contain files from the operating system,
coming from SUNW packages, for which we don't have the original srv4
files. (Even if we could, they are generally not accessible.) They
need to be specific to Solaris release and architecture, so we need
a way to link CswFile with a specific catalog.
Fake registered Srv4FileStats object would do, but we would have to
ensure that they can't be associated with a catalog. Also, we'd have
to generate fake md5 sums for them.
"""
basename = sqlobject.UnicodeCol(length=255, notNone=True)
path = sqlobject.UnicodeCol(notNone=True, length=900)
line = sqlobject.UnicodeCol(notNone=True, length=900)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
srv4_file = sqlobject.ForeignKey('Srv4FileStats')
basename_idx = sqlobject.DatabaseIndex('basename')
def __unicode__(self):
return u"File: %s" % os.path.join(self.path, self.basename)
class Srv4FileInCatalog(sqlobject.SQLObject):
"""Assignment of a particular srv4 file to a specific catalog.
There could be one more layer, to which arch and osrel could be moved.
But for now, it's going to be a not-normalized structure.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
osrel = sqlobject.ForeignKey('OsRelease', notNone=True)
catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True)
srv4file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
created_on = sqlobject.DateTimeCol(
notNone=True,
default=sqlobject.DateTimeCol.now)
created_by = sqlobject.UnicodeCol(length=50, notNone=True)
uniqueness_idx = sqlobject.DatabaseIndex(
'arch', 'osrel', 'catrel', 'srv4file',
unique=True)
# http://turbogears.org/1.0/docs/SQLObject/Caching.html#what-does-cache-false-do
class sqlmeta:
cacheValues = False
def __unicode__(self):
return (u"%s is in catalog %s %s %s"
% (self.srv4file,
self.arch.name, self.osrel.full_name, self.catrel.name))
class CswFile(sqlobject.SQLObject):
"""Represents a file in a catalog.
There can be multiple files with the same basename and the same path,
belonging to different packages.
This class needs to also contain files from the operating system,
coming from SUNW packages, for which we don't have the original srv4
files. (Even if we could, they are generally not accessible.) They
need to be specific to Solaris release and architecture, so we need
a way to link CswFile with a specific catalog.
Fake registered Srv4FileStats object would do, but we would have to
ensure that they can't be associated with a catalog. Also, we'd have
to generate fake md5 sums for them.
"""
class sqlmeta:
# MySQL uses case-insensitive collation by default, which doesn't make sense
# for file names. If the utf8_bin (or other case sensitive) collation is not
# used, it reports e.g. Zcat and zcat as a file collision, while it really
# isn't one.
createSQL = {
'mysql' : [
'ALTER TABLE csw_file CONVERT TO CHARACTER SET utf8 '
'COLLATE utf8_bin']
}
basename = sqlobject.UnicodeCol(length=255, notNone=True)
path = sqlobject.UnicodeCol(notNone=True, length=900)
line = sqlobject.UnicodeCol(notNone=True, length=900)
# Symlinks don't have permissions on their own
perm_user = sqlobject.UnicodeCol(notNone=False, length=255)
perm_group = sqlobject.UnicodeCol(notNone=False, length=255)
perm_mode = sqlobject.UnicodeCol(notNone=False, length=5)
target = sqlobject.UnicodeCol(notNone=False, length=900)
mimetype = sqlobject.UnicodeCol(notNone=False, length=255)
machine = sqlobject.UnicodeCol(notNone=False, length=255)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
srv4_file = sqlobject.ForeignKey('Srv4FileStats')
basename_idx = sqlobject.DatabaseIndex('basename')
path_idx = sqlobject.DatabaseIndex({'column': 'path', 'length': 255})
def FullPath(self):
return os.path.join(self.path, self.basename)
def __unicode__(self):
return u"%s %s %s %s %s" % (self.perm_user, self.perm_group, self.perm_mode,
self.FullPath(), self.mimetype)
class ProfileCategory(so.SQLObject):
"""
Model the many-to-many relationship between Profile and Category records.
Attributes are based on a recommendation in the SQLObject docs.
"""
profile = so.ForeignKey("Profile", notNull=True, cascade=True)
category = so.ForeignKey("Category", notNull=True, cascade=True)
uniqueIdx = so.DatabaseIndex(profile, category, unique=True)
class Town(Place):
"""
Place which falls into Town or Unknown category in Twitter API.
"""
_inheritable = False
# Country which this Town belongs. Optional and defaults to None.
country = so.ForeignKey("Country", default=None)
countryIdx = so.DatabaseIndex(country)
class TweetCampaign(so.SQLObject):
"""
Model the many-to-many relationship between Tweet and Campaign records.
Attributes are based on a recommendation in the SQLObject docs for doing
this relationship.
"""
tweet = so.ForeignKey("Tweet", notNull=True, cascade=True)
campaign = so.ForeignKey("Campaign", notNull=True, cascade=True)
uniqueIdx = so.DatabaseIndex(tweet, campaign, unique=True)
class PageLabel(so.SQLObject):
"""
Model the many-to-many relationship between Page and Label records.
A Label may be applied to many Pages and a Page may have many Labels.
But a paired relationship must be unique.
Attributes here are based on a recommendation in the SQLObject docs.
"""
page = so.ForeignKey('Page', notNull=True, cascade=True)
label = so.ForeignKey('Label', notNull=True, cascade=True)
unique_idx = so.DatabaseIndex(page, label, unique=True)
class Page(so.SQLObject):
"""
Model a URI for a webpage on the internet.
Do not worry about duplicate pairs of domain and path, since we want
to allow those to occur on an import and to clean them up later.
A page may have a null Folder (as unsorted), though a folder must always
have a parent folder, even if the top folder is "root".
"""
# The host website for the page.
# TODO: Ensure this is always converted lowercase rather than raising
# an error.
domain = so.ForeignKey('Domain', notNull=True)
# The location of the webpage relative to the domain.
# TODO: Should this start with forwardslash? Check what happens when
# splitting and joining.
# TODO: Create custom validator?
path = so.UnicodeCol(notNull=True)
# Webpage title, usually taken from the metadata of the HTML head section.
title = so.UnicodeCol(default=None)
# The date and time when the record was created. Defaults to the
# current time.
created_at = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# Optional preview image for the link, scraped from the metadata.
image_url = so.UnicodeCol(default=None)
description = so.UnicodeCol(default=None)
# The folder this link is placed into. If null then the link must still
# be sorted. Domain and path pairs must be unique in a folder.
folder = so.ForeignKey('Folder')
unique_idx = so.DatabaseIndex(domain, path, folder, unique=True)
source = so.ForeignKey('Source', notNull=True)
# Link to labels which this page is assigned to.
labels = so.SQLRelatedJoin('Labels',
intermediateTable='page_label',
createRelatedTable=False)
def get_url(self):
return "".join((self.domain.value, self.path))
class ObservationCalibration(sqlobject.SQLObject):
cluster = sqlobject.StringCol(length=55)
filter = sqlobject.StringCol(length=55)
mangledSpecification = sqlobject.StringCol(length=250)
calibration = sqlobject.ForeignKey('ZeropointEntry')
ClusterFilterIndex = sqlobject.DatabaseIndex('cluster',
'filter',
'mangledSpecification',
unique=True)
def _get_specification(self):
return unmangleSpecification(self.mangledSpecification)
def _set_specification(self, **specification):
self.mangledSpecification = mangleSpecification(specification)
class Srv4FileInCatalog(sqlobject.SQLObject):
"""Assignment of a particular srv4 file to a specific catalog.
There could be one more layer, to which arch and osrel could be moved.
But for now, it's going to be a not-normalized structure.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
osrel = sqlobject.ForeignKey('OsRelease', notNone=True)
catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True)
srv4file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
uniqueness_idx = sqlobject.DatabaseIndex('arch',
'osrel',
'catrel',
'srv4file',
unique=True)
def __unicode__(self):
return (u"%s is in catalog %s %s %s" %
(self.srv4file, self.arch.name, self.osrel.full_name,
self.catrel.name))
class Source(so.SQLObject):
"""
Model a datasource of exported or manually created webpage data.
"""
# Creation date of source. This could be the last day that a source
# was used and that could be related to moving away from using a device
# or exporting data from a company computer before leaving a job.
date_created = so.DateCol(notNull=True)
date_created_idx = so.DatabaseIndex(date_created)
# The format of the datasource.
format_ = so.ForeignKey('Format', notNull=True)
# The web browser where the data originated.
browser = so.ForeignKey('Browser')
# The location where one lived and worked when creating the datasource.
location = so.ForeignKey('Location')
# True if it was work related, false if it was personal.
is_work = so.BoolCol(notNull=True)
class Profile(so.SQLObject):
"""
Models a user profile on Twitter.
Note that URL columns are named as 'Url', since SQLOlbject converts
'imageURL' to db column named 'image_ur_l'.
Twitter screen name and username rules:
https://help.twitter.com/en/managing-your-account/twitter-username-rules
We use slightly higher values than the ones there, to be safe.
Notes on screen name:
- This should not have unique restriction as users can edit their
screen name so others can take an older screen name. Or someone
could delete and recreate their account.
- Twitter itself enforces uniqueness across case.
"""
# Profile's ID (integer), as assigned by Twitter when the Profile was
# created. This is a global ID, rather than an ID specific to our local db.
guid = so.IntCol(alternateID=True)
# Profile screen name.
screenName = so.StringCol(notNull=True, length=30)
# Profile display Name.
name = so.StringCol(notNull=True, length=60)
# Description, as set in profile's bio.
description = so.StringCol(default=None)
# Location, as set in profile's bio.
location = so.StringCol(default=None)
# Link to the profile's image online. This will only be thumbnail size.
imageUrl = so.StringCol(default=None, validator=URL)
# Count of profile's followers.
followersCount = so.IntCol(notNull=True)
# Count of profile's statuses (tweets) posted by this profile.
statusesCount = so.IntCol(notNull=True)
# Profile's verified status.
verified = so.BoolCol(notNull=True, default=False)
# Join the Profile with its created tweets in the Tweet table.
tweets = so.MultipleJoin("Tweet")
# Date and time when follower and status counts were last updated.
modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
modifiedIdx = so.DatabaseIndex(modified)
# Get Category objects which this Profile has been assigned to, if any.
categories = so.SQLRelatedJoin("Category",
intermediateTable="profile_category",
createRelatedTable=False)
def set(self, **kwargs):
"""
Override the update hook to update the modified field if necessary.
"""
if ("followersCount" in kwargs
or "statusesCount" in kwargs) and "modified" not in kwargs:
kwargs["modified"] = so.DateTimeCol.now()
super(Profile, self).set(**kwargs)
def getFlatDescription(self):
"""
Return the description with newline characters replaced with spaces.
"""
if self.description is not None:
return lib.text_handling.flattenText(self.description)
return None
def getProfileUrl(self):
"""
Get link to the profile's page online.
:return: Twitter profile's URL, as a string.
"""
return "https://twitter.com/{0}".format(self.screenName)
def getLargeImageUrl(self):
"""
Get link to a large version profile's image, based on thumbnail URL.
The image URL comes from the API as '..._normal.jpeg', but
from API calls on loading a twitter.com page, it is possible to
see that the image media server allows variations of the last part,
to return a large image. Such as
- '..._bigger.jpeg' (which is not much bigger than the normal
thumbnail)
- '..._400x400.jpeg' (which is much bigger).
:return: image URL using 400x400 size parameter, or None if value
was not set.
"""
if self.imageUrl:
return self.imageUrl.replace("_normal", "_400x400")
return None
def prettyPrint(self):
"""
Method to print the attributes of the Profile instance neatly.
:return: dictionary of data which was printed.
"""
output = """\
Screen name : @{screenName}
Name : {name}
Verified : {verified}
Followers : {followers:,d}
Statuses : {statuses:,d}
DB tweets : {tweetCount}
Description : {description}
Profile URL : {url}
Image URL : {imageUrl}
Stats modified : {statsModified}
"""
data = dict(
screenName=self.screenName,
name=self.name,
verified=self.verified,
followers=self.followersCount,
statuses=self.statusesCount,
tweetCount=len(self.tweets),
description=self.getFlatDescription(),
url=self.getProfileUrl(),
imageUrl=self.getLargeImageUrl(),
statsModified=self.modified,
)
print(output.format(**data))
return data
class Tweet(so.SQLObject):
"""
Models a tweet on Twitter.
If we are inserting the Tweet in our db, we expect to always have the
author's profile in the Profile table. If the tweet is a reply, we will
have references to the target Profile and original Tweet as GUID integers.
But we are unlikely to have those object stored in our db. Use
the `.getInReplyToTweet` and `.getInReplyToProfile` methods to see if those
exist in the db, otherwise use the GUIDs to look up data from the
Twitter API and then store them locally as db records.
For relating a Tweet to its author Profile with a foreign key, a
`setProfileByGuid` method could be implemented to set the profile
foreign key using a given GUID, but that would require doing a search
each time. So, when creating a Tweet object or multiple objects for one
Profile, it is preferable to get the Profile object's ID once
and then repeately pass that in as an argument for each Tweet object
that is created for that Profile.
For ordering, the '-guid' syntax is here is preferred, since
'guid DESC' results in an error when getting tweets of a Profile object,
even though doing a query on Tweet class itself is fine.
`AttributeError: 'Tweet' object has no attribute 'guid DESC'`
The error is also raised for multiple names e.g. '-guid, message'.
"""
class sqlmeta:
# Show recent Tweets (with higher GUID values) first.
defaultOrder = "-guid"
# Tweet ID (integer), as assigned by Twitter when the Tweet was posted.
# This is a global ID, rather than specific to our local db.
guid = so.IntCol(alternateID=True)
# Link to Tweet's author in the Profile table. Delete Tweet if
# the Profile is deleted.
profile = so.ForeignKey("Profile", notNull=True, cascade=True)
profileIdx = so.DatabaseIndex(profile)
# Date and time the tweet was posted.
createdAt = so.DateTimeCol(notNull=True)
createdAtIdx = so.DatabaseIndex(createdAt)
# Tweet message text. Length is not validated since expanded tweets can
# be longer than the standard 280 (previously 140) characters.
message = so.StringCol(notNull=True)
# Count of favorites on this Tweet.
favoriteCount = so.IntCol(notNull=True)
# Count of retweets of this Tweet.
retweetCount = so.IntCol(notNull=True)
# If the tweet is a reply, the GUID of the Tweet which the reply is
# directed at (from reply_to_status_id field). This does not require
# the Tweet to be in the local db.
inReplyToTweetGuid = so.IntCol(default=None)
# If the tweet is a reply, the GUID of the Profile which the reply is
# directed at (from reply_to_user_id field). This does not require
# the Tweet to be in the local db.
inReplyToProfileGuid = so.IntCol(default=None)
# Date and time when favorite and retweet counts where last updated.
modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
modifiedIdx = so.DatabaseIndex(modified)
# Get Campaign objects which this Profile has been assigned to, if any.
campaigns = so.SQLRelatedJoin("Campaign",
intermediateTable="tweet_campaign",
createRelatedTable=False)
def set(self, **kwargs):
"""
Override the update hook to update the modified field if necessary.
"""
if ("favoriteCount" in kwargs
or "retweetCount" in kwargs) and "modified" not in kwargs:
kwargs["modified"] = so.DateTimeCol.now()
super(Tweet, self).set(**kwargs)
def isRT(self):
return self.message.startswith("RT ")
def isReply(self):
return self.inReplyToProfileGuid is not None
def getFlatMessage(self):
"""
Return the message with newline characters replaced with spaces.
"""
return lib.text_handling.flattenText(self.message)
def getInReplyToTweet(self):
"""
If this Tweet is a reply, get the original Tweet it was directed at.
:return: single Tweet object. Return None if this is not a reply. Raise
an error if the Tweet is not in the local db.
"""
if self.inReplyToTweetGuid:
try:
return Tweet.byGuid(self.inReplyToTweetGuid)
except SQLObjectNotFound as e:
raise type(e)(
"Could not find Tweet in db with GUID: {0}".format(
self.inReplyToTweetGuid))
return None
def getInReplyToProfile(self):
"""
If this Tweet is a reply, get the Profile which it was directed at.
:return: single Profile object. Return None if this is not a reply.
Raise an error if the Tweet is not in the local db.
"""
if self.inReplyToProfileGuid:
try:
return Profile.byGuid(self.inReplyToProfileGuid)
except SQLObjectNotFound as e:
raise type(e)(
"Could not find Profile in db with GUID: {0}".format(
self.inReplyToProfileGuid))
return None
def getTweetURL(self):
"""
Return URL for the tweet as a string, using tweet author's screen name
and the tweet's GUID.
"""
return "https://twitter.com/{screenName}/status/{tweetID}".format(
screenName=self.profile.screenName, tweetID=self.guid)
def prettyPrint(self):
"""
Method to print the attributes of the Tweet instance neatly.
:return: dictionary of data which was printed.
"""
output = """\
Author : @{screenName} - {name} - {followers:,d} followers
Created at : {createdAt}
Message : {message}
Favorites : {favoriteCount:,d}
Retweets : {retweetCount:,d}
Reply To User ID : {replyProf}
Reply To Tweet ID : {replyTweet}
URL : {url}
Stats modified : {statsModified}
"""
author = self.profile
data = dict(
screenName=author.screenName,
createdAt=self.createdAt,
name=author.name,
followers=author.followersCount,
message=self.getFlatMessage(),
favoriteCount=self.favoriteCount,
retweetCount=self.retweetCount,
replyProf=self.inReplyToProfileGuid,
replyTweet=self.inReplyToTweetGuid,
url=self.getTweetURL(),
statsModified=self.modified,
)
print(output.format(**data))
return data
def report(self):
"""
Return Tweet and Profile data as dict for writing a CSV report.
"""
author = self.profile
return {
"Screen name": author.screenName,
"Followers": author.followersCount,
"Tweet URL": self.getTweetURL(),
"Tweet ID": self.guid,
"Tweeted at": str(self.createdAt),
"Is reply": "Y" if self.isReply() else "N",
"Is RT": "Y" if self.isRT() else "N",
"Message": self.message,
"Favs": self.favoriteCount,
"RTs": self.retweetCount,
}
class Trend(so.SQLObject):
"""
A trending topic on Twitter, meaning a lot of Twitter accounts were talking
about that topic.
A topic exists at a point in time and maps to a specific place. It's term
can either be a hashtag (starts with '#' and has no spaces) or a keyword
phrase (no '#' and can have multiple words).
Note that the topic is not unique, since the topic can be repeated in
many locations and it can be repeated in one location across time.
The topic has a trending volume figure, which is how many tweets there are
about the topic in the past 24 hours (according to Twitter API docs).
Adding up trends for a Place taken at the SAME time each day should give
an accurate total of tweets for the period, since there should not be any
overlap in tweets across two consecutive 24-hour periods. One might use
the earliest record available for the day, assuming the cron job
runs soon after midnight, so that any ad hoc data will not skew the
results. However, the value will for the 24 hours of the PREVIOUS day.
Note that the topic volume shown is always GLOBAL total volume i.e.
independent of the location used to look up the topic. Volume usually
ranges from around 10,000 to 1 million and smaller values are returned
as null by Twitter API.
However, it is still useful to count the number of places which a tppic
is trending in as an indication of how widespread it is.
"""
class sqlmeta:
defaultOrder = "-timestamp"
_connection = conn
# The topic which trended.
topic = so.StringCol(length=64)
topicIdx = so.DatabaseIndex(topic)
# Whether the topic is a hashtag i.e. starts with '#'.
hashtag = so.BoolCol(default=False)
# Number of global tweets about topic in past 24 hours. Required since
# there no default set here. Null values are allowed.
volume = so.IntCol(notNull=False)
# The place associated with this trend record. See `setPlace` for why
# this is an optional field.
place = so.ForeignKey("Place", notNull=False, default=None)
placeIdx = so.DatabaseIndex(place)
# Date and time when record was created.
timestamp = so.DateTimeCol(default=so.DateTimeCol.now)
timestampIdx = so.DatabaseIndex(timestamp)
def setPlace(self, woeid):
"""
Link an existing Trend and Place records, given a Place WOEID.
Expects a WOEID int, gets ID for the Place, then stores it as the
foreign key for the Trend.
This doesn't work to be placed in __init__ since then its called
on a select and doen't work for modelCreate because the input kwargs
are validated before the method is called.
:param woeid: integer value for WOEID of the Place to link to.
:return self: returns object instance.
"""
assert isinstance(
woeid,
int), "Expected WOEID as an `int`, but " "got type `{0}`.".format(
type(woeid).__name__)
try:
self.placeID = Place.byWoeid(woeid).id
except so.SQLObjectNotFound as e:
raise type(e)(
"Place with WOEID {0} could not be found in the db.".format(
woeid))
return self
def _set_topic(self, value):
"""
Override the topic setting method, so that hashtag boolean is updated
automatically whenever topic is set.
:param value: string value to set as the topic.
"""
self._SO_set_topic(value)
if value.startswith("#"):
self._SO_set_hashtag(True)
else:
self._SO_set_hashtag(False)
@classmethod
def getColumnNames(cls):
"""
Return a list of column names for the class, as strings. This is
created from a dictionary, so the order is not guaranteed.
"""
return list(cls.sqlmeta.columns.keys())
def getData(self, quiet=True):
"""
Output the current record with key:value pairs for column name
and value. Note that this is not suitable to converted to JSON
because of the data types of values.
"""
data = {col: getattr(self, col) for col in self.getColumnNames()}
if not quiet:
for k, v in data.items():
# Align key to the right.
print("{0:>15} : {1}".format(k, v))
class PlaceJob(so.SQLObject):
"""
Listing of places which we want to regularly get trend data for.
The WOEID of a place is used to look up trends at the location,
for records which have status set to enabled. An item can be marked
as disabled, so that it will be skipped by a procedure but kept in the
table so it can be enabled again easily.
A place can appear only once in the table. Only towns or countries can be
added to the job list, due to how Twitter API works - this is enforced
when a record is created.
The table starts off empty and desired places can be added or removed
depending on admin user's preferences.
"""
class sqlmeta:
# Order as enabled (True) items first, then by jobs with oldest
# (or null) last completed timestamps first and then by oldest
# (or null) last attempted timestamps. Therefore when running jobs, the
# ones which have not been completed before or for the longest time
# are given priority over ones which were recently completed.
# And any disabled jobs will be at the bottom when viewing a report.
defaultOrder = "enabled DESC, last_completed ASC, last_attempted ASC"
_connection = conn
# Create a reference to Place table. Place IDs cannot be repeated in
# this job table.
place = so.ForeignKey("Place", unique=True)
# Create an index on place.
placeIdx = so.DatabaseIndex(place)
# Date and time when record was created.
created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now)
# When the job was last attempted regardless of outcome.
lastAttempted = so.DateTimeCol(notNull=False, default=None)
# When the job item was last completed successfully. Defaults to null.
lastCompleted = so.DateTimeCol(notNull=False, default=None)
# Create an index on last completed.
lastCompletedIdx = so.DatabaseIndex(lastCompleted)
# Boolean flag for whether the job item is enabled or should be skipped.
enabled = so.BoolCol(notNull=True, default=True)
def start(self):
"""
Use this function to update the last attempted time.
"""
self._set_lastAttempted(so.DateTimeCol.now())
def end(self):
"""
Use this function to update the last run time, if successful.
"""
self._set_lastCompleted(so.DateTimeCol.now())
def setEnabled(self):
"""
Set the job to enabled.
"""
self.enabled = True
def setDisabled(self):
"""
Set the job to disabled.
"""
self.enabled = False
def getStatus(self, asText=False):
"""
Get the status from when the job was last run.
If last attempted time is None, then return None since we cannot
confirm success or failure.
Return True for success, if last completed time is not None and
is after last attempted time. Otherwise return False for failure.
:param asText: Default False. If True, return status as human-readable
string.
:return status: job status as OK (True) or failed (False) or not
run (None). Returned as human-readable string if asText is True.
"""
if self.lastAttempted:
if self.lastCompleted and self.lastCompleted > self.lastAttempted:
status = "OK" if asText else True
else:
status = "failed" if asText else False
else:
status = "not run" if asText else None
return status
class Place(InheritableSQLObject):
"""
A place in the world. This is created from the Yahoo Where On Earth
locations as returned by Twitter API.
This table has childName to indicate which table the object is in and
therefore the parent Place's location type.
Name is *not* an alternateID, since place names can be duplicated around
the world e.g. Barcelona in Venezuela and Spain.
Therefore `.byName` is not available, but we can do a `.selectBy` with
both town name and the country's ID set in the where clause, to ensure
we get one result.
Default order by ID is omitted as it causes ambiguity issues on some
selects. And timestamp is not recognised as a column on the subclasses
so cannot be used either.
"""
_connection = conn
# WOEID integer value from Yahoo system.
# Note that `.byWoeid` can be used on Place class, but cannot be used
# on any subclasses. This is because of the `id` in the order by statement
# having ambiguous meaning in `where (Place.id = {subclass}.id)`.
woeid = so.IntCol(alternateID=True)
# Name of the place.
name = so.StringCol(length=64, default=None)
nameIdx = so.DatabaseIndex(name)
# Date and time when record was created.
timestamp = so.DateTimeCol(default=so.DateTimeCol.now)
timestampIdx = so.DatabaseIndex(timestamp)
# Get all the trend records relating to this Place.
hasTrends = so.MultipleJoin("Trend")
@classmethod
def getColumnNames(cls):
"""
Return a list of column names for the class, as strings. This is
created from a dictionary, so the order is not guaranteed.
"""
return list(cls.sqlmeta.columns.keys())
def getData(self, quiet=True):
"""
Output the current record with key:value pairs for column name
and value. Note that this is not suitable to be converted directly to
JSON because of the data types of some values.
TODO: Ensure that attributes of the parent and child are all accessed,
to get more use out of this.
"""
data = {col: getattr(self, col) for col in self.getColumnNames()}
if not quiet:
for k, v in data.items():
print(f"{k:>15} : {v}")
return data
class Srv4FileStats(sqlobject.SQLObject):
"""Represents a srv4 file.
It focuses on the stats, but it can as well represent just a srv4 file.
"""
arch = sqlobject.ForeignKey('Architecture', notNone=True)
basename = sqlobject.UnicodeCol(notNone=True, length=250)
catalogname = sqlobject.UnicodeCol(notNone=True, length=250)
filename_arch = sqlobject.ForeignKey('Architecture', notNone=True)
maintainer = sqlobject.ForeignKey('Maintainer', notNone=False)
md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32)
size = sqlobject.IntCol()
mtime = sqlobject.DateTimeCol(notNone=False)
os_rel = sqlobject.ForeignKey('OsRelease', notNone=True)
osrel_str = sqlobject.UnicodeCol(notNone=True, length=9) # "SunOS5.10"
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
pkginst_str = sqlobject.UnicodeCol(notNone=True, length=255)
registered_level_one = sqlobject.BoolCol(notNone=True)
registered_level_two = sqlobject.BoolCol(notNone=True)
use_to_generate_catalogs = sqlobject.BoolCol(notNone=True)
rev = sqlobject.UnicodeCol(notNone=False, length=250)
stats_version = sqlobject.IntCol(notNone=True)
version_string = sqlobject.UnicodeCol(notNone=True, length=250)
bundle = sqlobject.UnicodeCol(length=250)
in_catalogs = sqlobject.MultipleJoin(
'Srv4FileInCatalog',
joinColumn='srv4file_id')
files = sqlobject.MultipleJoin('CswFile',
joinColumn='id')
catalog_idx = sqlobject.DatabaseIndex('catalogname')
basename_idx = sqlobject.DatabaseIndex('basename')
pkginst_idx = sqlobject.DatabaseIndex('pkginst')
def __init__(self, *args, **kwargs):
super(Srv4FileStats, self).__init__(*args, **kwargs)
def __unicode__(self):
return u'%s/%s, %s' % (self.pkginst.pkgname, self.catalogname, self.md5_sum)
def __str__(self):
return str(unicode(self))
def DeleteAllDependentObjects(self):
"""Prepares the object to be deleted.
Use this function with caution.
"""
self.DeleteDependentObjectsPopulatedFromPackageItself()
logger.debug('Removing all dependent objects from %s; it will cause the '
'package to be removed from all catalogs.', self)
self.RemoveCatalogAssignments()
self.RemoveAllCheckpkgResults()
def DeleteDependentObjectsPopulatedFromPackageItself(self):
"""Removing all the objects that only depend on the package contents.
It doesn't touch rows that are created for other reasons, e.g. assignments
of packages to catalogs.
"""
logger.debug('%s - Deleting objects that only depend on the package '
'contents', self)
self.RemoveAllCswFiles()
self.RemoveOverrides()
self.RemoveDepends()
self.RemoveIncompatibles()
def RemoveAllCswFiles(self):
# Removing existing files, using sqlbuilder to use sql-level
# mechanisms without interacting with Python.
# http://www.mail-archive.com/[email protected]/msg00520.html
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CswFile.sqlmeta.table,
CswFile.q.srv4_file==self)))
def RemoveDepends(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4DependsOn.sqlmeta.table,
Srv4DependsOn.q.srv4_file==self)))
def RemoveIncompatibles(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4IncompatibleWith.sqlmeta.table,
Srv4IncompatibleWith.q.srv4_file==self)))
def RemoveCatalogAssignments(self):
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
Srv4FileInCatalog.sqlmeta.table,
Srv4FileInCatalog.q.srv4file==self)))
def GetOverridesResult(self):
return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file==self)
def GetErrorTagsResult(self, os_rel, arch, catrel):
assert arch.name != 'all', (
"Asked for the 'all' architecture, this is not valid "
"for GetErrorTagsResult().")
return CheckpkgErrorTag.select(
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))
def RemoveCheckpkgResults(self, os_rel, arch, catrel):
logger.debug("%s: RemoveCheckpkgResults(%s, %s, %s)",
self, os_rel, arch, catrel)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
sqlobject.AND(
CheckpkgErrorTag.q.srv4_file==self,
CheckpkgErrorTag.q.os_rel==os_rel,
CheckpkgErrorTag.q.arch==arch,
CheckpkgErrorTag.q.catrel==catrel))))
def RemoveAllCheckpkgResults(self):
logger.debug("%s: RemoveAllCheckpkgResults()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgErrorTag.sqlmeta.table,
CheckpkgErrorTag.q.srv4_file==self)))
def RemoveOverrides(self):
logger.debug("%s: RemoveOverrides()", self)
sqlobject.sqlhub.processConnection.query(
sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete(
CheckpkgOverride.sqlmeta.table,
CheckpkgOverride.q.srv4_file==self)))
def GetUnicodeOrNone(self, s):
"""Tries to decode UTF-8.
If the object does not decode as UTF-8, it's forced to do so, while
ignoring any potential errors.
Returns: a unicode object or a None type.
"""
if s is None:
return None
if type(s) != unicode:
try:
s = unicode(s, 'utf-8')
except UnicodeDecodeError, e:
s = s.decode("utf-8", "ignore")
s = s + u" (bad unicode detected)"
return s
class UrlMapping(sqlobject.SQLObject):
shortUrl = sqlobject.StringCol(length=8, unique=True)
originalUrl = sqlobject.StringCol()
ctime = sqlobject.IntCol(default=None)
shortUrl_index = sqlobject.DatabaseIndex(shortUrl)
ctime_index = sqlobject.DatabaseIndex(ctime)
class Srv4IncompatibleWith(sqlobject.SQLObject):
"""Models dependencies."""
srv4_file = sqlobject.ForeignKey('Srv4FileStats', notNone=True)
pkginst = sqlobject.ForeignKey('Pkginst', notNone=True)
dep_uniq_idx = sqlobject.DatabaseIndex('srv4_file', 'pkginst')
class Node(sqlobject.SQLObject):
label = sqlobject.StringCol()
next = sqlobject.ForeignKey("Node")
next_index = sqlobject.DatabaseIndex("next")
