class CswFile(sqlobject.SQLObject): """Represents a file in a catalog. There can be multiple files with the same basename and the same path, belonging to different packages. This class needs to also contain files from the operating system, coming from SUNW packages, for which we don't have the original srv4 files. (Even if we could, they are generally not accessible.) They need to be specific to Solaris release and architecture, so we need a way to link CswFile with a specific catalog. Fake registered Srv4FileStats object would do, but we would have to ensure that they can't be associated with a catalog. Also, we'd have to generate fake md5 sums for them. """ basename = sqlobject.UnicodeCol(length=255, notNone=True) path = sqlobject.UnicodeCol(notNone=True, length=900) line = sqlobject.UnicodeCol(notNone=True, length=900) pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) srv4_file = sqlobject.ForeignKey('Srv4FileStats') basename_idx = sqlobject.DatabaseIndex('basename') def __unicode__(self): return u"File: %s" % os.path.join(self.path, self.basename)
class Srv4FileInCatalog(sqlobject.SQLObject): """Assignment of a particular srv4 file to a specific catalog. There could be one more layer, to which arch and osrel could be moved. But for now, it's going to be a not-normalized structure. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) osrel = sqlobject.ForeignKey('OsRelease', notNone=True) catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True) srv4file = sqlobject.ForeignKey('Srv4FileStats', notNone=True) created_on = sqlobject.DateTimeCol( notNone=True, default=sqlobject.DateTimeCol.now) created_by = sqlobject.UnicodeCol(length=50, notNone=True) uniqueness_idx = sqlobject.DatabaseIndex( 'arch', 'osrel', 'catrel', 'srv4file', unique=True) # http://turbogears.org/1.0/docs/SQLObject/Caching.html#what-does-cache-false-do class sqlmeta: cacheValues = False def __unicode__(self): return (u"%s is in catalog %s %s %s" % (self.srv4file, self.arch.name, self.osrel.full_name, self.catrel.name))
class CswFile(sqlobject.SQLObject): """Represents a file in a catalog. There can be multiple files with the same basename and the same path, belonging to different packages. This class needs to also contain files from the operating system, coming from SUNW packages, for which we don't have the original srv4 files. (Even if we could, they are generally not accessible.) They need to be specific to Solaris release and architecture, so we need a way to link CswFile with a specific catalog. Fake registered Srv4FileStats object would do, but we would have to ensure that they can't be associated with a catalog. Also, we'd have to generate fake md5 sums for them. """ class sqlmeta: # MySQL uses case-insensitive collation by default, which doesn't make sense # for file names. If the utf8_bin (or other case sensitive) collation is not # used, it reports e.g. Zcat and zcat as a file collision, while it really # isn't one. createSQL = { 'mysql' : [ 'ALTER TABLE csw_file CONVERT TO CHARACTER SET utf8 ' 'COLLATE utf8_bin'] } basename = sqlobject.UnicodeCol(length=255, notNone=True) path = sqlobject.UnicodeCol(notNone=True, length=900) line = sqlobject.UnicodeCol(notNone=True, length=900) # Symlinks don't have permissions on their own perm_user = sqlobject.UnicodeCol(notNone=False, length=255) perm_group = sqlobject.UnicodeCol(notNone=False, length=255) perm_mode = sqlobject.UnicodeCol(notNone=False, length=5) target = sqlobject.UnicodeCol(notNone=False, length=900) mimetype = sqlobject.UnicodeCol(notNone=False, length=255) machine = sqlobject.UnicodeCol(notNone=False, length=255) pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) srv4_file = sqlobject.ForeignKey('Srv4FileStats') basename_idx = sqlobject.DatabaseIndex('basename') path_idx = sqlobject.DatabaseIndex({'column': 'path', 'length': 255}) def FullPath(self): return os.path.join(self.path, self.basename) def __unicode__(self): return u"%s %s %s %s %s" % (self.perm_user, self.perm_group, self.perm_mode, self.FullPath(), self.mimetype)
class ProfileCategory(so.SQLObject): """ Model the many-to-many relationship between Profile and Category records. Attributes are based on a recommendation in the SQLObject docs. """ profile = so.ForeignKey("Profile", notNull=True, cascade=True) category = so.ForeignKey("Category", notNull=True, cascade=True) uniqueIdx = so.DatabaseIndex(profile, category, unique=True)
class Town(Place): """ Place which falls into Town or Unknown category in Twitter API. """ _inheritable = False # Country which this Town belongs. Optional and defaults to None. country = so.ForeignKey("Country", default=None) countryIdx = so.DatabaseIndex(country)
class TweetCampaign(so.SQLObject): """ Model the many-to-many relationship between Tweet and Campaign records. Attributes are based on a recommendation in the SQLObject docs for doing this relationship. """ tweet = so.ForeignKey("Tweet", notNull=True, cascade=True) campaign = so.ForeignKey("Campaign", notNull=True, cascade=True) uniqueIdx = so.DatabaseIndex(tweet, campaign, unique=True)
class PageLabel(so.SQLObject): """ Model the many-to-many relationship between Page and Label records. A Label may be applied to many Pages and a Page may have many Labels. But a paired relationship must be unique. Attributes here are based on a recommendation in the SQLObject docs. """ page = so.ForeignKey('Page', notNull=True, cascade=True) label = so.ForeignKey('Label', notNull=True, cascade=True) unique_idx = so.DatabaseIndex(page, label, unique=True)
class Page(so.SQLObject): """ Model a URI for a webpage on the internet. Do not worry about duplicate pairs of domain and path, since we want to allow those to occur on an import and to clean them up later. A page may have a null Folder (as unsorted), though a folder must always have a parent folder, even if the top folder is "root". """ # The host website for the page. # TODO: Ensure this is always converted lowercase rather than raising # an error. domain = so.ForeignKey('Domain', notNull=True) # The location of the webpage relative to the domain. # TODO: Should this start with forwardslash? Check what happens when # splitting and joining. # TODO: Create custom validator? path = so.UnicodeCol(notNull=True) # Webpage title, usually taken from the metadata of the HTML head section. title = so.UnicodeCol(default=None) # The date and time when the record was created. Defaults to the # current time. created_at = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now) # Optional preview image for the link, scraped from the metadata. image_url = so.UnicodeCol(default=None) description = so.UnicodeCol(default=None) # The folder this link is placed into. If null then the link must still # be sorted. Domain and path pairs must be unique in a folder. folder = so.ForeignKey('Folder') unique_idx = so.DatabaseIndex(domain, path, folder, unique=True) source = so.ForeignKey('Source', notNull=True) # Link to labels which this page is assigned to. labels = so.SQLRelatedJoin('Labels', intermediateTable='page_label', createRelatedTable=False) def get_url(self): return "".join((self.domain.value, self.path))
class ObservationCalibration(sqlobject.SQLObject): cluster = sqlobject.StringCol(length=55) filter = sqlobject.StringCol(length=55) mangledSpecification = sqlobject.StringCol(length=250) calibration = sqlobject.ForeignKey('ZeropointEntry') ClusterFilterIndex = sqlobject.DatabaseIndex('cluster', 'filter', 'mangledSpecification', unique=True) def _get_specification(self): return unmangleSpecification(self.mangledSpecification) def _set_specification(self, **specification): self.mangledSpecification = mangleSpecification(specification)
class Srv4FileInCatalog(sqlobject.SQLObject): """Assignment of a particular srv4 file to a specific catalog. There could be one more layer, to which arch and osrel could be moved. But for now, it's going to be a not-normalized structure. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) osrel = sqlobject.ForeignKey('OsRelease', notNone=True) catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True) srv4file = sqlobject.ForeignKey('Srv4FileStats', notNone=True) uniqueness_idx = sqlobject.DatabaseIndex('arch', 'osrel', 'catrel', 'srv4file', unique=True) def __unicode__(self): return (u"%s is in catalog %s %s %s" % (self.srv4file, self.arch.name, self.osrel.full_name, self.catrel.name))
class Source(so.SQLObject): """ Model a datasource of exported or manually created webpage data. """ # Creation date of source. This could be the last day that a source # was used and that could be related to moving away from using a device # or exporting data from a company computer before leaving a job. date_created = so.DateCol(notNull=True) date_created_idx = so.DatabaseIndex(date_created) # The format of the datasource. format_ = so.ForeignKey('Format', notNull=True) # The web browser where the data originated. browser = so.ForeignKey('Browser') # The location where one lived and worked when creating the datasource. location = so.ForeignKey('Location') # True if it was work related, false if it was personal. is_work = so.BoolCol(notNull=True)
class Profile(so.SQLObject): """ Models a user profile on Twitter. Note that URL columns are named as 'Url', since SQLOlbject converts 'imageURL' to db column named 'image_ur_l'. Twitter screen name and username rules: https://help.twitter.com/en/managing-your-account/twitter-username-rules We use slightly higher values than the ones there, to be safe. Notes on screen name: - This should not have unique restriction as users can edit their screen name so others can take an older screen name. Or someone could delete and recreate their account. - Twitter itself enforces uniqueness across case. """ # Profile's ID (integer), as assigned by Twitter when the Profile was # created. This is a global ID, rather than an ID specific to our local db. guid = so.IntCol(alternateID=True) # Profile screen name. screenName = so.StringCol(notNull=True, length=30) # Profile display Name. name = so.StringCol(notNull=True, length=60) # Description, as set in profile's bio. description = so.StringCol(default=None) # Location, as set in profile's bio. location = so.StringCol(default=None) # Link to the profile's image online. This will only be thumbnail size. imageUrl = so.StringCol(default=None, validator=URL) # Count of profile's followers. followersCount = so.IntCol(notNull=True) # Count of profile's statuses (tweets) posted by this profile. statusesCount = so.IntCol(notNull=True) # Profile's verified status. verified = so.BoolCol(notNull=True, default=False) # Join the Profile with its created tweets in the Tweet table. tweets = so.MultipleJoin("Tweet") # Date and time when follower and status counts were last updated. modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now) modifiedIdx = so.DatabaseIndex(modified) # Get Category objects which this Profile has been assigned to, if any. categories = so.SQLRelatedJoin("Category", intermediateTable="profile_category", createRelatedTable=False) def set(self, **kwargs): """ Override the update hook to update the modified field if necessary. """ if ("followersCount" in kwargs or "statusesCount" in kwargs) and "modified" not in kwargs: kwargs["modified"] = so.DateTimeCol.now() super(Profile, self).set(**kwargs) def getFlatDescription(self): """ Return the description with newline characters replaced with spaces. """ if self.description is not None: return lib.text_handling.flattenText(self.description) return None def getProfileUrl(self): """ Get link to the profile's page online. :return: Twitter profile's URL, as a string. """ return "https://twitter.com/{0}".format(self.screenName) def getLargeImageUrl(self): """ Get link to a large version profile's image, based on thumbnail URL. The image URL comes from the API as '..._normal.jpeg', but from API calls on loading a twitter.com page, it is possible to see that the image media server allows variations of the last part, to return a large image. Such as - '..._bigger.jpeg' (which is not much bigger than the normal thumbnail) - '..._400x400.jpeg' (which is much bigger). :return: image URL using 400x400 size parameter, or None if value was not set. """ if self.imageUrl: return self.imageUrl.replace("_normal", "_400x400") return None def prettyPrint(self): """ Method to print the attributes of the Profile instance neatly. :return: dictionary of data which was printed. """ output = """\ Screen name : @{screenName} Name : {name} Verified : {verified} Followers : {followers:,d} Statuses : {statuses:,d} DB tweets : {tweetCount} Description : {description} Profile URL : {url} Image URL : {imageUrl} Stats modified : {statsModified} """ data = dict( screenName=self.screenName, name=self.name, verified=self.verified, followers=self.followersCount, statuses=self.statusesCount, tweetCount=len(self.tweets), description=self.getFlatDescription(), url=self.getProfileUrl(), imageUrl=self.getLargeImageUrl(), statsModified=self.modified, ) print(output.format(**data)) return data
class Tweet(so.SQLObject): """ Models a tweet on Twitter. If we are inserting the Tweet in our db, we expect to always have the author's profile in the Profile table. If the tweet is a reply, we will have references to the target Profile and original Tweet as GUID integers. But we are unlikely to have those object stored in our db. Use the `.getInReplyToTweet` and `.getInReplyToProfile` methods to see if those exist in the db, otherwise use the GUIDs to look up data from the Twitter API and then store them locally as db records. For relating a Tweet to its author Profile with a foreign key, a `setProfileByGuid` method could be implemented to set the profile foreign key using a given GUID, but that would require doing a search each time. So, when creating a Tweet object or multiple objects for one Profile, it is preferable to get the Profile object's ID once and then repeately pass that in as an argument for each Tweet object that is created for that Profile. For ordering, the '-guid' syntax is here is preferred, since 'guid DESC' results in an error when getting tweets of a Profile object, even though doing a query on Tweet class itself is fine. `AttributeError: 'Tweet' object has no attribute 'guid DESC'` The error is also raised for multiple names e.g. '-guid, message'. """ class sqlmeta: # Show recent Tweets (with higher GUID values) first. defaultOrder = "-guid" # Tweet ID (integer), as assigned by Twitter when the Tweet was posted. # This is a global ID, rather than specific to our local db. guid = so.IntCol(alternateID=True) # Link to Tweet's author in the Profile table. Delete Tweet if # the Profile is deleted. profile = so.ForeignKey("Profile", notNull=True, cascade=True) profileIdx = so.DatabaseIndex(profile) # Date and time the tweet was posted. createdAt = so.DateTimeCol(notNull=True) createdAtIdx = so.DatabaseIndex(createdAt) # Tweet message text. Length is not validated since expanded tweets can # be longer than the standard 280 (previously 140) characters. message = so.StringCol(notNull=True) # Count of favorites on this Tweet. favoriteCount = so.IntCol(notNull=True) # Count of retweets of this Tweet. retweetCount = so.IntCol(notNull=True) # If the tweet is a reply, the GUID of the Tweet which the reply is # directed at (from reply_to_status_id field). This does not require # the Tweet to be in the local db. inReplyToTweetGuid = so.IntCol(default=None) # If the tweet is a reply, the GUID of the Profile which the reply is # directed at (from reply_to_user_id field). This does not require # the Tweet to be in the local db. inReplyToProfileGuid = so.IntCol(default=None) # Date and time when favorite and retweet counts where last updated. modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now) modifiedIdx = so.DatabaseIndex(modified) # Get Campaign objects which this Profile has been assigned to, if any. campaigns = so.SQLRelatedJoin("Campaign", intermediateTable="tweet_campaign", createRelatedTable=False) def set(self, **kwargs): """ Override the update hook to update the modified field if necessary. """ if ("favoriteCount" in kwargs or "retweetCount" in kwargs) and "modified" not in kwargs: kwargs["modified"] = so.DateTimeCol.now() super(Tweet, self).set(**kwargs) def isRT(self): return self.message.startswith("RT ") def isReply(self): return self.inReplyToProfileGuid is not None def getFlatMessage(self): """ Return the message with newline characters replaced with spaces. """ return lib.text_handling.flattenText(self.message) def getInReplyToTweet(self): """ If this Tweet is a reply, get the original Tweet it was directed at. :return: single Tweet object. Return None if this is not a reply. Raise an error if the Tweet is not in the local db. """ if self.inReplyToTweetGuid: try: return Tweet.byGuid(self.inReplyToTweetGuid) except SQLObjectNotFound as e: raise type(e)( "Could not find Tweet in db with GUID: {0}".format( self.inReplyToTweetGuid)) return None def getInReplyToProfile(self): """ If this Tweet is a reply, get the Profile which it was directed at. :return: single Profile object. Return None if this is not a reply. Raise an error if the Tweet is not in the local db. """ if self.inReplyToProfileGuid: try: return Profile.byGuid(self.inReplyToProfileGuid) except SQLObjectNotFound as e: raise type(e)( "Could not find Profile in db with GUID: {0}".format( self.inReplyToProfileGuid)) return None def getTweetURL(self): """ Return URL for the tweet as a string, using tweet author's screen name and the tweet's GUID. """ return "https://twitter.com/{screenName}/status/{tweetID}".format( screenName=self.profile.screenName, tweetID=self.guid) def prettyPrint(self): """ Method to print the attributes of the Tweet instance neatly. :return: dictionary of data which was printed. """ output = """\ Author : @{screenName} - {name} - {followers:,d} followers Created at : {createdAt} Message : {message} Favorites : {favoriteCount:,d} Retweets : {retweetCount:,d} Reply To User ID : {replyProf} Reply To Tweet ID : {replyTweet} URL : {url} Stats modified : {statsModified} """ author = self.profile data = dict( screenName=author.screenName, createdAt=self.createdAt, name=author.name, followers=author.followersCount, message=self.getFlatMessage(), favoriteCount=self.favoriteCount, retweetCount=self.retweetCount, replyProf=self.inReplyToProfileGuid, replyTweet=self.inReplyToTweetGuid, url=self.getTweetURL(), statsModified=self.modified, ) print(output.format(**data)) return data def report(self): """ Return Tweet and Profile data as dict for writing a CSV report. """ author = self.profile return { "Screen name": author.screenName, "Followers": author.followersCount, "Tweet URL": self.getTweetURL(), "Tweet ID": self.guid, "Tweeted at": str(self.createdAt), "Is reply": "Y" if self.isReply() else "N", "Is RT": "Y" if self.isRT() else "N", "Message": self.message, "Favs": self.favoriteCount, "RTs": self.retweetCount, }
class Trend(so.SQLObject): """ A trending topic on Twitter, meaning a lot of Twitter accounts were talking about that topic. A topic exists at a point in time and maps to a specific place. It's term can either be a hashtag (starts with '#' and has no spaces) or a keyword phrase (no '#' and can have multiple words). Note that the topic is not unique, since the topic can be repeated in many locations and it can be repeated in one location across time. The topic has a trending volume figure, which is how many tweets there are about the topic in the past 24 hours (according to Twitter API docs). Adding up trends for a Place taken at the SAME time each day should give an accurate total of tweets for the period, since there should not be any overlap in tweets across two consecutive 24-hour periods. One might use the earliest record available for the day, assuming the cron job runs soon after midnight, so that any ad hoc data will not skew the results. However, the value will for the 24 hours of the PREVIOUS day. Note that the topic volume shown is always GLOBAL total volume i.e. independent of the location used to look up the topic. Volume usually ranges from around 10,000 to 1 million and smaller values are returned as null by Twitter API. However, it is still useful to count the number of places which a tppic is trending in as an indication of how widespread it is. """ class sqlmeta: defaultOrder = "-timestamp" _connection = conn # The topic which trended. topic = so.StringCol(length=64) topicIdx = so.DatabaseIndex(topic) # Whether the topic is a hashtag i.e. starts with '#'. hashtag = so.BoolCol(default=False) # Number of global tweets about topic in past 24 hours. Required since # there no default set here. Null values are allowed. volume = so.IntCol(notNull=False) # The place associated with this trend record. See `setPlace` for why # this is an optional field. place = so.ForeignKey("Place", notNull=False, default=None) placeIdx = so.DatabaseIndex(place) # Date and time when record was created. timestamp = so.DateTimeCol(default=so.DateTimeCol.now) timestampIdx = so.DatabaseIndex(timestamp) def setPlace(self, woeid): """ Link an existing Trend and Place records, given a Place WOEID. Expects a WOEID int, gets ID for the Place, then stores it as the foreign key for the Trend. This doesn't work to be placed in __init__ since then its called on a select and doen't work for modelCreate because the input kwargs are validated before the method is called. :param woeid: integer value for WOEID of the Place to link to. :return self: returns object instance. """ assert isinstance( woeid, int), "Expected WOEID as an `int`, but " "got type `{0}`.".format( type(woeid).__name__) try: self.placeID = Place.byWoeid(woeid).id except so.SQLObjectNotFound as e: raise type(e)( "Place with WOEID {0} could not be found in the db.".format( woeid)) return self def _set_topic(self, value): """ Override the topic setting method, so that hashtag boolean is updated automatically whenever topic is set. :param value: string value to set as the topic. """ self._SO_set_topic(value) if value.startswith("#"): self._SO_set_hashtag(True) else: self._SO_set_hashtag(False) @classmethod def getColumnNames(cls): """ Return a list of column names for the class, as strings. This is created from a dictionary, so the order is not guaranteed. """ return list(cls.sqlmeta.columns.keys()) def getData(self, quiet=True): """ Output the current record with key:value pairs for column name and value. Note that this is not suitable to converted to JSON because of the data types of values. """ data = {col: getattr(self, col) for col in self.getColumnNames()} if not quiet: for k, v in data.items(): # Align key to the right. print("{0:>15} : {1}".format(k, v))
class PlaceJob(so.SQLObject): """ Listing of places which we want to regularly get trend data for. The WOEID of a place is used to look up trends at the location, for records which have status set to enabled. An item can be marked as disabled, so that it will be skipped by a procedure but kept in the table so it can be enabled again easily. A place can appear only once in the table. Only towns or countries can be added to the job list, due to how Twitter API works - this is enforced when a record is created. The table starts off empty and desired places can be added or removed depending on admin user's preferences. """ class sqlmeta: # Order as enabled (True) items first, then by jobs with oldest # (or null) last completed timestamps first and then by oldest # (or null) last attempted timestamps. Therefore when running jobs, the # ones which have not been completed before or for the longest time # are given priority over ones which were recently completed. # And any disabled jobs will be at the bottom when viewing a report. defaultOrder = "enabled DESC, last_completed ASC, last_attempted ASC" _connection = conn # Create a reference to Place table. Place IDs cannot be repeated in # this job table. place = so.ForeignKey("Place", unique=True) # Create an index on place. placeIdx = so.DatabaseIndex(place) # Date and time when record was created. created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now) # When the job was last attempted regardless of outcome. lastAttempted = so.DateTimeCol(notNull=False, default=None) # When the job item was last completed successfully. Defaults to null. lastCompleted = so.DateTimeCol(notNull=False, default=None) # Create an index on last completed. lastCompletedIdx = so.DatabaseIndex(lastCompleted) # Boolean flag for whether the job item is enabled or should be skipped. enabled = so.BoolCol(notNull=True, default=True) def start(self): """ Use this function to update the last attempted time. """ self._set_lastAttempted(so.DateTimeCol.now()) def end(self): """ Use this function to update the last run time, if successful. """ self._set_lastCompleted(so.DateTimeCol.now()) def setEnabled(self): """ Set the job to enabled. """ self.enabled = True def setDisabled(self): """ Set the job to disabled. """ self.enabled = False def getStatus(self, asText=False): """ Get the status from when the job was last run. If last attempted time is None, then return None since we cannot confirm success or failure. Return True for success, if last completed time is not None and is after last attempted time. Otherwise return False for failure. :param asText: Default False. If True, return status as human-readable string. :return status: job status as OK (True) or failed (False) or not run (None). Returned as human-readable string if asText is True. """ if self.lastAttempted: if self.lastCompleted and self.lastCompleted > self.lastAttempted: status = "OK" if asText else True else: status = "failed" if asText else False else: status = "not run" if asText else None return status
class Place(InheritableSQLObject): """ A place in the world. This is created from the Yahoo Where On Earth locations as returned by Twitter API. This table has childName to indicate which table the object is in and therefore the parent Place's location type. Name is *not* an alternateID, since place names can be duplicated around the world e.g. Barcelona in Venezuela and Spain. Therefore `.byName` is not available, but we can do a `.selectBy` with both town name and the country's ID set in the where clause, to ensure we get one result. Default order by ID is omitted as it causes ambiguity issues on some selects. And timestamp is not recognised as a column on the subclasses so cannot be used either. """ _connection = conn # WOEID integer value from Yahoo system. # Note that `.byWoeid` can be used on Place class, but cannot be used # on any subclasses. This is because of the `id` in the order by statement # having ambiguous meaning in `where (Place.id = {subclass}.id)`. woeid = so.IntCol(alternateID=True) # Name of the place. name = so.StringCol(length=64, default=None) nameIdx = so.DatabaseIndex(name) # Date and time when record was created. timestamp = so.DateTimeCol(default=so.DateTimeCol.now) timestampIdx = so.DatabaseIndex(timestamp) # Get all the trend records relating to this Place. hasTrends = so.MultipleJoin("Trend") @classmethod def getColumnNames(cls): """ Return a list of column names for the class, as strings. This is created from a dictionary, so the order is not guaranteed. """ return list(cls.sqlmeta.columns.keys()) def getData(self, quiet=True): """ Output the current record with key:value pairs for column name and value. Note that this is not suitable to be converted directly to JSON because of the data types of some values. TODO: Ensure that attributes of the parent and child are all accessed, to get more use out of this. """ data = {col: getattr(self, col) for col in self.getColumnNames()} if not quiet: for k, v in data.items(): print(f"{k:>15} : {v}") return data
class Srv4FileStats(sqlobject.SQLObject): """Represents a srv4 file. It focuses on the stats, but it can as well represent just a srv4 file. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) basename = sqlobject.UnicodeCol(notNone=True, length=250) catalogname = sqlobject.UnicodeCol(notNone=True, length=250) filename_arch = sqlobject.ForeignKey('Architecture', notNone=True) maintainer = sqlobject.ForeignKey('Maintainer', notNone=False) md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32) size = sqlobject.IntCol() mtime = sqlobject.DateTimeCol(notNone=False) os_rel = sqlobject.ForeignKey('OsRelease', notNone=True) osrel_str = sqlobject.UnicodeCol(notNone=True, length=9) # "SunOS5.10" pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) pkginst_str = sqlobject.UnicodeCol(notNone=True, length=255) registered_level_one = sqlobject.BoolCol(notNone=True) registered_level_two = sqlobject.BoolCol(notNone=True) use_to_generate_catalogs = sqlobject.BoolCol(notNone=True) rev = sqlobject.UnicodeCol(notNone=False, length=250) stats_version = sqlobject.IntCol(notNone=True) version_string = sqlobject.UnicodeCol(notNone=True, length=250) bundle = sqlobject.UnicodeCol(length=250) in_catalogs = sqlobject.MultipleJoin( 'Srv4FileInCatalog', joinColumn='srv4file_id') files = sqlobject.MultipleJoin('CswFile', joinColumn='id') catalog_idx = sqlobject.DatabaseIndex('catalogname') basename_idx = sqlobject.DatabaseIndex('basename') pkginst_idx = sqlobject.DatabaseIndex('pkginst') def __init__(self, *args, **kwargs): super(Srv4FileStats, self).__init__(*args, **kwargs) def __unicode__(self): return u'%s/%s, %s' % (self.pkginst.pkgname, self.catalogname, self.md5_sum) def __str__(self): return str(unicode(self)) def DeleteAllDependentObjects(self): """Prepares the object to be deleted. Use this function with caution. """ self.DeleteDependentObjectsPopulatedFromPackageItself() logger.debug('Removing all dependent objects from %s; it will cause the ' 'package to be removed from all catalogs.', self) self.RemoveCatalogAssignments() self.RemoveAllCheckpkgResults() def DeleteDependentObjectsPopulatedFromPackageItself(self): """Removing all the objects that only depend on the package contents. It doesn't touch rows that are created for other reasons, e.g. assignments of packages to catalogs. """ logger.debug('%s - Deleting objects that only depend on the package ' 'contents', self) self.RemoveAllCswFiles() self.RemoveOverrides() self.RemoveDepends() self.RemoveIncompatibles() def RemoveAllCswFiles(self): # Removing existing files, using sqlbuilder to use sql-level # mechanisms without interacting with Python. # http://www.mail-archive.com/[email protected]/msg00520.html sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CswFile.sqlmeta.table, CswFile.q.srv4_file==self))) def RemoveDepends(self): sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( Srv4DependsOn.sqlmeta.table, Srv4DependsOn.q.srv4_file==self))) def RemoveIncompatibles(self): sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( Srv4IncompatibleWith.sqlmeta.table, Srv4IncompatibleWith.q.srv4_file==self))) def RemoveCatalogAssignments(self): sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( Srv4FileInCatalog.sqlmeta.table, Srv4FileInCatalog.q.srv4file==self))) def GetOverridesResult(self): return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file==self) def GetErrorTagsResult(self, os_rel, arch, catrel): assert arch.name != 'all', ( "Asked for the 'all' architecture, this is not valid " "for GetErrorTagsResult().") return CheckpkgErrorTag.select( sqlobject.AND( CheckpkgErrorTag.q.srv4_file==self, CheckpkgErrorTag.q.os_rel==os_rel, CheckpkgErrorTag.q.arch==arch, CheckpkgErrorTag.q.catrel==catrel)) def RemoveCheckpkgResults(self, os_rel, arch, catrel): logger.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel, arch, catrel) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, sqlobject.AND( CheckpkgErrorTag.q.srv4_file==self, CheckpkgErrorTag.q.os_rel==os_rel, CheckpkgErrorTag.q.arch==arch, CheckpkgErrorTag.q.catrel==catrel)))) def RemoveAllCheckpkgResults(self): logger.debug("%s: RemoveAllCheckpkgResults()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, CheckpkgErrorTag.q.srv4_file==self))) def RemoveOverrides(self): logger.debug("%s: RemoveOverrides()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CheckpkgOverride.sqlmeta.table, CheckpkgOverride.q.srv4_file==self))) def GetUnicodeOrNone(self, s): """Tries to decode UTF-8. If the object does not decode as UTF-8, it's forced to do so, while ignoring any potential errors. Returns: a unicode object or a None type. """ if s is None: return None if type(s) != unicode: try: s = unicode(s, 'utf-8') except UnicodeDecodeError, e: s = s.decode("utf-8", "ignore") s = s + u" (bad unicode detected)" return s
class UrlMapping(sqlobject.SQLObject): shortUrl = sqlobject.StringCol(length=8, unique=True) originalUrl = sqlobject.StringCol() ctime = sqlobject.IntCol(default=None) shortUrl_index = sqlobject.DatabaseIndex(shortUrl) ctime_index = sqlobject.DatabaseIndex(ctime)
class Srv4IncompatibleWith(sqlobject.SQLObject): """Models dependencies.""" srv4_file = sqlobject.ForeignKey('Srv4FileStats', notNone=True) pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) dep_uniq_idx = sqlobject.DatabaseIndex('srv4_file', 'pkginst')
class Node(sqlobject.SQLObject): label = sqlobject.StringCol() next = sqlobject.ForeignKey("Node") next_index = sqlobject.DatabaseIndex("next")