class Message(orm.SQLObject): """The class generates a Message table containing information about user messages. Args: text (str, optional): time (int, optional): filePicture (byte, optional): fileVideo (byte, optional): fileAudio (byte, optional): fileDocument (byte, optional): emoji (str, optional): editedTime (int, optional): editedStatus (bool, optional): Returns: None """ text = orm.StringCol(default=None) time = orm.IntCol(default=None) filePicture = orm.BLOBCol(default=None) fileVideo = orm.BLOBCol(default=None) fileAudio = orm.BLOBCol(default=None) fileDocument = orm.BLOBCol(default=None) emoji = orm.BLOBCol(default=None) editedTime = orm.IntCol(default=None) editedStatus = orm.BoolCol(default=False) userConfig = orm.ForeignKey('UserConfig', refColumn="uuid") flow = orm.ForeignKey('Flow', refColumn="flowId")
class PhotometricCalibration(ZeropointEntry): zperr = sqlobject.FloatCol() colorterm = sqlobject.FloatCol() colorerr = sqlobject.FloatCol(default=0.) fixedcolor = sqlobject.BoolCol(default=True) table_name = 'photometric_calibration'
class Job(sqlobject.SQLObject): job_id = sqlobject.StringCol() vault_name = sqlobject.StringCol(length=20, default=None) action = sqlobject.StringCol(length=20, default="") creation_date = sqlobject.DateCol(default=None) has_completed = sqlobject.BoolCol(default=False) completion_date = sqlobject.DateCol(default=None) status_code = sqlobject.StringCol(length=10, default="") response_text = sqlobject.StringCol()
class Guest(DictSQLObject): """ Collects information about Guests Refrenced to by the ChatMsg Table """ name = so.UnicodeCol() admin = so.BoolCol(default=False) parprop = so.IntCol(default=-1) token = so.StringCol(length=8) event = so.ForeignKey('Event')
class CheckpkgErrorTag(CheckpkgErrorTagMixin, sqlobject.SQLObject): srv4_file = sqlobject.ForeignKey('Srv4FileStats', notNone=True) pkgname = sqlobject.UnicodeCol(default=None, length=250) tag_name = sqlobject.UnicodeCol(notNone=True, length=250) tag_info = sqlobject.UnicodeCol(default=None, length=250) msg = sqlobject.UnicodeCol(default=None, length=250) # To cache results from checkpkg overridden = sqlobject.BoolCol(default=False) # The same package might have different sets of errors for different # catalogs or Solaris releases. os_rel = sqlobject.ForeignKey('OsRelease', notNone=True) arch = sqlobject.ForeignKey('Architecture', notNone=True) catrel = sqlobject.ForeignKey('CatalogRelease', notNone=True) def __unicode__(self): return (u"CheckpkgErrorTag: %s %s %s" % (self.pkgname, self.tag_name, self.tag_info))
class Log(sqlobject.SQLObject): timestamp = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now()) status = sqlobject.BoolCol(default=True) section = sqlobject.StringCol() message = sqlobject.StringCol() def __init__(self, *args, **kwargs): sqlobject.SQLObject.__init__(self, *args, **kwargs) def __setattr__(self, name, value): sqlobject.SQLObject.__setattr__(self, name, value) def dict(self): result = { "id": self.id, "timestamp": try_strftime(self.timestamp, default='never'), "section": self.section, "status": self.status, "message": self.message } return result
class Source(so.SQLObject): """ Model a datasource of exported or manually created webpage data. """ # Creation date of source. This could be the last day that a source # was used and that could be related to moving away from using a device # or exporting data from a company computer before leaving a job. date_created = so.DateCol(notNull=True) date_created_idx = so.DatabaseIndex(date_created) # The format of the datasource. format_ = so.ForeignKey('Format', notNull=True) # The web browser where the data originated. browser = so.ForeignKey('Browser') # The location where one lived and worked when creating the datasource. location = so.ForeignKey('Location') # True if it was work related, false if it was personal. is_work = so.BoolCol(notNull=True)
class UserConfig(orm.SQLObject): """The class generates a table containing data about the user and his settings. Args: uuid (int, required): login (str, required): password (str, required): hash_password (str, optional) username (str, optional): isBot (bool, optional): default False authId (str, optional): email (str, optional): avatar (str, optional): bio (str, optional): salt (str, optional): key (str, optional): Returns: None """ # added alternateID for added class method @byUUID # which will return that object uuid = orm.IntCol(alternateID=True, unique=True, notNone=True) login = orm.StringCol() password = orm.StringCol() hashPassword = orm.StringCol(default=None) username = orm.StringCol(default=None) isBot = orm.BoolCol(default=False) authId = orm.StringCol(default=None) email = orm.StringCol(default=None) avatar = orm.BLOBCol(default=None) bio = orm.StringCol(default=None) salt = orm.BLOBCol(default=None) key = orm.BLOBCol(default=None) # Connection to the Message table message = orm.MultipleJoin('Message')
class Srv4FileStats(sqlobject.SQLObject): """Represents a srv4 file. It focuses on the stats, but it can as well represent just a srv4 file. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) basename = sqlobject.UnicodeCol(notNone=True, length=250) catalogname = sqlobject.UnicodeCol(notNone=True, length=250) filename_arch = sqlobject.ForeignKey('Architecture', notNone=True) maintainer = sqlobject.ForeignKey('Maintainer', notNone=False) md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32) size = sqlobject.IntCol() mtime = sqlobject.DateTimeCol(notNone=False) os_rel = sqlobject.ForeignKey('OsRelease', notNone=True) osrel_str = sqlobject.UnicodeCol(notNone=True, length=9) # "SunOS5.10" pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) pkginst_str = sqlobject.UnicodeCol(notNone=True, length=255) registered_level_one = sqlobject.BoolCol(notNone=True) registered_level_two = sqlobject.BoolCol(notNone=True) use_to_generate_catalogs = sqlobject.BoolCol(notNone=True) rev = sqlobject.UnicodeCol(notNone=False, length=250) stats_version = sqlobject.IntCol(notNone=True) version_string = sqlobject.UnicodeCol(notNone=True, length=250) bundle = sqlobject.UnicodeCol(length=250) in_catalogs = sqlobject.MultipleJoin( 'Srv4FileInCatalog', joinColumn='srv4file_id') files = sqlobject.MultipleJoin('CswFile', joinColumn='id') catalog_idx = sqlobject.DatabaseIndex('catalogname') basename_idx = sqlobject.DatabaseIndex('basename') pkginst_idx = sqlobject.DatabaseIndex('pkginst') def __init__(self, *args, **kwargs): super(Srv4FileStats, self).__init__(*args, **kwargs) def __unicode__(self): return u'%s/%s, %s' % (self.pkginst.pkgname, self.catalogname, self.md5_sum) def __str__(self): return str(unicode(self)) def DeleteAllDependentObjects(self): """Prepares the object to be deleted. Use this function with caution. """ self.DeleteDependentObjectsPopulatedFromPackageItself() logger.debug('Removing all dependent objects from %s; it will cause the ' 'package to be removed from all catalogs.', self) self.RemoveCatalogAssignments() self.RemoveAllCheckpkgResults() def DeleteDependentObjectsPopulatedFromPackageItself(self): """Removing all the objects that only depend on the package contents. It doesn't touch rows that are created for other reasons, e.g. assignments of packages to catalogs. """ logger.debug('%s - Deleting objects that only depend on the package ' 'contents', self) self.RemoveAllCswFiles() self.RemoveOverrides() self.RemoveDepends() self.RemoveIncompatibles() def RemoveAllCswFiles(self): # Removing existing files, using sqlbuilder to use sql-level # mechanisms without interacting with Python. # http://www.mail-archive.com/[email protected]/msg00520.html sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CswFile.sqlmeta.table, CswFile.q.srv4_file==self))) def RemoveDepends(self): sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( Srv4DependsOn.sqlmeta.table, Srv4DependsOn.q.srv4_file==self))) def RemoveIncompatibles(self): sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( Srv4IncompatibleWith.sqlmeta.table, Srv4IncompatibleWith.q.srv4_file==self))) def RemoveCatalogAssignments(self): sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( Srv4FileInCatalog.sqlmeta.table, Srv4FileInCatalog.q.srv4file==self))) def GetOverridesResult(self): return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file==self) def GetErrorTagsResult(self, os_rel, arch, catrel): assert arch.name != 'all', ( "Asked for the 'all' architecture, this is not valid " "for GetErrorTagsResult().") return CheckpkgErrorTag.select( sqlobject.AND( CheckpkgErrorTag.q.srv4_file==self, CheckpkgErrorTag.q.os_rel==os_rel, CheckpkgErrorTag.q.arch==arch, CheckpkgErrorTag.q.catrel==catrel)) def RemoveCheckpkgResults(self, os_rel, arch, catrel): logger.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel, arch, catrel) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, sqlobject.AND( CheckpkgErrorTag.q.srv4_file==self, CheckpkgErrorTag.q.os_rel==os_rel, CheckpkgErrorTag.q.arch==arch, CheckpkgErrorTag.q.catrel==catrel)))) def RemoveAllCheckpkgResults(self): logger.debug("%s: RemoveAllCheckpkgResults()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, CheckpkgErrorTag.q.srv4_file==self))) def RemoveOverrides(self): logger.debug("%s: RemoveOverrides()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr(sqlbuilder.Delete( CheckpkgOverride.sqlmeta.table, CheckpkgOverride.q.srv4_file==self))) def GetUnicodeOrNone(self, s): """Tries to decode UTF-8. If the object does not decode as UTF-8, it's forced to do so, while ignoring any potential errors. Returns: a unicode object or a None type. """ if s is None: return None if type(s) != unicode: try: s = unicode(s, 'utf-8') except UnicodeDecodeError, e: s = s.decode("utf-8", "ignore") s = s + u" (bad unicode detected)" return s
class Profile(so.SQLObject): """ Models a user profile on Twitter. Note that URL columns are named as 'Url', since SQLOlbject converts 'imageURL' to db column named 'image_ur_l'. Twitter screen name and username rules: https://help.twitter.com/en/managing-your-account/twitter-username-rules We use slightly higher values than the ones there, to be safe. Notes on screen name: - This should not have unique restriction as users can edit their screen name so others can take an older screen name. Or someone could delete and recreate their account. - Twitter itself enforces uniqueness across case. """ # Profile's ID (integer), as assigned by Twitter when the Profile was # created. This is a global ID, rather than an ID specific to our local db. guid = so.IntCol(alternateID=True) # Profile screen name. screenName = so.StringCol(notNull=True, length=30) # Profile display Name. name = so.StringCol(notNull=True, length=60) # Description, as set in profile's bio. description = so.StringCol(default=None) # Location, as set in profile's bio. location = so.StringCol(default=None) # Link to the profile's image online. This will only be thumbnail size. imageUrl = so.StringCol(default=None, validator=URL) # Count of profile's followers. followersCount = so.IntCol(notNull=True) # Count of profile's statuses (tweets) posted by this profile. statusesCount = so.IntCol(notNull=True) # Profile's verified status. verified = so.BoolCol(notNull=True, default=False) # Join the Profile with its created tweets in the Tweet table. tweets = so.MultipleJoin("Tweet") # Date and time when follower and status counts were last updated. modified = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now) modifiedIdx = so.DatabaseIndex(modified) # Get Category objects which this Profile has been assigned to, if any. categories = so.SQLRelatedJoin("Category", intermediateTable="profile_category", createRelatedTable=False) def set(self, **kwargs): """ Override the update hook to update the modified field if necessary. """ if ("followersCount" in kwargs or "statusesCount" in kwargs) and "modified" not in kwargs: kwargs["modified"] = so.DateTimeCol.now() super(Profile, self).set(**kwargs) def getFlatDescription(self): """ Return the description with newline characters replaced with spaces. """ if self.description is not None: return lib.text_handling.flattenText(self.description) return None def getProfileUrl(self): """ Get link to the profile's page online. :return: Twitter profile's URL, as a string. """ return "https://twitter.com/{0}".format(self.screenName) def getLargeImageUrl(self): """ Get link to a large version profile's image, based on thumbnail URL. The image URL comes from the API as '..._normal.jpeg', but from API calls on loading a twitter.com page, it is possible to see that the image media server allows variations of the last part, to return a large image. Such as - '..._bigger.jpeg' (which is not much bigger than the normal thumbnail) - '..._400x400.jpeg' (which is much bigger). :return: image URL using 400x400 size parameter, or None if value was not set. """ if self.imageUrl: return self.imageUrl.replace("_normal", "_400x400") return None def prettyPrint(self): """ Method to print the attributes of the Profile instance neatly. :return: dictionary of data which was printed. """ output = """\ Screen name : @{screenName} Name : {name} Verified : {verified} Followers : {followers:,d} Statuses : {statuses:,d} DB tweets : {tweetCount} Description : {description} Profile URL : {url} Image URL : {imageUrl} Stats modified : {statsModified} """ data = dict( screenName=self.screenName, name=self.name, verified=self.verified, followers=self.followersCount, statuses=self.statusesCount, tweetCount=len(self.tweets), description=self.getFlatDescription(), url=self.getProfileUrl(), imageUrl=self.getLargeImageUrl(), statsModified=self.modified, ) print(output.format(**data)) return data
class Srv4FileStats(sqlobject.SQLObject): """Represents a srv4 file. It focuses on the stats, but it can as well represent just a srv4 file. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) basename = sqlobject.UnicodeCol(notNone=True) catalogname = sqlobject.UnicodeCol(notNone=True) # The data structure can be missing - necessary for fake SUNW # packages. data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False) filename_arch = sqlobject.ForeignKey('Architecture', notNone=True) latest = sqlobject.BoolCol(notNone=True) maintainer = sqlobject.ForeignKey('Maintainer', notNone=False) md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32) size = sqlobject.IntCol() mtime = sqlobject.DateTimeCol(notNone=False) os_rel = sqlobject.ForeignKey('OsRelease', notNone=True) pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) registered = sqlobject.BoolCol(notNone=True) use_to_generate_catalogs = sqlobject.BoolCol(notNone=True) rev = sqlobject.UnicodeCol(notNone=False) stats_version = sqlobject.IntCol(notNone=True) version_string = sqlobject.UnicodeCol(notNone=True) in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog', joinColumn='srv4file_id') files = sqlobject.MultipleJoin('CswFile', joinColumn='id') def __init__(self, *args, **kwargs): super(Srv4FileStats, self).__init__(*args, **kwargs) self._cached_pkgstats = None def DeleteAllDependentObjects(self): data_obj = self.data_obj self.data_obj = None if data_obj: # It could be already missing data_obj.destroySelf() self.RemoveAllCswFiles() self.RemoveAllCheckpkgResults() self.RemoveOverrides() def RemoveAllCswFiles(self): # Removing existing files, using sqlbuilder to use sql-level # mechanisms without interacting with Python. # http://www.mail-archive.com/[email protected]/msg00520.html sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CswFile.sqlmeta.table, CswFile.q.srv4_file == self))) def GetOverridesResult(self): return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self) def GetErrorTagsResult(self, os_rel, arch, catrel): assert arch.name != 'all', ( "Asked for the 'all' architecture, this is not valid " "for GetErrorTagsResult().") return CheckpkgErrorTag.select( sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self, CheckpkgErrorTag.q.os_rel == os_rel, CheckpkgErrorTag.q.arch == arch, CheckpkgErrorTag.q.catrel == catrel)) def RemoveCheckpkgResults(self, os_rel, arch, catrel): logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel, arch, catrel) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self, CheckpkgErrorTag.q.os_rel == os_rel, CheckpkgErrorTag.q.arch == arch, CheckpkgErrorTag.q.catrel == catrel)))) def RemoveAllCheckpkgResults(self): logging.debug("%s: RemoveAllCheckpkgResults()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table, CheckpkgErrorTag.q.srv4_file == self))) def RemoveOverrides(self): logging.debug("%s: RemoveOverrides()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table, CheckpkgOverride.q.srv4_file == self))) def __unicode__(self): return (u"Package: %s-%s, %s" % (self.catalogname, self.version_string, self.arch.name)) def GetStatsStruct(self): if not self._cached_pkgstats: self._cached_pkgstats = cPickle.loads(str(self.data_obj.pickle)) return self._cached_pkgstats def _GetBuildSource(self): data = self.GetStatsStruct() build_src = None if "OPENCSW_REPOSITORY" in data["pkginfo"]: build_src = data["pkginfo"]["OPENCSW_REPOSITORY"] return build_src def GetSvnUrl(self): build_src = self._GetBuildSource() svn_url = None if build_src: svn_url = re.sub(r'([^@]*).*', r'\1/Makefile', build_src) return svn_url def GetTracUrl(self): build_src = self._GetBuildSource() trac_url = None if build_src: trac_url = re.sub( r'https://gar.svn.(sf|sourceforge).net/svnroot/gar/([^@]+)@(.*)', r'http://sourceforge.net/apps/trac/gar/browser/\2/Makefile?rev=\3', build_src) return trac_url def GetVendorUrl(self): data = self.GetStatsStruct() vendor_url = None if "VENDOR" in data["pkginfo"]: vendor_url = re.split(r"\s+", data["pkginfo"]["VENDOR"])[0] return vendor_url def GetRestRepr(self): mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail" data = { 'arch': self.arch.name, 'basename': self.basename, # For compatibility with the catalog parser from catalog.py 'file_basename': self.basename, 'catalogname': self.catalogname, 'filename_arch': self.filename_arch.name, 'maintainer_email': self.maintainer.email, 'maintainer_full_name': self.maintainer.full_name, 'md5_sum': self.md5_sum, 'mtime': unicode(self.mtime), 'osrel': self.os_rel.short_name, 'pkgname': self.pkginst.pkgname, 'rev': self.rev, 'size': self.size, 'version_string': self.version_string, # For compatibility with the catalog parser from catalog.py 'version': self.version_string, # 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]), 'vendor_url': self.GetVendorUrl(), 'repository_url': self.GetSvnUrl(), } return mimetype, data
class Srv4FileStats(sqlobject.SQLObject): """Represents a srv4 file. It focuses on the stats, but it can as well represent just a srv4 file. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) basename = sqlobject.UnicodeCol(notNone=True) catalogname = sqlobject.UnicodeCol(notNone=True) # The data structure can be missing - necessary for fake SUNW # packages. data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False) filename_arch = sqlobject.ForeignKey('Architecture', notNone=True) latest = sqlobject.BoolCol(notNone=True) maintainer = sqlobject.ForeignKey('Maintainer', notNone=False) md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32) size = sqlobject.IntCol() mtime = sqlobject.DateTimeCol(notNone=False) os_rel = sqlobject.ForeignKey('OsRelease', notNone=True) pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) registered = sqlobject.BoolCol(notNone=True) use_to_generate_catalogs = sqlobject.BoolCol(notNone=True) rev = sqlobject.UnicodeCol(notNone=False) stats_version = sqlobject.IntCol(notNone=True) version_string = sqlobject.UnicodeCol(notNone=True) in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog', joinColumn='srv4file_id') files = sqlobject.MultipleJoin('CswFile', joinColumn='id') def __init__(self, *args, **kwargs): super(Srv4FileStats, self).__init__(*args, **kwargs) self._cached_pkgstats = None def DeleteAllDependentObjects(self): data_obj = self.data_obj self.data_obj = None if data_obj: # It could be already missing data_obj.destroySelf() self.RemoveAllCswFiles() self.RemoveAllCheckpkgResults() self.RemoveOverrides() def RemoveAllCswFiles(self): # Removing existing files, using sqlbuilder to use sql-level # mechanisms without interacting with Python. # http://www.mail-archive.com/[email protected]/msg00520.html sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CswFile.sqlmeta.table, CswFile.q.srv4_file == self))) def GetOverridesResult(self): return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self) def GetErrorTagsResult(self, os_rel, arch, catrel): assert arch.name != 'all', ( "Asked for the 'all' architecture, this is not valid " "for GetErrorTagsResult().") return CheckpkgErrorTag.select( sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self, CheckpkgErrorTag.q.os_rel == os_rel, CheckpkgErrorTag.q.arch == arch, CheckpkgErrorTag.q.catrel == catrel)) def RemoveCheckpkgResults(self, os_rel, arch, catrel): logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel, arch, catrel) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self, CheckpkgErrorTag.q.os_rel == os_rel, CheckpkgErrorTag.q.arch == arch, CheckpkgErrorTag.q.catrel == catrel)))) def RemoveAllCheckpkgResults(self): logging.debug("%s: RemoveAllCheckpkgResults()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table, CheckpkgErrorTag.q.srv4_file == self))) def RemoveOverrides(self): logging.debug("%s: RemoveOverrides()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table, CheckpkgOverride.q.srv4_file == self))) def __unicode__(self): return (u"Package: %s-%s, %s" % (self.catalogname, self.version_string, self.arch.name)) def GetUnicodeOrNone(self, s): """Tries to decode UTF-8. If the object does not decode as UTF-8, it's forced to do so, while ignoring any potential errors. Returns: a unicode object or a None type. """ if s is None: return None if type(s) != unicode: try: s = unicode(s, 'utf-8') except UnicodeDecodeError, e: s = s.decode("utf-8", "ignore") s = s + u" (bad unicode detected)" return s
class PlaceJob(so.SQLObject): """ Listing of places which we want to regularly get trend data for. The WOEID of a place is used to look up trends at the location, for records which have status set to enabled. An item can be marked as disabled, so that it will be skipped by a procedure but kept in the table so it can be enabled again easily. A place can appear only once in the table. Only towns or countries can be added to the job list, due to how Twitter API works - this is enforced when a record is created. The table starts off empty and desired places can be added or removed depending on admin user's preferences. """ class sqlmeta: # Order as enabled (True) items first, then by jobs with oldest # (or null) last completed timestamps first and then by oldest # (or null) last attempted timestamps. Therefore when running jobs, the # ones which have not been completed before or for the longest time # are given priority over ones which were recently completed. # And any disabled jobs will be at the bottom when viewing a report. defaultOrder = "enabled DESC, last_completed ASC, last_attempted ASC" _connection = conn # Create a reference to Place table. Place IDs cannot be repeated in # this job table. place = so.ForeignKey("Place", unique=True) # Create an index on place. placeIdx = so.DatabaseIndex(place) # Date and time when record was created. created = so.DateTimeCol(notNull=True, default=so.DateTimeCol.now) # When the job was last attempted regardless of outcome. lastAttempted = so.DateTimeCol(notNull=False, default=None) # When the job item was last completed successfully. Defaults to null. lastCompleted = so.DateTimeCol(notNull=False, default=None) # Create an index on last completed. lastCompletedIdx = so.DatabaseIndex(lastCompleted) # Boolean flag for whether the job item is enabled or should be skipped. enabled = so.BoolCol(notNull=True, default=True) def start(self): """ Use this function to update the last attempted time. """ self._set_lastAttempted(so.DateTimeCol.now()) def end(self): """ Use this function to update the last run time, if successful. """ self._set_lastCompleted(so.DateTimeCol.now()) def setEnabled(self): """ Set the job to enabled. """ self.enabled = True def setDisabled(self): """ Set the job to disabled. """ self.enabled = False def getStatus(self, asText=False): """ Get the status from when the job was last run. If last attempted time is None, then return None since we cannot confirm success or failure. Return True for success, if last completed time is not None and is after last attempted time. Otherwise return False for failure. :param asText: Default False. If True, return status as human-readable string. :return status: job status as OK (True) or failed (False) or not run (None). Returned as human-readable string if asText is True. """ if self.lastAttempted: if self.lastCompleted and self.lastCompleted > self.lastAttempted: status = "OK" if asText else True else: status = "failed" if asText else False else: status = "not run" if asText else None return status
class Trend(so.SQLObject): """ A trending topic on Twitter, meaning a lot of Twitter accounts were talking about that topic. A topic exists at a point in time and maps to a specific place. It's term can either be a hashtag (starts with '#' and has no spaces) or a keyword phrase (no '#' and can have multiple words). Note that the topic is not unique, since the topic can be repeated in many locations and it can be repeated in one location across time. The topic has a trending volume figure, which is how many tweets there are about the topic in the past 24 hours (according to Twitter API docs). Adding up trends for a Place taken at the SAME time each day should give an accurate total of tweets for the period, since there should not be any overlap in tweets across two consecutive 24-hour periods. One might use the earliest record available for the day, assuming the cron job runs soon after midnight, so that any ad hoc data will not skew the results. However, the value will for the 24 hours of the PREVIOUS day. Note that the topic volume shown is always GLOBAL total volume i.e. independent of the location used to look up the topic. Volume usually ranges from around 10,000 to 1 million and smaller values are returned as null by Twitter API. However, it is still useful to count the number of places which a tppic is trending in as an indication of how widespread it is. """ class sqlmeta: defaultOrder = "-timestamp" _connection = conn # The topic which trended. topic = so.StringCol(length=64) topicIdx = so.DatabaseIndex(topic) # Whether the topic is a hashtag i.e. starts with '#'. hashtag = so.BoolCol(default=False) # Number of global tweets about topic in past 24 hours. Required since # there no default set here. Null values are allowed. volume = so.IntCol(notNull=False) # The place associated with this trend record. See `setPlace` for why # this is an optional field. place = so.ForeignKey("Place", notNull=False, default=None) placeIdx = so.DatabaseIndex(place) # Date and time when record was created. timestamp = so.DateTimeCol(default=so.DateTimeCol.now) timestampIdx = so.DatabaseIndex(timestamp) def setPlace(self, woeid): """ Link an existing Trend and Place records, given a Place WOEID. Expects a WOEID int, gets ID for the Place, then stores it as the foreign key for the Trend. This doesn't work to be placed in __init__ since then its called on a select and doen't work for modelCreate because the input kwargs are validated before the method is called. :param woeid: integer value for WOEID of the Place to link to. :return self: returns object instance. """ assert isinstance( woeid, int), "Expected WOEID as an `int`, but " "got type `{0}`.".format( type(woeid).__name__) try: self.placeID = Place.byWoeid(woeid).id except so.SQLObjectNotFound as e: raise type(e)( "Place with WOEID {0} could not be found in the db.".format( woeid)) return self def _set_topic(self, value): """ Override the topic setting method, so that hashtag boolean is updated automatically whenever topic is set. :param value: string value to set as the topic. """ self._SO_set_topic(value) if value.startswith("#"): self._SO_set_hashtag(True) else: self._SO_set_hashtag(False) @classmethod def getColumnNames(cls): """ Return a list of column names for the class, as strings. This is created from a dictionary, so the order is not guaranteed. """ return list(cls.sqlmeta.columns.keys()) def getData(self, quiet=True): """ Output the current record with key:value pairs for column name and value. Note that this is not suitable to converted to JSON because of the data types of values. """ data = {col: getattr(self, col) for col in self.getColumnNames()} if not quiet: for k, v in data.items(): # Align key to the right. print("{0:>15} : {1}".format(k, v))
class Srv4FileStats(sqlobject.SQLObject): """Represents a srv4 file. It focuses on the stats, but it can as well represent just a srv4 file. """ arch = sqlobject.ForeignKey('Architecture', notNone=True) basename = sqlobject.UnicodeCol(notNone=True) catalogname = sqlobject.UnicodeCol(notNone=True) # The data structure can be missing - necessary for fake SUNW # packages. data_obj = sqlobject.ForeignKey('Srv4FileStatsBlob', notNone=False) filename_arch = sqlobject.ForeignKey('Architecture', notNone=True) latest = sqlobject.BoolCol(notNone=True) maintainer = sqlobject.ForeignKey('Maintainer', notNone=False) md5_sum = sqlobject.UnicodeCol(notNone=True, unique=True, length=32) size = sqlobject.IntCol() mtime = sqlobject.DateTimeCol(notNone=False) os_rel = sqlobject.ForeignKey('OsRelease', notNone=True) pkginst = sqlobject.ForeignKey('Pkginst', notNone=True) registered = sqlobject.BoolCol(notNone=True) use_to_generate_catalogs = sqlobject.BoolCol(notNone=True) rev = sqlobject.UnicodeCol(notNone=False) stats_version = sqlobject.IntCol(notNone=True) version_string = sqlobject.UnicodeCol(notNone=True) in_catalogs = sqlobject.MultipleJoin('Srv4FileInCatalog', joinColumn='srv4file_id') files = sqlobject.MultipleJoin('CswFile', joinColumn='id') def DeleteAllDependentObjects(self): data_obj = self.data_obj self.data_obj = None if data_obj: # It could be already missing data_obj.destroySelf() self.RemoveAllCswFiles() self.RemoveAllCheckpkgResults() self.RemoveOverrides() def RemoveAllCswFiles(self): # Removing existing files, using sqlbuilder to use sql-level # mechanisms without interacting with Python. # http://www.mail-archive.com/[email protected]/msg00520.html sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CswFile.sqlmeta.table, CswFile.q.srv4_file == self))) def GetOverridesResult(self): return CheckpkgOverride.select(CheckpkgOverride.q.srv4_file == self) def GetErrorTagsResult(self, os_rel, arch, catrel): assert arch.name != 'all', ( "Asked for the 'all' architecture, this is not valid " "for GetErrorTagsResult().") return CheckpkgErrorTag.select( sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self, CheckpkgErrorTag.q.os_rel == os_rel, CheckpkgErrorTag.q.arch == arch, CheckpkgErrorTag.q.catrel == catrel)) def RemoveCheckpkgResults(self, os_rel, arch, catrel): logging.debug("%s: RemoveCheckpkgResults(%s, %s, %s)", self, os_rel, arch, catrel) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete( CheckpkgErrorTag.sqlmeta.table, sqlobject.AND(CheckpkgErrorTag.q.srv4_file == self, CheckpkgErrorTag.q.os_rel == os_rel, CheckpkgErrorTag.q.arch == arch, CheckpkgErrorTag.q.catrel == catrel)))) def RemoveAllCheckpkgResults(self): logging.debug("%s: RemoveAllCheckpkgResults()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CheckpkgErrorTag.sqlmeta.table, CheckpkgErrorTag.q.srv4_file == self))) def RemoveOverrides(self): logging.debug("%s: RemoveOverrides()", self) sqlobject.sqlhub.processConnection.query( sqlobject.sqlhub.processConnection.sqlrepr( sqlbuilder.Delete(CheckpkgOverride.sqlmeta.table, CheckpkgOverride.q.srv4_file == self))) def __unicode__(self): return (u"Package: %s-%s, %s" % (self.catalogname, self.version_string, self.arch.name)) def GetStatsStruct(self): return cPickle.loads(str(self.data_obj.pickle)) def GetRestRepr(self): mimetype = "application/x-vnd.opencsw.pkg;type=srv4-detail" data = { 'arch': self.arch.name, 'basename': self.basename, # For compatibility with the catalog parser from catalog.py 'file_basename': self.basename, 'catalogname': self.catalogname, 'filename_arch': self.filename_arch.name, 'maintainer_email': self.maintainer.email, 'maintainer_full_name': self.maintainer.full_name, 'md5_sum': self.md5_sum, 'mtime': unicode(self.mtime), 'osrel': self.os_rel.short_name, 'pkgname': self.pkginst.pkgname, 'rev': self.rev, 'size': self.size, 'version_string': self.version_string, # For compatibility with the catalog parser from catalog.py 'version': self.version_string, # 'in_catalogs': unicode([unicode(x) for x in self.in_catalogs]), } return mimetype, data