def postconditions_34(self): store = Store(create_database(GLSettings.db_uri)) notification_l10n = NotificationL10NFactory(store) x = notification_l10n.get_val(u'export_template', u'it') self.assertNotEqual(x, 'unmodifiable') store.commit() store.close()
def test_check_unmodifiable_strings(self): # This test case asserts that data migration updates unmodifiable l10n strings self._initStartDB(34) notification_l10n = NotificationL10NFactory(self.store) t0 = notification_l10n.get_val('export_template', 'it') notification_l10n.set_val('export_template', 'it', '') t1 = notification_l10n.get_val('export_template', 'it') self.assertEqual(t1, '') self.store.commit() # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val('version', self.dummy_ver) self.assertEqual(prv.get_val('version'), self.dummy_ver) store.commit() store.close() migration.perform_data_update(self.db_file) store = Store(create_database(GLSettings.db_uri)) notification_l10n = NotificationL10NFactory(store) t2 = notification_l10n.get_val('export_template', 'it') self.assertEqual(t2, t0) store.commit() store.close() shutil.rmtree(GLSettings.db_path)
def setUp(self): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % DATABASE_VERSION db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val('version', self.dummy_ver) self.assertEqual(prv.get_val('version'), self.dummy_ver) store.commit() store.close() # backup various mocks that we will use self._bck_f = config.is_cfg_valid GLConfig['private']['xx_smtp_password'] = GLConfig['private'].pop('smtp_password') self.dp = u'yes_you_really_should_change_me'
def setUp(self): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') shutil.rmtree(GLSettings.db_path, True) os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % DATABASE_VERSION db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val(u'version', self.dummy_ver) self.assertEqual(prv.get_val(u'version'), self.dummy_ver) store.commit() store.close() # backup various mocks that we will use self._bck_f = config.is_cfg_valid GLConfig['private']['xx_smtp_password'] = GLConfig['private'].pop( 'smtp_password') self.dp = u'yes_you_really_should_change_me'
def initialize(self, debug=None): """See `IDatabase`.""" # Calculate the engine url. url = expand(config.database.url, config.paths) log.debug('Database url: %s', url) # XXX By design of SQLite, database file creation does not honor # umask. See their ticket #1193: # http://www.sqlite.org/cvstrac/tktview?tn=1193,31 # # This sucks for us because the mailman.db file /must/ be group # writable, however even though we guarantee our umask is 002 here, it # still gets created without the necessary g+w permission, due to # SQLite's policy. This should only affect SQLite engines because its # the only one that creates a little file on the local file system. # This kludges around their bug by "touch"ing the database file before # SQLite has any chance to create it, thus honoring the umask and # ensuring the right permissions. We only try to do this for SQLite # engines, and yes, we could have chmod'd the file after the fact, but # half dozen and all... self.url = url self._prepare(url) database = create_database(url) store = Store(database, GenerationalCache()) database.DEBUG = (as_boolean(config.database.debug) if debug is None else debug) self.store = store store.commit()
def test_check_unmodifiable_strings(self): # This test case asserts that data migration updates unmodifiable l10n strings self._initStartDB(34) notification_l10n = NotificationL10NFactory(self.store) t0 = notification_l10n.get_val("export_template", "it") notification_l10n.set_val("export_template", "it", "") t1 = notification_l10n.get_val("export_template", "it") self.assertEqual(t1, "") self.store.commit() # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = "2.XX.XX" prv.set_val("version", self.dummy_ver) self.assertEqual(prv.get_val("version"), self.dummy_ver) store.commit() store.close() migration.perform_data_update(self.db_file) store = Store(create_database(GLSettings.db_uri)) notification_l10n = NotificationL10NFactory(store) t2 = notification_l10n.get_val("export_template", "it") self.assertEqual(t2, t0) store.commit() store.close() shutil.rmtree(GLSettings.db_path)
def test_migration_error_with_removed_language(self): store = Store(create_database(GLSettings.db_uri)) zyx = EnabledLanguage('zyx') store.add(zyx) store.commit() store.close() self.assertRaises(Exception, migration.perform_data_update, self.db_file)
def preconditions_34(self): store = Store(create_database(self.start_db_uri)) notification_l10n = NotificationL10NFactory(store) notification_l10n.set_val(u'export_template', u'it', 'unmodifiable') x = notification_l10n.get_val(u'export_template', u'it') self.assertTrue(x, 'unmodifiable') store.commit() store.close()
def perform_data_update(dbfile): new_tmp_store = Store(create_database(GLSettings.make_db_uri(dbfile))) try: db_perform_data_update(new_tmp_store) new_tmp_store.commit() except: new_tmp_store.rollback() raise finally: new_tmp_store.close()
def do_statspollute(dbfile): # source gl_database = create_database("sqlite:%s" % dbfile) source_store = Store(gl_database) stats = source_store.find(models.Stats) counter = 0 for s in stats: source_store.remove(s) counter += 1 print "removed %d entry in stats" % counter counter = 0 # 21 days in the past for past_hours in xrange(24 * 7 * 3): past_hours += 4 when = utc_past_date(hours=past_hours) newstat = models.Stats() newstat.freemb = randint(1000, 1050) newstat.year = when.isocalendar()[0] newstat.week = when.isocalendar()[1] level = round((randint(0, 1000) / 240.0), 1) - 2 def random_pollution(): return int(randint(0,11) + (5 * level)) activity_fake = { 'successfull_logins': random_pollution(), 'failed_logins': random_pollution(), 'started_submissions': random_pollution(), 'completed_submissions': random_pollution(), 'uploaded_files': int(randint(0,11) + (5 * level)), 'appended_files': random_pollution(), 'wb_comments': random_pollution(), 'wb_messages': random_pollution(), 'receiver_comments': random_pollution(), 'receiver_messages': random_pollution() } for k, v in activity_fake.iteritems(): if v < 0: activity_fake[k] = 0 newstat.start = when newstat.summary = activity_fake counter += 1 source_store.add(newstat) print "Committing %d stats" % counter source_store.commit()
def test_trim_value_to_range(self): store = Store(create_database(GLSettings.db_uri)) nf = config.NodeFactory(store) fake_cfg = nf.get_cfg(u'wbtip_timetolive') self.assertRaises(errors.InvalidModelInput, fake_cfg.set_v, 3650) fake_cfg.value = {'v': 3650} store.commit() MigrationBase.trim_value_to_range(nf, u'wbtip_timetolive') self.assertEqual(nf.get_val(u'wbtip_timetolive'), 365 * 2)
def testCreate(self): """Test creation of new VideoFile""" store = Store(self.db) videofile = models.VideoFile() videofile.filename = u"/home/user/foo.mpg" store.add(videofile) store.commit() self.assertTrue(Store.of(videofile) is store) videofile_from_database = store.find(models.VideoFile, models.VideoFile.filename == u"/home/user/foo.mpg").one() self.assertTrue(videofile is videofile_from_database)
def testCreate(self): """Test creation of new MusicAlbum""" store = Store(self.db) musicalbum = models.MusicAlbum() musicalbum.title = u"The Lady Dance" store.add(musicalbum) store.commit() self.assertTrue(Store.of(musicalbum) is store) musicalbum_from_database = store.find(models.MusicAlbum, models.MusicAlbum.title == u"The Lady Dance").one() self.assertTrue(musicalbum is musicalbum_from_database)
def testCreate(self): """Test creation of new MusicTrack""" store = Store(self.db) musictrack = models.MusicTrack() musictrack.title = u"The Beautiful Ones" store.add(musictrack) store.commit() self.assertTrue(Store.of(musictrack) is store) musictrack_from_database = store.find(models.MusicTrack, models.MusicTrack.title == u"The Beautiful Ones").one() self.assertTrue(musictrack is musictrack_from_database)
def testCreate(self): """Test creation of new PhotoAlbum""" store = Store(self.db) photoalbum = models.PhotoAlbum() photoalbum.title = u"Photo Album Title" photoalbum.description = u"This is a photo description" store.add(photoalbum) store.commit() self.assertTrue(Store.of(photoalbum) is store) photoalbum_from_database = store.find(models.PhotoAlbum, models.PhotoAlbum.title == u"Photo Album Title").one() self.assertTrue(photoalbum is photoalbum_from_database)
def testCreate(self): '''Test creation of new MusicTrack''' store = Store(self.db) musictrack = models.MusicTrack() musictrack.title = u'The Beautiful Ones' store.add(musictrack) store.commit() self.assertTrue(Store.of(musictrack) is store) musictrack_from_database = store.find( models.MusicTrack, models.MusicTrack.title == u'The Beautiful Ones').one() self.assertTrue(musictrack is musictrack_from_database)
def testCreate(self): '''Test creation of new MusicAlbum''' store = Store(self.db) musicalbum = models.MusicAlbum() musicalbum.title = u'The Lady Dance' store.add(musicalbum) store.commit() self.assertTrue(Store.of(musicalbum) is store) musicalbum_from_database = store.find( models.MusicAlbum, models.MusicAlbum.title == u'The Lady Dance').one() self.assertTrue(musicalbum is musicalbum_from_database)
def testCreate(self): '''Test creation of new PhotoImage''' store = Store(self.db) photoimage = models.PhotoImage() photoimage.filename = u'/home/user/photo.jpg' store.add(photoimage) store.commit() self.assertTrue(Store.of(photoimage) is store) photoimage_from_database = store.find( models.PhotoImage, models.PhotoImage.filename == u'/home/user/photo.jpg').one() self.assertTrue(photoimage is photoimage_from_database)
def testCreate(self): '''Test creation of new VideoFile''' store = Store(self.db) videofile = models.VideoFile() videofile.filename = u'/home/user/foo.mpg' store.add(videofile) store.commit() self.assertTrue(Store.of(videofile) is store) videofile_from_database = store.find( models.VideoFile, models.VideoFile.filename == u'/home/user/foo.mpg').one() self.assertTrue(videofile is videofile_from_database)
def testCreate(self): '''Test creation of new MusicPlaylist''' store = Store(self.db) musicplaylist = models.MusicPlaylist() musicplaylist.title = u'The Ultimate Heavy Metal Goth Emo' store.add(musicplaylist) store.commit() self.assertTrue(Store.of(musicplaylist) is store) musicplaylist_from_database = store.find(models.MusicPlaylist, models.MusicPlaylist.title == \ u'The Ultimate Heavy Metal Goth Emo').one() self.assertTrue(musicplaylist is musicplaylist_from_database)
def testCreate(self): """Test creation of new MusicPlaylist""" store = Store(self.db) musicplaylist = models.MusicPlaylist() musicplaylist.title = u"The Ultimate Heavy Metal Goth Emo" store.add(musicplaylist) store.commit() self.assertTrue(Store.of(musicplaylist) is store) musicplaylist_from_database = store.find( models.MusicPlaylist, models.MusicPlaylist.title == u"The Ultimate Heavy Metal Goth Emo" ).one() self.assertTrue(musicplaylist is musicplaylist_from_database)
def testCreate(self): """Test creation of new PhotoImage""" store = Store(self.db) photoimage = models.PhotoImage() photoimage.filename = u"/home/user/photo.jpg" store.add(photoimage) store.commit() self.assertTrue(Store.of(photoimage) is store) photoimage_from_database = store.find( models.PhotoImage, models.PhotoImage.filename == u"/home/user/photo.jpg" ).one() self.assertTrue(photoimage is photoimage_from_database)
def testCreate(self): '''Test creation of new PhotoAlbum''' store = Store(self.db) photoalbum = models.PhotoAlbum() photoalbum.title = u'Photo Album Title' photoalbum.description = u'This is a photo description' store.add(photoalbum) store.commit() self.assertTrue(Store.of(photoalbum) is store) photoalbum_from_database = store.find( models.PhotoAlbum, models.PhotoAlbum.title == u'Photo Album Title').one() self.assertTrue(photoalbum is photoalbum_from_database)
def commit(self): if self.transaction: self.transaction = None return self.transaction.commit() result = Store.commit(self) #Store.reset(self) return result
def _create(self): '''Create a new entertainer database Reads the current database schema dictionary, and creates the sqlite database based on that schema ''' store = Store(self._db) store.execute(""" CREATE TABLE `entertainer_data` ( name VARCHAR PRIMARY KEY, value VARCHAR);""") store.execute( "INSERT INTO `entertainer_data` VALUES ('version', '0.2a');") for query in SCHEMA.itervalues(): store.execute(query, noresult=True) store.commit()
class Controller : def __init__(self): self.DATABASE = None self.store = None self.openConnection() def openConnection(self): self.DATABASE = None self.store = None self.DATABASE = create_database('sqlite: mydata.db') self.store = Store(self.DATABASE) def closeConnection(self): self.store.commit() self.store.close()
def getStore(url, create = False): # register new Storm scheme register_scheme("sqlitefk", ForeignKeysSQLite) d = create_database(url) s = Store(d) if create: schema = file(os.path.join(os.path.dirname(__file__), "schema.sql"), "r").read().split("\n\n") for cmd in schema: s.execute(cmd) version = Meta() version.key = u"created" s.add(version) s.commit() return s
def test_schema_uri(self): """ It's possible to specify an alternate URI for applying the schema and cleaning up tables after a test. """ schema_uri = "sqlite:///%s" % self.makeFile() self.databases[0]["schema-uri"] = schema_uri zstorm = self.resource.make([]) store = zstorm.get("test") schema_store = Store(create_database(schema_uri)) # The schema was applied using the alternate schema URI statement = "SELECT name FROM sqlite_master WHERE name='patch'" self.assertEqual([], list(store.execute(statement))) self.assertEqual([("patch", )], list(schema_store.execute(statement))) # The cleanup is performed with the alternate schema URI store.commit() schema_store.execute("INSERT INTO test (foo) VALUES ('data')") schema_store.commit() self.resource.clean(zstorm) self.assertEqual([], list(schema_store.execute("SELECT * FROM test")))
def test_schema_uri(self): """ It's possible to specify an alternate URI for applying the schema and cleaning up tables after a test. """ schema_uri = "sqlite:///%s" % self.makeFile() self.databases[0]["schema-uri"] = schema_uri zstorm = self.resource.make([]) store = zstorm.get("test") schema_store = Store(create_database(schema_uri)) # The schema was applied using the alternate schema URI statement = "SELECT name FROM sqlite_master WHERE name='patch'" self.assertEqual([], list(store.execute(statement))) self.assertEqual([("patch",)], list(schema_store.execute(statement))) # The cleanup is performed with the alternate schema URI store.commit() schema_store.execute("INSERT INTO test (foo) VALUES ('data')") schema_store.commit() self.resource.clean(zstorm) self.assertEqual([], list(schema_store.execute("SELECT * FROM test")))
class ZStormResourceManagerTest(TestHelper): def is_supported(self): return has_transaction and has_zope_component and has_testresources def setUp(self): super(ZStormResourceManagerTest, self).setUp() self._package_dir = self.makeDir() sys.path.append(self._package_dir) patch_dir = os.path.join(self._package_dir, "patch_package") os.mkdir(patch_dir) self.makeFile(path=os.path.join(patch_dir, "__init__.py"), content="") self.makeFile(path=os.path.join(patch_dir, "patch_1.py"), content=PATCH) import patch_package create = ["CREATE TABLE test (foo TEXT UNIQUE, bar INT)"] drop = ["DROP TABLE test"] delete = ["DELETE FROM test"] uri = "sqlite:///%s" % self.makeFile() schema = ZSchema(create, drop, delete, patch_package) self.databases = [{"name": "test", "uri": uri, "schema": schema}] self.resource = ZStormResourceManager(self.databases) self.store = Store(create_database(uri)) def tearDown(self): del sys.modules["patch_package"] sys.path.remove(self._package_dir) if "patch_1" in sys.modules: del sys.modules["patch_1"] super(ZStormResourceManagerTest, self).tearDown() def test_make(self): """ L{ZStormResourceManager.make} returns a L{ZStorm} resource that can be used to get the registered L{Store}s. """ zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo, bar FROM test"))) def test_make_upgrade(self): """ L{ZStormResourceManager.make} upgrades the schema if needed. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT bar FROM test"))) def test_make_delete(self): """ L{ZStormResourceManager.make} deletes the data from all tables to make sure that tests run against a clean database. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.execute("INSERT INTO test (foo) VALUES ('data')") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo FROM test"))) def test_make_zstorm_overwritten(self): """ L{ZStormResourceManager.make} registers its own ZStorm again if a test has registered a new ZStorm utility overwriting the resource one. """ zstorm = self.resource.make([]) provideUtility(ZStorm()) self.resource.make([]) self.assertIs(zstorm, getUtility(IZStorm)) def test_clean_flush(self): """ L{ZStormResourceManager.clean} tries to flush the stores to make sure that they are all in a consistent state. """ class Test(object): __storm_table__ = "test" foo = Unicode() bar = Int(primary=True) def __init__(self, foo, bar): self.foo = foo self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(u"data", 1)) store.add(Test(u"data", 2)) self.assertRaises(IntegrityError, self.resource.clean, zstorm) def test_clean_delete(self): """ L{ZStormResourceManager.clean} cleans the database tables from the data created by the tests. """ zstorm = self.resource.make([]) store = zstorm.get("test") store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") store.commit() self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test"))) def test_clean_with_force_delete(self): """ If L{ZStormResourceManager.force_delete} is C{True}, L{Schema.delete} is always invoked upon test cleanup. """ zstorm = self.resource.make([]) self.store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") self.store.commit() self.resource.force_delete = True self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test"))) def test_wb_clean_clears_alive_cache_before_abort(self): """ L{ZStormResourceManager.clean} clears the alive cache before aborting the transaction. """ class Test(object): __storm_table__ = "test" bar = Int(primary=True) def __init__(self, bar): self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(1)) store.add(Test(2)) real_invalidate = store.invalidate def invalidate_proxy(): self.assertEqual(0, len(store._alive.values())) real_invalidate() store.invalidate = invalidate_proxy self.resource.clean(zstorm) def test_schema_uri(self): """ It's possible to specify an alternate URI for applying the schema and cleaning up tables after a test. """ schema_uri = "sqlite:///%s" % self.makeFile() self.databases[0]["schema-uri"] = schema_uri zstorm = self.resource.make([]) store = zstorm.get("test") schema_store = Store(create_database(schema_uri)) # The schema was applied using the alternate schema URI statement = "SELECT name FROM sqlite_master WHERE name='patch'" self.assertEqual([], list(store.execute(statement))) self.assertEqual([("patch",)], list(schema_store.execute(statement))) # The cleanup is performed with the alternate schema URI store.commit() schema_store.execute("INSERT INTO test (foo) VALUES ('data')") schema_store.commit() self.resource.clean(zstorm) self.assertEqual([], list(schema_store.execute("SELECT * FROM test"))) def test_deprecated_database_format(self): """ The old deprecated format of the 'database' constructor parameter is still supported. """ import patch_package uri = "sqlite:///%s" % self.makeFile() schema = ZSchema([], [], [], patch_package) resource = ZStormResourceManager({"test": (uri, schema)}) zstorm = resource.make([]) store = zstorm.get("test") self.assertIsNot(None, store)
class IssuesLog(): def __init__(self, backend_name): self.backend_name = backend_name self.connect() self.create_db() def connect(self): opts = Config() self.database = create_database('mysql://' + opts.db_user_out + ':' + opts.db_password_out + '@' + opts.db_hostname_out + ':' + opts.db_port_out + '/' + opts.db_database_out) self.store = Store(self.database) def create_db(self): print("self.backend_name = %s" % (self.backend_name)) if self.backend_is_bugzilla(): self.store.execute(__sql_table_bugzilla__) elif self.backend_is_jira(): self.store.execute(__sql_table_jira__) def copy_issue(self, db_ilog): """ This method creates a copy of DBBugzilla/JiraIssuesLog object """ if self.backend_is_bugzilla(): aux = DBBugzillaIssuesLog(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to #aux = DBBugzillaIssuesLog (db_ilog.issue_id) aux.alias = db_ilog.alias aux.delta_ts = db_ilog.delta_ts aux.reporter_accessible = db_ilog.reporter_accessible aux.cclist_accessible = db_ilog.cclist_accessible aux.classification_id = db_ilog.classification_id aux.classification = db_ilog.classification aux.product = db_ilog.product aux.component = db_ilog.component aux.version = db_ilog.version aux.rep_platform = db_ilog.rep_platform aux.op_sys = db_ilog.op_sys aux.dup_id = db_ilog.dup_id aux.bug_file_loc = db_ilog.bug_file_loc aux.status_whiteboard = db_ilog.status_whiteboard aux.target_milestone = db_ilog.target_milestone aux.votes = db_ilog.votes aux.everconfirmed = db_ilog.everconfirmed aux.qa_contact = db_ilog.qa_contact aux.estimated_time = db_ilog.estimated_time aux.remaining_time = db_ilog.remaining_time aux.actual_time = db_ilog.actual_time aux.deadline = db_ilog.deadline aux.keywords = db_ilog.keywords aux.cc = db_ilog.cc aux.group_bugzilla = db_ilog.group_bugzilla aux.flag = db_ilog.flag return aux elif self.backend_is_jira(): aux = DBJiraIssuesLog(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to aux.link = db_ilog.link aux.component = db_ilog.component aux.version = db_ilog.version aux.issue_key = db_ilog.issue_key aux.environment = db_ilog.environment aux.project = db_ilog.project aux.project_key = db_ilog.project_key aux.security = db_ilog.security return aux def get_people_id(self, email, tracker_id): """ Gets the id of an user """ p = self.store.find(DBPeople, DBPeople.email == email).one() ## ## the code below was created ad-hoc for KDE solid ## try: return p.id except AttributeError: p = self.store.find(DBPeople, DBPeople.user_id == email).one() try: return p.id except AttributeError: # no person was found in People with the email above, so # we include it printdbg("Person not found. Inserted with email %s " % (email)) dp = DBPeople(email, tracker_id) self.store.add(dp) self.store.commit() return dp.id def get_last_change_date(self): """ This method gets the date of the last change included in the log table """ if self.backend_is_bugzilla(): result = self.store.find(DBBugzillaIssuesLog) aux = result.order_by(Desc(DBBugzillaIssuesLog.date))[:1] for entry in aux: return entry.date elif self.backend_is_jira(): result = self.store.find(DBJiraIssuesLog) aux = result.order_by(Desc(DBJiraIssuesLog.date))[:1] for entry in aux: return entry.date return None def get_issues_changed_since(self, date): """ This method fetchs the issues changes since date """ #SELECT DISTINCT(issues.id) FROM issues, changes #WHERE issues.id = changes.issue_id #AND (issues.submitted_on >= '2012-02-28 12:34:44' # OR changes.changed_on >= '2012-02-28 12:34:44'); result = self.store.find(DBIssue, DBChange.issue_id == DBIssue.id, Or(DBIssue.submitted_on > date, DBChange.changed_on > date )).group_by(DBIssue.id) return result def get_previous_state(self, issue_id): """ This method returns a db_ilog object with the last row found in the log table """ db_ilog = None if self.backend_is_jira(): rows = self.store.find(DBJiraIssuesLog, DBJiraIssuesLog.issue_id == issue_id) lrow = rows.order_by(Desc(DBJiraIssuesLog.id))[:1] for aux in lrow: # FIXME it only contains an element! db_ilog = DBJiraIssuesLog(aux.issue, aux.tracker_id) db_ilog.issue_id = aux.issue_id db_ilog.type = aux.type db_ilog.summary = aux.summary db_ilog.description = aux.description db_ilog.status = aux.status db_ilog.resolution = aux.resolution db_ilog.priority = aux.priority db_ilog.submitted_by = aux.submitted_by db_ilog.date = aux.date db_ilog.assigned_to = aux.assigned_to db_ilog.issue_key = aux.issue_key db_ilog.link = aux.link db_ilog.environment = aux.environment db_ilog.security = aux.security db_ilog.updated = aux.updated db_ilog.version = aux.version db_ilog.component = aux.component db_ilog.votes = aux.votes db_ilog.project = aux.project db_ilog.project_id = aux.project_id db_ilog.project_key = aux.project_key else: # elif self.backend_is_bugzilla(): rows = self.store.find(DBBugzillaIssuesLog, DBBugzillaIssuesLog.issue_id == issue_id) lrow = rows.order_by(Desc(DBBugzillaIssuesLog.id))[:1] for aux in lrow: # FIXME it only contains an element! db_ilog = DBBugzillaIssuesLog(aux.issue, aux.tracker_id) db_ilog.issue_id = aux.issue_id db_ilog.type = aux.type db_ilog.summary = aux.summary db_ilog.description = aux.description db_ilog.status = aux.status db_ilog.resolution = aux.resolution db_ilog.priority = aux.priority db_ilog.submitted_by = aux.submitted_by db_ilog.date = aux.date db_ilog.assigned_to = aux.assigned_to db_ilog.alias = aux.alias db_ilog.delta_ts = aux.delta_ts db_ilog.reporter_accessible = aux.reporter_accessible db_ilog.cclist_accessible = aux.cclist_accessible db_ilog.classification_id = aux.classification_id db_ilog.classification = aux.classification db_ilog.product = aux.product db_ilog.component = aux.component db_ilog.version = aux.version db_ilog.rep_platform = aux.rep_platform db_ilog.op_sys = aux.op_sys db_ilog.dup_id = aux.dup_id db_ilog.bug_file_loc = aux.bug_file_loc db_ilog.status_whiteboard = aux.status_whiteboard db_ilog.target_milestone = aux.target_milestone db_ilog.votes = aux.votes db_ilog.everconfirmed = aux.everconfirmed db_ilog.qa_contact = aux.qa_contact db_ilog.estimated_time = aux.estimated_time db_ilog.remaining_time = aux.remaining_time db_ilog.actual_time = aux.actual_time db_ilog.deadline = aux.deadline db_ilog.keywords = aux.keywords db_ilog.cc = aux.cc db_ilog.group_bugzilla = aux.group_bugzilla db_ilog.flag = aux.flag return db_ilog def issue_is_new(self, issue_id): """ This method returns True if the issue is not logged in the log table """ if self.backend_is_jira(): result = self.store.find(DBJiraIssuesLog, DBJiraIssuesLog.issue_id == issue_id) elif self.backend_is_bugzilla(): result = self.store.find(DBBugzillaIssuesLog, DBBugzillaIssuesLog.issue_id == issue_id) return (result.count() == 0) def build_initial_state(self, db_ilog): """ This method gets the first changes of every field in order to get the initial state of the bug """ fields = self.store.execute("SELECT DISTINCT(field) FROM changes\ where issue_id=%s" % (db_ilog.issue_id)) for f in fields: value = self.store.execute("SELECT old_value FROM changes \ WHERE issue_id=%s AND field=\"%s\" ORDER BY changed_on LIMIT 1" % (db_ilog.issue_id, f[0])) for v in value: if self.backend_is_bugzilla(): # Bugzilla section # if f[0] in bg_issues_links: table_field = bg_issues_links[f[0]] if table_field == 'summary': db_ilog.summary = v[0] elif table_field == 'priority': db_ilog.priority = v[0] elif table_field == 'type': db_ilog.type = v[0] elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id( v[0], self.get_tracker_id(db_ilog.issue_id)) elif table_field == 'status': db_ilog.status = v[0] elif table_field == 'resolution': db_ilog.resolution = v[0] elif table_field == 'alias': db_ilog.alias = v[0] elif table_field == 'reporter_accessible': db_ilog.reporter_accessible = v[0] elif table_field == 'cclist_accessible': db_ilog.cclist_accessible = v[0] elif table_field == 'product': db_ilog.product = v[0] elif table_field == 'component': db_ilog.component = v[0] elif table_field == 'version': db_ilog.version = v[0] elif table_field == 'rep_platform': db_ilog.rep_platform = v[0] elif table_field == 'op_sys': db_ilog.op_sys = v[0] elif table_field == 'bug_file_loc': db_ilog.bug_file_loc = v[0] elif table_field == 'status_whiteboard': db_ilog.status_whiteboard = v[0] elif table_field == 'target_milestone': db_ilog.target_milestone = v[0] elif table_field == 'votes': db_ilog.votes = v[0] elif table_field == 'everconfirmed': db_ilog.everconfirmed = v[0] elif table_field == 'qa_contact': db_ilog.qa_contact = v[0] elif table_field == 'keywords': db_ilog.Keywords = v[0] elif table_field == 'cc': db_ilog.cc = v[0] if self.backend_is_jira(): # Jira section # if f[0] in jira_issues_links: table_field = jira_issues_links[f[0]] if table_field == 'summary': db_ilog.summary = v[0] elif table_field == 'priority': db_ilog.priority = v[0] elif table_field == 'type': db_ilog.type = v[0] elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id(v[0]) elif table_field == 'status': db_ilog.status = v[0] elif table_field == 'resolution': db_ilog.resolution = v[0] elif table_field == 'link': db_ilog.link = v[0] elif table_field == 'environment': db_ilog.environment = v[0] elif table_field == 'component': db_ilog.component = v[0] elif table_field == 'description': db_ilog.description = v[0] elif table_field == 'security': db_ilog.security = v[0] return db_ilog def backend_is_bugzilla(self): return self.backend_name == 'bg' def backend_is_jira(self): return self.backend_name == 'jira' def get_last_values(self, issue_row): i = issue_row db_ilog = None if self.backend_is_bugzilla(): db_ilog_bugzilla = DBBugzillaIssuesLog(i.issue, i.tracker_id) db_ilog_bugzilla.issue_id = i.id db_ilog_bugzilla.type = i.type db_ilog_bugzilla.summary = i.summary db_ilog_bugzilla.description = i.description db_ilog_bugzilla.status = i.status db_ilog_bugzilla.resolution = i.resolution db_ilog_bugzilla.priority = i.priority db_ilog_bugzilla.submitted_by = i.submitted_by db_ilog_bugzilla.date = i.submitted_on db_ilog_bugzilla.assigned_to = i.assigned_to ib = self.store.find(DBBugzillaIssueExt, \ DBBugzillaIssueExt.issue_id == db_ilog_bugzilla.issue_id).one() #### db_ilog_bugzilla.alias = ib.alias db_ilog_bugzilla.delta_ts = ib.delta_ts db_ilog_bugzilla.reporter_accessible = ib.reporter_accessible db_ilog_bugzilla.cclist_accessible = ib.cclist_accessible db_ilog_bugzilla.classification_id = ib.classification_id db_ilog_bugzilla.classification = ib.classification db_ilog_bugzilla.product = ib.product db_ilog_bugzilla.component = ib.component db_ilog_bugzilla.version = ib.version db_ilog_bugzilla.rep_platform = ib.rep_platform db_ilog_bugzilla.op_sys = ib.op_sys db_ilog_bugzilla.dup_id = ib.dup_id db_ilog_bugzilla.bug_file_loc = ib.bug_file_loc db_ilog_bugzilla.status_whiteboard = ib.status_whiteboard db_ilog_bugzilla.target_milestone = ib.target_milestone db_ilog_bugzilla.votes = ib.votes db_ilog_bugzilla.everconfirmed = ib.everconfirmed db_ilog_bugzilla.qa_contact = ib.qa_contact db_ilog_bugzilla.estimated_time = ib.estimated_time db_ilog_bugzilla.remaining_time = ib.remaining_time db_ilog_bugzilla.actual_time = ib.actual_time db_ilog_bugzilla.deadline = ib.deadline db_ilog_bugzilla.keywords = ib.keywords db_ilog_bugzilla.cc = ib.cc db_ilog_bugzilla.group_bugzilla = ib.group_bugzilla db_ilog_bugzilla.flag = ib.flag db_ilog = db_ilog_bugzilla elif self.backend_is_jira(): db_ilog = DBJiraIssuesLog(i.issue, i.tracker_id) db_ilog.issue_id = i.id db_ilog.type = i.type db_ilog.summary = i.summary db_ilog.description = i.description db_ilog.status = i.status db_ilog.resolution = i.resolution db_ilog.priority = i.priority db_ilog.submitted_by = i.submitted_by db_ilog.date = i.submitted_on db_ilog.assigned_to = i.assigned_to ib = self.store.find(DBJiraIssueExt, \ DBJiraIssueExt.issue_id == db_ilog.issue_id).one() db_ilog.issue_key = ib.issue_key db_ilog.link = ib.link db_ilog.environment = ib.environment db_ilog.security = ib.security db_ilog.updated = ib.updated db_ilog.version = ib.version db_ilog.component = ib.component db_ilog.votes = ib.votes db_ilog.project = ib.project db_ilog.project_id = ib.project_id db_ilog.project_key = ib.project_key return db_ilog def insert_new_bugs_created(self, date_from, date_to): """ This method inserts an entry with the data of the creation time """ if (not date_from) and (not date_to): issues = self.store.find(DBIssue) elif not date_from: issues = self.store.find(DBIssue, DBIssue.submitted_on < date_to) elif not date_to: issues = self.store.find(DBIssue, DBIssue.submitted_on > date_from) else: issues = self.store.find(DBIssue, And(DBIssue.submitted_on <= date_to, DBIssue.submitted_on > date_from)) issues = issues.order_by(Asc(DBIssue.submitted_on)) ## we store the initial data for each bug found for i in issues: db_ilog = self.get_last_values(i) # from issues and change tables db_ilog = self.build_initial_state(db_ilog) self.store.add(db_ilog) printdbg("Issue #%s created at %s - date_from = %s - date_to = %s" % (db_ilog.issue, db_ilog.date, date_from, date_to)) def get_tracker_id(self, issue_id): """ Returns tracker id from issues """ result = self.store.find(DBIssue.tracker_id, DBIssue.id == issue_id).one() return result def run(self): last_change_date = self.get_last_change_date() printdbg("Last change logged at %s" % (last_change_date)) date_from = None date_to = None if last_change_date: changes = self.store.find(DBChange, DBChange.changed_on > last_change_date) date_from = last_change_date else: changes = self.store.find(DBChange) changes = changes.order_by(Asc(DBChange.changed_on)) for ch in changes: # insert creation if needed date_to = ch.changed_on self.insert_new_bugs_created(date_from, date_to) date_from = date_to field = ch.field new_value = ch.new_value changed_by = ch.changed_by date = ch.changed_on issue_id = ch.issue_id #print("field = %s, new_value = %s, changed_by = %s, date = %s" # % (field, new_value, str(changed_by), str(date))) db_ilog = self.get_previous_state(issue_id) printdbg("Issue #%s modified at %s" % (db_ilog.issue, date)) if self.backend_is_bugzilla(): # Bugzilla section # # if (field in bg_issues_links): table_field = bg_issues_links[field] db_ilog.submitted_by = changed_by db_ilog.date = date if table_field == 'summary': db_ilog.summary = new_value elif table_field == 'priority': db_ilog.priority = new_value elif table_field == 'type': db_ilog.type = new_value elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id( new_value, self.get_tracker_id(db_ilog.issue_id)) elif table_field == 'status': db_ilog.status = new_value elif table_field == 'resolution': db_ilog.resolution = new_value elif table_field == 'alias': db_ilog.alias = new_value elif table_field == 'reporter_accessible': db_ilog.reporter_accessible = new_value elif table_field == 'cclist_accessible': db_ilog.cclist_accessible = new_value elif table_field == 'product': db_ilog.product = new_value elif table_field == 'component': db_ilog.component = new_value elif table_field == 'version': db_ilog.version = new_value elif table_field == 'rep_platform': db_ilog.rep_platform = new_value elif table_field == 'op_sys': db_ilog.op_sys = new_value elif table_field == 'bug_file_loc': db_ilog.bug_file_loc = new_value elif table_field == 'status_whiteboard': db_ilog.status_whiteboard = new_value elif table_field == 'target_milestone': db_ilog.target_milestone = new_value elif table_field == 'votes': db_ilog.votes = new_value elif table_field == 'everconfirmed': db_ilog.everconfirmed = new_value elif table_field == 'qa_contact': db_ilog.qa_contact = new_value elif table_field == 'keywords': db_ilog.Keywords = new_value elif table_field == 'cc': db_ilog.cc = new_value try: self.store.add(db_ilog) except: traceback.print_exc() elif self.backend_is_jira(): # Jira section # # if (field in jira_issues_links): table_field = jira_issues_links[field] db_ilog.submitted_by = changed_by db_ilog.date = date if table_field == 'summary': db_ilog.summary = new_value elif table_field == 'priority': db_ilog.priority = new_value elif table_field == 'type': db_ilog.type = new_value elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id( new_value, self.get_tracker_id(db_ilog.issue_id)) elif table_field == 'status': db_ilog.status = new_value elif table_field == 'resolution': db_ilog.resolution = new_value elif table_field == 'description': db_ilog.description = new_value elif table_field == 'link': db_ilog.link = new_value elif table_field == 'component': db_ilog.component = new_value elif table_field == 'version': db_ilog.version = new_value elif table_field == 'security': db_ilog.security = new_value try: self.store.add(db_ilog) except: traceback.print_exc() # if there are changes, it stores the last bugs after the last # change. If there are no changes, insert all the created bugs self.insert_new_bugs_created(date_from, None) self.store.commit()
class TableReplacer: """ This is the base class used by every Updater """ def __init__(self, old_db_file, new_db_file, start_ver): from globaleaks.db.update_5_6 import User_version_5, Comment_version_5, Node_version_5 from globaleaks.db.update_6_7 import Node_version_6, Context_version_6 from globaleaks.db.update_7_8 import Node_version_7, Notification_version_7, Context_version_7, \ Receiver_version_7, InternalFile_version_7 from globaleaks.db.update_8_9 import Context_version_8, Receiver_version_8, Notification_version_8 from globaleaks.db.update_9_10 import Node_version_9, ApplicationData_version_10, \ Receiver_version_9, User_version_9 from globaleaks.db.update_10_11 import InternalTip_version_10, InternalFile_version_10 from globaleaks.db.update_11_12 import Node_version_11, ApplicationData_version_11, Context_version_11 self.old_db_file = old_db_file self.new_db_file = new_db_file self.start_ver = start_ver self.std_fancy = " ł " self.debug_info = " [%d => %d] " % (start_ver, start_ver + 1) self.table_history = { 'Node' : [ Node_version_5, Node_version_6, Node_version_7, Node_version_9, None, Node_version_11, None, models.Node], 'User' : [ User_version_5, User_version_9, None, None, None, models.User, None, None], 'Context' : [ Context_version_6, None, Context_version_7, Context_version_8, Context_version_11, None, None, models.Context], 'Receiver': [ Receiver_version_7, None, None, Receiver_version_8, Receiver_version_9, models.Receiver, None, None], 'ReceiverFile' : [ models.ReceiverFile, None, None, None, None, None, None, None], 'Notification': [ Notification_version_7, None, None, Notification_version_8, models.Notification, None, None, None], 'Comment': [ Comment_version_5, models.Comment, None, None, None, None, None, None], 'InternalTip' : [ InternalTip_version_10, None, None, None, None, None, models.InternalTip, None], 'InternalFile' : [ InternalFile_version_7, None, None, InternalFile_version_10, None, None, models.InternalFile, None], 'WhistleblowerTip' : [ models.WhistleblowerTip, None, None, None, None, None, None, None], 'ReceiverTip' : [ models.ReceiverTip, None, None, None, None, None, None , None], 'ReceiverInternalTip' : [ models.ReceiverInternalTip, None, None, None, None, None, None, None], 'ReceiverContext' : [ models.ReceiverContext, None, None, None, None, None, None, None], 'Message' : [ models.Message, None, None, None, None, None, None, None], 'Stats' : [models.Stats, None, None, None, None, None, None, None], 'ApplicationData' : [ApplicationData_version_10, None, None, None, None, None, None, models.ApplicationData], } for k, v in self.table_history.iteritems(): # +1 because count start from 0, # -5 because the relase 0,1,2,3,4 are not supported anymore assert len(v) == (DATABASE_VERSION + 1 - 5), \ "I'm expecting a table with %d statuses (%s)" % (DATABASE_VERSION, k) print "%s Opening old DB: %s" % (self.debug_info, old_db_file) old_database = create_database("sqlite:%s" % self.old_db_file) self.store_old = Store(old_database) GLSetting.db_file = new_db_file new_database = create_database("sqlite:%s" % new_db_file) self.store_new = Store(new_database) if self.start_ver + 1 == DATABASE_VERSION: print "%s Acquire SQL schema %s" % (self.debug_info, GLSetting.db_schema_file) if not os.access(GLSetting.db_schema_file, os.R_OK): print "Unable to access %s" % GLSetting.db_schema_file raise Exception("Unable to access db schema file") with open(GLSetting.db_schema_file) as f: create_queries = ''.join(f.readlines()).split(';') for create_query in create_queries: try: self.store_new.execute(create_query+';') except OperationalError: print "OperationalError in [%s]" % create_query self.store_new.commit() return # return here and manage the migrant versions here: for k, v in self.table_history.iteritems(): create_query = self.get_right_sql_version(k, self.start_ver +1) if not create_query: # table not present in the version continue try: self.store_new.execute(create_query+';') except OperationalError as excep: print "%s OperationalError in [%s]" % (self.debug_info, create_query) raise excep self.store_new.commit() def close(self): self.store_old.close() self.store_new.close() def initialize(self): pass def epilogue(self): pass def get_right_model(self, table_name, version): table_index = (version - 5) if not self.table_history.has_key(table_name): print "Not implemented usage of get_right_model %s (%s %d)" % ( __file__, table_name, self.start_ver) raise NotImplementedError assert version <= DATABASE_VERSION, "wrong developer brainsync" if self.table_history[table_name][table_index]: # print "Immediate return %s = %s at version %d" % \ # ( table_name, self.table_history[table_name][table_index], version ) return self.table_history[table_name][table_index] # else, it's none, and we've to take the previous valid version # # print "Requested version %d of %s need to be collected in the past" %\ # (version, table_name) while version >= 0: if self.table_history[table_name][table_index]: # print ".. returning %s = %s" %\ # ( table_name, self.table_history[table_name][table_index] ) return self.table_history[table_name][table_index] table_index -= 1 # This never want happen return None def get_right_sql_version(self, model_name, version): """ @param model_name: @param version: @return: The SQL right for the stuff we've """ modelobj = self.get_right_model(model_name, version) if not modelobj: return None right_query = generateCreateQuery(modelobj) return right_query def _perform_copy_list(self, table_name): print "%s default %s migration assistant: #%d" % ( self.debug_info, table_name, self.store_old.find(self.get_right_model(table_name, self.start_ver)).count()) old_objects = self.store_old.find(self.get_right_model(table_name, self.start_ver)) for old_obj in old_objects: new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for k, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name) ) self.store_new.add(new_obj) self.store_new.commit() def _perform_copy_single(self, table_name): print "%s default %s migration assistant" % (self.debug_info, table_name) old_obj = self.store_old.find(self.get_right_model(table_name, self.start_ver)).one() new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for k, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name) ) self.store_new.add(new_obj) self.store_new.commit() def migrate_Context(self): self._perform_copy_list("Context") def migrate_Node(self): self._perform_copy_single("Node") def migrate_User(self): self._perform_copy_list("User") def migrate_ReceiverTip(self): self._perform_copy_list("ReceiverTip") def migrate_WhistleblowerTip(self): self._perform_copy_list("WhistleblowerTip") def migrate_Comment(self): self._perform_copy_list("Comment") def migrate_InternalTip(self): self._perform_copy_list("InternalTip") def migrate_Receiver(self): self._perform_copy_list("Receiver") def migrate_InternalFile(self): self._perform_copy_list("InternalFile") def migrate_ReceiverFile(self): self._perform_copy_list("ReceiverFile") def migrate_Notification(self): self._perform_copy_single("Notification") def migrate_ReceiverContext(self): self._perform_copy_list("ReceiverContext") def migrate_ReceiverInternalTip(self): self._perform_copy_list("ReceiverInternalTip") def migrate_Message(self): """ has been created between 7 and 8! """ if self.start_ver < 8: return self._perform_copy_list("Message") def migrate_Stats(self): """ has been created between 9 and 10! """ if self.start_ver < 10: return self._perform_copy_list("Stats") def migrate_ApplicationData(self): """ has been created between 9 and 10! """ if self.start_ver < 10: return self._perform_copy_list("ApplicationData")
class IssuesLog(): def __init__(self, backend_name): self.backend_name = backend_name self.connect() self.create_db() def connect(self): opts = Config() self.database = create_database('mysql://' + opts.db_user_out + ':' + opts.db_password_out + '@' + opts.db_hostname_out + ':' + opts.db_port_out + '/' + opts.db_database_out) self.store = Store(self.database) def create_db(self): print("self.backend_name = %s" % (self.backend_name)) if self.backend_is_bugzilla(): self.store.execute(__sql_table_bugzilla__) elif self.backend_is_jira(): self.store.execute(__sql_table_jira__) def copy_issue(self, db_ilog): """ This method creates a copy of DBBugzilla/JiraIssuesLog object """ if self.backend_is_bugzilla(): aux = DBBugzillaIssuesLog(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to #aux = DBBugzillaIssuesLog (db_ilog.issue_id) aux.alias = db_ilog.alias aux.delta_ts = db_ilog.delta_ts aux.reporter_accessible = db_ilog.reporter_accessible aux.cclist_accessible = db_ilog.cclist_accessible aux.classification_id = db_ilog.classification_id aux.classification = db_ilog.classification aux.product = db_ilog.product aux.component = db_ilog.component aux.version = db_ilog.version aux.rep_platform = db_ilog.rep_platform aux.op_sys = db_ilog.op_sys aux.dup_id = db_ilog.dup_id aux.bug_file_loc = db_ilog.bug_file_loc aux.status_whiteboard = db_ilog.status_whiteboard aux.target_milestone = db_ilog.target_milestone aux.votes = db_ilog.votes aux.everconfirmed = db_ilog.everconfirmed aux.qa_contact = db_ilog.qa_contact aux.estimated_time = db_ilog.estimated_time aux.remaining_time = db_ilog.remaining_time aux.actual_time = db_ilog.actual_time aux.deadline = db_ilog.deadline aux.keywords = db_ilog.keywords aux.cc = db_ilog.cc aux.group_bugzilla = db_ilog.group_bugzilla aux.flag = db_ilog.flag return aux elif self.backend_is_jira(): aux = DBJiraIssuesLog(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to aux.link = db_ilog.link aux.component = db_ilog.component aux.version = db_ilog.version aux.issue_key = db_ilog.issue_key aux.environment = db_ilog.environment aux.project = db_ilog.project aux.project_key = db_ilog.project_key aux.security = db_ilog.security return aux def get_people_id(self, email, tracker_id): """ Gets the id of an user """ p = self.store.find(DBPeople, DBPeople.email == email).one() ## ## the code below was created ad-hoc for KDE solid ## try: return p.id except AttributeError: p = self.store.find(DBPeople, DBPeople.user_id == email).one() try: return p.id except AttributeError: # no person was found in People with the email above, so # we include it printdbg("Person not found. Inserted with email %s " % (email)) dp = DBPeople(email, tracker_id) self.store.add(dp) self.store.commit() return dp.id def get_last_change_date(self): """ This method gets the date of the last change included in the log table """ if self.backend_is_bugzilla(): result = self.store.find(DBBugzillaIssuesLog) aux = result.order_by(Desc(DBBugzillaIssuesLog.date))[:1] for entry in aux: return entry.date elif self.backend_is_jira(): result = self.store.find(DBJiraIssuesLog) aux = result.order_by(Desc(DBJiraIssuesLog.date))[:1] for entry in aux: return entry.date return None def get_issues_changed_since(self, date): """ This method fetchs the issues changes since date """ #SELECT DISTINCT(issues.id) FROM issues, changes #WHERE issues.id = changes.issue_id #AND (issues.submitted_on >= '2012-02-28 12:34:44' # OR changes.changed_on >= '2012-02-28 12:34:44'); result = self.store.find( DBIssue, DBChange.issue_id == DBIssue.id, Or(DBIssue.submitted_on > date, DBChange.changed_on > date)).group_by(DBIssue.id) return result def get_previous_state(self, issue_id): """ This method returns a db_ilog object with the last row found in the log table """ db_ilog = None if self.backend_is_jira(): rows = self.store.find(DBJiraIssuesLog, DBJiraIssuesLog.issue_id == issue_id) lrow = rows.order_by(Desc(DBJiraIssuesLog.id))[:1] for aux in lrow: # FIXME it only contains an element! db_ilog = DBJiraIssuesLog(aux.issue, aux.tracker_id) db_ilog.issue_id = aux.issue_id db_ilog.type = aux.type db_ilog.summary = aux.summary db_ilog.description = aux.description db_ilog.status = aux.status db_ilog.resolution = aux.resolution db_ilog.priority = aux.priority db_ilog.submitted_by = aux.submitted_by db_ilog.date = aux.date db_ilog.assigned_to = aux.assigned_to db_ilog.issue_key = aux.issue_key db_ilog.link = aux.link db_ilog.environment = aux.environment db_ilog.security = aux.security db_ilog.updated = aux.updated db_ilog.version = aux.version db_ilog.component = aux.component db_ilog.votes = aux.votes db_ilog.project = aux.project db_ilog.project_id = aux.project_id db_ilog.project_key = aux.project_key else: # elif self.backend_is_bugzilla(): rows = self.store.find(DBBugzillaIssuesLog, DBBugzillaIssuesLog.issue_id == issue_id) lrow = rows.order_by(Desc(DBBugzillaIssuesLog.id))[:1] for aux in lrow: # FIXME it only contains an element! db_ilog = DBBugzillaIssuesLog(aux.issue, aux.tracker_id) db_ilog.issue_id = aux.issue_id db_ilog.type = aux.type db_ilog.summary = aux.summary db_ilog.description = aux.description db_ilog.status = aux.status db_ilog.resolution = aux.resolution db_ilog.priority = aux.priority db_ilog.submitted_by = aux.submitted_by db_ilog.date = aux.date db_ilog.assigned_to = aux.assigned_to db_ilog.alias = aux.alias db_ilog.delta_ts = aux.delta_ts db_ilog.reporter_accessible = aux.reporter_accessible db_ilog.cclist_accessible = aux.cclist_accessible db_ilog.classification_id = aux.classification_id db_ilog.classification = aux.classification db_ilog.product = aux.product db_ilog.component = aux.component db_ilog.version = aux.version db_ilog.rep_platform = aux.rep_platform db_ilog.op_sys = aux.op_sys db_ilog.dup_id = aux.dup_id db_ilog.bug_file_loc = aux.bug_file_loc db_ilog.status_whiteboard = aux.status_whiteboard db_ilog.target_milestone = aux.target_milestone db_ilog.votes = aux.votes db_ilog.everconfirmed = aux.everconfirmed db_ilog.qa_contact = aux.qa_contact db_ilog.estimated_time = aux.estimated_time db_ilog.remaining_time = aux.remaining_time db_ilog.actual_time = aux.actual_time db_ilog.deadline = aux.deadline db_ilog.keywords = aux.keywords db_ilog.cc = aux.cc db_ilog.group_bugzilla = aux.group_bugzilla db_ilog.flag = aux.flag return db_ilog def issue_is_new(self, issue_id): """ This method returns True if the issue is not logged in the log table """ if self.backend_is_jira(): result = self.store.find(DBJiraIssuesLog, DBJiraIssuesLog.issue_id == issue_id) elif self.backend_is_bugzilla(): result = self.store.find(DBBugzillaIssuesLog, DBBugzillaIssuesLog.issue_id == issue_id) return (result.count() == 0) def build_initial_state(self, db_ilog): """ This method gets the first changes of every field in order to get the initial state of the bug """ fields = self.store.execute("SELECT DISTINCT(field) FROM changes\ where issue_id=%s" % (db_ilog.issue_id)) for f in fields: value = self.store.execute("SELECT old_value FROM changes \ WHERE issue_id=%s AND field=\"%s\" ORDER BY changed_on LIMIT 1" % (db_ilog.issue_id, f[0])) for v in value: if self.backend_is_bugzilla(): # Bugzilla section # if f[0] in bg_issues_links: table_field = bg_issues_links[f[0]] if table_field == 'summary': db_ilog.summary = v[0] elif table_field == 'priority': db_ilog.priority = v[0] elif table_field == 'type': db_ilog.type = v[0] elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id( v[0], self.get_tracker_id(db_ilog.issue_id)) elif table_field == 'status': db_ilog.status = v[0] elif table_field == 'resolution': db_ilog.resolution = v[0] elif table_field == 'alias': db_ilog.alias = v[0] elif table_field == 'reporter_accessible': db_ilog.reporter_accessible = v[0] elif table_field == 'cclist_accessible': db_ilog.cclist_accessible = v[0] elif table_field == 'product': db_ilog.product = v[0] elif table_field == 'component': db_ilog.component = v[0] elif table_field == 'version': db_ilog.version = v[0] elif table_field == 'rep_platform': db_ilog.rep_platform = v[0] elif table_field == 'op_sys': db_ilog.op_sys = v[0] elif table_field == 'bug_file_loc': db_ilog.bug_file_loc = v[0] elif table_field == 'status_whiteboard': db_ilog.status_whiteboard = v[0] elif table_field == 'target_milestone': db_ilog.target_milestone = v[0] elif table_field == 'votes': db_ilog.votes = v[0] elif table_field == 'everconfirmed': db_ilog.everconfirmed = v[0] elif table_field == 'qa_contact': db_ilog.qa_contact = v[0] elif table_field == 'keywords': db_ilog.Keywords = v[0] elif table_field == 'cc': db_ilog.cc = v[0] if self.backend_is_jira(): # Jira section # if f[0] in jira_issues_links: table_field = jira_issues_links[f[0]] if table_field == 'summary': db_ilog.summary = v[0] elif table_field == 'priority': db_ilog.priority = v[0] elif table_field == 'type': db_ilog.type = v[0] elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id(v[0]) elif table_field == 'status': db_ilog.status = v[0] elif table_field == 'resolution': db_ilog.resolution = v[0] elif table_field == 'link': db_ilog.link = v[0] elif table_field == 'environment': db_ilog.environment = v[0] elif table_field == 'component': db_ilog.component = v[0] elif table_field == 'description': db_ilog.description = v[0] elif table_field == 'security': db_ilog.security = v[0] return db_ilog def backend_is_bugzilla(self): return self.backend_name == 'bg' def backend_is_jira(self): return self.backend_name == 'jira' def get_last_values(self, issue_row): i = issue_row db_ilog = None if self.backend_is_bugzilla(): db_ilog_bugzilla = DBBugzillaIssuesLog(i.issue, i.tracker_id) db_ilog_bugzilla.issue_id = i.id db_ilog_bugzilla.type = i.type db_ilog_bugzilla.summary = i.summary db_ilog_bugzilla.description = i.description db_ilog_bugzilla.status = i.status db_ilog_bugzilla.resolution = i.resolution db_ilog_bugzilla.priority = i.priority db_ilog_bugzilla.submitted_by = i.submitted_by db_ilog_bugzilla.date = i.submitted_on db_ilog_bugzilla.assigned_to = i.assigned_to ib = self.store.find(DBBugzillaIssueExt, \ DBBugzillaIssueExt.issue_id == db_ilog_bugzilla.issue_id).one() #### db_ilog_bugzilla.alias = ib.alias db_ilog_bugzilla.delta_ts = ib.delta_ts db_ilog_bugzilla.reporter_accessible = ib.reporter_accessible db_ilog_bugzilla.cclist_accessible = ib.cclist_accessible db_ilog_bugzilla.classification_id = ib.classification_id db_ilog_bugzilla.classification = ib.classification db_ilog_bugzilla.product = ib.product db_ilog_bugzilla.component = ib.component db_ilog_bugzilla.version = ib.version db_ilog_bugzilla.rep_platform = ib.rep_platform db_ilog_bugzilla.op_sys = ib.op_sys db_ilog_bugzilla.dup_id = ib.dup_id db_ilog_bugzilla.bug_file_loc = ib.bug_file_loc db_ilog_bugzilla.status_whiteboard = ib.status_whiteboard db_ilog_bugzilla.target_milestone = ib.target_milestone db_ilog_bugzilla.votes = ib.votes db_ilog_bugzilla.everconfirmed = ib.everconfirmed db_ilog_bugzilla.qa_contact = ib.qa_contact db_ilog_bugzilla.estimated_time = ib.estimated_time db_ilog_bugzilla.remaining_time = ib.remaining_time db_ilog_bugzilla.actual_time = ib.actual_time db_ilog_bugzilla.deadline = ib.deadline db_ilog_bugzilla.keywords = ib.keywords db_ilog_bugzilla.cc = ib.cc db_ilog_bugzilla.group_bugzilla = ib.group_bugzilla db_ilog_bugzilla.flag = ib.flag db_ilog = db_ilog_bugzilla elif self.backend_is_jira(): db_ilog = DBJiraIssuesLog(i.issue, i.tracker_id) db_ilog.issue_id = i.id db_ilog.type = i.type db_ilog.summary = i.summary db_ilog.description = i.description db_ilog.status = i.status db_ilog.resolution = i.resolution db_ilog.priority = i.priority db_ilog.submitted_by = i.submitted_by db_ilog.date = i.submitted_on db_ilog.assigned_to = i.assigned_to ib = self.store.find(DBJiraIssueExt, \ DBJiraIssueExt.issue_id == db_ilog.issue_id).one() db_ilog.issue_key = ib.issue_key db_ilog.link = ib.link db_ilog.environment = ib.environment db_ilog.security = ib.security db_ilog.updated = ib.updated db_ilog.version = ib.version db_ilog.component = ib.component db_ilog.votes = ib.votes db_ilog.project = ib.project db_ilog.project_id = ib.project_id db_ilog.project_key = ib.project_key return db_ilog def insert_new_bugs_created(self, date_from, date_to): """ This method inserts an entry with the data of the creation time """ if (not date_from) and (not date_to): issues = self.store.find(DBIssue) elif not date_from: issues = self.store.find(DBIssue, DBIssue.submitted_on < date_to) elif not date_to: issues = self.store.find(DBIssue, DBIssue.submitted_on > date_from) else: issues = self.store.find( DBIssue, And(DBIssue.submitted_on <= date_to, DBIssue.submitted_on > date_from)) issues = issues.order_by(Asc(DBIssue.submitted_on)) ## we store the initial data for each bug found for i in issues: db_ilog = self.get_last_values(i) # from issues and change tables db_ilog = self.build_initial_state(db_ilog) self.store.add(db_ilog) printdbg( "Issue #%s created at %s - date_from = %s - date_to = %s" % (db_ilog.issue, db_ilog.date, date_from, date_to)) def get_tracker_id(self, issue_id): """ Returns tracker id from issues """ result = self.store.find(DBIssue.tracker_id, DBIssue.id == issue_id).one() return result def run(self): last_change_date = self.get_last_change_date() printdbg("Last change logged at %s" % (last_change_date)) date_from = None date_to = None if last_change_date: changes = self.store.find(DBChange, DBChange.changed_on > last_change_date) date_from = last_change_date else: changes = self.store.find(DBChange) changes = changes.order_by(Asc(DBChange.changed_on)) for ch in changes: # insert creation if needed date_to = ch.changed_on self.insert_new_bugs_created(date_from, date_to) date_from = date_to field = ch.field new_value = ch.new_value changed_by = ch.changed_by date = ch.changed_on issue_id = ch.issue_id #print("field = %s, new_value = %s, changed_by = %s, date = %s" # % (field, new_value, str(changed_by), str(date))) db_ilog = self.get_previous_state(issue_id) printdbg("Issue #%s modified at %s" % (db_ilog.issue, date)) if self.backend_is_bugzilla(): # Bugzilla section # # if (field in bg_issues_links): table_field = bg_issues_links[field] db_ilog.submitted_by = changed_by db_ilog.date = date if table_field == 'summary': db_ilog.summary = new_value elif table_field == 'priority': db_ilog.priority = new_value elif table_field == 'type': db_ilog.type = new_value elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id( new_value, self.get_tracker_id(db_ilog.issue_id)) elif table_field == 'status': db_ilog.status = new_value elif table_field == 'resolution': db_ilog.resolution = new_value elif table_field == 'alias': db_ilog.alias = new_value elif table_field == 'reporter_accessible': db_ilog.reporter_accessible = new_value elif table_field == 'cclist_accessible': db_ilog.cclist_accessible = new_value elif table_field == 'product': db_ilog.product = new_value elif table_field == 'component': db_ilog.component = new_value elif table_field == 'version': db_ilog.version = new_value elif table_field == 'rep_platform': db_ilog.rep_platform = new_value elif table_field == 'op_sys': db_ilog.op_sys = new_value elif table_field == 'bug_file_loc': db_ilog.bug_file_loc = new_value elif table_field == 'status_whiteboard': db_ilog.status_whiteboard = new_value elif table_field == 'target_milestone': db_ilog.target_milestone = new_value elif table_field == 'votes': db_ilog.votes = new_value elif table_field == 'everconfirmed': db_ilog.everconfirmed = new_value elif table_field == 'qa_contact': db_ilog.qa_contact = new_value elif table_field == 'keywords': db_ilog.Keywords = new_value elif table_field == 'cc': db_ilog.cc = new_value try: self.store.add(db_ilog) except: traceback.print_exc() elif self.backend_is_jira(): # Jira section # # if (field in jira_issues_links): table_field = jira_issues_links[field] db_ilog.submitted_by = changed_by db_ilog.date = date if table_field == 'summary': db_ilog.summary = new_value elif table_field == 'priority': db_ilog.priority = new_value elif table_field == 'type': db_ilog.type = new_value elif table_field == 'assigned_to': db_ilog.assigned_to = self.get_people_id( new_value, self.get_tracker_id(db_ilog.issue_id)) elif table_field == 'status': db_ilog.status = new_value elif table_field == 'resolution': db_ilog.resolution = new_value elif table_field == 'description': db_ilog.description = new_value elif table_field == 'link': db_ilog.link = new_value elif table_field == 'component': db_ilog.component = new_value elif table_field == 'version': db_ilog.version = new_value elif table_field == 'security': db_ilog.security = new_value try: self.store.add(db_ilog) except: traceback.print_exc() # if there are changes, it stores the last bugs after the last # change. If there are no changes, insert all the created bugs self.insert_new_bugs_created(date_from, None) self.store.commit()
class TableReplacer(object): """ This is the base class used by every Updater """ def __init__(self, old_db_file, new_db_file, start_ver): from globaleaks.db.update_8_9 import Context_v_8, Receiver_v_8, Notification_v_8 from globaleaks.db.update_9_10 import Node_v_9, Receiver_v_9, User_v_9 from globaleaks.db.update_10_11 import InternalTip_v_10, InternalFile_v_10 from globaleaks.db.update_11_12 import Node_v_11, Context_v_11 from globaleaks.db.update_12_13 import Node_v_12, Context_v_12 from globaleaks.db.update_13_14 import Node_v_13, Context_v_13 from globaleaks.db.update_14_15 import Node_v_14, User_v_14, Context_v_14, Receiver_v_14, \ InternalTip_v_14, Notification_v_14, Stats_v_14, Comment_v_14 from globaleaks.db.update_15_16 import Receiver_v_15, Notification_v_15 from globaleaks.db.update_16_17 import Node_v_16, Receiver_v_16, Notification_v_16, Stats_v_16 from globaleaks.db.update_17_18 import Node_v_17 from globaleaks.db.update_18_19 import Node_v_18 self.old_db_file = old_db_file self.new_db_file = new_db_file self.start_ver = start_ver self.std_fancy = " ł " self.debug_info = " [%d => %d] " % (start_ver, start_ver + 1) self.table_history = { 'Node': [ Node_v_9, None, Node_v_11, None, Node_v_12, Node_v_13, Node_v_14, Node_v_16, None, Node_v_17, Node_v_18, models.Node ], 'User': [ User_v_9, None, User_v_14, None, None, None, None, models.User, None, None, None, None ], 'Context': [ Context_v_8, Context_v_11, None, None, Context_v_12, Context_v_13, Context_v_14, models.Context, None, None, None, None ], 'Receiver': [ Receiver_v_8, Receiver_v_9, Receiver_v_14, None, None, None, None, Receiver_v_15, Receiver_v_16, models.Receiver, None, None ], 'ReceiverFile': [ models.ReceiverFile, None, None, None, None, None, None, None, None, None, None, None ], 'Notification': [ Notification_v_8, Notification_v_14, None, None, None, None, None, Notification_v_15, Notification_v_16, models.Notification, None, None ], 'Comment': [ Comment_v_14, None, None, None, None, None, None, models.Comment, None, None, None, None ], 'InternalTip': [ InternalTip_v_10, None, None, InternalTip_v_14, None, None, None, models.InternalTip, None, None, None, None ], 'InternalFile': [ InternalFile_v_10, None, None, models.InternalFile, None, None, None, None, None, None, None, None ], 'WhistleblowerTip': [ models.WhistleblowerTip, None, None, None, None, None, None, None, None, None, None, None ], 'ReceiverTip': [ models.ReceiverTip, None, None, None, None, None, None, None, None, None, None, None ], 'ReceiverInternalTip': [ models.ReceiverInternalTip, None, None, None, None, None, None, None, None, None, None, None ], 'ReceiverContext': [ models.ReceiverContext, None, None, None, None, None, None, None, None, None, None, None ], 'Message': [ models.Message, None, None, None, None, None, None, None, None, None, None, None ], 'Stats': [ Stats_v_14, None, None, None, None, None, None, Stats_v_16, None, models.Stats, None, None ], 'ApplicationData': [ models.ApplicationData, None, None, None, None, None, None, None, None, None, None, None ], 'Field': [ models.Field, None, None, None, None, None, None, None, None, None, None, None ], 'FieldOption': [ models.FieldOption, None, None, None, None, None, None, None, None, None, None, None ], 'FieldField': [ models.FieldField, None, None, None, None, None, None, None, None, None, None, None ], 'Step': [ models.Step, None, None, None, None, None, None, None, None, None, None, None ], 'StepField': [ models.StepField, None, None, None, None, None, None, None, None, None, None, None ], 'Anomalies': [ models.Anomalies, None, None, None, None, None, None, None, None, None, None, None ], 'EventLogs': [ models.EventLogs, None, None, None, None, None, None, None, None, None, None, None ], } for k, v in self.table_history.iteritems(): # +1 because count start from 0, # -8 because the relase befor the 8th are not supported anymore length = DATABASE_VERSION + 1 - 8 if len(v) != length: msg = 'Expecting a table with {} statuses ({})'.format( length, k) raise TypeError(msg) log.msg('{} Opening old DB: {}'.format(self.debug_info, old_db_file)) old_database = create_database('sqlite:' + self.old_db_file) self.store_old = Store(old_database) GLSetting.db_file = new_db_file new_database = create_database('sqlite:' + new_db_file) self.store_new = Store(new_database) if self.start_ver + 1 == DATABASE_VERSION: log.msg('{} Acquire SQL schema {}'.format( self.debug_info, GLSetting.db_schema_file)) if not os.access(GLSetting.db_schema_file, os.R_OK): log.msg('Unable to access', GLSetting.db_schema_file) raise IOError('Unable to access db schema file') with open(GLSetting.db_schema_file) as f: create_queries = ''.join(f).split(';') for create_query in create_queries: try: self.store_new.execute(create_query + ';') except OperationalError: log.msg( 'OperationalError in "{}"'.format(create_query)) self.store_new.commit() return # return here and manage the migrant versions here: for k, v in self.table_history.iteritems(): create_query = self.get_right_sql_version(k, self.start_ver + 1) if not create_query: # table not present in the version continue try: self.store_new.execute(create_query + ';') except OperationalError as excep: log.msg('{} OperationalError in [{}]'.format( self.debug_info, create_query)) raise excep self.store_new.commit() def close(self): self.store_old.close() self.store_new.close() def initialize(self): pass def epilogue(self): pass def get_right_model(self, table_name, version): table_index = (version - 8) if table_name not in self.table_history: msg = 'Not implemented usage of get_right_model {} ({} {})'.format( __file__, table_name, self.start_ver) raise NotImplementedError(msg) if version > DATABASE_VERSION: raise ValueError( 'Version supplied must be less or equal to {}'.format( DATABASE_VERSION)) if self.table_history[table_name][table_index]: return self.table_history[table_name][table_index] # else, it's none, and we've to take the previous valid version while version >= 0: if self.table_history[table_name][table_index]: return self.table_history[table_name][table_index] table_index -= 1 # This never want happen return None def get_right_sql_version(self, model_name, version): """ @param model_name: @param version: @return: The SQL right for the stuff we've """ modelobj = self.get_right_model(model_name, version) if not modelobj: return None right_query = generateCreateQuery(modelobj) return right_query def _perform_copy_list(self, table_name): models_count = self.store_old.find( self.get_right_model(table_name, self.start_ver)).count() log.msg('{} default {} migration assistant: #{}'.format( self.debug_info, table_name, models_count)) old_objects = self.store_old.find( self.get_right_model(table_name, self.start_ver)) for old_obj in old_objects: new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for _, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def _perform_copy_single(self, table_name): log.msg('{} default {} migration assistant'.format( self.debug_info, table_name)) old_obj = self.store_old.find( self.get_right_model(table_name, self.start_ver)).one() new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for _, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def migrate_Context(self): self._perform_copy_list("Context") def migrate_Node(self): self._perform_copy_single("Node") def migrate_User(self): self._perform_copy_list("User") def migrate_ReceiverTip(self): self._perform_copy_list("ReceiverTip") def migrate_WhistleblowerTip(self): self._perform_copy_list("WhistleblowerTip") def migrate_Comment(self): self._perform_copy_list("Comment") def migrate_InternalTip(self): self._perform_copy_list("InternalTip") def migrate_Receiver(self): self._perform_copy_list("Receiver") def migrate_InternalFile(self): self._perform_copy_list("InternalFile") def migrate_ReceiverFile(self): self._perform_copy_list("ReceiverFile") def migrate_Notification(self): self._perform_copy_single("Notification") def migrate_ReceiverContext(self): self._perform_copy_list("ReceiverContext") def migrate_ReceiverInternalTip(self): self._perform_copy_list("ReceiverInternalTip") def migrate_Message(self): """ has been created between 7 and 8! """ if self.start_ver < 8: return self._perform_copy_list("Message") def migrate_Stats(self): """ has been created between 14 and 15 and is not migrated since 17 """ if self.start_ver < 17: return self._perform_copy_list("Stats") def migrate_ApplicationData(self): """ There is no need to migrate it the application data. Default application data is loaded by the application and stored onto the db at each new start. """ return def migrate_Field(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Field") def migrate_FieldOption(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("FieldOption") def migrate_FieldField(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("FieldField") def migrate_Step(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Step") def migrate_StepField(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("StepField") def migrate_Anomalies(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Anomalies") def migrate_EventLogs(self): """ has been created between 15 and 16! """ if self.start_ver < 16: return self._perform_copy_list("EventLogs")
class StormManager(Singleton): log = logging.getLogger('{}.StormManager'.format(__name__)) def __init__(self): pass @loggingInfo def init(self, *args): self.dbOK = False self.openDB() @loggingInfo def reset(self): self.closeDB() self.openDB() @loggingInfo def openDB(self): try: self._config = ConfigManager() self.db = self._config.config[self._config.database]["database"] create_db = False if self.db == self._config.Sqlite: folder = self._config.config[self._config.database]["folder"] loc = folder + '/icepapcms.db' print("Using Sqlite database at %s" % loc) create_db = not os.path.exists(loc) if create_db: print("No database file found, creating it") if not os.path.exists(folder): os.mkdir(folder) self._database = create_database("%s:%s" % (self.db, loc)) else: server = self._config.config[self._config.database]["server"] user = self._config.config[self._config.database]["user"] pwd = self._config.config[self._config.database]["password"] scheme = "{}://{}:{}@{}/icepapcms".format( self.db, user, pwd, server) if self.db == 'mysql': self._database = MySQL(scheme) else: self._database = create_database(scheme) self._store = Store(self._database) if create_db: self.dbOK = self.createSqliteDB() else: self.dbOK = True except Exception as e: self.log.error("Unexpected error on openDB: %s", e) self.dbOK = False @loggingInfo def createSqliteDB(self): try: sql_file = resource_filename('icepapcms.db', 'creates_sqlite.sql') with open(sql_file) as f: sql_script = f.read() statements = re.compile(r";[ \t]*$", re.M) for statement in statements.split(sql_script): # Remove any comments from the file statement = re.sub(r"--.*[\n\\Z]", "", statement) if statement.strip(): create = statement + ";" self._store.execute(create) self._store.commit() return True except Exception as e: self.log.error("Unexpected error on createSqliteDB: %s", e) return False @loggingInfo def closeDB(self): try: if self.dbOK: self._store.close() return True except Exception as e: self.log.error("Unexpected error on closeDB:", e) self.dbOK = False return False @loggingInfo def store(self, obj): self._store.add(obj) @loggingInfo def remove(self, obj): self._store.remove(obj) @loggingInfo def addIcepapSystem(self, icepap_system): try: self._store.add(icepap_system) self.commitTransaction() return True except Exception as e: self.log.error( "some exception trying to store the icepap system " "%s: %s", icepap_system, e) return False @loggingInfo def deleteLocation(self, location): if self.db == self._config.Sqlite: for system in location.systems: self.deleteIcepapSystem(system) self._store.remove(location) self.commitTransaction() @loggingInfo def deleteIcepapSystem(self, icepap_system): if self.db == self._config.Sqlite: for driver in icepap_system.drivers: self.deleteDriver(driver) self._store.remove(icepap_system) self.commitTransaction() @loggingInfo def deleteDriver(self, driver): for cfg in driver.historic_cfgs: for par in cfg.parameters: self._store.remove(par) self._store.remove(cfg) self._store.remove(driver) self.commitTransaction() @loggingInfo def getAllLocations(self): try: locations = self._store.find(Location) location_dict = {} for location in locations: location_dict[location.name] = location return location_dict except Exception as e: self.log.error("Unexpected error on getAllLocations: %s", e) return {} @loggingInfo def getLocation(self, name): return self._store.get(Location, name) @loggingInfo def getIcepapSystem(self, icepap_name): return self._store.get(IcepapSystem, icepap_name) @loggingInfo def existsDriver(self, mydriver, id): drivers = self._store.find( IcepapDriver, IcepapDriver.addr == IcepapDriverCfg.driver_addr, IcepapDriverCfg.id == CfgParameter.cfg_id, CfgParameter.name == str("ID"), CfgParameter.value == id) if drivers: for driver in drivers: if driver.addr != mydriver.addr: return driver return None else: return None @loggingInfo def getLocationIcepapSystem(self, location): try: icepaps = self._store.find(IcepapSystem, IcepapSystem.location_name == location) icepaps.order_by(IcepapSystem.name) ipapdict = {} for ipap_sys in icepaps: ipapdict[ipap_sys.name] = ipap_sys return ipapdict except Exception as e: self.log.error( "Unexpected error on getLocationIcepapSystem: " "%s", e) return {} @loggingInfo def rollback(self): self._store.rollback() @loggingInfo def commitTransaction(self): try: self._store.commit() return True except Exception: return False
class StormORM(ORM): """ Storm implementation of ORM super class. """ def __init__(self, uri=None, store=None): ''' @param uri: Database URI following storm rules. @param store: Storm store. If uri is given a new store is instanciated and it is used to execute the statements. If both parameters are given the early created store overrides the store given. ''' from storm.locals import create_database, Store self.uri = uri self.store = store if self.uri: database = create_database(self.uri) self.store = Store(database) if not self.store: raise Exception('None storm store') self.attrParser = StormAttributeParser() def _getObject(self, csvType, csvStatement): """ Retrieves the object to be used at statement execution. @param csvType: The CSVType @param csvStatement: The CSVStatement @return: The object early instanciated (for insert statement) or retrieved from database (for update or delete statements). """ typo = csvType.type keys = csvType.keys attributes = csvStatement.attributes if csvStatement.action in [DELETE, UPDATE]: if csvType.hasPrimaryKey: return self.store.get(typo, attributes[ csvType.primaryKey[0] ]) else: pred = And([Eq(typo, key, attributes[i]) for i,key in keys.iteritems()]) result = self.store.find(typo, pred) if result.count() == 0: return None elif result.count() == 1: return result.one() else: return [r for r in result] elif csvStatement.action is INSERT: return typo() def executeStatement(self, csvType, csvStatement): """ Executes csv statements matched by the pair csvType, csvStatement. @param csvType: The CSVType @param csvStatement: The CSVStatement @return: Total statements executed or raises a ValueError if the object retrieved with the pair csvType, csvStatement is None. """ obj = self._getObject(csvType, csvStatement) if not obj: msg = 'Statement return None in line %d: %s' % (csvStatement.lineNumber, csvStatement.lineContent) raise ValueError(msg) objs = [] if type(obj) is list: objs += obj else: objs.append(obj) i = 0 for _obj in objs: self._executeStatement(_obj, csvType, csvStatement) i += 1 return i def _executeStatement(self, obj, csvType, csvStatement): """ Executes a single csv statement @param csvType: The CSVType @param csvStatement: The CSVStatement """ keys = csvType.keys attributes = csvType.attributes values = csvStatement.attributes if csvStatement.action is INSERT: pairs = [(key, values[i]) for i,key in keys.iteritems()] pairs += [(key, values[i]) for i,key in attributes.iteritems()] for key, value in pairs: setattr(obj, key, value) self.store.add(obj) elif csvStatement.action is UPDATE: pairs = [(key, values[i]) for i,key in attributes.iteritems()] for key, value in pairs: setattr(obj, key, value) elif csvStatement.action is DELETE: self.store.remove(obj) self.store.commit()
class TestMigrationRegression(unittest.TestCase): def _initStartDB(self, target_ver): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % target_ver db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) self.store = Store(create_database(GLSettings.db_uri)) def test_check_field_constraints(self): # This test case asserts that a migration from db ver 32 up to 34 with # fields that fail the constraints still functions. self._initStartDB(32) field_dict = helpers.get_dummy_field() field_dict['instance'] = 'reference' field_dict['step_id'] = None field_dict['field_id'] = None db_create_field(self.store, field_dict, u'en') field_dict = helpers.get_dummy_field() field_dict['instance'] = 'instance' db_create_field(self.store, field_dict, u'en') field_dict = helpers.get_dummy_field() field_dict['instance'] = 'template' field_dict['step_id'] = None fld_grp_id = self.store.find(Field, Field.fieldgroup_id is not None)[0].fieldgroup_id field_dict['field_id'] = fld_grp_id db_create_field(self.store, field_dict, u'en') self.store.commit() ret = perform_system_update() shutil.rmtree(GLSettings.db_path) self.assertNotEqual(ret, -1) def test_check_unmodifiable_strings(self): # This test case asserts that data migration updates unmodifiable l10n strings self._initStartDB(34) notification_l10n = NotificationL10NFactory(self.store) t0 = notification_l10n.get_val('export_template', 'ar') #print notification_l10n.get_val('export_template', 'ar') notification_l10n.set_val('export_template', 'ar', '') t1 = notification_l10n.get_val('export_template', 'ar') self.assertEqual(t1, '') self.store.commit() # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val('version', self.dummy_ver) self.assertEqual(prv.get_val('version'), self.dummy_ver) store.commit() store.close() migration.perform_data_update(self.db_file) # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) notification_l10n = NotificationL10NFactory(store) t2 = notification_l10n.get_val('export_template', 'ar') self.assertEqual(t2, t0) store.commit() store.close() shutil.rmtree(GLSettings.db_path)
class IssuesLog: def __init__(self, backend_name): self.backend_name = backend_name self.connect() self.create_db() def connect(self): opts = Config() self.database = create_database( "mysql://" + opts.db_user_out + ":" + opts.db_password_out + "@" + opts.db_hostname_out + ":" + opts.db_port_out + "/" + opts.db_database_out ) self.store = Store(self.database) def create_db(self): self.store.execute(__sql_table__) def copy_issue(self, db_ilog): """ This method is create a copy of a DBIssueLog object """ aux = DBIssuesLog(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to return aux def build_initial_state(self, db_ilog): """ This method gets the first changes of every field in order to get the initial state of the bug """ fields = self.store.execute("SELECT DISTINCT(field) FROM changes where issue_id=%s" % (db_ilog.issue_id)) for f in fields: value = self.store.execute( 'SELECT old_value FROM changes WHERE issue_id=%s AND field="%s" ORDER BY changed_on LIMIT 1' % (db_ilog.issue_id, f[0]) ) for v in value: # Bugzilla section # if f[0] in bg_issues_links: table_field = bg_issues_links[f[0]] if table_field == "summary": db_ilog.summary = v[0] elif table_field == "priority": db_ilog.priority = v[0] elif table_field == "assigned_to": db_ilog.assigned_to = v[0] elif table_field == "status": db_ilog.status = v[0] elif table_field == "resolution": db_ilog.resolution = v[0] return db_ilog def run(self): issues = self.store.find(DBIssue) for i in issues: db_ilog = DBIssuesLog(i.issue, i.tracker_id) db_ilog.issue_id = i.id db_ilog.type = i.type db_ilog.summary = i.summary db_ilog.description = i.description db_ilog.status = i.status db_ilog.resolution = i.resolution db_ilog.priority = i.priority db_ilog.submitted_by = i.submitted_by db_ilog.date = i.submitted_on db_ilog.assigned_to = i.assigned_to db_ilog = self.build_initial_state(db_ilog) self.store.add(db_ilog) # the code below gets all the changes and insert a row per change changes = self.store.execute( "SELECT field, new_value, changed_by, changed_on FROM changes where issue_id=%s" % (db_ilog.issue_id) ) for ch in changes: field = ch[0] new_value = ch[1] changed_by = ch[2] date = ch[3] db_ilog = self.copy_issue(db_ilog) # Bugzilla section # if field in bg_issues_links: table_field = bg_issues_links[field] if table_field == "summary": db_ilog.summary = new_value elif table_field == "priority": db_ilog.priority = new_value elif table_field == "assignted_to": db_ilog.assigned_to = new_value elif table_field == "status": db_ilog.status = new_value elif table_field == "resolution": db_ilog.resolution = new_value db_ilog.submitted_by = changed_by db_ilog.date = date try: self.store.add(db_ilog) except: traceback.print_exc() self.store.commit()
class ZStormResourceManagerTest(TestHelper): def is_supported(self): return has_transaction and has_zope_component and has_testresources def setUp(self): super(ZStormResourceManagerTest, self).setUp() package_dir = self.makeDir() sys.path.append(package_dir) self.patch_dir = os.path.join(package_dir, "patch_package") os.mkdir(self.patch_dir) self.makeFile(path=os.path.join(self.patch_dir, "__init__.py"), content="") self.makeFile(path=os.path.join(self.patch_dir, "patch_1.py"), content=PATCH) import patch_package create = ["CREATE TABLE test (foo TEXT UNIQUE, bar INT)"] drop = ["DROP TABLE test"] delete = ["DELETE FROM test"] uri = "sqlite:///%s" % self.makeFile() schema = ZSchema(create, drop, delete, patch_package) self.databases = [{"name": "test", "uri": uri, "schema": schema}] self.resource = ZStormResourceManager(self.databases) self.store = Store(create_database(uri)) def tearDown(self): global_zstorm._reset() del sys.modules["patch_package"] if "patch_1" in sys.modules: del sys.modules["patch_1"] super(ZStormResourceManagerTest, self).tearDown() def test_make(self): """ L{ZStormResourceManager.make} returns a L{ZStorm} resource that can be used to get the registered L{Store}s. """ zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo, bar FROM test"))) def test_make_lazy(self): """ L{ZStormResourceManager.make} does not create all stores upfront, but only when they're actually used, likewise L{ZStorm.get}. """ zstorm = self.resource.make([]) self.assertEqual([], list(zstorm.iterstores())) store = zstorm.get("test") self.assertEqual([("test", store)], list(zstorm.iterstores())) def test_make_upgrade(self): """ L{ZStormResourceManager.make} upgrades the schema if needed. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT bar FROM test"))) def test_make_upgrade_unknown_patch(self): """ L{ZStormResourceManager.make} resets the schema if an unknown patch is found """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("INSERT INTO patch VALUES (2)") self.store.execute("CREATE TABLE test (foo TEXT, egg BOOL)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo, bar FROM test"))) self.assertEqual([(1,)], list(store.execute("SELECT version FROM patch"))) def test_make_delete(self): """ L{ZStormResourceManager.make} deletes the data from all tables to make sure that tests run against a clean database. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.execute("INSERT INTO test (foo) VALUES ('data')") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo FROM test"))) def test_make_commits_transaction_once(self): """ L{ZStormResourceManager.make} commits schema changes only once across all stores, after all patch and delete statements have been executed. """ database2 = {"name": "test2", "uri": "sqlite:///%s" % self.makeFile(), "schema": self.databases[0]["schema"]} self.databases.append(database2) other_store = Store(create_database(database2["uri"])) for store in [self.store, other_store]: store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") store.execute("CREATE TABLE test (foo TEXT)") store.execute("INSERT INTO test (foo) VALUES ('data')") store.commit() with CaptureTracer() as tracer: zstorm = self.resource.make([]) self.assertEqual(["COMMIT", "COMMIT"], tracer.queries[-2:]) store1 = zstorm.get("test") store2 = zstorm.get("test2") self.assertEqual([], list(store1.execute("SELECT foo FROM test"))) self.assertEqual([], list(store2.execute("SELECT foo FROM test"))) def test_make_zstorm_overwritten(self): """ L{ZStormResourceManager.make} registers its own ZStorm again if a test has registered a new ZStorm utility overwriting the resource one. """ zstorm = self.resource.make([]) provideUtility(ZStorm()) self.resource.make([]) self.assertIs(zstorm, getUtility(IZStorm)) def test_clean_flush(self): """ L{ZStormResourceManager.clean} tries to flush the stores to make sure that they are all in a consistent state. """ class Test(object): __storm_table__ = "test" foo = Unicode() bar = Int(primary=True) def __init__(self, foo, bar): self.foo = foo self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(u"data", 1)) store.add(Test(u"data", 2)) self.assertRaises(IntegrityError, self.resource.clean, zstorm) def test_clean_delete(self): """ L{ZStormResourceManager.clean} cleans the database tables from the data created by the tests. """ zstorm = self.resource.make([]) store = zstorm.get("test") store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") store.commit() self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test"))) def test_clean_with_force_delete(self): """ If L{ZStormResourceManager.force_delete} is C{True}, L{Schema.delete} is always invoked upon test cleanup. """ zstorm = self.resource.make([]) zstorm.get("test") # Force the creation of the store self.store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") self.store.commit() self.resource.force_delete = True self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test"))) def test_wb_clean_clears_alive_cache_before_abort(self): """ L{ZStormResourceManager.clean} clears the alive cache before aborting the transaction. """ class Test(object): __storm_table__ = "test" bar = Int(primary=True) def __init__(self, bar): self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(1)) store.add(Test(2)) real_invalidate = store.invalidate def invalidate_proxy(): self.assertEqual(0, len(store._alive.values())) real_invalidate() store.invalidate = invalidate_proxy self.resource.clean(zstorm) def test_schema_uri(self): """ It's possible to specify an alternate URI for applying the schema and cleaning up tables after a test. """ schema_uri = "sqlite:///%s" % self.makeFile() self.databases[0]["schema-uri"] = schema_uri zstorm = self.resource.make([]) store = zstorm.get("test") schema_store = Store(create_database(schema_uri)) # The schema was applied using the alternate schema URI statement = "SELECT name FROM sqlite_master WHERE name='patch'" self.assertEqual([], list(store.execute(statement))) self.assertEqual([("patch",)], list(schema_store.execute(statement))) # The cleanup is performed with the alternate schema URI store.commit() schema_store.execute("INSERT INTO test (foo) VALUES ('data')") schema_store.commit() self.resource.clean(zstorm) self.assertEqual([], list(schema_store.execute("SELECT * FROM test"))) def test_schema_uri_with_schema_stamp_dir(self): """ If a schema stamp directory is set, and the stamp indicates there's no need to update the schema, the resource clean up code will still connect as schema user if it needs to run the schema delete statements because of a commit. """ self.resource.schema_stamp_dir = self.makeFile() self.databases[0]["schema-uri"] = self.databases[0]["uri"] self.resource.make([]) # Simulate a second test run that initializes the zstorm resource # from scratch, using the same schema stamp directory resource2 = ZStormResourceManager(self.databases) resource2.schema_stamp_dir = self.resource.schema_stamp_dir zstorm = resource2.make([]) store = zstorm.get("test") store.execute("INSERT INTO test (foo) VALUES ('data')") store.commit() # Committing will force a schema.delete() run resource2.clean(zstorm) self.assertEqual([], list(store.execute("SELECT * FROM test"))) def test_no_schema(self): """ A particular database may have no schema associated. """ self.databases[0]["schema"] = None zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT * FROM sqlite_master"))) def test_no_schema_clean(self): """ A particular database may have no schema associated. If it's committed during tests, it will just be skipped when cleaning up tables. """ self.databases[0]["schema"] = None zstorm = self.resource.make([]) store = zstorm.get("test") store.commit() with CaptureTracer() as tracer: self.resource.clean(zstorm) self.assertEqual([], tracer.queries) def test_deprecated_database_format(self): """ The old deprecated format of the 'database' constructor parameter is still supported. """ import patch_package uri = "sqlite:///%s" % self.makeFile() schema = ZSchema([], [], [], patch_package) resource = ZStormResourceManager({"test": (uri, schema)}) zstorm = resource.make([]) store = zstorm.get("test") self.assertIsNot(None, store) def test_use_global_zstorm(self): """ If the C{use_global_zstorm} attribute is C{True} then the global L{ZStorm} will be used. """ self.resource.use_global_zstorm = True zstorm = self.resource.make([]) self.assertIs(global_zstorm, zstorm) def test_provide_utility_before_patches(self): """ The L{IZStorm} utility is provided before patches are applied, in order to let them get it if they need. """ content = ("from zope.component import getUtility\n" "from storm.zope.interfaces import IZStorm\n" "def apply(store):\n" " getUtility(IZStorm)\n") self.makeFile(path=os.path.join(self.patch_dir, "patch_2.py"), content=content) self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([(1,), (2,)], sorted(store.execute("SELECT version FROM patch"))) def test_create_schema_stamp_dir(self): """ If a schema stamp directory is set, it's created automatically if it doesn't exist yet. """ self.resource.schema_stamp_dir = self.makeFile() self.resource.make([]) self.assertTrue(os.path.exists(self.resource.schema_stamp_dir)) def test_use_schema_stamp(self): """ If a schema stamp directory is set, then it's used to decide whether to upgrade the schema or not. In case the patch directory hasn't been changed since the last known upgrade, no schema upgrade is run. """ self.resource.schema_stamp_dir = self.makeFile() self.resource.make([]) # Simulate a second test run that initializes the zstorm resource # from scratch, using the same schema stamp directory resource2 = ZStormResourceManager(self.databases) resource2.schema_stamp_dir = self.resource.schema_stamp_dir with CaptureTracer() as tracer: resource2.make([]) self.assertEqual([], tracer.queries) def test_use_schema_stamp_out_of_date(self): """ If a schema stamp directory is set, then it's used to decide whether to upgrade the schema or not. In case the patch directory has changed a schema upgrade is run. """ self.resource.schema_stamp_dir = self.makeFile() self.resource.make([]) # Simulate a second test run that initializes the zstorm resource # from scratch, using the same schema stamp directory resource2 = ZStormResourceManager(self.databases) resource2.schema_stamp_dir = self.resource.schema_stamp_dir self.makeFile(path=os.path.join(self.patch_dir, "patch_2.py"), content="def apply(store): pass") class FakeStat(object): st_mtime = os.stat(self.patch_dir).st_mtime + 1 stat_mock = self.mocker.replace(os.stat) stat_mock(self.patch_dir) self.mocker.result(FakeStat()) self.mocker.replay() resource2.make([]) result = self.store.execute("SELECT version FROM patch") self.assertEqual([(1,), (2,)], sorted(result.get_all()))
class StormStorageBackend(StorageBackend): """Storage back-end based on the Storm ORM framework.""" def __init__(self): self.store = None def set_config(self, **kwargs): """Set the configuration of this back-end.""" uri = kwargs['uri'] database = create_database(uri) self.store = Store(database) self.logger = logging.getLogger('StormStorageBackend') handler = logging.StreamHandler() formatter = logging.Formatter(kwargs['log_format']) handler.setFormatter(formatter) self.logger.addHandler(handler) self.logger.setLevel( logging.__getattribute__(kwargs['log_level'])) def create_node(self, node, jid, node_config): """Create a PubSub node with the given configuration. Creates the Node, NodeConfig, Affiliation and Subscription model for the given node. """ self.logger.debug('Creating node %s for jid %s with config %s' % (node, jid, node_config)) new_node = Node(node) self.store.add(new_node) config = copy.deepcopy(DEFAULT_CONFIG) config.update(node_config) for key, value in config.items(): new_node_config = NodeConfig(node, key, value) new_node_config.updated = datetime.utcnow() self.store.add(new_node_config) affiliation = Affiliation(node, jid, u'owner', datetime.utcnow()) self.store.add(affiliation) subscription = Subscription(node, jid, jid, u'subscribed', datetime.utcnow()) self.store.add(subscription) def create_channel(self, jid): """Create a channel for the given JID. Creates all the required PubSub nodes that constitute a channel, with the appropriate permissions. """ self.logger.debug('Creating channel for %s' % jid) creation_date = unicode(datetime.utcnow().isoformat()) self.create_node(u'/user/%s/posts' % jid, jid, {u'channelType': u'personal', u'creationDate': creation_date, u'defaultAffiliation': u'publisher', u'description': u'buddycloud channel for %s' % jid, u'title': jid}) self.create_node(u'/user/%s/geo/current' % jid, jid, {u'creationDate': creation_date, u'description': u'Where %s is at now' % jid, u'title': u'%s Current Location' % jid}) self.create_node(u'/user/%s/geo/next' % jid, jid, {u'creationDate': creation_date, u'description': u'Where %s intends to go' % jid, u'title': u'%s Next Location' % jid}) self.create_node(u'/user/%s/geo/previous' % jid, jid, {u'creationDate': creation_date, u'description': u'Where %s has been before' % jid, u'title': u'%s Previous Location' % jid}) self.create_node(u'/user/%s/status' % jid, jid, {u'creationDate': creation_date, u'description': u'M000D', u'title': u'%s status updates' % jid}) self.create_node(u'/user/%s/subscriptions' % jid, jid, {u'creationDate': creation_date, u'description': u'Browse my interests', u'title': u'%s subscriptions' % jid}) self.store.commit() def get_node(self, node): """Get the requested PubSub node.""" self.logger.debug('Getting node %s' % node) the_node = self.store.get(Node, node) self.logger.debug('Returning node %s' % the_node) return the_node def get_nodes(self): """Get a list of all the available PubSub nodes.""" self.logger.debug('Getting list of available nodes.') node_list = self.store.find(Node) self.logger.debug('Returning list of available node %s' % node_list) return node_list def add_item(self, node, item_id, item): """Add an item to the requested PubSub node.""" new_item = Item(node, unicode(item_id), datetime.utcnow(), item) self.store.add(new_item) self.store.commit() def shutdown(self): """Shut down this storage module - flush, commit and close the store.""" self.store.flush() self.store.commit() self.store.close()
class TableReplacer: """ This is the base class used by every Updater """ def __init__(self, old_db_file, new_db_file, start_ver): from globaleaks.db.update_5_6 import User_version_5, Comment_version_5, Node_version_5 from globaleaks.db.update_6_7 import Node_version_6, Context_version_6 from globaleaks.db.update_7_8 import Node_version_7, Notification_version_7, Context_version_7, \ Receiver_version_7, InternalFile_version_7 from globaleaks.db.update_8_9 import Context_version_8, Receiver_version_8, Notification_version_8 from globaleaks.db.update_9_10 import Node_version_9, ApplicationData_version_10, \ Receiver_version_9, User_version_9 from globaleaks.db.update_10_11 import InternalTip_version_10, InternalFile_version_10 from globaleaks.db.update_11_12 import Node_version_11, ApplicationData_version_11, Context_version_11 self.old_db_file = old_db_file self.new_db_file = new_db_file self.start_ver = start_ver self.std_fancy = " ł " self.debug_info = " [%d => %d] " % (start_ver, start_ver + 1) self.table_history = { 'Node': [ Node_version_5, Node_version_6, Node_version_7, Node_version_9, None, Node_version_11, None, models.Node ], 'User': [ User_version_5, User_version_9, None, None, None, models.User, None, None ], 'Context': [ Context_version_6, None, Context_version_7, Context_version_8, Context_version_11, None, None, models.Context ], 'Receiver': [ Receiver_version_7, None, None, Receiver_version_8, Receiver_version_9, models.Receiver, None, None ], 'ReceiverFile': [models.ReceiverFile, None, None, None, None, None, None, None], 'Notification': [ Notification_version_7, None, None, Notification_version_8, models.Notification, None, None, None ], 'Comment': [ Comment_version_5, models.Comment, None, None, None, None, None, None ], 'InternalTip': [ InternalTip_version_10, None, None, None, None, None, models.InternalTip, None ], 'InternalFile': [ InternalFile_version_7, None, None, InternalFile_version_10, None, None, models.InternalFile, None ], 'WhistleblowerTip': [ models.WhistleblowerTip, None, None, None, None, None, None, None ], 'ReceiverTip': [models.ReceiverTip, None, None, None, None, None, None, None], 'ReceiverInternalTip': [ models.ReceiverInternalTip, None, None, None, None, None, None, None ], 'ReceiverContext': [models.ReceiverContext, None, None, None, None, None, None, None], 'Message': [models.Message, None, None, None, None, None, None, None], 'Stats': [models.Stats, None, None, None, None, None, None, None], 'ApplicationData': [ ApplicationData_version_10, None, None, None, None, None, None, models.ApplicationData ], } for k, v in self.table_history.iteritems(): # +1 because count start from 0, # -5 because the relase 0,1,2,3,4 are not supported anymore assert len(v) == (DATABASE_VERSION + 1 - 5), \ "I'm expecting a table with %d statuses (%s)" % (DATABASE_VERSION, k) print "%s Opening old DB: %s" % (self.debug_info, old_db_file) old_database = create_database("sqlite:%s" % self.old_db_file) self.store_old = Store(old_database) GLSetting.db_file = new_db_file new_database = create_database("sqlite:%s" % new_db_file) self.store_new = Store(new_database) if self.start_ver + 1 == DATABASE_VERSION: print "%s Acquire SQL schema %s" % (self.debug_info, GLSetting.db_schema_file) if not os.access(GLSetting.db_schema_file, os.R_OK): print "Unable to access %s" % GLSetting.db_schema_file raise Exception("Unable to access db schema file") with open(GLSetting.db_schema_file) as f: create_queries = ''.join(f.readlines()).split(';') for create_query in create_queries: try: self.store_new.execute(create_query + ';') except OperationalError: print "OperationalError in [%s]" % create_query self.store_new.commit() return # return here and manage the migrant versions here: for k, v in self.table_history.iteritems(): create_query = self.get_right_sql_version(k, self.start_ver + 1) if not create_query: # table not present in the version continue try: self.store_new.execute(create_query + ';') except OperationalError as excep: print "%s OperationalError in [%s]" % (self.debug_info, create_query) raise excep self.store_new.commit() def close(self): self.store_old.close() self.store_new.close() def initialize(self): pass def epilogue(self): pass def get_right_model(self, table_name, version): table_index = (version - 5) if not self.table_history.has_key(table_name): print "Not implemented usage of get_right_model %s (%s %d)" % ( __file__, table_name, self.start_ver) raise NotImplementedError assert version <= DATABASE_VERSION, "wrong developer brainsync" if self.table_history[table_name][table_index]: # print "Immediate return %s = %s at version %d" % \ # ( table_name, self.table_history[table_name][table_index], version ) return self.table_history[table_name][table_index] # else, it's none, and we've to take the previous valid version # # print "Requested version %d of %s need to be collected in the past" %\ # (version, table_name) while version >= 0: if self.table_history[table_name][table_index]: # print ".. returning %s = %s" %\ # ( table_name, self.table_history[table_name][table_index] ) return self.table_history[table_name][table_index] table_index -= 1 # This never want happen return None def get_right_sql_version(self, model_name, version): """ @param model_name: @param version: @return: The SQL right for the stuff we've """ modelobj = self.get_right_model(model_name, version) if not modelobj: return None right_query = generateCreateQuery(modelobj) return right_query def _perform_copy_list(self, table_name): print "%s default %s migration assistant: #%d" % ( self.debug_info, table_name, self.store_old.find( self.get_right_model(table_name, self.start_ver)).count()) old_objects = self.store_old.find( self.get_right_model(table_name, self.start_ver)) for old_obj in old_objects: new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for k, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def _perform_copy_single(self, table_name): print "%s default %s migration assistant" % (self.debug_info, table_name) old_obj = self.store_old.find( self.get_right_model(table_name, self.start_ver)).one() new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for k, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def migrate_Context(self): self._perform_copy_list("Context") def migrate_Node(self): self._perform_copy_single("Node") def migrate_User(self): self._perform_copy_list("User") def migrate_ReceiverTip(self): self._perform_copy_list("ReceiverTip") def migrate_WhistleblowerTip(self): self._perform_copy_list("WhistleblowerTip") def migrate_Comment(self): self._perform_copy_list("Comment") def migrate_InternalTip(self): self._perform_copy_list("InternalTip") def migrate_Receiver(self): self._perform_copy_list("Receiver") def migrate_InternalFile(self): self._perform_copy_list("InternalFile") def migrate_ReceiverFile(self): self._perform_copy_list("ReceiverFile") def migrate_Notification(self): self._perform_copy_single("Notification") def migrate_ReceiverContext(self): self._perform_copy_list("ReceiverContext") def migrate_ReceiverInternalTip(self): self._perform_copy_list("ReceiverInternalTip") def migrate_Message(self): """ has been created between 7 and 8! """ if self.start_ver < 8: return self._perform_copy_list("Message") def migrate_Stats(self): """ has been created between 9 and 10! """ if self.start_ver < 10: return self._perform_copy_list("Stats") def migrate_ApplicationData(self): """ has been created between 9 and 10! """ if self.start_ver < 10: return self._perform_copy_list("ApplicationData")
class TestMigrationRegression(unittest.TestCase): def _initStartDB(self, target_ver): helpers.init_glsettings_for_unit_tests() GLSettings.db_path = os.path.join(GLSettings.ramdisk_path, 'db_test') shutil.rmtree(GLSettings.db_path, True) os.mkdir(GLSettings.db_path) db_name = 'glbackend-%d.db' % target_ver db_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'db', 'populated', db_name) shutil.copyfile(db_path, os.path.join(GLSettings.db_path, db_name)) self.db_file = os.path.join(GLSettings.db_path, db_name) GLSettings.db_uri = GLSettings.make_db_uri(self.db_file) self.store = Store(create_database(GLSettings.db_uri)) def test_check_field_constraints(self): # This test case asserts that a migration from db ver 32 up to the latest # db with fields that fail the constraints still functions. self._initStartDB(32) field_dict = helpers.get_dummy_field() field_dict['instance'] = 'reference' field_dict['step_id'] = None field_dict['field_id'] = None db_create_field(self.store, field_dict, u'en') field_dict = helpers.get_dummy_field() field_dict['instance'] = 'instance' db_create_field(self.store, field_dict, u'en') field_dict = helpers.get_dummy_field() field_dict['instance'] = 'template' field_dict['step_id'] = None fld_grp_id = self.store.find(Field, Field.fieldgroup_id is not None)[0].fieldgroup_id field_dict['field_id'] = fld_grp_id db_create_field(self.store, field_dict, u'en') self.store.commit() ret = update_db() shutil.rmtree(GLSettings.db_path) self.assertNotEqual(ret, -1) def test_check_unmodifiable_strings(self): # This test case asserts that data migration updates unmodifiable l10n strings self._initStartDB(34) notification_l10n = NotificationL10NFactory(self.store) t0 = notification_l10n.get_val('export_template', 'it') notification_l10n.set_val('export_template', 'it', '') t1 = notification_l10n.get_val('export_template', 'it') self.assertEqual(t1, '') self.store.commit() # place a dummy version in the current db store = Store(create_database(GLSettings.db_uri)) prv = config.PrivateFactory(store) self.dummy_ver = '2.XX.XX' prv.set_val('version', self.dummy_ver) self.assertEqual(prv.get_val('version'), self.dummy_ver) store.commit() store.close() migration.perform_data_update(self.db_file) store = Store(create_database(GLSettings.db_uri)) notification_l10n = NotificationL10NFactory(store) t2 = notification_l10n.get_val('export_template', 'it') self.assertEqual(t2, t0) store.commit() store.close() shutil.rmtree(GLSettings.db_path) def test_mig_37_valid_tor_hs_key(self): self._initStartDB(36) from globaleaks.db.migrations import update_37 t = update_37.TOR_DIR update_37.TOR_DIR = GLSettings.db_path pk_path = os.path.join(update_37.TOR_DIR, 'private_key') hn_path = os.path.join(update_37.TOR_DIR, 'hostname') shutil.copy(os.path.join(helpers.DATA_DIR, 'tor/private_key'), pk_path) shutil.copy(os.path.join(helpers.DATA_DIR, 'tor/hostname'), hn_path) ret = update_db() self.assertEqual(ret, None) new_uri = GLSettings.make_db_uri( os.path.join(GLSettings.db_path, GLSettings.db_file_name)) store = Store(create_database(new_uri)) hs = config.NodeFactory(store).get_val('onionservice') pk = config.PrivateFactory(store).get_val('tor_onion_key') self.assertEqual('lftx7dbyvlc5txtl.onion', hs) with open(os.path.join(helpers.DATA_DIR, 'tor/ephemeral_service_key')) as f: saved_key = f.read().strip() self.assertEqual(saved_key, pk) store.close() shutil.rmtree(GLSettings.db_path) update_37.TOR_DIR = t
class IssuesLog(): def __init__(self): self._connect() # it is not incremental so we first drop the table self._drop_db() self._create_db() def _connect(self): opts = Config() self.database = create_database('mysql://' + opts.db_user_out + ':' + opts.db_password_out + '@' + opts.db_hostname_out + ':' + opts.db_port_out + '/' + opts.db_database_out) self.store = Store(self.database) def _create_db(self): self.store.execute(self._get_sql_create()) def _drop_db(self): self.store.execute(self._get_sql_drop()) def _get_people_id(self, email): """ Gets the id of an user """ try: p = self.store.find(DBPeople, DBPeople.email == email).one() return p.id except (AttributeError, NotOneError): p = self.store.find(DBPeople, DBPeople.user_id == email).one() try: return p.id except AttributeError: # no person was found in People with the email above, so # we include it printdbg("Person not found. Inserted with email %s " % (email)) dp = DBPeople(email) self.store.add(dp) self.store.commit() return dp.id def _get_sql_drop(self): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _get_sql_create(self): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _get_tracker_id(self, issue_id): """ Returns tracker id from issues """ result = self.store.find(DBIssue.tracker_id, DBIssue.id == issue_id).one() return result def _copy_issue_ext(self, aux, db_ilog): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError # TODO: reuse _copy_standard_values def _copy_issue(self, db_ilog): """ This method returns a copy of the DB*Log object """ aux = self._get_dbissues_object(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.change_id = db_ilog.change_id aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to aux = self._copy_issue_ext(aux, db_ilog) return aux def _assign_values(self, db_ilog, field, value): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _build_initial_state(self, db_ilog): """ This method gets the first changes of every field in order to get the initial state of the bug """ fields = self.store.execute("SELECT DISTINCT(field) FROM changes " + "WHERE issue_id=%s" % (db_ilog.issue_id)) for f in fields: values = self.store.execute( "SELECT old_value FROM changes WHERE issue_id=%s AND \ field=\"%s\" ORDER BY changed_on LIMIT 1" % (db_ilog.issue_id, f[0])) for v in values: db_ilog = self._assign_values(db_ilog, f[0], v[0]) # Initial status does not have a real change db_ilog.change_id = 0 return db_ilog def _get_dbissues_object(self, issue_name, tracker_id): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _copy_standard_values(self, issue, issue_log): """ Copy the standard values from the issue object to the issue_log object """ issue_log.issue_id = issue.id issue_log.type = issue.type issue_log.summary = issue.summary issue_log.description = issue.description issue_log.status = issue.status issue_log.resolution = issue.resolution issue_log.priority = issue.priority issue_log.submitted_by = issue.submitted_by issue_log.date = issue.submitted_on issue_log.assigned_to = issue.assigned_to return issue_log def _print_final_msg(self): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _get_changes(self, issue_id): aux = self.store.execute("SELECT id, field, new_value, changed_by, \ changed_on FROM changes where issue_id=%s" % (issue_id)) return aux def _post_history(self, issue_id): """ Abstract method for inserting extra data usign full issue history """ pass def run(self): ndone = 0 issues = self.store.find(DBIssue) total = str(issues.count()) print("[IssuesLog] Total issues to analyze: " + str(issues.count())) for i in issues: if (ndone % 1000 == 0): print("[IssuesLog] Analyzed " + str(ndone) + "/" + str(total)) db_ilog = self._get_dbissues_object(i.issue, i.tracker_id) db_ilog = self._copy_standard_values(i, db_ilog) final_status = db_ilog.status db_ilog = self._build_initial_state(db_ilog) self.store.add(db_ilog) self.store.flush() # the code below gets all the changes and insert a row per change changes = self._get_changes(db_ilog.issue_id) for ch in changes: change_id = ch[0] field = ch[1] new_value = ch[2] changed_by = ch[3] date = ch[4] # we need a new object to be inserted in the database db_ilog = self._copy_issue(db_ilog) db_ilog.date = date db_ilog.change_id = change_id db_ilog.submitted_by = changed_by db_ilog = self._assign_values(db_ilog, field, new_value) try: self.store.add(db_ilog) self.store.flush() except: # self.store.rollback() # is this useful in this context? traceback.print_exc() ##self._post_history(db_ilog, final_status) self.store.commit() ndone += 1 self._print_final_msg()
class ZStormResourceManagerTest(TestHelper): def is_supported(self): return has_transaction and has_zope_component and has_testresources def setUp(self): super(ZStormResourceManagerTest, self).setUp() package_dir = self.makeDir() sys.path.append(package_dir) self.patch_dir = os.path.join(package_dir, "patch_package") os.mkdir(self.patch_dir) self.makeFile(path=os.path.join(self.patch_dir, "__init__.py"), content="") self.makeFile(path=os.path.join(self.patch_dir, "patch_1.py"), content=PATCH) import patch_package create = ["CREATE TABLE test (foo TEXT UNIQUE, bar INT)"] drop = ["DROP TABLE test"] delete = ["DELETE FROM test"] uri = "sqlite:///%s" % self.makeFile() schema = ZSchema(create, drop, delete, PatchSet(patch_package)) self.databases = [{"name": "test", "uri": uri, "schema": schema}] self.resource = ZStormResourceManager(self.databases) self.resource.vertical_patching = False self.store = Store(create_database(uri)) def tearDown(self): global_zstorm._reset() del sys.modules["patch_package"] sys.modules.pop("patch_package.patch_1", None) super(ZStormResourceManagerTest, self).tearDown() def test_make(self): """ L{ZStormResourceManager.make} returns a L{ZStorm} resource that can be used to get the registered L{Store}s. """ zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo, bar FROM test"))) def test_make_lazy(self): """ L{ZStormResourceManager.make} does not create all stores upfront, but only when they're actually used, likewise L{ZStorm.get}. """ zstorm = self.resource.make([]) self.assertEqual([], list(zstorm.iterstores())) store = zstorm.get("test") self.assertEqual([("test", store)], list(zstorm.iterstores())) def test_make_upgrade(self): """ L{ZStormResourceManager.make} upgrades the schema if needed. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT bar FROM test"))) def test_make_upgrade_unknown_patch(self): """ L{ZStormResourceManager.make} resets the schema if an unknown patch is found """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("INSERT INTO patch VALUES (2)") self.store.execute("CREATE TABLE test (foo TEXT, egg BOOL)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo, bar FROM test"))) self.assertEqual([(1, )], list(store.execute("SELECT version FROM patch"))) def test_make_delete(self): """ L{ZStormResourceManager.make} deletes the data from all tables to make sure that tests run against a clean database. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.execute("INSERT INTO test (foo) VALUES ('data')") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo FROM test"))) def test_make_commits_transaction_once(self): """ L{ZStormResourceManager.make} commits schema changes only once across all stores, after all patch and delete statements have been executed. """ database2 = { "name": "test2", "uri": "sqlite:///%s" % self.makeFile(), "schema": self.databases[0]["schema"] } self.databases.append(database2) other_store = Store(create_database(database2["uri"])) for store in [self.store, other_store]: store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") store.execute("CREATE TABLE test (foo TEXT)") store.execute("INSERT INTO test (foo) VALUES ('data')") store.commit() with CaptureTracer() as tracer: zstorm = self.resource.make([]) self.assertEqual(["COMMIT", "COMMIT"], tracer.queries[-2:]) store1 = zstorm.get("test") store2 = zstorm.get("test2") self.assertEqual([], list(store1.execute("SELECT foo FROM test"))) self.assertEqual([], list(store2.execute("SELECT foo FROM test"))) def test_make_zstorm_overwritten(self): """ L{ZStormResourceManager.make} registers its own ZStorm again if a test has registered a new ZStorm utility overwriting the resource one. """ zstorm = self.resource.make([]) provideUtility(ZStorm()) self.resource.make([]) self.assertIs(zstorm, getUtility(IZStorm)) def test_clean_flush(self): """ L{ZStormResourceManager.clean} tries to flush the stores to make sure that they are all in a consistent state. """ class Test(object): __storm_table__ = "test" foo = Unicode() bar = Int(primary=True) def __init__(self, foo, bar): self.foo = foo self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(u"data", 1)) store.add(Test(u"data", 2)) self.assertRaises(IntegrityError, self.resource.clean, zstorm) def test_clean_delete(self): """ L{ZStormResourceManager.clean} cleans the database tables from the data created by the tests. """ zstorm = self.resource.make([]) store = zstorm.get("test") store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") store.commit() self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test"))) def test_clean_with_force_delete(self): """ If L{ZStormResourceManager.force_delete} is C{True}, L{Schema.delete} is always invoked upon test cleanup. """ zstorm = self.resource.make([]) zstorm.get("test") # Force the creation of the store self.store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") self.store.commit() self.resource.force_delete = True self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test"))) def test_wb_clean_clears_alive_cache_before_abort(self): """ L{ZStormResourceManager.clean} clears the alive cache before aborting the transaction. """ class Test(object): __storm_table__ = "test" bar = Int(primary=True) def __init__(self, bar): self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(1)) store.add(Test(2)) real_invalidate = store.invalidate def invalidate_proxy(): self.assertEqual(0, len(list(store._alive.values()))) real_invalidate() store.invalidate = invalidate_proxy self.resource.clean(zstorm) def test_schema_uri(self): """ It's possible to specify an alternate URI for applying the schema and cleaning up tables after a test. """ schema_uri = "sqlite:///%s" % self.makeFile() self.databases[0]["schema-uri"] = schema_uri zstorm = self.resource.make([]) store = zstorm.get("test") schema_store = Store(create_database(schema_uri)) # The schema was applied using the alternate schema URI statement = "SELECT name FROM sqlite_master WHERE name='patch'" self.assertEqual([], list(store.execute(statement))) self.assertEqual([("patch", )], list(schema_store.execute(statement))) # The cleanup is performed with the alternate schema URI store.commit() schema_store.execute("INSERT INTO test (foo) VALUES ('data')") schema_store.commit() self.resource.clean(zstorm) self.assertEqual([], list(schema_store.execute("SELECT * FROM test"))) def test_schema_uri_with_schema_stamp_dir(self): """ If a schema stamp directory is set, and the stamp indicates there's no need to update the schema, the resource clean up code will still connect as schema user if it needs to run the schema delete statements because of a commit. """ self.resource.schema_stamp_dir = self.makeFile() self.databases[0]["schema-uri"] = self.databases[0]["uri"] self.resource.make([]) # Simulate a second test run that initializes the zstorm resource # from scratch, using the same schema stamp directory resource2 = ZStormResourceManager(self.databases) resource2.schema_stamp_dir = self.resource.schema_stamp_dir zstorm = resource2.make([]) store = zstorm.get("test") store.execute("INSERT INTO test (foo) VALUES ('data')") store.commit() # Committing will force a schema.delete() run resource2.clean(zstorm) self.assertEqual([], list(store.execute("SELECT * FROM test"))) def test_no_schema(self): """ A particular database may have no schema associated. """ self.databases[0]["schema"] = None zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT * FROM sqlite_master"))) def test_no_schema_clean(self): """ A particular database may have no schema associated. If it's committed during tests, it will just be skipped when cleaning up tables. """ self.databases[0]["schema"] = None zstorm = self.resource.make([]) store = zstorm.get("test") store.commit() with CaptureTracer() as tracer: self.resource.clean(zstorm) self.assertEqual([], tracer.queries) def test_deprecated_database_format(self): """ The old deprecated format of the 'database' constructor parameter is still supported. """ import patch_package uri = "sqlite:///%s" % self.makeFile() schema = ZSchema([], [], [], patch_package) resource = ZStormResourceManager({"test": (uri, schema)}) zstorm = resource.make([]) store = zstorm.get("test") self.assertIsNot(None, store) def test_use_global_zstorm(self): """ If the C{use_global_zstorm} attribute is C{True} then the global L{ZStorm} will be used. """ self.resource.use_global_zstorm = True zstorm = self.resource.make([]) self.assertIs(global_zstorm, zstorm) def test_provide_utility_before_patches(self): """ The L{IZStorm} utility is provided before patches are applied, in order to let them get it if they need. """ content = ("from zope.component import getUtility\n" "from storm.zope.interfaces import IZStorm\n" "def apply(store):\n" " getUtility(IZStorm)\n") self.makeFile(path=os.path.join(self.patch_dir, "patch_2.py"), content=content) self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([(1, ), (2, )], sorted(store.execute("SELECT version FROM patch"))) def test_create_schema_stamp_dir(self): """ If a schema stamp directory is set, it's created automatically if it doesn't exist yet. """ self.resource.schema_stamp_dir = self.makeFile() self.resource.make([]) self.assertTrue(os.path.exists(self.resource.schema_stamp_dir)) def test_use_schema_stamp(self): """ If a schema stamp directory is set, then it's used to decide whether to upgrade the schema or not. In case the patch directory hasn't been changed since the last known upgrade, no schema upgrade is run. """ self.resource.schema_stamp_dir = self.makeFile() self.resource.make([]) # Simulate a second test run that initializes the zstorm resource # from scratch, using the same schema stamp directory resource2 = ZStormResourceManager(self.databases) resource2.schema_stamp_dir = self.resource.schema_stamp_dir with CaptureTracer() as tracer: resource2.make([]) self.assertEqual([], tracer.queries) def test_use_schema_stamp_out_of_date(self): """ If a schema stamp directory is set, then it's used to decide whether to upgrade the schema or not. In case the patch directory has changed a schema upgrade is run. """ self.resource.schema_stamp_dir = self.makeFile() self.resource.make([]) # Simulate a second test run that initializes the zstorm resource # from scratch, using the same schema stamp directory resource2 = ZStormResourceManager(self.databases) resource2.schema_stamp_dir = self.resource.schema_stamp_dir self.makeFile(path=os.path.join(self.patch_dir, "patch_2.py"), content="def apply(store): pass") class FakeStat(object): st_mtime = os.stat(self.patch_dir).st_mtime + 1 stat_mock = self.mocker.replace(os.stat) stat_mock(self.patch_dir) self.mocker.result(FakeStat()) self.mocker.replay() resource2.make([]) result = self.store.execute("SELECT version FROM patch") self.assertEqual([(1, ), (2, )], sorted(result.get_all()))
class PatchApplierTest(MockerTestCase): def setUp(self): super(PatchApplierTest, self).setUp() self.patchdir = self.makeDir() self.pkgdir = os.path.join(self.patchdir, "mypackage") os.makedirs(self.pkgdir) f = open(os.path.join(self.pkgdir, "__init__.py"), "w") f.write("shared_data = []") f.close() # Order of creation here is important to try to screw up the # patch ordering, as os.listdir returns in order of mtime (or # something). for pname, data in [("patch_380.py", patch_test_1), ("patch_42.py", patch_test_0)]: self.add_module(pname, data) sys.path.append(self.patchdir) self.filename = self.makeFile() self.uri = "sqlite:///%s" % self.filename self.store = Store(create_database(self.uri)) self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.assertFalse(self.store.get(Patch, (42))) self.assertFalse(self.store.get(Patch, (380))) import mypackage self.mypackage = mypackage self.patch_set = PatchSet(mypackage) # Create another connection just to keep track of the state of the # whole transaction manager. See the assertion functions below. self.another_store = Store(create_database("sqlite:")) self.another_store.execute("CREATE TABLE test (id INT)") self.another_store.commit() self.prepare_for_transaction_check() class Committer(object): def commit(committer): self.store.commit() self.another_store.commit() def rollback(committer): self.store.rollback() self.another_store.rollback() self.committer = Committer() self.patch_applier = PatchApplier(self.store, self.patch_set, self.committer) def tearDown(self): super(PatchApplierTest, self).tearDown() self.committer.rollback() sys.path.remove(self.patchdir) for name in list(sys.modules): if name == "mypackage" or name.startswith("mypackage."): del sys.modules[name] def add_module(self, module_filename, contents): filename = os.path.join(self.pkgdir, module_filename) file = open(filename, "w") file.write(contents) file.close() def remove_all_modules(self): for filename in os.listdir(self.pkgdir): os.unlink(os.path.join(self.pkgdir, filename)) def prepare_for_transaction_check(self): self.another_store.execute("DELETE FROM test") self.another_store.execute("INSERT INTO test VALUES (1)") def assert_transaction_committed(self): self.another_store.rollback() result = self.another_store.execute("SELECT * FROM test").get_one() self.assertEquals(result, (1, ), "Transaction manager wasn't committed.") def assert_transaction_aborted(self): self.another_store.commit() result = self.another_store.execute("SELECT * FROM test").get_one() self.assertEquals(result, None, "Transaction manager wasn't aborted.") def test_apply(self): """ L{PatchApplier.apply} executes the patch with the given version. """ self.patch_applier.apply(42) x = getattr(self.mypackage, "patch_42").x self.assertEquals(x, 42) self.assertTrue(self.store.get(Patch, (42))) self.assertTrue("mypackage.patch_42" in sys.modules) self.assert_transaction_committed() def test_apply_with_patch_directory(self): """ If the given L{PatchSet} uses sub-level patches, then the L{PatchApplier.apply} method will look at the per-patch directory and apply the relevant sub-level patch. """ path = os.path.join(self.pkgdir, "patch_99") self.makeDir(path=path) self.makeFile(content="", path=os.path.join(path, "__init__.py")) self.makeFile(content=patch_test_0, path=os.path.join(path, "foo.py")) self.patch_set._sub_level = "foo" self.add_module("patch_99/foo.py", patch_test_0) self.patch_applier.apply(99) self.assertTrue(self.store.get(Patch, (99))) def test_apply_all(self): """ L{PatchApplier.apply_all} executes all unapplied patches. """ self.patch_applier.apply_all() self.assertTrue("mypackage.patch_42" in sys.modules) self.assertTrue("mypackage.patch_380" in sys.modules) x = getattr(self.mypackage, "patch_42").x y = getattr(self.mypackage, "patch_380").y self.assertEquals(x, 42) self.assertEquals(y, 380) self.assert_transaction_committed() def test_apply_exploding_patch(self): """ L{PatchApplier.apply} aborts the transaction if the patch fails. """ self.remove_all_modules() self.add_module("patch_666.py", patch_explosion) self.assertRaises(StormError, self.patch_applier.apply, 666) self.assert_transaction_aborted() def test_wb_apply_all_exploding_patch(self): """ When a patch explodes the store is rolled back to make sure that any changes the patch made to the database are removed. Any other patches that have been applied successfully before it should not be rolled back. Any patches pending after the exploding patch should remain unapplied. """ self.add_module("patch_666.py", patch_explosion) self.add_module("patch_667.py", patch_after_explosion) self.assertEquals(list(self.patch_applier.get_unapplied_versions()), [42, 380, 666, 667]) self.assertRaises(StormError, self.patch_applier.apply_all) self.assertEquals(list(self.patch_applier.get_unapplied_versions()), [666, 667]) def test_mark_applied(self): """ L{PatchApplier.mark} marks a patch has applied by inserting a new row in the patch table. """ self.patch_applier.mark_applied(42) self.assertFalse("mypackage.patch_42" in sys.modules) self.assertFalse("mypackage.patch_380" in sys.modules) self.assertTrue(self.store.get(Patch, 42)) self.assertFalse(self.store.get(Patch, 380)) self.assert_transaction_committed() def test_mark_applied_all(self): """ L{PatchApplier.mark_applied_all} marks all pending patches as applied. """ self.patch_applier.mark_applied_all() self.assertFalse("mypackage.patch_42" in sys.modules) self.assertFalse("mypackage.patch_380" in sys.modules) self.assertTrue(self.store.get(Patch, 42)) self.assertTrue(self.store.get(Patch, 380)) self.assert_transaction_committed() def test_application_order(self): """ L{PatchApplier.apply_all} applies the patches in increasing version order. """ self.patch_applier.apply_all() self.assertEquals(self.mypackage.shared_data, [42, 380]) def test_has_pending_patches(self): """ L{PatchApplier.has_pending_patches} returns C{True} if there are patches to be applied, C{False} otherwise. """ self.assertTrue(self.patch_applier.has_pending_patches()) self.patch_applier.apply_all() self.assertFalse(self.patch_applier.has_pending_patches()) def test_abort_if_unknown_patches(self): """ L{PatchApplier.mark_applied} raises and error if the patch table contains patches without a matching file in the patch module. """ self.patch_applier.mark_applied(381) self.assertRaises(UnknownPatchError, self.patch_applier.apply_all) def test_get_unknown_patch_versions(self): """ L{PatchApplier.get_unknown_patch_versions} returns the versions of all unapplied patches. """ patches = [Patch(42), Patch(380), Patch(381)] my_store = MockPatchStore("database", patches=patches) patch_applier = PatchApplier(my_store, self.mypackage) self.assertEqual(set([381]), patch_applier.get_unknown_patch_versions()) def test_no_unknown_patch_versions(self): """ L{PatchApplier.get_unknown_patch_versions} returns an empty set if no patches are unapplied. """ patches = [Patch(42), Patch(380)] my_store = MockPatchStore("database", patches=patches) patch_applier = PatchApplier(my_store, self.mypackage) self.assertEqual(set(), patch_applier.get_unknown_patch_versions()) def test_patch_with_incorrect_apply(self): """ L{PatchApplier.apply_all} raises an error as soon as one of the patches to be applied fails. """ self.add_module("patch_999.py", patch_no_args_apply) try: self.patch_applier.apply_all() except BadPatchError as e: self.assertTrue("mypackage/patch_999.py" in str(e)) self.assertTrue("takes no arguments" in str(e)) self.assertTrue("TypeError" in str(e)) else: self.fail("BadPatchError not raised") def test_patch_with_missing_apply(self): """ L{PatchApplier.apply_all} raises an error if one of the patches to to be applied has no 'apply' function defined. """ self.add_module("patch_999.py", patch_missing_apply) try: self.patch_applier.apply_all() except BadPatchError as e: self.assertTrue("mypackage/patch_999.py" in str(e)) self.assertTrue("no attribute" in str(e)) self.assertTrue("AttributeError" in str(e)) else: self.fail("BadPatchError not raised") def test_patch_with_syntax_error(self): """ L{PatchApplier.apply_all} raises an error if one of the patches to to be applied contains a syntax error. """ self.add_module("patch_999.py", "that's not python") try: self.patch_applier.apply_all() except BadPatchError as e: self.assertTrue(" 999 " in str(e)) self.assertTrue("SyntaxError" in str(e)) else: self.fail("BadPatchError not raised") def test_patch_error_includes_traceback(self): """ The exception raised by L{PatchApplier.apply_all} when a patch fails include the relevant traceback from the patch. """ self.add_module("patch_999.py", patch_name_error) try: self.patch_applier.apply_all() except BadPatchError as e: self.assertTrue("mypackage/patch_999.py" in str(e)) self.assertTrue("NameError" in str(e)) self.assertTrue("blah" in str(e)) formatted = traceback.format_exc() self.assertTrue("# Comment" in formatted) else: self.fail("BadPatchError not raised")
class TableReplacer(object): """ This is the base class used by every Updater """ def __init__(self, old_db_file, new_db_file, start_ver): from globaleaks.db.update_8_9 import Context_v_8, Receiver_v_8, Notification_v_8 from globaleaks.db.update_9_10 import Node_v_9, Receiver_v_9, User_v_9 from globaleaks.db.update_10_11 import InternalTip_v_10, InternalFile_v_10 from globaleaks.db.update_11_12 import Node_v_11, Context_v_11 from globaleaks.db.update_12_13 import Node_v_12, Context_v_12 from globaleaks.db.update_13_14 import Node_v_13, Context_v_13 from globaleaks.db.update_14_15 import Node_v_14, User_v_14, Context_v_14, Receiver_v_14, \ InternalTip_v_14, Notification_v_14, Stats_v_14, Comment_v_14 from globaleaks.db.update_15_16 import Receiver_v_15, Notification_v_15 from globaleaks.db.update_16_17 import Node_v_16, Receiver_v_16, Notification_v_16, Stats_v_16 from globaleaks.db.update_17_18 import Node_v_17 from globaleaks.db.update_18_19 import Node_v_18 from globaleaks.db.update_19_20 import Node_v_19, Notification_v_19, Comment_v_19, Message_v_19, \ InternalTip_v_19, ReceiverTip_v_19, InternalFile_v_19, ReceiverFile_v_19, Receiver_v_19, \ Context_v_19 self.old_db_file = old_db_file self.new_db_file = new_db_file self.start_ver = start_ver self.std_fancy = " ł " self.debug_info = " [%d => %d] " % (start_ver, start_ver + 1) self.table_history = { 'Node': [Node_v_9, None, Node_v_11, None, Node_v_12, Node_v_13, Node_v_14, Node_v_16, None, Node_v_17, Node_v_18, Node_v_19, models.Node], 'User': [User_v_9, None, User_v_14, None, None, None, None, models.User, None, None, None, None, None], 'Context': [Context_v_8, Context_v_11, None, None, Context_v_12, Context_v_13, Context_v_14, Context_v_19, None, None, None, None, models.Context], 'Receiver': [Receiver_v_8, Receiver_v_9, Receiver_v_14, None, None, None, None, Receiver_v_15, Receiver_v_16, Receiver_v_19, None, None, models.Receiver], 'ReceiverFile': [ReceiverFile_v_19, None, None, None, None, None, None, None, None, None, None, None, models.ReceiverFile], 'Notification': [Notification_v_8, Notification_v_14, None, None, None, None, None, Notification_v_15, Notification_v_16, Notification_v_19, None, None, models.Notification], 'Comment': [Comment_v_14, None, None, None, None, None, None, Comment_v_19, None, None, None, None, models.Comment], 'InternalTip': [InternalTip_v_10, None, None, InternalTip_v_14, None, None, None, InternalTip_v_19, None, None, None, None, models.InternalTip], 'InternalFile': [InternalFile_v_10, None, None, InternalFile_v_19, None, None, None, None, None, None, None, None, models.InternalFile], 'WhistleblowerTip': [models.WhistleblowerTip, None, None, None, None, None, None, None, None, None, None, None, None], 'ReceiverTip': [ReceiverTip_v_19, None, None, None, None, None, None, None, None, None, None, None, models.ReceiverTip], 'ReceiverInternalTip': [models.ReceiverInternalTip, None, None, None, None, None, None, None, None, None, None, None, None], 'ReceiverContext': [models.ReceiverContext, None, None, None, None, None, None, None, None, None, None, None, None], 'Message': [Message_v_19, None, None, None, None, None, None, None, None, None, None, None, models.Message], 'Stats': [Stats_v_14, None, None, None, None, None, None, Stats_v_16, None, models.Stats, None, None, None], 'ApplicationData': [models.ApplicationData, None, None, None, None, None, None, None, None, None, None, None, None], 'Field': [models.Field, None, None, None, None, None, None, None, None, None, None, None, None], 'FieldOption': [models.FieldOption, None, None, None, None, None, None, None, None, None, None, None, None], 'FieldField': [models.FieldField, None, None, None, None, None, None, None, None, None, None, None, None], 'Step': [models.Step, None, None, None, None, None, None, None, None, None, None, None, None], 'StepField': [models.StepField, None, None, None, None, None, None, None, None, None, None, None, None], 'Anomalies': [models.Anomalies, None, None, None, None, None, None, None, None, None, None, None, None], 'EventLogs': [models.EventLogs, None, None, None, None, None, None, None, None, None, None, None, None], } for k, v in self.table_history.iteritems(): # +1 because count start from 0, # -8 because the relase befor the 8th are not supported anymore length = DATABASE_VERSION + 1 - 8 if len(v) != length: msg = 'Expecting a table with {} statuses ({})'.format(length, k) raise TypeError(msg) log.msg('{} Opening old DB: {}'.format(self.debug_info, old_db_file)) old_database = create_database('sqlite:' + self.old_db_file) self.store_old = Store(old_database) GLSetting.db_file = new_db_file new_database = create_database('sqlite:' + new_db_file) self.store_new = Store(new_database) if self.start_ver + 1 == DATABASE_VERSION: log.msg('{} Acquire SQL schema {}'.format(self.debug_info, GLSetting.db_schema_file)) if not os.access(GLSetting.db_schema_file, os.R_OK): log.msg('Unable to access', GLSetting.db_schema_file) raise IOError('Unable to access db schema file') with open(GLSetting.db_schema_file) as f: create_queries = ''.join(f).split(';') for create_query in create_queries: try: self.store_new.execute(create_query + ';') except OperationalError: log.msg('OperationalError in "{}"'.format(create_query)) self.store_new.commit() return # return here and manage the migrant versions here: for k, v in self.table_history.iteritems(): create_query = self.get_right_sql_version(k, self.start_ver + 1) if not create_query: # table not present in the version continue try: self.store_new.execute(create_query + ';') except OperationalError as excep: log.msg('{} OperationalError in [{}]'.format(self.debug_info, create_query)) raise excep self.store_new.commit() def close(self): self.store_old.close() self.store_new.close() def initialize(self): pass def epilogue(self): pass def get_right_model(self, table_name, version): table_index = (version - 8) if table_name not in self.table_history: msg = 'Not implemented usage of get_right_model {} ({} {})'.format( __file__, table_name, self.start_ver) raise NotImplementedError(msg) if version > DATABASE_VERSION: raise ValueError('Version supplied must be less or equal to {}'.format( DATABASE_VERSION)) if self.table_history[table_name][table_index]: return self.table_history[table_name][table_index] # else, it's none, and we've to take the previous valid version while version >= 0: if self.table_history[table_name][table_index]: return self.table_history[table_name][table_index] table_index -= 1 # This never want happen return None def get_right_sql_version(self, model_name, version): """ @param model_name: @param version: @return: The SQL right for the stuff we've """ modelobj = self.get_right_model(model_name, version) if not modelobj: return None right_query = generateCreateQuery(modelobj) return right_query def _perform_copy_list(self, table_name): models_count = self.store_old.find( self.get_right_model(table_name, self.start_ver) ).count() log.msg('{} default {} migration assistant: #{}'.format( self.debug_info, table_name, models_count)) old_objects = self.store_old.find(self.get_right_model(table_name, self.start_ver)) for old_obj in old_objects: new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for _, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def _perform_copy_single(self, table_name): log.msg('{} default {} migration assistant'.format(self.debug_info, table_name)) old_obj = self.store_old.find(self.get_right_model(table_name, self.start_ver)).one() new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for _, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def migrate_Context(self): self._perform_copy_list("Context") def migrate_Node(self): self._perform_copy_single("Node") def migrate_User(self): self._perform_copy_list("User") def migrate_ReceiverTip(self): self._perform_copy_list("ReceiverTip") def migrate_WhistleblowerTip(self): self._perform_copy_list("WhistleblowerTip") def migrate_Comment(self): self._perform_copy_list("Comment") def migrate_InternalTip(self): self._perform_copy_list("InternalTip") def migrate_Receiver(self): self._perform_copy_list("Receiver") def migrate_InternalFile(self): self._perform_copy_list("InternalFile") def migrate_ReceiverFile(self): self._perform_copy_list("ReceiverFile") def migrate_Notification(self): self._perform_copy_single("Notification") def migrate_ReceiverContext(self): self._perform_copy_list("ReceiverContext") def migrate_ReceiverInternalTip(self): self._perform_copy_list("ReceiverInternalTip") def migrate_Message(self): """ has been created between 7 and 8! """ if self.start_ver < 8: return self._perform_copy_list("Message") def migrate_Stats(self): """ has been created between 14 and 15 and is not migrated since 17 """ if self.start_ver < 17: return self._perform_copy_list("Stats") def migrate_ApplicationData(self): """ There is no need to migrate it the application data. Default application data is loaded by the application and stored onto the db at each new start. """ return def migrate_Field(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Field") def migrate_FieldOption(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("FieldOption") def migrate_FieldField(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("FieldField") def migrate_Step(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Step") def migrate_StepField(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("StepField") def migrate_Anomalies(self): """ has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Anomalies") def migrate_EventLogs(self): """ has been created between 15 and 16! should be dropped befor 20 """ if self.start_ver < 20: return self._perform_copy_list("EventLogs")
class ZStormResourceManagerTest(TestHelper): def is_supported(self): return has_zope and has_testresources def setUp(self): super(ZStormResourceManagerTest, self).setUp() self._package_dir = self.makeDir() sys.path.append(self._package_dir) patch_dir = os.path.join(self._package_dir, "patch_package") os.mkdir(patch_dir) self.makeFile(path=os.path.join(patch_dir, "__init__.py"), content="") self.makeFile(path=os.path.join(patch_dir, "patch_1.py"), content=PATCH) import patch_package create = ["CREATE TABLE test (foo TEXT UNIQUE, bar INT)"] drop = ["DROP TABLE test"] delete = ["DELETE FROM test"] schema = ZSchema(create, drop, delete, patch_package) uri = "sqlite:///%s" % self.makeFile() self.resource = ZStormResourceManager({"test": (uri, schema)}) self.store = Store(create_database(uri)) def tearDown(self): del sys.modules["patch_package"] sys.path.remove(self._package_dir) if "patch_1" in sys.modules: del sys.modules["patch_1"] super(ZStormResourceManagerTest, self).tearDown() def test_make(self): """ L{ZStormResourceManager.make} returns a L{ZStorm} resource that can be used to get the registered L{Store}s. """ zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo, bar FROM test"))) def test_make_upgrade(self): """ L{ZStormResourceManager.make} upgrades the schema if needed. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT bar FROM test"))) def test_make_delete(self): """ L{ZStormResourceManager.make} deletes the data from all tables to make sure that tests run against a clean database. """ self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.store.execute("CREATE TABLE test (foo TEXT)") self.store.execute("INSERT INTO test (foo) VALUES ('data')") self.store.commit() zstorm = self.resource.make([]) store = zstorm.get("test") self.assertEqual([], list(store.execute("SELECT foo FROM test"))) def test_clean_flush(self): """ L{ZStormResourceManager.clean} tries to flush the stores to make sure that they are all in a consistent state. """ class Test(object): __storm_table__ = "test" foo = Unicode() bar = Int(primary=True) def __init__(self, foo, bar): self.foo = foo self.bar = bar zstorm = self.resource.make([]) store = zstorm.get("test") store.add(Test(u"data", 1)) store.add(Test(u"data", 2)) self.assertRaises(IntegrityError, self.resource.clean, zstorm) def test_clean_delete(self): """ L{ZStormResourceManager.clean} cleans the database tables from the data created by the tests. """ zstorm = self.resource.make([]) store = zstorm.get("test") store.execute("INSERT INTO test (foo, bar) VALUES ('data', 123)") store.commit() self.resource.clean(zstorm) self.assertEqual([], list(self.store.execute("SELECT * FROM test")))
class NCBITaxnomyInserter(object): def __init__(self, divisions_file_path, taxonomy_divisions_file_path): self.included_divisions= {0:"Bacteria",3:"Phages",9:"Viruses", 11:"Environmental samples", 1:"Invertebrates", 4:"Plants and Fungi"} self.divisions_file_path= divisions_file_path self.taxonomy_divisions_file_path= taxonomy_divisions_file_path self.__init_database() if not self.init_tables(): self.create_tables() def __init_database(self): """ creates the sqlite database instance and checks if the database exists in biodb. """ database= create_database("sqlite:%s" % biodb_sql_db_path) print "Created storm database from %s." % biodb_sql_db_path self.store= Store(database) def init_tables(self): self.biodb_table= "biodb_ncbi" self.taxonomy_division_table = "biodb_ncbi_taxonomy_division" self.division_table= "biodb_ncbi_division" #### check if the db_name exists in the database table_list= [table[0] for table in self.store.execute('select tbl_name from SQLITE_MASTER')] return 0 if self.taxonomy_division_table not in table_list else 1 def create_tables(self): self.create_taxonomy_division_string='CREATE TABLE '+ self.taxonomy_division_table +' (taxonID INTEGER PRIMARY KEY, divisionID INTEGER, FOREIGN KEY (taxonID) REFERENCES '+ self.biodb_table+'(id), FOREIGN KEY (divisionID) REFERENCES '+ self.division_table +'(id) )' self.create_division_string='CREATE TABLE '+ self.division_table +' (id INTEGER PRIMARY KEY, name VARCHAR)' self.store.execute(self.create_taxonomy_division_string) self.store.execute(self.create_division_string) def insert_division(self, div_id, name): div= NCBIDivision() div.id = int(div_id) div.name= unicode(name) self.store.add(div) def insert_taxonomy_division(self, taxon_id, div_id): n_tax_div= NCBITaxonomyDivision() n_tax_div.taxonID= int(taxon_id) n_tax_div.divisionID= int(div_id) self.store.add(n_tax_div) def insert_divisions_from_file(self): with open(self.divisions_file_path) as div_file: for line in div_file: cols= line.rstrip('\n').split(sep) div_id= cols[0] name= cols[2] self.insert_division(div_id, name) self.store.commit() def insert_taxonomy_divisions_from_file(self): i=0 with open(self.taxonomy_divisions_file_path) as tax_div_file: for line in tax_div_file: cols= line.rstrip('\n').split(sep) division_id= int(cols[4].strip()) if division_id in self.included_divisions: tax_id= cols[0].strip() self.insert_taxonomy_division(tax_id, division_id) self.store.commit() i+=1 if i % 10000 == 0: print "%d taxa inserted!" %i
class PatchTest(MockerTestCase): def setUp(self): super(PatchTest, self).setUp() self.patchdir = self.makeDir() self.pkgdir = os.path.join(self.patchdir, "mypackage") os.makedirs(self.pkgdir) f = open(os.path.join(self.pkgdir, "__init__.py"), "w") f.write("shared_data = []") f.close() # Order of creation here is important to try to screw up the # patch ordering, as os.listdir returns in order of mtime (or # something). for pname, data in [("patch_380.py", patch_test_1), ("patch_42.py", patch_test_0)]: self.add_module(pname, data) sys.path.append(self.patchdir) self.filename = self.makeFile() self.uri = "sqlite:///%s" % self.filename self.store = Store(create_database(self.uri)) self.store.execute("CREATE TABLE patch " "(version INTEGER NOT NULL PRIMARY KEY)") self.assertFalse(self.store.get(Patch, (42))) self.assertFalse(self.store.get(Patch, (380))) import mypackage self.mypackage = mypackage # Create another connection just to keep track of the state of the # whole transaction manager. See the assertion functions below. self.another_store = Store(create_database("sqlite:")) self.another_store.execute("CREATE TABLE test (id INT)") self.another_store.commit() self.prepare_for_transaction_check() class Committer(object): def commit(committer): self.store.commit() self.another_store.commit() def rollback(committer): self.store.rollback() self.another_store.rollback() self.committer = Committer() self.patch_applier = PatchApplier(self.store, self.mypackage, self.committer) def tearDown(self): super(PatchTest, self).tearDown() self.committer.rollback() sys.path.remove(self.patchdir) for name in list(sys.modules): if name == "mypackage" or name.startswith("mypackage."): del sys.modules[name] def add_module(self, module_filename, contents): filename = os.path.join(self.pkgdir, module_filename) file = open(filename, "w") file.write(contents) file.close() def remove_all_modules(self): for filename in os.listdir(self.pkgdir): os.unlink(os.path.join(self.pkgdir, filename)) def prepare_for_transaction_check(self): self.another_store.execute("DELETE FROM test") self.another_store.execute("INSERT INTO test VALUES (1)") def assert_transaction_committed(self): self.another_store.rollback() result = self.another_store.execute("SELECT * FROM test").get_one() self.assertEquals(result, (1,), "Transaction manager wasn't committed.") def assert_transaction_aborted(self): self.another_store.commit() result = self.another_store.execute("SELECT * FROM test").get_one() self.assertEquals(result, None, "Transaction manager wasn't aborted.") def test_apply(self): """ L{PatchApplier.apply} executes the patch with the given version. """ self.patch_applier.apply(42) x = getattr(self.mypackage, "patch_42").x self.assertEquals(x, 42) self.assertTrue(self.store.get(Patch, (42))) self.assertTrue("mypackage.patch_42" in sys.modules) self.assert_transaction_committed() def test_apply_all(self): """ L{PatchApplier.apply_all} executes all unapplied patches. """ self.patch_applier.apply_all() self.assertTrue("mypackage.patch_42" in sys.modules) self.assertTrue("mypackage.patch_380" in sys.modules) x = getattr(self.mypackage, "patch_42").x y = getattr(self.mypackage, "patch_380").y self.assertEquals(x, 42) self.assertEquals(y, 380) self.assert_transaction_committed() def test_apply_exploding_patch(self): """ L{PatchApplier.apply} aborts the transaction if the patch fails. """ self.remove_all_modules() self.add_module("patch_666.py", patch_explosion) self.assertRaises(StormError, self.patch_applier.apply, 666) self.assert_transaction_aborted() def test_wb_apply_all_exploding_patch(self): """ When a patch explodes the store is rolled back to make sure that any changes the patch made to the database are removed. Any other patches that have been applied successfully before it should not be rolled back. Any patches pending after the exploding patch should remain unapplied. """ self.add_module("patch_666.py", patch_explosion) self.add_module("patch_667.py", patch_after_explosion) self.assertEquals(list(self.patch_applier._get_unapplied_versions()), [42, 380, 666, 667]) self.assertRaises(StormError, self.patch_applier.apply_all) self.assertEquals(list(self.patch_applier._get_unapplied_versions()), [666, 667]) def test_mark_applied(self): """ L{PatchApplier.mark} marks a patch has applied by inserting a new row in the patch table. """ self.patch_applier.mark_applied(42) self.assertFalse("mypackage.patch_42" in sys.modules) self.assertFalse("mypackage.patch_380" in sys.modules) self.assertTrue(self.store.get(Patch, 42)) self.assertFalse(self.store.get(Patch, 380)) self.assert_transaction_committed() def test_mark_applied_all(self): """ L{PatchApplier.mark_applied_all} marks all pending patches as applied. """ self.patch_applier.mark_applied_all() self.assertFalse("mypackage.patch_42" in sys.modules) self.assertFalse("mypackage.patch_380" in sys.modules) self.assertTrue(self.store.get(Patch, 42)) self.assertTrue(self.store.get(Patch, 380)) self.assert_transaction_committed() def test_application_order(self): """ L{PatchApplier.apply_all} applies the patches in increasing version order. """ self.patch_applier.apply_all() self.assertEquals(self.mypackage.shared_data, [42, 380]) def test_has_pending_patches(self): """ L{PatchApplier.has_pending_patches} returns C{True} if there are patches to be applied, C{False} otherwise. """ self.assertTrue(self.patch_applier.has_pending_patches()) self.patch_applier.apply_all() self.assertFalse(self.patch_applier.has_pending_patches()) def test_abort_if_unknown_patches(self): """ L{PatchApplier.mark_applied} raises and error if the patch table contains patches without a matching file in the patch module. """ self.patch_applier.mark_applied(381) self.assertRaises(UnknownPatchError, self.patch_applier.apply_all) def test_get_unknown_patch_versions(self): """ L{PatchApplier.get_unknown_patch_versions} returns the versions of all unapplied patches. """ patches = [Patch(42), Patch(380), Patch(381)] my_store = MockPatchStore("database", patches=patches) patch_applier = PatchApplier(my_store, self.mypackage) self.assertEqual(set([381]), patch_applier.get_unknown_patch_versions()) def test_no_unknown_patch_versions(self): """ L{PatchApplier.get_unknown_patch_versions} returns an empty set if no patches are unapplied. """ patches = [Patch(42), Patch(380)] my_store = MockPatchStore("database", patches=patches) patch_applier = PatchApplier(my_store, self.mypackage) self.assertEqual(set(), patch_applier.get_unknown_patch_versions()) def test_patch_with_incorrect_apply(self): """ L{PatchApplier.apply_all} raises an error as soon as one of the patches to be applied fails. """ self.add_module("patch_999.py", patch_no_args_apply) try: self.patch_applier.apply_all() except BadPatchError, e: self.assertTrue("mypackage/patch_999.py" in str(e)) self.assertTrue("takes no arguments" in str(e)) self.assertTrue("TypeError" in str(e)) else:
class TableReplacer(object): """ This is the base class used by every Updater """ def __init__(self, table_history, old_db_file, new_db_file, start_ver): self.table_history = table_history self.old_db_file = old_db_file self.new_db_file = new_db_file self.start_ver = start_ver self.std_fancy = " ł " self.debug_info = " [%d => %d] " % (start_ver, start_ver + 1) for k, v in table_history.iteritems(): length = DATABASE_VERSION + 1 - FIRST_DATABASE_VERSION_SUPPORTED if len(v) != length: msg = 'Expecting a table with {} statuses ({})'.format(length, k) raise TypeError(msg) log.msg('{} Opening old DB: {}'.format(self.debug_info, old_db_file)) old_database = create_database('sqlite:' + self.old_db_file) self.store_old = Store(old_database) GLSettings.db_file = new_db_file new_database = create_database('sqlite:' + new_db_file) self.store_new = Store(new_database) if self.start_ver + 1 == DATABASE_VERSION: log.msg('{} Acquire SQL schema {}'.format(self.debug_info, GLSettings.db_schema_file)) if not os.access(GLSettings.db_schema_file, os.R_OK): log.msg('Unable to access', GLSettings.db_schema_file) raise IOError('Unable to access db schema file') with open(GLSettings.db_schema_file) as f: create_queries = ''.join(f).split(';') for create_query in create_queries: try: self.store_new.execute(create_query + ';') except OperationalError: log.msg('OperationalError in "{}"'.format(create_query)) self.store_new.commit() return # return here and manage the migrant versions here: for k, v in self.table_history.iteritems(): create_query = self.get_right_sql_version(k, self.start_ver + 1) if not create_query: # table not present in the version continue try: self.store_new.execute(create_query + ';') except OperationalError as excep: log.msg('{} OperationalError in [{}]'.format(self.debug_info, create_query)) raise excep self.store_new.commit() def close(self): self.store_old.close() self.store_new.close() def initialize(self): pass def epilogue(self): pass def get_right_model(self, table_name, version): table_index = (version - FIRST_DATABASE_VERSION_SUPPORTED) if table_name not in self.table_history: msg = 'Not implemented usage of get_right_model {} ({} {})'.format( __file__, table_name, self.start_ver) raise NotImplementedError(msg) if version > DATABASE_VERSION: raise ValueError('Version supplied must be less or equal to {}'.format( DATABASE_VERSION)) if self.table_history[table_name][table_index]: return self.table_history[table_name][table_index] # else, it's none, and we've to take the previous valid version while version >= 0: if self.table_history[table_name][table_index]: return self.table_history[table_name][table_index] table_index -= 1 # This never want happen return None def get_right_sql_version(self, model_name, version): """ @param model_name: @param version: @return: The SQL right for the stuff we've """ modelobj = self.get_right_model(model_name, version) if not modelobj: return None right_query = generateCreateQuery(modelobj) return right_query def _perform_copy_list(self, table_name): objs_count = self.store_old.find( self.get_right_model(table_name, self.start_ver) ).count() log.msg('{} default {} migration assistant: #{}'.format( self.debug_info, table_name, objs_count)) old_objects = self.store_old.find(self.get_right_model(table_name, self.start_ver)) for old_obj in old_objects: new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for _, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def _perform_copy_single(self, table_name): log.msg('{} default {} migration assistant'.format(self.debug_info, table_name)) old_obj = self.store_old.find(self.get_right_model(table_name, self.start_ver)).one() new_obj = self.get_right_model(table_name, self.start_ver + 1)() # Storm internals simply reversed for _, v in new_obj._storm_columns.iteritems(): setattr(new_obj, v.name, getattr(old_obj, v.name)) self.store_new.add(new_obj) self.store_new.commit() def migrate_Context(self): self._perform_copy_list("Context") def migrate_Node(self): self._perform_copy_single("Node") def migrate_User(self): self._perform_copy_list("User") def migrate_ReceiverTip(self): self._perform_copy_list("ReceiverTip") def migrate_WhistleblowerTip(self): self._perform_copy_list("WhistleblowerTip") def migrate_Comment(self): self._perform_copy_list("Comment") def migrate_InternalTip(self): self._perform_copy_list("InternalTip") def migrate_Receiver(self): self._perform_copy_list("Receiver") def migrate_InternalFile(self): self._perform_copy_list("InternalFile") def migrate_ReceiverFile(self): self._perform_copy_list("ReceiverFile") def migrate_Notification(self): self._perform_copy_single("Notification") def migrate_ReceiverContext(self): self._perform_copy_list("ReceiverContext") def migrate_ReceiverInternalTip(self): self._perform_copy_list("ReceiverInternalTip") def migrate_Message(self): self._perform_copy_list("Message") def migrate_Stats(self): """ Stats has been created between 14 and 15 and is not migrated since 17 """ if self.start_ver < 17: return self._perform_copy_list("Stats") def migrate_ApplicationData(self): """ There is no need to migrate it the application data. Default application data is loaded by the application and stored onto the db at each new start. """ return def migrate_Field(self): """ Field has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Field") def migrate_FieldAttr(self): """ FieldAttr has been created between 22 and 23! """ if self.start_ver < 23: return self._perform_copy_list("FieldAttr") def migrate_FieldOption(self): """ FieldOption has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("FieldOption") def migrate_FieldField(self): """ FieldField has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("FieldField") def migrate_Step(self): """ Step has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("Step") def migrate_StepField(self): """ StepField has been created between 14 and 15! """ if self.start_ver < 15: return self._perform_copy_list("StepField") def migrate_Anomalies(self): """ Anomalies has been created between 14 and 15! should be dropped before 22 """ if self.start_ver < 23: return self._perform_copy_list("Anomalies") def migrate_EventLogs(self): """ EventLogs has been created between 15 and 16! should be dropped before 20 """ if self.start_ver < 20: return self._perform_copy_list("EventLogs")
class IssuesLog(): def __init__(self): self._connect() # it is not incremental so we first drop the table self._drop_db() self._create_db() def _connect(self): opts = Config() self.database = create_database('mysql://' + opts.db_user_out + ':' + opts.db_password_out + '@' + opts.db_hostname_out + ':' + opts.db_port_out + '/' + opts.db_database_out) self.store = Store(self.database) def _create_db(self): self.store.execute(self._get_sql_create()) def _drop_db(self): self.store.execute(self._get_sql_drop()) def _get_people_id(self, email): """ Gets the id of an user """ try: p = self.store.find(DBPeople, DBPeople.email == email).one() return p.id except (AttributeError, NotOneError): p = self.store.find(DBPeople, DBPeople.user_id == email).one() try: return p.id except AttributeError: # no person was found in People with the email above, so # we include it printdbg("Person not found. Inserted with email %s " % (email)) dp = DBPeople(email) self.store.add(dp) self.store.commit() return dp.id def _get_sql_drop(self): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _get_sql_create(self): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _get_tracker_id(self, issue_id): """ Returns tracker id from issues """ result = self.store.find(DBIssue.tracker_id, DBIssue.id == issue_id).one() return result def _copy_issue_ext(self, aux, db_ilog): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError # TODO: reuse _copy_standard_values def _copy_issue(self, db_ilog): """ This method returns a copy of the DB*Log object """ aux = self._get_dbissues_object(db_ilog.issue, db_ilog.tracker_id) aux.issue_id = db_ilog.issue_id aux.change_id = db_ilog.change_id aux.changed_by = db_ilog.changed_by aux.type = db_ilog.type aux.summary = db_ilog.summary aux.description = db_ilog.description aux.status = db_ilog.status aux.resolution = db_ilog.resolution aux.priority = db_ilog.priority aux.submitted_by = db_ilog.submitted_by aux.date = db_ilog.date aux.assigned_to = db_ilog.assigned_to aux = self._copy_issue_ext(aux, db_ilog) return aux def _assign_values(self, db_ilog, field, value): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _build_initial_state(self, db_ilog): """ This method gets the first changes of every field in order to get the initial state of the bug """ fields = self.store.execute("SELECT DISTINCT(field) FROM changes " + "WHERE issue_id=%s" % (db_ilog.issue_id)) for f in fields: values = self.store.execute( "SELECT old_value FROM changes WHERE issue_id=%s AND \ field=\"%s\" ORDER BY changed_on LIMIT 1" % (db_ilog.issue_id, f[0])) for v in values: db_ilog = self._assign_values(db_ilog, f[0], v[0]) return db_ilog def _get_dbissues_object(self, issue_name, tracker_id): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _copy_standard_values(self, issue, issue_log): """ Copy the standard values from the issue object to the issue_log object """ issue_log.issue_id = issue.id issue_log.type = issue.type issue_log.summary = issue.summary issue_log.description = issue.description issue_log.status = issue.status issue_log.resolution = issue.resolution issue_log.priority = issue.priority issue_log.submitted_by = issue.submitted_by issue_log.date = issue.submitted_on issue_log.assigned_to = issue.assigned_to return issue_log def _print_final_msg(self): """ Abstract method for inserting extra data related to a change """ raise NotImplementedError def _get_changes(self, issue_id): aux = self.store.execute("SELECT id, field, new_value, changed_by, \ changed_on FROM changes where issue_id=%s" % (issue_id)) return aux def _post_history(self, issue_id): """ Abstract method for inserting extra data usign full issue history """ pass def run(self): ndone = 0 issues = self.store.find(DBIssue) total = str(issues.count()) print ("[IssuesLog] Total issues to analyze: " + str(issues.count())) for i in issues: if (ndone % 1000 == 0): print ("[IssuesLog] Analyzed " + str(ndone) + "/" + str(total)) db_ilog = self._get_dbissues_object(i.issue, i.tracker_id) db_ilog = self._copy_standard_values(i, db_ilog) final_status = db_ilog.status db_ilog = self._build_initial_state(db_ilog) self.store.add(db_ilog) self.store.flush() # the code below gets all the changes and insert a row per change changes = self._get_changes(db_ilog.issue_id) for ch in changes: change_id = ch[0] field = ch[1] new_value = ch[2] changed_by = ch[3] date = ch[4] # we need a new object to be inserted in the database db_ilog = self._copy_issue(db_ilog) db_ilog.date = date db_ilog.change_id = change_id db_ilog.changed_by = changed_by db_ilog = self._assign_values(db_ilog, field, new_value) try: self.store.add(db_ilog) self.store.flush() except: # self.store.rollback() # is this useful in this context? traceback.print_exc() self._post_history(db_ilog, final_status) self.store.commit() ndone += 1 self._print_final_msg()
def perform_version_update(version): """ @param version: @return: """ to_delete_on_fail = [] to_delete_on_success = [] if version < FIRST_DATABASE_VERSION_SUPPORTED: GLSettings.print_msg("Migrations from DB version lower than %d are no more supported!" % FIRST_DATABASE_VERSION_SUPPORTED) GLSettings.print_msg("If you can't create your Node from scratch, contact us asking for support.") quit() tmpdir = os.path.abspath(os.path.join(GLSettings.db_path, 'tmp')) orig_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % version)) final_db_file = os.path.abspath(os.path.join(GLSettings.db_path, 'glbackend-%d.db' % DATABASE_VERSION)) shutil.rmtree(tmpdir, True) os.mkdir(tmpdir) shutil.copy2(orig_db_file, tmpdir) old_db_file = None new_db_file = None try: while version < DATABASE_VERSION: old_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % version)) new_db_file = os.path.abspath(os.path.join(tmpdir, 'glbackend-%d.db' % (version + 1))) GLSettings.db_file = new_db_file GLSettings.enable_input_length_checks = False to_delete_on_fail.append(new_db_file) to_delete_on_success.append(old_db_file) GLSettings.print_msg("Updating DB from version %d to version %d" % (version, version + 1)) store_old = Store(create_database('sqlite:' + old_db_file)) store_new = Store(create_database('sqlite:' + new_db_file)) # Here is instanced the migration script MigrationModule = importlib.import_module("globaleaks.db.migrations.update_%d" % (version + 1)) migration_script = MigrationModule.MigrationScript(migration_mapping, version, store_old, store_new) GLSettings.print_msg("Migrating table:") try: try: migration_script.prologue() except Exception as exception: GLSettings.print_msg("Failure while executing migration prologue: %s" % exception) raise exception for model_name, _ in migration_mapping.iteritems(): if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None: try: migration_script.migrate_model(model_name) # Commit at every table migration in order to be able to detect # the precise migration that may fail. migration_script.commit() except Exception as exception: GLSettings.print_msg("Failure while migrating table %s: %s " % (model_name, exception)) raise exception try: migration_script.epilogue() migration_script.commit() except Exception as exception: GLSettings.print_msg("Failure while executing migration epilogue: %s " % exception) raise exception finally: # the database should bee always closed before leaving the application # in order to not keep leaking journal files. migration_script.close() GLSettings.print_msg("Migration stats:") # we open a new db in order to verify integrity of the generated file store_verify = Store(create_database('sqlite:' + new_db_file)) for model_name, _ in migration_mapping.iteritems(): if model_name == 'ApplicationData': continue if migration_script.model_from[model_name] is not None and migration_script.model_to[model_name] is not None: count = store_verify.find(migration_script.model_to[model_name]).count() if migration_script.entries_count[model_name] != count: if migration_script.fail_on_count_mismatch[model_name]: raise AssertionError("Integrity check failed on count equality for table %s: %d != %d" % \ (model_name, count, migration_script.entries_count[model_name])) else: GLSettings.print_msg(" * %s table migrated (entries count changed from %d to %d)" % \ (model_name, migration_script.entries_count[model_name], count)) else: GLSettings.print_msg(" * %s table migrated (%d entry(s))" % \ (model_name, migration_script.entries_count[model_name])) version += 1 store_verify.close() store_appdata = Store(create_database('sqlite:' + new_db_file)) db_update_appdata(store_appdata) db_fix_fields_attrs(store_appdata) store_appdata.commit() store_appdata.close() except Exception as exception: print exception # simply propagage the exception raise exception else: # in case of success first copy the new migrated db, then as last action delete the original db file shutil.copy(new_db_file, final_db_file) os.remove(orig_db_file) finally: # always cleanup the temporary directory used for the migration shutil.rmtree(tmpdir, True)