def get_transform_instance(type, isopen, config, database, destination_collection, run_until_timestamp=None): if type == TRANSFORM_TYPE_COMPILE_RELEASES: return CompileReleasesTransform( config, database, destination_collection, run_until_timestamp=run_until_timestamp) elif type == TRANSFORM_TYPE_UPGRADE_1_0_TO_1_1: return Upgrade10To11Transform(config, database, destination_collection, run_until_timestamp=run_until_timestamp) elif type == TRANSFORM_TYPE_RECORD and isopen: return RecordTransform(config, database, destination_collection, run_until_timestamp=run_until_timestamp) elif type == TRANSFORM_TYPE_RECORD_NEW_ONLY: return RecordTransformNewOnly(config, database, destination_collection, run_until_timestamp=run_until_timestamp) else: raise Exception("That transform type is not known")
def test_release_1(self): # Make source collection source_collection_id = self.database.get_or_create_collection_id("test", datetime.datetime.now(), False) source_collection = self.database.get_collection(source_collection_id) # Load some data store = Store(self.config, self.database) store.set_collection(source_collection) json_filename = os.path.join(os.path.dirname( os.path.realpath(__file__)), 'fixtures', 'sample_1_0_releases.json' ) store.store_file_from_local("test.json", "http://example.com", "release_package", "utf-8", json_filename) # Make destination collection destination_collection_id = self.database.get_or_create_collection_id( source_collection.source_id, source_collection.data_version, source_collection.sample, transform_from_collection_id=source_collection_id, transform_type=TRANSFORM_TYPE_UPGRADE_1_0_TO_1_1) destination_collection = self.database.get_collection(destination_collection_id) # transform! transform = Upgrade10To11Transform(self.config, self.database, destination_collection) transform.process() # check with self.database.get_engine().begin() as connection: s = sa.sql.select([self.database.release_table]) result = connection.execute(s) assert 4 == result.rowcount s = sa.sql.select([self.database.record_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.transform_upgrade_1_0_to_1_1_status_record_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.transform_upgrade_1_0_to_1_1_status_release_table]) result = connection.execute(s) assert 2 == result.rowcount # transform again! This should be fine transform = Upgrade10To11Transform(self.config, self.database, destination_collection) transform.process() # check with self.database.get_engine().begin() as connection: s = sa.sql.select([self.database.release_table]) result = connection.execute(s) assert 4 == result.rowcount s = sa.sql.select([self.database.record_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.transform_upgrade_1_0_to_1_1_status_record_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.transform_upgrade_1_0_to_1_1_status_release_table]) result = connection.execute(s) assert 2 == result.rowcount # destination collection will not be closed (because source is still open!) destination_collection = self.database.get_collection(destination_collection_id) assert destination_collection.store_end_at is None # Mark source collection as finished self.database.mark_collection_store_done(source_collection_id) # transform! transform = Upgrade10To11Transform(self.config, self.database, destination_collection) transform.process() # destination collection should be closed destination_collection = self.database.get_collection(destination_collection_id) assert destination_collection.store_end_at is not None
def test_two_collections_with_upgrade_1_0_to_1_1(self): # Source Collection source_collection_id = self.database.get_or_create_collection_id("test", datetime.datetime.now(), False) source_collection = self.database.get_collection(source_collection_id) store = Store(self.config, self.database) store.set_collection(source_collection) json_filename = os.path.join(os.path.dirname( os.path.realpath(__file__)), 'data', 'sample_1_0_record.json' ) store.store_file_from_local("test.json", "http://example.com", "record", "utf-8", json_filename) self.database.mark_collection_store_done(source_collection_id) # Destination Collection destination_collection_id = self.database.get_or_create_collection_id("test", datetime.datetime.now(), False, transform_from_collection_id=source_collection_id, transform_type=TRANSFORM_TYPE_UPGRADE_1_0_TO_1_1) destination_collection = self.database.get_collection(destination_collection_id) transform = Upgrade10To11Transform(self.config, self.database, destination_collection) transform.process() # Check Number of rows in various tables with self.database.get_engine().begin() as connection: s = sa.sql.select([self.database.collection_table]) result = connection.execute(s) assert 2 == result.rowcount s = sa.sql.select([self.database.collection_file_table]) result = connection.execute(s) assert 2 == result.rowcount s = sa.sql.select([self.database.collection_file_item_table]) result = connection.execute(s) assert 2 == result.rowcount s = sa.sql.select([self.database.record_table]) result = connection.execute(s) assert 2 == result.rowcount s = sa.sql.select([self.database.transform_upgrade_1_0_to_1_1_status_record_table]) result = connection.execute(s) assert 1 == result.rowcount s = sa.sql.select([self.database.data_table]) result = connection.execute(s) assert 0 < result.rowcount s = sa.sql.select([self.database.package_data_table]) result = connection.execute(s) assert 0 < result.rowcount # Delete self.database.mark_collection_deleted_at(source_collection_id) self.database.delete_collection(source_collection_id) self.database.mark_collection_deleted_at(destination_collection_id) self.database.delete_collection(destination_collection_id) # Check Number of rows in various tables with self.database.get_engine().begin() as connection: s = sa.sql.select([self.database.collection_table]) result = connection.execute(s) assert 2 == result.rowcount s = sa.sql.select([self.database.collection_file_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.collection_file_item_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.record_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.transform_upgrade_1_0_to_1_1_status_record_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.data_table]) result = connection.execute(s) assert 0 == result.rowcount s = sa.sql.select([self.database.package_data_table]) result = connection.execute(s) assert 0 == result.rowcount