def setUp(self): db_configs = [] for engine in get_testable_engines(): config = TestDatabaseConfiguration(protocol=engine) db_configs.append(config) self.database_name = 'test_database' self.dbs = [] for config in db_configs: try: server = DatabaseServer(config) if server.has_database(self.database_name): server.drop_database(self.database_name) server.create_database(self.database_name) self.assertTrue( server.has_database(database_name=self.database_name)) db = OpusDatabase(database_server_configuration=config, database_name=self.database_name) storage = sql_storage(storage_location=db) self.dbs.append((db, server, storage)) self.storage = storage except: import traceback traceback.print_exc() print 'WARNING: could not start server for protocol %s' % config.protocol
def _get_db(self): #db_server = DatabaseServer(self.database_server_config) #db = db_server.get_database(self.database_name) #db_server.close() #return db db = OpusDatabase(self.database_server_config, self.database_name) return db
def get_database(self, database_name, create_if_doesnt_exist=True): """ Returns an object connecting to this database on this database server. """ from opus_core.database_management.opus_database import OpusDatabase if create_if_doesnt_exist: self.create_database(database_name=database_name) database = OpusDatabase(database_server_configuration=self.config, database_name=database_name) if database_name in self.open_databases: self.open_databases[database_name].append(database) else: self.open_databases[database_name] = [database] return database
def _test_export_one_table(self, table_name): optional_args = [ '-c', os.path.join(self.temp_dir, str(self.year)), '-d', self.test_db, '-t', table_name, '--database_configuration=%s' % self.db_config_node ] self._call_script(self.export_from_cache_opus_path, args=optional_args) self.assertTrue( self.db_server.has_database(database_name=self.test_db)) db = OpusDatabase(database_server_configuration=self.db_config, database_name=self.test_db) existing_tables = db.get_tables_in_database() self.assertEqual(set(existing_tables), set([table_name])) export_year = str(self.year + 100) exp_dir = os.path.join(self.temp_dir, export_year) optional_args = [ '-d', self.test_db, '-c', self.temp_dir, '-y', export_year, '-t', table_name, '--database_configuration=%s' % self.db_config_node ] self._call_script(self.export_to_cache_opus_path, args=optional_args) exported_datasets = [ os.path.split(f)[1] for f in glob(os.path.join(self.temp_dir, export_year) + '/*') ] self.assertEqual(set(exported_datasets), set([table_name])) org_dir = os.path.join(self.temp_dir, str(self.year)) self._two_caches_are_identical(org_dir, exp_dir, table_names=[table_name]) db.close() self.db_server.drop_database(self.test_db) rmtree(exp_dir)
def test_export_all_tables(self): logger.log_status("Test export all tables for %s with %s" % (self.protocol, self.__class__)) optional_args = [ '-c', os.path.join(self.temp_dir, str(self.year)), '-d', self.test_db, '--database_configuration=%s' % self.db_config_node ] self._call_script(self.export_from_cache_opus_path, args=optional_args) self.assertTrue( self.db_server.has_database(database_name=self.test_db)) db = OpusDatabase(database_server_configuration=self.db_config, database_name=self.test_db) table_names = self.test_data[self.year].keys() existing_tables = db.get_tables_in_database() self.assertEqual(set(existing_tables), set(table_names)) ## export data from db to cache export_year = str(self.year + 100) exp_dir = os.path.join(self.temp_dir, export_year) optional_args = [ '-d', self.test_db, '-c', self.temp_dir, '-y', export_year, '--database_configuration=%s' % self.db_config_node ] self._call_script(self.export_to_cache_opus_path, args=optional_args) exported_datasets = [os.path.split(f)[1] for f in glob(exp_dir + '/*')] self.assertEqual(set(exported_datasets), set(table_names)) org_dir = os.path.join(self.temp_dir, str(self.year)) self._two_caches_are_identical(org_dir, exp_dir) db.close() self.db_server.drop_database(self.test_db) rmtree(exp_dir)
attribute_cache_directory = options.attribute_cache_directory table_name = options.table_name cache_year = options.cache_year if (db_name is None or attribute_cache_directory is None or cache_year is None): parser.print_help() sys.exit(1) if table_name is None: table_name = 'ALL' dbserverconfig = EstimationDatabaseConfiguration( database_configuration=options.database_configuration) opusdb = OpusDatabase(dbserverconfig, db_name) input_storage = sql_storage(storage_location=opusdb) attribute_cache = AttributeCache(cache_directory=attribute_cache_directory) output_storage = attribute_cache.get_flt_storage_for_year(cache_year) SimulationState().set_current_time(cache_year) SessionConfiguration(new_instance=True, package_order=[], in_storage=AttributeCache()) if table_name == 'ALL': logger.log_status('Caching all tables in database...') lst = input_storage.get_table_names() for i in lst: ExportStorage().export_dataset(