def _test_export_one_table(self, table_name):
        optional_args = ['-c', os.path.join(self.temp_dir, str(self.year)), 
                         '-d', self.test_db,
                         '-t', table_name,
                         '--database_configuration=%s' % self.db_config_node ]
        self._call_script(self.export_from_cache_opus_path,
                          args = optional_args)
        
        self.assertTrue(self.db_server.has_database(database_name = self.test_db))
        db = OpusDatabase(database_server_configuration = self.db_config, 
                          database_name = self.test_db)
        existing_tables = db.get_tables_in_database()
        self.assertEqual( set(existing_tables), set([table_name]) )
        
        export_year = str(self.year + 100)        
        exp_dir = os.path.join(self.temp_dir, export_year)
        
        optional_args = ['-d', self.test_db, 
                         '-c', self.temp_dir, 
                         '-y', export_year,
                         '-t', table_name,
                         '--database_configuration=%s' % self.db_config_node ]
        self._call_script(self.export_to_cache_opus_path,
                          args = optional_args)

        exported_datasets = [os.path.split(f)[1] for f in glob(os.path.join(self.temp_dir, export_year) + '/*')]
        self.assertEqual( set(exported_datasets), set([table_name]))
        
        org_dir = os.path.join(self.temp_dir, str(self.year))
        self._two_caches_are_identical(org_dir, exp_dir, table_names=[table_name])

        db.close()
        self.db_server.drop_database(self.test_db)
        rmtree(exp_dir)
예제 #2
0
        def setUp(self):

            db_configs = []
            for engine in get_testable_engines():
                config = TestDatabaseConfiguration(protocol=engine)
                db_configs.append(config)

            self.database_name = 'test_database'
            self.dbs = []
            for config in db_configs:
                try:
                    server = DatabaseServer(config)
                    if server.has_database(self.database_name):
                        server.drop_database(self.database_name)
                    server.create_database(self.database_name)
                    self.assertTrue(
                        server.has_database(database_name=self.database_name))
                    db = OpusDatabase(database_server_configuration=config,
                                      database_name=self.database_name)
                    storage = sql_storage(storage_location=db)
                    self.dbs.append((db, server, storage))
                    self.storage = storage
                except:
                    import traceback
                    traceback.print_exc()

                    print 'WARNING: could not start server for protocol %s' % config.protocol
예제 #3
0
 def _get_db(self):
     #db_server = DatabaseServer(self.database_server_config)
     #db = db_server.get_database(self.database_name)
     #db_server.close()
     #return db
     db = OpusDatabase(self.database_server_config, self.database_name)
     return db
    def _test_export_one_table(self, table_name):
        optional_args = [
            '-c',
            os.path.join(self.temp_dir, str(self.year)), '-d', self.test_db,
            '-t', table_name,
            '--database_configuration=%s' % self.db_config_node
        ]
        self._call_script(self.export_from_cache_opus_path, args=optional_args)

        self.assertTrue(
            self.db_server.has_database(database_name=self.test_db))
        db = OpusDatabase(database_server_configuration=self.db_config,
                          database_name=self.test_db)
        existing_tables = db.get_tables_in_database()
        self.assertEqual(set(existing_tables), set([table_name]))

        export_year = str(self.year + 100)
        exp_dir = os.path.join(self.temp_dir, export_year)

        optional_args = [
            '-d', self.test_db, '-c', self.temp_dir, '-y', export_year, '-t',
            table_name,
            '--database_configuration=%s' % self.db_config_node
        ]
        self._call_script(self.export_to_cache_opus_path, args=optional_args)

        exported_datasets = [
            os.path.split(f)[1]
            for f in glob(os.path.join(self.temp_dir, export_year) + '/*')
        ]
        self.assertEqual(set(exported_datasets), set([table_name]))

        org_dir = os.path.join(self.temp_dir, str(self.year))
        self._two_caches_are_identical(org_dir,
                                       exp_dir,
                                       table_names=[table_name])

        db.close()
        self.db_server.drop_database(self.test_db)
        rmtree(exp_dir)
    def test_export_all_tables(self):
        logger.log_status("Test export all tables for %s with %s" %
                          (self.protocol, self.__class__))
        optional_args = [
            '-c',
            os.path.join(self.temp_dir, str(self.year)), '-d', self.test_db,
            '--database_configuration=%s' % self.db_config_node
        ]
        self._call_script(self.export_from_cache_opus_path, args=optional_args)

        self.assertTrue(
            self.db_server.has_database(database_name=self.test_db))
        db = OpusDatabase(database_server_configuration=self.db_config,
                          database_name=self.test_db)

        table_names = self.test_data[self.year].keys()
        existing_tables = db.get_tables_in_database()
        self.assertEqual(set(existing_tables), set(table_names))

        ## export data from db to cache
        export_year = str(self.year + 100)
        exp_dir = os.path.join(self.temp_dir, export_year)

        optional_args = [
            '-d', self.test_db, '-c', self.temp_dir, '-y', export_year,
            '--database_configuration=%s' % self.db_config_node
        ]
        self._call_script(self.export_to_cache_opus_path, args=optional_args)

        exported_datasets = [os.path.split(f)[1] for f in glob(exp_dir + '/*')]
        self.assertEqual(set(exported_datasets), set(table_names))

        org_dir = os.path.join(self.temp_dir, str(self.year))
        self._two_caches_are_identical(org_dir, exp_dir)

        db.close()
        self.db_server.drop_database(self.test_db)
        rmtree(exp_dir)
    def test_export_all_tables(self):
        logger.log_status("Test export all tables for %s with %s" % (self.protocol, self.__class__))
        optional_args = ['-c', os.path.join(self.temp_dir, str(self.year)), 
                         '-d', self.test_db, 
                         '--database_configuration=%s' % self.db_config_node ]
        self._call_script(self.export_from_cache_opus_path,
                          args = optional_args)

        
        self.assertTrue(self.db_server.has_database(database_name = self.test_db))
        db = OpusDatabase(database_server_configuration = self.db_config, 
                          database_name = self.test_db)
        
        table_names = self.test_data[self.year].keys()
        existing_tables = db.get_tables_in_database()        
        self.assertEqual( set(existing_tables), set(table_names) )

        ## export data from db to cache
        export_year = str(self.year + 100)        
        exp_dir = os.path.join(self.temp_dir, export_year)

        optional_args = ['-d', self.test_db, 
                         '-c', self.temp_dir, 
                         '-y', export_year,
                         '--database_configuration=%s' % self.db_config_node ]
        self._call_script(self.export_to_cache_opus_path,
                          args = optional_args)

        exported_datasets = [os.path.split(f)[1] for f in glob(exp_dir + '/*')]
        self.assertEqual( set(exported_datasets), set(table_names))
        
        org_dir = os.path.join(self.temp_dir, str(self.year))
        self._two_caches_are_identical(org_dir, exp_dir)

        db.close()
        self.db_server.drop_database(self.test_db)
        rmtree(exp_dir)
예제 #7
0
    def get_database(self, database_name, create_if_doesnt_exist=True):
        """
        Returns an object connecting to this database on this database server.
        """
        from opus_core.database_management.opus_database import OpusDatabase

        if create_if_doesnt_exist:
            self.create_database(database_name=database_name)

        database = OpusDatabase(database_server_configuration=self.config,
                                database_name=database_name)

        if database_name in self.open_databases:
            self.open_databases[database_name].append(database)
        else:
            self.open_databases[database_name] = [database]

        return database
    attribute_cache_directory = options.attribute_cache_directory
    table_name = options.table_name
    cache_year = options.cache_year

    if (db_name is None or attribute_cache_directory is None
            or cache_year is None):

        parser.print_help()
        sys.exit(1)

    if table_name is None:
        table_name = 'ALL'

    dbserverconfig = EstimationDatabaseConfiguration(
        database_configuration=options.database_configuration)
    opusdb = OpusDatabase(dbserverconfig, db_name)

    input_storage = sql_storage(storage_location=opusdb)

    attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
    output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
    SimulationState().set_current_time(cache_year)
    SessionConfiguration(new_instance=True,
                         package_order=[],
                         in_storage=AttributeCache())

    if table_name == 'ALL':
        logger.log_status('Caching all tables in database...')
        lst = input_storage.get_table_names()
        for i in lst:
            ExportStorage().export_dataset(