def handle_postgresql_db(database_url, table_name, pycsw_home):
_wait_for_postgresql_db(database_url)
try:
admin.setup_db(database=database_url, table=table_name,
home=pycsw_home)
except ProgrammingError:
pass # database tables are already created
def setup(self) -> None:
"""
Initialises the backing database for the catalogue
Convenience method to call the PyCSW admin task for setting up the required database components (tables,
indexes, triggers, etc.)
Note: There are currently limitations with using multiple catalogues within one schema. The specific errors
this causes (which are not fatal) are detected by this method and treated as a false positive. See the project
README for more information.
"""
if self._is_initialised:
raise CSWDatabaseAlreadyInitialisedException()
csw_database = create_engine(
self._csw_config["repository"]["database"])
csw_database.execute("SELECT version();")
try:
admin.setup_db(
database=self._csw_config["repository"]["database"],
table=self._csw_config["repository"]["table"],
home=None,
)
except ProgrammingError as e:
# Ignore errors related to PyCSW's limitations with non-namespaced indexes
if 'ERROR: relation "fts_gin_idx" already exists' not in e.orig.pgerror:
raise CSWDatabaseAlreadyInitialisedException()
pass
def handle_postgresql_db(database_url, table_name, pycsw_home):
_wait_for_postgresql_db(database_url)
try:
admin.setup_db(database=database_url, table=table_name,
home=pycsw_home)
except ProgrammingError:
pass # database tables are already created
def _initialize_database(repository_url, table_name, data_dir, test_dir,
export_dir):
"""Initialize database for tests.
This function will create the database and load any test data that
the suite may require.
Parameters
----------
repository_url: str
URL for the repository, as used by SQLAlchemy engines
table_name: str
Name of the table that is to be used to store pycsw records
data_dir: str
Path to a directory that contains sample data records to be loaded
into the database
test_dir: str
Directory where the database is to be created, in case of sqlite.
export_dir: str
Diretory where the exported records are to be saved, if any
"""
print("Setting up {0!r} repository...".format(repository_url))
if repository_url.startswith("postgresql"):
extra_kwargs = {
"create_sfsql_tables": True,
"create_plpythonu_functions": False
}
else:
extra_kwargs = {}
admin.setup_db(database=repository_url,
table=table_name,
home=test_dir,
**extra_kwargs)
if len(os.listdir(data_dir)) > 0:
print("Loading database data...")
loaded = admin.load_records(context=StaticContext(),
database=repository_url,
table=table_name,
xml_dirpath=data_dir,
recursive=True)
admin.optimize_db(context=StaticContext(),
database=repository_url,
table=table_name)
if export_dir is not None:
# Attempt to export files
exported = admin.export_records(context=StaticContext(),
database=repository_url,
table=table_name,
xml_dirpath=export_dir)
if len(loaded) != len(exported):
raise ValueError(
"Loaded records (%s) is different from exported records (%s)"
% (len(loaded), len(exported)))
# Remove the files that were exported since this was just a test
for toremove in exported:
os.remove(toremove)
def _initialize_database(repository_url, table_name, data_dir, test_dir, export_dir):
"""Initialize database for tests.
This function will create the database and load any test data that
the suite may require.
Parameters
----------
repository_url: str
URL for the repository, as used by SQLAlchemy engines
table_name: str
Name of the table that is to be used to store pycsw records
data_dir: str
Path to a directory that contains sample data records to be loaded
into the database
test_dir: str
Directory where the database is to be created, in case of sqlite.
export_dir: str
Diretory where the exported records are to be saved, if any
"""
print("Setting up {0!r} repository...".format(repository_url))
if repository_url.startswith("postgresql"):
extra_kwargs = {
"create_sfsql_tables": True,
"create_plpythonu_functions": False
}
else:
extra_kwargs = {}
admin.setup_db(database=repository_url, table=table_name, home=test_dir,
**extra_kwargs)
if len(os.listdir(data_dir)) > 0:
print("Loading database data...")
loaded = admin.load_records(
context=StaticContext(),
database=repository_url,
table=table_name,
xml_dirpath=data_dir,
recursive=True
)
admin.optimize_db(context=StaticContext(), database=repository_url, table=table_name)
if export_dir is not None:
# Attempt to export files
exported = admin.export_records(
context=StaticContext(),
database=repository_url,
table=table_name,
xml_dirpath=export_dir
)
if len(loaded) != len(exported):
raise ValueError(
"Loaded records (%s) is different from exported records (%s)" %
(len(loaded), len(exported))
)
# Remove the files that were exported since this was just a test
for toremove in exported:
os.remove(toremove)
def handle_sqlite_db(database_url, table_name, pycsw_home):
db_path = database_url.rpartition(":///")[-1]
if not os.path.isfile(db_path):
try:
os.makedirs(os.path.dirname(db_path))
except OSError as exc:
if exc.args[0] == 17: # directory already exists
pass
admin.setup_db(database=database_url, table=table_name,
home=pycsw_home)
def handle_sqlite_db(database_url, table_name, pycsw_home):
db_path = database_url.rpartition(":///")[-1]
if not os.path.isfile(db_path):
try:
os.makedirs(os.path.dirname(db_path))
except OSError as exc:
if exc.args[0] == 17: # directory already exists
pass
admin.setup_db(database=database_url, table=table_name,
home=pycsw_home)
def handle_db(self, args):
database, table = self._get_db_settings()
if args.db_command == "create":
home = self.config.get('server', 'home')
admin.setup_db(database, table, home)
elif args.db_command == "optimize":
admin.optimize_db(self.context, database, table)
elif args.db_command == "rebuild":
admin.rebuild_db_indexes(database, table)
elif args.db_command == "clean":
force = args.accept_changes
if not force:
msg = "This will delete all records! Continue [Y/n] "
if raw_input(msg) == 'Y':
force = True
if force:
admin.delete_records(self.context, database, table)
def handle_db(self, args):
database, table = self._get_db_settings()
if args.db_command == "create":
home = self.config.get('server', 'home')
admin.setup_db(database, table, home)
elif args.db_command == "optimize":
admin.optimize_db(self.context, database, table)
elif args.db_command == "rebuild":
admin.rebuild_db_indexes(database, table)
elif args.db_command == "clean":
force = args.accept_changes
if not force:
msg = "This will delete all records! Continue [Y/n] "
if raw_input(msg) == 'Y':
force = True
if force:
admin.delete_records(self.context, database, table)
def _initialize_database(repository_url, table_name, data_dir, test_dir):
"""Initialize database for tests.
This function will create the database and load any test data that
the suite may require.
Parameters
----------
repository_url: str
URL for the repository, as used by SQLAlchemy engines
table_name: str
Name of the table that is to be used to store pycsw records
data_dir: str
Path to a directory that contains sample data records to be loaded
into the database
test_dir: str
Directory where the database is to be created, in case of sqlite.
"""
print("Setting up {0!r} repository...".format(repository_url))
if repository_url.startswith("postgresql"):
extra_kwargs = {
"create_sfsql_tables": True,
"create_plpythonu_functions": False
}
else:
extra_kwargs = {}
admin.setup_db(database=repository_url,
table=table_name,
home=test_dir,
**extra_kwargs)
if len(os.listdir(data_dir)) > 0:
print("Loading database data...")
admin.load_records(context=StaticContext(),
database=repository_url,
table=table_name,
xml_dirpath=data_dir)
if XML is None:
print('ERROR: -x /path/to/request.xml is a required argument')
sys.exit(10)
elif COMMAND == 'validate_xml':
if XML is None:
print('ERROR: -x /path/to/file.xml is a required argument')
sys.exit(11)
if XSD is None:
print('ERROR: -s /path/to/file.xsd is a required argument')
sys.exit(12)
if LOGGING is not None:
logging.basicConfig(stream=sys.stdout, level=getattr(logging, LOGGING))
if COMMAND == 'setup_db':
try:
admin.setup_db(DATABASE, TABLE, HOME, mappings_filepath=MAPPINGS)
except Exception as err:
print(err)
print('ERROR: DB creation error. Database tables already exist')
print('Delete tables or database to reinitialize')
elif COMMAND == 'load_records':
admin.load_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH, RECURSIVE, FORCE_CONFIRM)
elif COMMAND == 'export_records':
admin.export_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH)
elif COMMAND == 'rebuild_db_indexes':
admin.rebuild_db_indexes(DATABASE, TABLE)
elif COMMAND == 'optimize_db':
admin.optimize_db(CONTEXT, DATABASE, TABLE)
elif COMMAND == 'refresh_harvested_records':
admin.refresh_harvested_records(CONTEXT, DATABASE, TABLE, URL)
elif COMMAND == 'gen_sitemap':
print 'ERROR: -u <http://host/csw> is a required argument'
sys.exit(9)
if XML is None:
print 'ERROR: -x /path/to/request.xml is a required argument'
sys.exit(10)
elif COMMAND == 'validate_xml':
if XML is None:
print 'ERROR: -x /path/to/file.xml is a required argument'
sys.exit(11)
if XSD is None:
print 'ERROR: -s /path/to/file.xsd is a required argument'
sys.exit(12)
if COMMAND == 'setup_db':
try:
admin.setup_db(DATABASE, TABLE, HOME)
except Exception as err:
print err
print 'ERROR: DB creation error. Database tables already exist'
print 'Delete tables or database to reinitialize'
elif COMMAND == 'load_records':
admin.load_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH, RECURSIVE, FORCE_CONFIRM)
elif COMMAND == 'export_records':
admin.export_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH)
elif COMMAND == 'rebuild_db_indexes':
admin.rebuild_db_indexes(DATABASE, TABLE)
elif COMMAND == 'optimize_db':
admin.optimize_db(CONTEXT, DATABASE, TABLE)
elif COMMAND == 'refresh_harvested_records':
admin.refresh_harvested_records(CONTEXT, DATABASE, TABLE, URL)
elif COMMAND == 'gen_sitemap':
print('ERROR: -u <http://host/csw> is a required argument')
sys.exit(9)
if XML is None:
print('ERROR: -x /path/to/request.xml is a required argument')
sys.exit(10)
elif COMMAND == 'validate_xml':
if XML is None:
print('ERROR: -x /path/to/file.xml is a required argument')
sys.exit(11)
if XSD is None:
print('ERROR: -s /path/to/file.xsd is a required argument')
sys.exit(12)
if COMMAND == 'setup_db':
try:
admin.setup_db(DATABASE, TABLE, HOME)
except Exception as err:
print(err)
print('ERROR: DB creation error. Database tables already exist')
print('Delete tables or database to reinitialize')
elif COMMAND == 'load_records':
admin.load_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH, RECURSIVE,
FORCE_CONFIRM)
elif COMMAND == 'export_records':
admin.export_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH)
elif COMMAND == 'rebuild_db_indexes':
admin.rebuild_db_indexes(DATABASE, TABLE)
elif COMMAND == 'optimize_db':
admin.optimize_db(CONTEXT, DATABASE, TABLE)
elif COMMAND == 'refresh_harvested_records':
admin.refresh_harvested_records(CONTEXT, DATABASE, TABLE, URL)
print('ERROR: -u <http://host/csw> is a required argument')
sys.exit(9)
if XML is None:
print('ERROR: -x /path/to/request.xml is a required argument')
sys.exit(10)
elif COMMAND == 'validate_xml':
if XML is None:
print('ERROR: -x /path/to/file.xml is a required argument')
sys.exit(11)
if XSD is None:
print('ERROR: -s /path/to/file.xsd is a required argument')
sys.exit(12)
if COMMAND == 'setup_db':
try:
admin.setup_db(DATABASE, TABLE, HOME, extra_columns=EXTRA_COLUMNS)
except Exception as err:
print(err)
print('ERROR: DB creation error. Database tables already exist')
print('Delete tables or database to reinitialize')
elif COMMAND == 'load_records':
admin.load_records(CONTEXT, DATABASE, TABLE,
XML_DIRPATH, RECURSIVE, FORCE_CONFIRM)
elif COMMAND == 'export_records':
admin.export_records(CONTEXT, DATABASE, TABLE, XML_DIRPATH)
elif COMMAND == 'rebuild_db_indexes':
admin.rebuild_db_indexes(DATABASE, TABLE)
elif COMMAND == 'optimize_db':
admin.optimize_db(CONTEXT, DATABASE, TABLE)
elif COMMAND == 'refresh_harvested_records':
admin.refresh_harvested_records(CONTEXT, DATABASE, TABLE, URL)
os.environ['DJANGO_SETTINGS_MODULE'] = 'registry'
if 'pycsw' in sys.argv[:2]:
OPTS, ARGS = getopt.getopt(sys.argv[2:], 'c:f:ho:p:ru:x:s:t:y')
for o, a in OPTS:
if o == '-c':
COMMAND = a
database = PYCSW['repository']['database']
table = PYCSW['repository']['table']
home = PYCSW['server']['home']
available_commands = ['setup_db', 'get_sysprof']
if COMMAND not in available_commands:
print('pycsw supports only the following commands: %s' %
available_commands)
sys.exit(1)
if COMMAND == 'setup_db':
pycsw_admin.setup_db(database, table, home)
elif COMMAND == 'get_sysprof':
print(pycsw_admin.get_sysprof())
sys.exit(0)
management.execute_from_command_line()
def create():
database = pycsw_config['repository']['database']
table = pycsw_config['repository']['table']
home = pycsw_config['server']['home']
admin.setup_db(database, table, home)
def run(self, argv):
if len(argv) == 0:
grass.error('Nothing to do. Set args')
return
print argv
try:
OPTS, ARGS = getopt.getopt(argv, 'c:f:ho:p:ru:x:s:t:y')
except getopt.GetoptError as err:
grass.error('\nERROR: %s' % err)
#print usage()
for o, a in OPTS:
if o == '-c':
self.COMMAND = a
if o == '-f':
self.CFG = a
if o == '-o':
self.OUTPUT_FILE = a
if o == '-p':
self.XML_DIRPATH = a
if o == '-r':
self.RECURSIVE = True
if o == '-u':
self.CSW_URL = a
if o == '-x':
self.XML = a
if o == '-t':
self.TIMEOUT = int(a)
if o == '-y':
self.FORCE_CONFIRM = True
if self.CFG is None and self.COMMAND not in ['post_xml']:
print 'ERROR: -f <cfg> is a required argument'
if self.COMMAND not in ['post_xml']:
SCP = ConfigParser.SafeConfigParser()
SCP.readfp(open(self.CFG))
self.DATABASE = SCP.get('repository', 'database')
self.URL = SCP.get('server', 'url')
self.HOME = SCP.get('server', 'home')
self.METADATA = dict(SCP.items('metadata:main'))
try:
self.TABLE = SCP.get('repository', 'table')
except ConfigParser.NoOptionError:
self.TABLE = 'records'
if self.COMMAND == 'setup_db':
try:
admin.setup_db(self.DATABASE, self.TABLE, self.HOME)
except Exception as err:
print err
print 'ERROR: DB creation error. Database tables already exist'
print 'Delete tables or database to reinitialize'
elif self.COMMAND == 'load_records':
admin.load_records(self.CONTEXT, self.DATABASE, self.TABLE,
self.XML_DIRPATH, self.RECURSIVE,
self.FORCE_CONFIRM)
elif self.COMMAND == 'export_records':
admin.export_records(self.CONTEXT, self.DATABASE, self.TABLE,
self.XML_DIRPATH)
elif self.COMMAND == 'rebuild_db_indexes':
admin.rebuild_db_indexes(self.DATABASE, self.TABLE)
elif self.COMMAND == 'optimize_db':
admin.optimize_db(self.CONTEXT, self.DATABASE, self.TABLE)
elif self.COMMAND == 'refresh_harvested_records':
admin.refresh_harvested_records(self.CONTEXT, self.DATABASE,
self.TABLE, self.URL)
elif self.COMMAND == 'gen_sitemap':
admin.gen_sitemap(self.CONTEXT, self.DATABASE, self.TABLE,
self.URL, self.OUTPUT_FILE)
elif self.COMMAND == 'post_xml':
grass.message(admin.post_xml(self.CSW_URL, self.XML, self.TIMEOUT))
elif self.COMMAND == 'delete_records':
admin.delete_records(self.CONTEXT, self.DATABASE, self.TABLE)
