def test_film_version_03(self):
"Update schema to version 0.3"
self.execute_script(__file__, 'film-schema-0.3a.sql')
self.execute_script(__file__, 'film-schema-0.3b.sql')
# Run pg_dump against source database
srcdump = self.tempfile_path('film-0.3-src.dump')
self.run_pg_dump(srcdump, True, True)
# Create source YAML file
usercfg = self.tempfile_path("usercfg.yaml")
with open(usercfg, 'w') as f:
f.write(yamldump({'repository': {'path': self.tempfile_path('')}}))
os.environ["PYRSEAS_USER_CONFIG"] = usercfg
with open(self.tempfile_path("config.yaml"), 'w') as f:
f.write(yamldump({'datacopy': {'schema sd': ['genre']}}))
srcyaml = self.tempfile_path('film-0.3-src.yaml')
self.create_yaml(srcyaml, True)
# Migrate the target database
self.migrate_target(srcyaml, self.tempfile_path('film-0.3.sql'))
# Run pg_dump against target database
targdump = self.tempfile_path('film-0.3.dump')
self.run_pg_dump(targdump, False, True)
# Create target YAML file
targyaml = self.tempfile_path('film-0.3.yaml')
self.create_yaml(targyaml)
# diff film-0.3-src.dump against film-0.3.dump
assert self.lines(srcdump) == self.lines(targdump)
# diff film-0.3-src.yaml against film-0.3.yaml
assert self.lines(srcyaml) == self.lines(targyaml)
def test_film_version_03(self):
"Update schema to version 0.3"
self.execute_script(__file__, 'film-schema-0.3a.sql')
self.execute_script(__file__, 'film-schema-0.3b.sql')
# Run pg_dump against source database
srcdump = self.tempfile_path('film-0.3-src.dump')
self.run_pg_dump(srcdump, True, True)
# Create source YAML file
usercfg = self.tempfile_path("usercfg.yaml")
with open(usercfg, 'w') as f:
f.write(yamldump({'repository': {'path': self.tempfile_path('')}}))
os.environ["PYRSEAS_USER_CONFIG"] = usercfg
with open(self.tempfile_path("config.yaml"), 'w') as f:
f.write(yamldump({'datacopy': {'schema sd': ['genre']}}))
srcyaml = self.tempfile_path('film-0.3-src.yaml')
self.create_yaml(srcyaml, True)
# Migrate the target database
self.migrate_target(srcyaml, self.tempfile_path('film-0.3.sql'))
# Run pg_dump against target database
targdump = self.tempfile_path('film-0.3.dump')
self.run_pg_dump(targdump, False, True)
# Create target YAML file
targyaml = self.tempfile_path('film-0.3.yaml')
self.create_yaml(targyaml)
# diff film-0.3-src.dump against film-0.3.dump
assert self.lines(srcdump) == self.lines(targdump)
# diff film-0.3-src.yaml against film-0.3.yaml
assert self.lines(srcyaml) == self.lines(targyaml)
def test_repo_config(tmpdir):
"Test a repository configuration file"
ucfg = tmpdir.join(CFG_FILE)
ucfg.write(yamldump({'repository': {'path': tmpdir.strpath}}))
f = tmpdir.join("config.yaml")
f.write(yamldump(CFG_DATA))
os.environ["PYRSEAS_USER_CONFIG"] = ucfg.strpath
cfg = Config()
assert cfg['datacopy'] == CFG_TABLE_DATA
def test_repo_config(tmpdir):
"Test a repository configuration file"
ucfg = tmpdir.join(CFG_FILE)
ucfg.write(yamldump({'repository': {'path': tmpdir.strpath}}))
f = tmpdir.join("config.yaml")
f.write(yamldump(CFG_DATA))
os.environ["PYRSEAS_USER_CONFIG"] = ucfg.strpath
cfg = Config()
assert cfg['datacopy'] == CFG_TABLE_DATA
def test_repo_user_config(tmpdir):
"Test a repository path specified in the user config"
usercfg = {'repository': {'path': tmpdir.strpath}}
userf = tmpdir.join("usercfg.yaml")
userf.write(yamldump(usercfg))
os.environ["PYRSEAS_USER_CONFIG"] = userf.strpath
repof = tmpdir.join("config.yaml")
repof.write(yamldump(CFG_DATA))
cfg = Config()
assert cfg['datacopy'] == CFG_TABLE_DATA
def test_repo_user_config(tmpdir):
"Test a repository path specified in the user config"
usercfg = {'repository': {'path': tmpdir.strpath}}
userf = tmpdir.join("usercfg.yaml")
userf.write(yamldump(usercfg))
os.environ["PYRSEAS_USER_CONFIG"] = userf.strpath
repof = tmpdir.join("config.yaml")
repof.write(yamldump(CFG_DATA))
cfg = Config()
assert cfg['datacopy'] == CFG_TABLE_DATA
def main():
"""Augment database specifications"""
parser = cmd_parser("Generate a modified schema for a PostgreSQL "
"database, in YAML format, augmented with specified "
"attributes and procedures", __version__)
# TODO: processing of multiple files, owner and privileges
parser.add_argument('-m', '--multiple-files', action='store_true',
help='multiple files (metadata directory)')
parser.add_argument('-O', '--no-owner', action='store_true',
help='exclude object ownership information')
parser.add_argument('-x', '--no-privileges', action='store_true',
dest='no_privs',
help='exclude privilege (GRANT/REVOKE) information')
parser.add_argument('spec', nargs='?', type=FileType('r'),
default=sys.stdin, help='YAML augmenter specification')
cfg = parse_args(parser)
output = cfg['files']['output']
options = cfg['options']
augdb = AugmentDatabase(cfg)
augmap = yaml.safe_load(options.spec)
try:
outmap = augdb.apply(augmap)
except BaseException as exc:
if type(exc) != KeyError:
raise
sys.exit("ERROR: %s" % str(exc))
print(yamldump(outmap), file=output or sys.stdout)
if output:
output.close()
def to_map(self, db, opts):
"""Convert the object dictionary to a regular dictionary
:param db: db used to tie the objects together
:param opts: options to include/exclude information, etc.
:return: dictionary
Invokes the `to_map` method of each object to construct the
dictionary. If `opts` specifies a directory, the objects are
written to files in that directory.
"""
objdict = {}
for objkey in sorted(self.keys()):
obj = self[objkey]
objmap = obj.to_map(db, opts.no_owner, opts.no_privs)
if objmap is not None:
extkey = obj.extern_key()
outobj = {extkey: objmap}
if opts.multiple_files:
filepath = obj.extern_filename()
with open(os.path.join(opts.metadata_dir, filepath),
'a') as f:
f.write(yamldump(outobj))
outobj = {extkey: filepath}
objdict.update(outobj)
return objdict
def to_map(self, opts):
"""Convert the object dictionary to a regular dictionary
:param opts: options to include/exclude information, etc.
:return: dictionary
Invokes the `to_map` method of each object to construct the
dictionary. If opts specifies a directory, the objects are
written to files in that directory.
"""
objdict = {}
for objkey in sorted(self.keys()):
obj = self[objkey]
objmap = obj.to_map(opts.no_owner, opts.no_privs)
if objmap is not None:
extkey = obj.extern_key()
outobj = {extkey: objmap}
if opts.multiple_files:
filepath = os.path.join(opts.metadata_dir,
obj.extern_filename())
with open(filepath, 'a') as f:
f.write(yamldump(outobj))
outobj = {extkey: filepath}
objdict.update(outobj)
return objdict
def test_user_config(tmpdir):
"Test a user configuration file"
f = tmpdir.join(CFG_FILE)
f.write(yamldump(USER_CFG_DATA))
os.environ["PYRSEAS_USER_CONFIG"] = f.strpath
cfg = Config()
assert cfg['database'] == {'port': 5433}
assert cfg['output'] == {'version_comment': True}
def test_user_config(tmpdir):
"Test a user configuration file"
f = tmpdir.join(CFG_FILE)
f.write(yamldump(USER_CFG_DATA))
os.environ["PYRSEAS_USER_CONFIG"] = f.strpath
cfg = Config()
assert cfg['database'] == {'port': 5433}
assert cfg['output'] == {'version_comment': True}
def to_map(self):
"""Convert the db maps to a single hierarchy suitable for YAML
:return: a YAML-suitable dictionary (without Python objects)
"""
if not self.db:
self.from_catalog(True)
opts = self.config['options']
def mkdir_parents(dir):
head, tail = os.path.split(dir)
if head and not os.path.isdir(head):
mkdir_parents(head)
if tail:
os.mkdir(dir)
if opts.multiple_files:
opts.metadata_dir = self.config['files']['metadata_path']
if not os.path.exists(opts.metadata_dir):
mkdir_parents(opts.metadata_dir)
dbfilepath = os.path.join(opts.metadata_dir, 'database.%s.yaml' %
self.dbconn.dbname)
if os.path.exists(dbfilepath):
with open(dbfilepath, 'r') as f:
objmap = yaml.safe_load(f)
for obj, val in list(objmap.items()):
if isinstance(val, dict):
dirpath = ''
for schobj, fpath in list(val.items()):
filepath = os.path.join(opts.metadata_dir, fpath)
if os.path.exists(filepath):
os.remove(filepath)
if schobj == 'schema':
(dirpath, ext) = os.path.splitext(filepath)
if os.path.exists(dirpath):
os.rmdir(dirpath)
else:
filepath = os.path.join(opts.metadata_dir, val)
if (os.path.exists(filepath)):
os.remove(filepath)
dbmap = self.db.extensions.to_map(self.db, opts)
dbmap.update(self.db.languages.to_map(self.db, opts))
dbmap.update(self.db.casts.to_map(self.db, opts))
dbmap.update(self.db.fdwrappers.to_map(self.db, opts))
dbmap.update(self.db.eventtrigs.to_map(self.db, opts))
if 'datacopy' in self.config:
opts.data_dir = self.config['files']['data_path']
if not os.path.exists(opts.data_dir):
mkdir_parents(opts.data_dir)
dbmap.update(self.db.schemas.to_map(self.db, opts))
if opts.multiple_files:
with open(dbfilepath, 'w') as f:
f.write(yamldump(dbmap))
return dbmap
def to_map(self):
"""Convert the db maps to a single hierarchy suitable for YAML
:return: a YAML-suitable dictionary (without Python objects)
"""
if not self.db:
self.from_catalog()
opts = self.config['options']
def mkdir_parents(dir):
head, tail = os.path.split(dir)
if head and not os.path.isdir(head):
mkdir_parents(head)
if tail:
os.mkdir(dir)
if opts.multiple_files:
opts.metadata_dir = self.config['files']['metadata_path']
if not os.path.exists(opts.metadata_dir):
mkdir_parents(opts.metadata_dir)
dbfilepath = os.path.join(opts.metadata_dir, 'database.%s.yaml' %
self.dbconn.dbname)
if os.path.exists(dbfilepath):
with open(dbfilepath, 'r') as f:
objmap = yaml.safe_load(f)
for obj, val in objmap.items():
if isinstance(val, dict):
dirpath = ''
for schobj, fpath in val.items():
filepath = os.path.join(opts.metadata_dir, fpath)
if os.path.exists(filepath):
os.remove(filepath)
if schobj == 'schema':
(dirpath, ext) = os.path.splitext(filepath)
if os.path.exists(dirpath):
os.rmdir(dirpath)
else:
filepath = os.path.join(opts.metadata_dir, val)
if (os.path.exists(filepath)):
os.remove(filepath)
dbmap = self.db.extensions.to_map(opts)
dbmap.update(self.db.languages.to_map(opts))
dbmap.update(self.db.casts.to_map(opts))
dbmap.update(self.db.fdwrappers.to_map(opts))
dbmap.update(self.db.eventtrigs.to_map(opts))
if 'datacopy' in self.config:
opts.data_dir = self.config['files']['data_path']
if not os.path.exists(opts.data_dir):
mkdir_parents(opts.data_dir)
dbmap.update(self.db.schemas.to_map(opts))
if opts.multiple_files:
with open(dbfilepath, 'w') as f:
f.write(yamldump(dbmap))
return dbmap
def test_cmd_parser(tmpdir):
"Test parsing a configuration file specified on the command line"
f = tmpdir.join(CFG_FILE)
f.write(yamldump(CFG_DATA))
sys.argv = ['testprog', 'testdb', '--config', f.strpath]
os.environ["PYRSEAS_USER_CONFIG"] = ''
parser = cmd_parser("Test description", '0.0.1')
cfg = parse_args(parser)
assert cfg['datacopy'] == CFG_TABLE_DATA
def test_cmd_parser(tmpdir):
"Test parsing a configuration file specified on the command line"
f = tmpdir.join(CFG_FILE)
f.write(yamldump(CFG_DATA))
sys.argv = ['testprog', 'testdb', '--config', f.strpath]
os.environ["PYRSEAS_USER_CONFIG"] = ''
parser = cmd_parser("Test description", '0.0.1')
cfg = parse_args(parser)
assert cfg['datacopy'] == CFG_TABLE_DATA
def test_parse_repo_config(tmpdir):
"Test parsing a repository configuration file in the current directory"
f = tmpdir.join('config.yaml')
f.write(yamldump(CFG_DATA))
os.chdir(tmpdir.strpath)
sys.argv = ['testprog', 'testdb']
os.environ["PYRSEAS_USER_CONFIG"] = ''
parser = cmd_parser("Test description", '0.0.1')
cfg = parse_args(parser)
assert cfg['datacopy'] == CFG_TABLE_DATA
def test_parse_repo_config(tmpdir):
"Test parsing a repository configuration file in the current directory"
f = tmpdir.join('config.yaml')
f.write(yamldump(CFG_DATA))
os.chdir(tmpdir.strpath)
sys.argv = ['testprog', 'testdb']
os.environ["PYRSEAS_USER_CONFIG"] = ''
parser = cmd_parser("Test description", '0.0.1')
cfg = parse_args(parser)
assert cfg['datacopy'] == CFG_TABLE_DATA
def main(schema=None):
"""Convert database table specifications to YAML."""
parser = cmd_parser("Extract the schema of a PostgreSQL database in "
"YAML format", __version__)
parser.add_argument('-m', '--multiple-files', action='store_true',
help='output to multiple files (metadata directory)')
parser.add_argument('-O', '--no-owner', action='store_true',
help='exclude object ownership information')
parser.add_argument('-x', '--no-privileges', action='store_true',
dest='no_privs',
help='exclude privilege (GRANT/REVOKE) information')
group = parser.add_argument_group("Object inclusion/exclusion options",
"(each can be given multiple times)")
group.add_argument('-n', '--schema', metavar='SCHEMA', dest='schemas',
action='append', default=[],
help="extract the named schema(s) (default all)")
group.add_argument('-N', '--exclude-schema', metavar='SCHEMA',
dest='excl_schemas', action='append', default=[],
help="do NOT extract the named schema(s) "
"(default none)")
group.add_argument('-t', '--table', metavar='TABLE', dest='tables',
action='append', default=[],
help="extract the named table(s) (default all)")
group.add_argument('-T', '--exclude-table', metavar='TABLE',
dest='excl_tables', action='append', default=[],
help="do NOT extract the named table(s) "
"(default none)")
parser.set_defaults(schema=schema)
cfg = parse_args(parser)
output = cfg['files']['output']
options = cfg['options']
if options.multiple_files and output:
parser.error("Cannot specify both --multiple-files and --output")
db = Database(cfg)
dbmap = db.to_map()
if not options.multiple_files:
print(yamldump(dbmap), file=output or sys.stdout)
if output:
output.close()
def to_map(self, dbschemas, opts):
"""Convert tables, etc., dictionaries to a YAML-suitable format
:param dbschemas: dictionary of schemas
:param opts: options to include/exclude schemas/tables, etc.
:return: dictionary
"""
if self.name == 'pyrseas':
return {}
no_owner = opts.no_owner
no_privs = opts.no_privs
schbase = {} if no_owner else {'owner': self.owner}
if not no_privs and self.privileges:
schbase.update({'privileges': self.map_privs()})
if self.description is not None:
schbase.update(description=self.description)
schobjs = []
seltbls = getattr(opts, 'tables', [])
if hasattr(self, 'tables'):
for objkey in self.tables:
if not seltbls or objkey in seltbls:
obj = self.tables[objkey]
schobjs.append((obj, obj.to_map(dbschemas, opts)))
def mapper(objtypes):
if hasattr(self, objtypes):
schemadict = getattr(self, objtypes)
for objkey in schemadict:
if objtypes == 'sequences' or (
not seltbls or objkey in seltbls):
obj = schemadict[objkey]
schobjs.append((obj, obj.to_map(opts)))
for objtypes in ['ftables', 'sequences', 'views', 'matviews']:
mapper(objtypes)
def mapper2(objtypes):
if hasattr(self, objtypes):
schemadict = getattr(self, objtypes)
for objkey in schemadict:
obj = schemadict[objkey]
schobjs.append((obj, obj.to_map(no_owner)))
if hasattr(opts, 'tables') and not opts.tables or \
not hasattr(opts, 'tables'):
for objtypes in ['conversions', 'domains',
'operators', 'operclasses', 'operfams',
'tsconfigs', 'tsdicts', 'tsparsers', 'tstempls',
'types', 'collations']:
mapper2(objtypes)
if hasattr(self, 'functions'):
for objkey in self.functions:
obj = self.functions[objkey]
schobjs.append((obj, obj.to_map(no_owner, no_privs)))
# special case for pg_catalog schema
if self.name == 'pg_catalog' and not schobjs:
return {}
if hasattr(self, 'datacopy') and self.datacopy:
dir = self.extern_dir(opts.data_dir)
if not os.path.exists(dir):
os.mkdir(dir)
for tbl in self.datacopy:
self.tables[tbl].data_export(dbschemas.dbconn, dir)
if opts.multiple_files:
dir = self.extern_dir(opts.metadata_dir)
if not os.path.exists(dir):
os.mkdir(dir)
filemap = {}
for obj, objmap in schobjs:
if objmap is not None:
extkey = obj.extern_key()
filepath = os.path.join(dir, obj.extern_filename())
with open(filepath, 'a') as f:
f.write(yamldump({extkey: objmap}))
outobj = {extkey:
os.path.relpath(filepath, opts.metadata_dir)}
filemap.update(outobj)
# always write the schema YAML file
filepath = self.extern_filename()
extkey = self.extern_key()
with open(os.path.join(opts.metadata_dir, filepath), 'a') as f:
f.write(yamldump({extkey: schbase}))
filemap.update(schema=filepath)
return {extkey: filemap}
schmap = dict((obj.extern_key(), objmap) for obj, objmap in schobjs
if objmap is not None)
schmap.update(schbase)
return {self.extern_key(): schmap}
def to_map(self, db, dbschemas, opts):
"""Convert tables, etc., dictionaries to a YAML-suitable format
:param dbschemas: dictionary of schemas
:param opts: options to include/exclude schemas/tables, etc.
:return: dictionary
"""
if self.name == 'pyrseas':
return {}
no_owner = opts.no_owner
no_privs = opts.no_privs
schbase = {} if no_owner else {'owner': self.owner}
if not no_privs:
schbase.update({'privileges': self.map_privs()})
if self.description is not None:
schbase.update(description=self.description)
schobjs = []
seltbls = getattr(opts, 'tables', [])
if hasattr(self, 'tables'):
for objkey in self.tables:
if not seltbls or objkey in seltbls:
obj = self.tables[objkey]
schobjs.append((obj, obj.to_map(db, dbschemas, opts)))
def mapper(objtypes):
if hasattr(self, objtypes):
schemadict = getattr(self, objtypes)
for objkey in schemadict:
if objtypes == 'sequences' or (
not seltbls or objkey in seltbls):
obj = schemadict[objkey]
schobjs.append((obj, obj.to_map(db, opts)))
for objtypes in ['ftables', 'sequences', 'views', 'matviews']:
mapper(objtypes)
def mapper2(objtypes, privs=False):
if hasattr(self, objtypes):
schemadict = getattr(self, objtypes)
for objkey in schemadict:
obj = schemadict[objkey]
if privs:
dct = obj.to_map(db, no_owner, no_privs)
else:
dct = obj.to_map(db, no_owner)
schobjs.append((obj, dct))
if hasattr(opts, 'tables') and not opts.tables or \
not hasattr(opts, 'tables'):
for objtypes in ('conversions', 'collations', 'operators',
'operclasses', 'operfams', 'tsconfigs',
'tsdicts', 'tsparsers', 'tstempls'):
mapper2(objtypes)
for objtypes in ('types', 'domains'):
mapper2(objtypes, True)
if hasattr(self, 'functions'):
for objkey in self.functions:
obj = self.functions[objkey]
schobjs.append((obj, obj.to_map(db, no_owner, no_privs)))
# special case for pg_catalog schema
if self.name == 'pg_catalog' and not schobjs:
return {}
if hasattr(self, 'datacopy') and self.datacopy:
dir = self.extern_dir(opts.data_dir)
if not os.path.exists(dir):
os.mkdir(dir)
for tbl in self.datacopy:
self.tables[tbl].data_export(dbschemas.dbconn, dir)
if opts.multiple_files:
dir = self.extern_dir(opts.metadata_dir)
if not os.path.exists(dir):
os.mkdir(dir)
filemap = {}
for obj, objmap in schobjs:
if objmap is not None:
extkey = obj.extern_key()
filepath = os.path.join(dir, obj.extern_filename())
with open(filepath, 'a') as f:
f.write(yamldump({extkey: objmap}))
outobj = {extkey:
os.path.relpath(filepath, opts.metadata_dir)}
filemap.update(outobj)
# always write the schema YAML file
filepath = self.extern_filename()
extkey = self.extern_key()
with open(os.path.join(opts.metadata_dir, filepath), 'a') as f:
f.write(yamldump({extkey: schbase}))
filemap.update(schema=filepath)
return {extkey: filemap}
schmap = dict((obj.extern_key(), objmap) for obj, objmap in schobjs
if objmap is not None)
schmap.update(schbase)
return {self.extern_key(): schmap}
def main(schema=None):
"""Convert database table specifications to YAML."""
parser = cmd_parser(
"Extract the schema of a PostgreSQL database in "
"YAML format", __version__)
parser.add_argument('-m',
'--multiple-files',
action='store_true',
help='output to multiple files (metadata directory)')
parser.add_argument('-O',
'--no-owner',
action='store_true',
help='exclude object ownership information')
parser.add_argument('-x',
'--no-privileges',
action='store_true',
dest='no_privs',
help='exclude privilege (GRANT/REVOKE) information')
group = parser.add_argument_group("Object inclusion/exclusion options",
"(each can be given multiple times)")
group.add_argument('-n',
'--schema',
metavar='SCHEMA',
dest='schemas',
action='append',
default=[],
help="extract the named schema(s) (default all)")
group.add_argument('-N',
'--exclude-schema',
metavar='SCHEMA',
dest='excl_schemas',
action='append',
default=[],
help="do NOT extract the named schema(s) "
"(default none)")
group.add_argument('-t',
'--table',
metavar='TABLE',
dest='tables',
action='append',
default=[],
help="extract the named table(s) (default all)")
group.add_argument('-T',
'--exclude-table',
metavar='TABLE',
dest='excl_tables',
action='append',
default=[],
help="do NOT extract the named table(s) "
"(default none)")
parser.set_defaults(schema=schema)
cfg = parse_args(parser)
output = cfg['files']['output']
options = cfg['options']
if options.multiple_files and output:
parser.error("Cannot specify both --multiple-files and --output")
db = Database(cfg)
dbmap = db.to_map()
if not options.multiple_files:
print(yamldump(dbmap), file=output or sys.stdout)
if output:
output.close()
