def test_min_version_failure_raises(self):
min_ver = (constants.MIN_VER[0], constants.MIN_VER[1] + 10,
constants.MIN_VER[2])
LOGGER.debug('Setting pgdumplib.constants.MIN_VER to %s', min_ver)
with mock.patch('pgdumplib.constants.MIN_VER', min_ver):
with self.assertRaises(ValueError):
pgdumplib.load('build/data/dump.not-compressed')
def test_invalid_dump_file(self):
with tempfile.NamedTemporaryFile('wb') as temp:
temp.write(b'PGBAD')
with open('build/data/dump.not-compressed', 'rb') as handle:
handle.read(5)
temp.write(handle.read())
with self.assertRaises(ValueError):
pgdumplib.load(temp.name)
def test_invalid_block_type_in_data(self):
dmp = pgdumplib.new('test')
dmp.add_entry(constants.TABLE_DATA, '', 'block_table', dump_id=128)
with gzip.open(pathlib.Path(dmp._temp_dir.name) / '128.gz', 'wb') as h:
h.write(b'1\t\1\t\1\n')
with mock.patch('pgdumplib.constants.BLK_DATA', b'\x02'):
dmp.save('build/data/dump.test')
with self.assertRaises(RuntimeError):
pgdumplib.load('build/data/dump.test')
def test_runtime_error_when_pos_not_set(self):
dmp = pgdumplib.new('test')
dmp.add_entry(constants.TABLE_DATA, 'public', 'table', dump_id=32)
with gzip.open(pathlib.Path(dmp._temp_dir.name) / '32.gz', 'wb') as h:
h.write(b'1\t\1\t\1\n')
with mock.patch('pgdumplib.constants.K_OFFSET_POS_SET', 9):
dmp.save('build/data/dump.test')
with self.assertRaises(RuntimeError):
pgdumplib.load('build/data/dump.test')
def test_dump_id_mismatch_in_data(self):
dmp = pgdumplib.new('test')
dmp.add_entry(constants.TABLE_DATA, '', 'block_table', dump_id=1024)
with gzip.open(pathlib.Path(dmp._temp_dir.name) / '1024.gz',
'wb') as handle:
handle.write(b'1\t\1\t\1\n')
dmp.save('build/data/dump.test')
with mock.patch('pgdumplib.dump.Dump._read_block_header') as rbh:
rbh.return_value = constants.BLK_DATA, 2048
with self.assertRaises(RuntimeError):
pgdumplib.load('build/data/dump.test')
def test_encoding_no_entries(self):
dmp = pgdumplib.new('test', 'LATIN1')
dmp.entries = []
self.assertEqual(dmp.encoding, 'LATIN1')
dmp.save('build/data/dump.test')
dmp = pgdumplib.load('build/data/dump.test')
self.assertEqual(dmp.encoding, 'UTF8')
def test_bad_encoding(self):
dmp = pgdumplib.new('test')
dmp.entries[0].defn = 'BAD ENTRY WILL FAIL'
dmp.save('build/data/dump.test')
dmp = pgdumplib.load('build/data/dump.test')
self.assertEqual(dmp.encoding, 'UTF8')
def test_encoding_not_first_entry(self):
dmp = pgdumplib.new('test', 'LATIN1')
entries = dmp.entries
dmp.entries = [entries[1], entries[2], entries[0]]
self.assertEqual(dmp.encoding, 'LATIN1')
dmp.save('build/data/dump.test')
dmp = pgdumplib.load('build/data/dump.test')
self.assertEqual(dmp.encoding, 'LATIN1')
def test_no_data(self):
dmp = pgdumplib.new('test')
dmp.add_entry(constants.TABLE_DATA, '', 'empty_table', dump_id=5)
with gzip.open(pathlib.Path(dmp._temp_dir.name) / '5.gz', 'wb') as h:
h.write(b'')
dmp.save('build/data/dump.test')
dmp = pgdumplib.load('build/data/dump.test')
data = [line for line in dmp.table_data('', 'empty_table')]
self.assertEqual(len(data), 0)
def test_read_blobs(self):
conn = psycopg2.connect(os.environ['POSTGRES_URI'])
cursor = conn.cursor()
cursor.execute(BLOB_COUNT_SQL)
expectation = cursor.fetchone()[0]
conn.close()
dmp = pgdumplib.load(self.dump_path, self.CONVERTER)
blobs = []
for oid, blob in dmp.blobs():
self.assertIsInstance(oid, int)
self.assertIsInstance(blob, bytes)
blobs.append((oid, blob))
self.assertEqual(len(blobs), expectation)
def main():
args = parse_cli_arguments()
dump = pgdumplib.load(args.file[0])
if args.format == 'json':
out = sys.stdout if args.out == '-' else open(args.out, 'wt')
as_json(args, dump, out)
elif args.format == 'html':
if args.out == '-':
sys.stderr.write('Must specify a directory for HTML output\n')
sys.exit(1)
out_dir = pathlib.Path(args.out)
if out_dir.exists() and not out_dir.is_dir():
sys.stderr.write('Must specify a directory for HTML output\n')
sys.exit(1)
elif not out_dir.exists():
out_dir.mkdir()
buffer = io.StringIO()
as_json(args, dump, buffer, 0)
buffer.seek(0)
html.as_html(buffer.read(), out_dir)
def run(self) -> typing.NoReturn:
"""Implement as core logic for generating the project"""
if self.project_path.exists() and not self.args.force:
common.exit_application(
'{} already exists'.format(self.project_path), 3)
LOGGER.info('Generating project in %s', self.project_path)
LOGGER.debug('args: %r', self.args)
if self.args.ignore:
with open(self.args.ignore, 'r') as handle:
for line in handle:
self.ignore.add(line.strip())
LOGGER.info('Ignoring %i files', len(self.ignore))
if self.args.extract:
LOGGER.info('Dumping schema from postgresql://%s:%s/%s',
self.args.host, self.args.port, self.args.dbname)
pgdump.dump(self.args, self.dump_path)
LOGGER.debug('Loading dump from %s', self.dump_path)
self.dump = pgdumplib.load(self.dump_path)
self.structure = Structure(self.dump.entries)
self._create_directories()
self._create_project_file()
if self.args.extract_roles:
self._extract_roles()
self._process_acls()
self._create_role_files()
self._create_group_files()
self._create_user_files()
self._create_namespace_files(constants.CAST)
self._create_namespace_files(constants.COLLATION)
self._create_namespace_files(constants.CONVERSION)
self._create_files(constants.DOMAIN)
self._create_files(constants.EVENT_TRIGGER)
self._create_files(constants.FOREIGN_DATA_WRAPPER)
self._create_files(constants.FUNCTION)
self._create_namespace_files(constants.MATERIALIZED_VIEW)
self._create_operator_files()
self._create_namespace_files(constants.PROCEDURE)
self._create_namespace_files(constants.PUBLICATION)
self._create_schema_files()
self._create_files(constants.SEQUENCE)
self._create_namespace_files(constants.SUBSCRIPTION)
self._create_files(constants.SERVER)
self._create_files(constants.TABLE)
self._create_namespace_files(constants.TABLESPACE)
self._create_namespace_files(constants.TYPE)
self._create_namespace_files(constants.TEXT_SEARCH_CONFIGURATION)
self._create_namespace_files(constants.TEXT_SEARCH_DICTIONARY)
self._create_files(constants.USER_MAPPING)
self._create_files(constants.VIEW)
remaining = collections.Counter()
for entry in [
e for e in self.dump.entries if e.dump_id not in self.processed
and e.desc != constants.SEARCHPATH
]:
remaining['{}:{}'.format(entry.section, entry.desc)] += 1
for key in sorted(remaining.keys(), reverse=True):
LOGGER.info('Remaining %s: %i', key, remaining[key])
if self.args.save_remaining:
LOGGER.debug('Writing remaining.yaml')
with open(self.project_path / 'remaining.yaml', 'w') as handle:
storage.yaml_dump(handle, [
dataclasses.asdict(e) for e in self.dump.entries
if e.dump_id not in self.processed
])
if self.args.gitkeep:
storage.remove_unneeded_gitkeeps(self.project_path)
if self.args.remove_empty_dirs:
storage.remove_empty_directories(self.project_path)
def setUpClass(cls) -> None:
cls.dump = pgdumplib.load(
pathlib.Path('build') / 'data' / cls.PATH, cls.CONVERTER)
def _read_dump(self):
return pgdumplib.load(self.local_path)
def setUp(self):
dmp = pgdumplib.load('build/data/dump.compressed')
dmp.save('build/data/dump.test')
self.original = pgdumplib.load('build/data/dump.compressed')
self.saved = pgdumplib.load('build/data/dump.test')
def test_dump_expectations(self):
dmp = pgdumplib.new('test', 'UTF8')
database = dmp.add_entry(
desc=constants.DATABASE,
tag='postgres',
owner='postgres',
defn="""\
CREATE DATABASE postgres
WITH TEMPLATE = template0
ENCODING = 'UTF8'
LC_COLLATE = 'en_US.utf8'
LC_CTYPE = 'en_US.utf8';""",
drop_stmt='DROP DATABASE postgres')
dmp.add_entry(
constants.COMMENT,
tag='DATABASE postgres',
owner='postgres',
defn="""\
COMMENT ON DATABASE postgres
IS 'default administrative connection database';""",
dependencies=[database.dump_id])
example = dmp.add_entry(
constants.TABLE, 'public', 'example', 'postgres',
'CREATE TABLE public.example (\
id UUID NOT NULL PRIMARY KEY, \
created_at TIMESTAMP WITH TIME ZONE, \
value TEXT NOT NULL);',
'DROP TABLE public.example')
columns = 'id', 'created_at', 'value'
fake = faker.Faker()
fake.add_provider(date_time)
rows = [
(uuid.uuid4(), fake.date_time(tzinfo=tz.tzutc()), 'foo'),
(uuid.uuid4(), fake.date_time(tzinfo=tz.tzutc()), 'bar'),
(uuid.uuid4(), fake.date_time(tzinfo=tz.tzutc()), 'baz'),
(uuid.uuid4(), fake.date_time(tzinfo=tz.tzutc()), 'qux')
]
with dmp.table_data_writer(example, columns) as writer:
for row in rows:
writer.append(*row)
row = (uuid.uuid4(), fake.date_time(tzinfo=tz.tzutc()), None)
rows.append(row)
# Append a second time to get same writer
with dmp.table_data_writer(example, columns) as writer:
writer.append(*row)
dmp.save('build/data/dump.test')
test_file = pathlib.Path('build/data/dump.test')
self.assertTrue(test_file.exists())
dmp = pgdumplib.load(test_file, converters.SmartDataConverter)
entry = dmp.get_entry(database.dump_id)
self.assertEqual(entry.desc, 'DATABASE')
self.assertEqual(entry.owner, 'postgres')
self.assertEqual(entry.tag, 'postgres')
values = [row for row in dmp.table_data('public', 'example')]
self.assertListEqual(values, rows)
def test_missing_file_raises_value_error(self):
path = pathlib.Path(tempfile.gettempdir()) / str(uuid.uuid4())
with self.assertRaises(ValueError):
pgdumplib.load(path)
