def test_iterate_doc(self):
doc = MetapackDoc(test_data('example1.csv'))
if exists(test_database_path):
remove(test_database_path)
db = Database('sqlite:///' + test_database_path)
mm = MetatabManager(db)
mm.add_doc(doc)
with mm.session() as s:
db_doc = mm.document(identifier=doc.get_value('Root.Identifier'))
mt_doc = db_doc.mt_doc
self.assertEqual(23, len(mt_doc.terms))
self.assertEqual([
'http://example.com/example1.csv',
'http://example.com/example2.csv'
], [str(r.resolved_url) for r in mt_doc.find("Root.Datafile")])
with mm.session():
with mm.session():
db_doc = mm.document(
identifier=doc.get_value('Root.Identifier'))
mt_doc = db_doc.mt_doc
self.assertEqual(23, len(mt_doc.terms))
def load(self, url, load_all_resources=False):
"""Load a package and possibly one or all resources, from a url"""
u = parse_app_url(url)
d = MetapackDoc(u.clear_fragment())
db_doc = self.document(name=d.get_value('Root.Name'))
if not db_doc:
self.add_doc(d)
db_doc = self.document(name=d.get_value('Root.Name'))
assert db_doc
resources = []
if load_all_resources:
for r in self.resources(db_doc):
self.load_resource(r)
resources.append(r)
elif u.target_file:
r = self.resource(db_doc, u.target_file)
self.load_resource(r)
resources.append(d)
return (db_doc, resources)
def test_line_doc_parts(self):
doc = MetapackDoc(TextRowGenerator("Declare: metatab-latest"))
for fn in (
'line/line-oriented-doc-root.txt',
'line/line-oriented-doc-contacts.txt',
'line/line-oriented-doc-datafiles.txt',
'line/line-oriented-doc-references-1.txt',
'line/line-oriented-doc-references-2.txt',
'line/line-oriented-doc-bib.txt',
):
with open(test_data(fn)) as f:
text = f.read()
tp = TermParser(TextRowGenerator(text),
resolver=doc.resolver,
doc=doc)
doc.load_terms(tp)
self.assertEqual('47bc1089-7584-41f0-b804-602ec42f1249',
doc.get_value('Root.Identifier'))
self.assertEqual(157, len(doc.terms))
self.assertEqual(5, len(list(doc['References'])))
self.assertEqual(5,
len(list(doc['References'].find('Root.Reference'))))
self.assertEqual(5, len(list(doc['References'].find(
'Root.Resource')))) # References are Resources
rt = list(doc['References'].find('Root.Resource'))[0]
self.assertIsInstance(rt, Reference)
self.assertEqual(5, len(list(doc['Resources'])))
self.assertEqual(5, len(list(doc['Resources'].find('Root.Datafile'))))
self.assertEqual(5, len(list(doc['Resources'].find(
'Root.Resource')))) # References are Resources
rt = list(doc['Resources'].find('Root.Resource'))[0]
self.assertIsInstance(rt, Resource)
doc._repr_html_() # Check no exceptions
def test_gen_line_rows(self):
from metatab import parse_app_url
from metapack import MetapackDocumentUrl
from metatab.rowgenerators import TextRowGenerator
u = parse_app_url(test_data('line', 'line-oriented-doc.txt'),
proto='metapack')
self.assertIsInstance(u, MetapackDocumentUrl)
self.assertIsInstance(u.get_resource(), MetapackDocumentUrl)
self.assertIsInstance(u.get_resource().get_target(),
MetapackDocumentUrl)
self.assertIsInstance(u.generator, TextRowGenerator)
doc = MetapackDoc(u)
self.assertEqual('47bc1089-7584-41f0-b804-602ec42f1249',
doc.get_value('Root.Identifier'))
def test_line_oriented(self):
doc = MetapackDoc(
TextRowGenerator(test_data('line', 'line-oriented-doc.txt')))
self.assertEqual('47bc1089-7584-41f0-b804-602ec42f1249',
doc.get_value('Root.Identifier'))
self.assertEqual(153, len(doc.terms))
self.assertEqual(6, len(list(doc['References'])))
self.assertEqual(6,
len(list(doc['References'].find('Root.Reference'))))
self.assertEqual(6, len(list(doc['References'].find(
'Root.Resource')))) # References are Resources
rt = list(doc['References'].find('Root.Resource'))[0]
self.assertIsInstance(rt, Reference)
def test_create_and_delete_tables(self):
from os import remove
from os.path import exists
doc = MetapackDoc(test_data('example1.csv'))
if exists(test_database_path):
remove(test_database_path)
db = Database('sqlite:///' + test_database_path)
db.create_tables()
mm = MetatabManager(db)
mm.add_doc(doc)
with mm.session() as s:
self.assertEqual(1, len(list(s.query(Document))))
self.assertEqual(154, len(list(s.query(Term))))
with self.assertRaises(IntegrityError):
mm.add_doc(doc)
with mm.session() as s:
self.assertEqual(1, len(list(s.query(Document))))
self.assertEqual(154, len(list(s.query(Term))))
with mm.session() as s:
db_doc = mm.document(identifier=doc.get_value('Root.Identifier'))
self.assertIsNotNone(db_doc)
s.delete(db_doc)
with mm.session() as s:
self.assertEqual(0, len(list(s.query(Document))))
self.assertEqual(0, len(list(s.query(Term))))
def new_cmd(args):
from metapack import MetapackDoc
from metapack.util import make_metatab_file, datetime_now, ensure_dir
from metapack.cli.core import write_doc, prt, err
from os.path import exists, join, expanduser
from metatab import DEFAULT_METATAB_FILE
from os import getenv
if args.config:
config_file = args.config
elif getenv("METAPACK_CONFIG"):
config_file = getenv("METAPACK_DEFAULTS")
elif expanduser('~/.metapack-default.csv'):
config_file = expanduser('~/.metapack-defaults.csv')
else:
config_file = None
if config_file and exists(config_file):
prt(f"Using defaults file '{config_file}'")
config = MetapackDoc(config_file)
else:
config = MetapackDoc()
if args.jupyter:
import tempfile
with tempfile.NamedTemporaryFile(suffix='.ipynb', delete=False) as fp:
r = requests.get(TEMPLATE_NOTEBOOK, allow_redirects=True)
r.raise_for_status()
fp.write(r.content)
nb_path = Path(fp.name)
doc = MetapackDoc(str(nb_path))
else:
doc = make_metatab_file(args.template)
doc['Root']['Created'] = datetime_now()
origin = args.origin or config.get_value('Root.Origin')
if not origin:
err("Must specify a value for origin, either on command line or in defaults file"
)
(doc['Root'].find_first('Root.Origin') or et).value = origin
(doc['Root'].find_first('Root.Dataset') or et).value = args.dataset
(doc['Root'].find_first('Root.Space')
or et).value = args.space or config.get_value('Root.Space')
(doc['Root'].find_first('Root.Time')
or et).value = args.time or config.get_value('Root.Time')
(doc['Root'].find_first('Root.Grain')
or et).value = args.grain or config.get_value('Root.Grain')
(doc['Root'].find_first('Root.Variant')
or et).value = args.variant or config.get_value('Root.Variant')
v = doc['Root'].get_or_new_term('Root.Version')
v.get_or_new_child(
'Version.Major'
).value = args.revision or config.get_value('Root.Version')
v.get_or_new_child('Version.Minor').value = 1
v.get_or_new_child('Version.Patch').value = 1
# Copy contacts in
if 'Contacts' in config:
for c in config['Contacts']:
doc['Contacts'].add_term(c)
if args.title:
doc['Root'].find_first('Root.Title').value = args.title.strip()
nv_name = doc.as_version(None)
if args.example:
doc['Resources'].new_term(
'Root.Datafile',
'http://public.source.civicknowledge.com/example.com/sources/random-names.csv',
name='random_names')
doc['Documentation'].new_term('Root.Homepage',
'http://metatab.org',
title='Metatab Home Page')
doc.ensure_identifier()
doc.update_name(create_term=True)
if getattr(args, 'jupyter'): # b/c maybe metatab_jupyter is not installed
from metapack_jupyter.convert import write_metatab_notebook
from metapack_jupyter.core import edit_notebook, set_cell_source, get_cell_source
new_nb_path = Path(f'{nv_name}.ipynb')
doc['Resources'].new_term(
'Root.Datafile',
'./' + str(new_nb_path) + "#df",
name='local_dataframe',
description='Example of using a local Dataframe')
if new_nb_path.exists():
err(f"Directory {nb_path} already exists")
copyfile(nb_path, new_nb_path)
write_metatab_notebook(doc, new_nb_path)
with edit_notebook(new_nb_path) as nb:
init = get_cell_source(nb, 'init')
init += f"\nthis_package_name = '{str(new_nb_path.name)}'"
set_cell_source(nb, 'init', init)
nb_path.unlink()
else:
doc['Documentation'].new_term('Root.Documentation',
'file:README.md',
title='README')
if exists(nv_name):
err(f"Directory {nv_name} already exists")
if args.csv:
fn = doc.nonver_name + '.csv'
write_doc(doc, fn)
prt(f"Writing to {fn}")
else:
ensure_dir(nv_name)
pylib_dir = join(nv_name, 'pylib')
ensure_dir(pylib_dir)
with open(join(pylib_dir, '__init__.py'),
'w') as f_out, open(pylib.__file__) as f_in:
f_out.write(f_in.read())
if args.example:
doc['Resources'].new_term('Root.Datafile',
'python:pylib#row_generator',
name='row_generator')
prt(f"Writing to '{nv_name}'")
write_doc(doc, join(nv_name, DEFAULT_METATAB_FILE))
add_missing_files(nv_name)
if args.title:
readme = '# {}\n'.format(args.title)
else:
readme = '# {}\n'.format(doc.get_value('Root.Name'))
with open(join(nv_name, 'README.md'), 'w') as f:
f.write(readme)