def save(self, path=None):
from metapack import MetapackPackageUrl
# HACK ...
if not self.doc.ref:
self.doc._ref = self.package_path # Really should not do this but ...
self.check_is_ready()
self.load_declares()
self.doc.cleanse()
self._load_resources()
self._relink_documentation()
self._clean_doc()
if path is None:
if self.package_path.inner.proto == 'file':
path = self.package_path.path
else:
raise PackageError("Can't write doc to path: '{}'".format(path))
self.doc['Root'].get_or_new_term('Root.Issued').value = datetime_now()
self._last_write_path = path
self.doc.write_csv(path)
return parse_app_url(abspath(path)).as_type(MetapackPackageUrl)
def save(self):
from openpyxl import Workbook
from openpyxl.cell import WriteOnlyCell
from openpyxl.styles import PatternFill, Alignment
self.check_is_ready()
self.wb = Workbook(write_only=True)
meta_ws = self.wb.create_sheet()
meta_ws.title = "meta"
meta_ws.sheet_properties.tabColor = "8888ff"
meta_ws.column_dimensions['A'].width = 15
meta_ws.column_dimensions['B'].width = 40
meta_ws.column_dimensions['C'].width = 20
meta_ws.column_dimensions['D'].width = 20
meta_ws.column_dimensions['E'].width = 20
self.sections.resources.sort_by_term()
self.load_declares()
self.doc.cleanse()
self._load_resources()
self._clean_doc()
fill = PatternFill(
"solid", fgColor="acc0e0") # PatternFill(patternType='gray125')
table_fill = PatternFill(
"solid", fgColor="d9dce0") # PatternFill(patternType='gray125')
alignment = Alignment(wrap_text=False)
self._doc['Root'].get_or_new_term('Root.Issued').value = datetime_now()
for i, row in enumerate(self.doc.rows, 1):
if row[0] == 'Section' or row[0] == 'Table':
styled_row = []
for c in row + [''] * 5:
cell = WriteOnlyCell(meta_ws, value=c)
cell.fill = fill if row[0] == 'Section' else table_fill
styled_row.append(cell)
meta_ws.append(styled_row)
else:
meta_ws.append(row)
self.wb.save(self.package_path.path)
return self.package_path
def make_package(md):
name = md['name']
root = Path('packages').joinpath(name)
root.mkdir(parents=True, exist_ok=True)
print(json.dumps(md, indent=4))
if md['notes']:
root.joinpath('README.md').write_text(md['notes'])
doc = make_metatab_file('metatab_template.csv')
doc['Root']['Created'] = datetime_now()
doc['Documentation'].new_term('Root.Documentation',
'file:README.md',
title='README')
doc.ensure_identifier()
doc.update_name(create_term=True)
for r in md['resources']:
doc['Resources'].new_term(
'Root.Datafile',
r['url'],
name=slugify(r['name']),
description=r['description'] or r['name'],
)
if md.get('organization'):
doc['Root'].find_first(
'Root.Origin').value = md['organization']['name'].replace(
'-', '.')
doc['Root'].find_first('Root.Title').value = md['title']
doc.write_csv(str(root.joinpath('metadata.csv')))
return doc
def new_cmd(args):
from metapack import MetapackDoc
from metapack.util import make_metatab_file, datetime_now, ensure_dir
from metapack.cli.core import write_doc, prt, err
from os.path import exists, join, expanduser
from metatab import DEFAULT_METATAB_FILE
from os import getenv
if args.config:
config_file = args.config
elif getenv("METAPACK_CONFIG"):
config_file = getenv("METAPACK_DEFAULTS")
elif expanduser('~/.metapack-default.csv'):
config_file = expanduser('~/.metapack-defaults.csv')
else:
config_file = None
if config_file and exists(config_file):
prt(f"Using defaults file '{config_file}'")
config = MetapackDoc(config_file)
else:
config = MetapackDoc()
if args.jupyter:
import tempfile
with tempfile.NamedTemporaryFile(suffix='.ipynb', delete=False) as fp:
r = requests.get(TEMPLATE_NOTEBOOK, allow_redirects=True)
r.raise_for_status()
fp.write(r.content)
nb_path = Path(fp.name)
doc = MetapackDoc(str(nb_path))
else:
doc = make_metatab_file(args.template)
doc['Root']['Created'] = datetime_now()
origin = args.origin or config.get_value('Root.Origin')
if not origin:
err("Must specify a value for origin, either on command line or in defaults file"
)
(doc['Root'].find_first('Root.Origin') or et).value = origin
(doc['Root'].find_first('Root.Dataset') or et).value = args.dataset
(doc['Root'].find_first('Root.Space')
or et).value = args.space or config.get_value('Root.Space')
(doc['Root'].find_first('Root.Time')
or et).value = args.time or config.get_value('Root.Time')
(doc['Root'].find_first('Root.Grain')
or et).value = args.grain or config.get_value('Root.Grain')
(doc['Root'].find_first('Root.Variant')
or et).value = args.variant or config.get_value('Root.Variant')
v = doc['Root'].get_or_new_term('Root.Version')
v.get_or_new_child(
'Version.Major'
).value = args.revision or config.get_value('Root.Version')
v.get_or_new_child('Version.Minor').value = 1
v.get_or_new_child('Version.Patch').value = 1
# Copy contacts in
if 'Contacts' in config:
for c in config['Contacts']:
doc['Contacts'].add_term(c)
if args.title:
doc['Root'].find_first('Root.Title').value = args.title.strip()
nv_name = doc.as_version(None)
if args.example:
doc['Resources'].new_term(
'Root.Datafile',
'http://public.source.civicknowledge.com/example.com/sources/random-names.csv',
name='random_names')
doc['Documentation'].new_term('Root.Homepage',
'http://metatab.org',
title='Metatab Home Page')
doc.ensure_identifier()
doc.update_name(create_term=True)
if getattr(args, 'jupyter'): # b/c maybe metatab_jupyter is not installed
from metapack_jupyter.convert import write_metatab_notebook
from metapack_jupyter.core import edit_notebook, set_cell_source, get_cell_source
new_nb_path = Path(f'{nv_name}.ipynb')
doc['Resources'].new_term(
'Root.Datafile',
'./' + str(new_nb_path) + "#df",
name='local_dataframe',
description='Example of using a local Dataframe')
if new_nb_path.exists():
err(f"Directory {nb_path} already exists")
copyfile(nb_path, new_nb_path)
write_metatab_notebook(doc, new_nb_path)
with edit_notebook(new_nb_path) as nb:
init = get_cell_source(nb, 'init')
init += f"\nthis_package_name = '{str(new_nb_path.name)}'"
set_cell_source(nb, 'init', init)
nb_path.unlink()
else:
doc['Documentation'].new_term('Root.Documentation',
'file:README.md',
title='README')
if exists(nv_name):
err(f"Directory {nv_name} already exists")
if args.csv:
fn = doc.nonver_name + '.csv'
write_doc(doc, fn)
prt(f"Writing to {fn}")
else:
ensure_dir(nv_name)
pylib_dir = join(nv_name, 'pylib')
ensure_dir(pylib_dir)
with open(join(pylib_dir, '__init__.py'),
'w') as f_out, open(pylib.__file__) as f_in:
f_out.write(f_in.read())
if args.example:
doc['Resources'].new_term('Root.Datafile',
'python:pylib#row_generator',
name='row_generator')
prt(f"Writing to '{nv_name}'")
write_doc(doc, join(nv_name, DEFAULT_METATAB_FILE))
add_missing_files(nv_name)
if args.title:
readme = '# {}\n'.format(args.title)
else:
readme = '# {}\n'.format(doc.get_value('Root.Name'))
with open(join(nv_name, 'README.md'), 'w') as f:
f.write(readme)
def _write_doc(self):
self._doc['Root'].get_or_new_term('Root.Issued').value = datetime_now()
self._doc.write_csv(self.doc_file)
return MetapackUrl(self.doc_file, downloader=self._downloader)
def run_s3(args):
m = MetapackCliMemo(args)
if m.args.credentials:
show_credentials(m.args.profile)
exit(0)
# upload packages uploads the FS ( individual files ) and XLSX packages,
# but does not create the CSV package file
dist_urls, fs_p = upload_packages(m)
writes = 0
csv_url = None
if dist_urls:
# Create the CSV package, with links into the filesystem package
if fs_p:
access_url, dist_urls, csv_url = create_s3_csv_package(
m, dist_urls, fs_p)
else:
# If this happens, then no packages were created, because an FS package
# is always built first
prt("Not creating CSV package; no FS package was uploaded")
add_to_index(open_package(access_url))
else:
access_url = None
if dist_urls:
rows = [[path, url, reason]
for what, reason, url, path in fs_p.files_processed
if what != 'skip']
if rows:
prt("\nWrote these files:")
writes = len(rows)
prt(tabulate(rows, headers='path url reason'.split()))
rows = [[path, url, reason]
for what, reason, url, path in fs_p.files_processed
if what == 'skip']
if rows:
prt("\nSkipped these files:")
prt(tabulate(rows, headers='path url reason'.split()))
prt("\nSynchronized these Package Urls")
prt("-------------------------------")
for au in dist_urls:
prt(au)
prt("-------------------------------")
else:
prt("тЪая╕П Did not find any packages to upload to S3")
m.doc['Root'].get_or_new_term('Root.Issued').value = datetime_now()
if fs_p:
clear_cache(m, fs_p.files_processed)
csv_pkg = open_package(csv_url)
# Write the last distribution marker
dist_info = {
'name': m.doc.name,
'version': m.doc.version,
'access_url': access_url,
'path': csv_pkg.path,
'issued': datetime_now(),
'distributions': {}
}
for d in csv_pkg['Distributions'].find('Root.Distribution'):
dist_info['distributions'][d.type] = str(d.metadata_url)
Path(last_dist_marker_path(m)).write_text(yaml.safe_dump(dist_info))
if m.args.result:
if writes > 0:
print(f"тЬЕ Wrote {writes} files to {args.s3}")
else:
print(f"ЁЯЪл Did not write anything to {args.s3}")