def test_write_line_doc(self): """Convert CSV files to text lines and back to text lines""" all = [ 'example1.csv', 'example2.csv', 'example1-web.csv', 'children.csv', 'children2.csv', 'issue1.csv' ] self.maxDiff = None for f in all: path = test_data(f) doc1 = MetatabDoc(path) doc1_lines = doc1.as_lines() print(doc1_lines) doc2 = MetatabDoc(TextRowGenerator(doc1_lines)) doc2_lines = doc2.as_lines() self.assertEqual(doc1_lines, doc2_lines) self.compare_dict(doc1.as_dict(), doc2.as_dict()) self.assertEqual(doc1_lines, doc2_lines) self.assertEqual(doc1.as_csv(), doc2.as_csv())
def test_datapackage_declare(self): import datapackage doc = MetatabDoc(test_data('datapackage_ex2.csv')) d = doc.as_dict() f = open('/tmp/package.json', 'w') # NamedTemporaryFile(delete=False) f.write(json.dumps(d, indent=4)) f.close() try: dp = datapackage.DataPackage(f.name) dp.validate() except: with open(f.name) as f2: print(f2.read()) raise print(f.name) # unlink(f.name) doc = MetatabDoc(test_data('example1.csv')) from metatab.datapackage import convert_to_datapackage print(json.dumps(convert_to_datapackage(doc), indent=4))
def test_children(self): doc = MetatabDoc(test_data('children.csv')) for t in doc.terms: print(t) import json print(json.dumps(doc.as_dict(), indent=4)) for t in doc.as_dict()['parent']: self.assertEquals( { 'prop1': 'prop1', 'prop2': 'prop2', '@value': 'parent' }, t)
def test_includes(self): doc = MetatabDoc(test_data('include1.csv')) d = doc.as_dict() for t in doc['root'].terms: print(t) print(d) self.assertEquals( ['Include File 1', 'Include File 2', 'Include File 3'], d['note']) self.assertTrue(any('include2.csv' in e for e in d['include'])) self.assertTrue(any('include3.csv' in e for e in d['include']))
def test_parse_everything(self): import json all = [ 'example1.csv', 'example2.csv', 'example1-web.csv', 'include1.csv', 'include2.csv', 'include3.csv', 'children.csv', 'children2.csv', 'issue1.csv' ] # These are currently broken -- as_dict doesn't work properly with the # datapackage-latest decl. datapackages = [ 'datapackage_ex1.csv', 'datapackage_ex1_web.csv', 'datapackage_ex2.csv' ] for fn in all: print('Testing ', fn) path = test_data(fn) json_path = test_data('json', fn.replace('.csv', '.json')) doc = MetatabDoc(path) d = doc.as_dict() if not exists(json_path): with open(json_path, 'w') as f: print("Writing", json_path) json.dump(d, f, indent=4) with open(json_path) as f: d2 = json.load(f) self.compare_dict(d, d2)
def metatab(): import argparse parser = argparse.ArgumentParser( prog='metatab', description='Matatab file parser, version {}'.format(_meta.__version__)) parser.add_argument('-C', '--clean-cache', default=False, action='store_true', help="Clean the download cache") g = parser.add_mutually_exclusive_group(required=True) g.add_argument('-i', '--info', default=False, action='store_true', help="Show configuration information") g.add_argument('-c', '--create', action='store', nargs='?', default=False, help="Create a new metatab file, from named template. With no argument, uses the 'metatab' template ") g.add_argument('-t', '--terms', default=False, action='store_true', help='Parse a file and print out the stream of terms, before interpretation') g.add_argument('-I', '--interp', default=False, action='store_true', help='Parse a file and print out the stream of terms, after interpretation') g.add_argument('-j', '--json', default=False, action='store_true', help='Parse a file and print out a JSON representation') g.add_argument('-y', '--yaml', default=False, action='store_true', help='Parse a file and print out a YAML representation') g.add_argument('-R', '--resource', default=False, action='store_true', help='If the URL has no fragment, dump the resources listed in the metatab file. With a fragment, dump a resource as a CSV') g.add_argument('-H', '--head', default=False, action='store_true', help="Dump the first 20 lines of a resoruce ") g.add_argument('-S', '--schema', help='Dump the schema for one named resource') parser.add_argument('-d', '--show-declaration', default=False, action='store_true', help='Parse a declaration file and print out declaration dict. Use -j or -y for the format') parser.add_argument('-D', '--declare', help='Parse and incorporate a declaration before parsing the file.' + ' (Adds the declaration to the start of the file as the first term. )') parser.add_argument('file', nargs='?', default=DEFAULT_METATAB_FILE, help='Path to a Metatab file') args = parser.parse_args(sys.argv[1:]) # Specing a fragment screws up setting the default metadata file name if args.file.startswith('#'): args.file = DEFAULT_METATAB_FILE + args.file cache = get_cache('metapack') if args.info: prt('Version : {}'.format(_meta.__version__)) prt('Cache dir: {}'.format(str(cache.getsyspath('/')))) exit(0) if args.clean_cache: clean_cache(cache) if args.create is not False: new_metatab_file(args.file, args.create) exit(0) if args.resource or args.head: limit = 20 if args.head else None u = Url(args.file) resource = u.parts.fragment metadata_url = u.rebuild_url(False, False) package_url, metadata_url = resolve_package_metadata_url(metadata_url) try: doc = MetatabDoc(metadata_url, cache=cache) except OSError as e: err("Failed to open Metatab doc: {}".format(e)) return # Never reached if resource: dump_resource(doc, resource, limit) else: dump_resources(doc) exit(0) if args.show_declaration: doc = MetatabDoc() doc.load_declarations([args.file]) print(json.dumps({ 'terms': doc.decl_terms, 'sections': doc.decl_sections }, indent=4)) exit(0) else: package_url, metadata_url = resolve_package_metadata_url(args.file) try: doc = MetatabDoc(metadata_url, cache=cache) except IOError as e: raise err("Failed to open '{}': {}".format(metadata_url, e)) if args.terms: for t in doc._term_parser: print(t) elif args.json: print(json.dumps(doc.as_dict(), indent=4)) elif args.yaml: import yaml print(yaml.safe_dump(doc.as_dict(), default_flow_style=False, indent=4)) elif args.schema: dump_schema(doc, args.schema) exit(0)
def metatab(): import argparse parser = argparse.ArgumentParser(prog='metatab', description='Matatab file parser', epilog='Cache dir: {}\n'.format( str(cache.getsyspath('/')))) g = parser.add_mutually_exclusive_group() g.add_argument( '-C', '--create', action='store', nargs='?', default=False, help= "Create a new metatab file, from named template. With no argument, uses the 'metatab' template " ) g.add_argument( '-t', '--terms', default=False, action='store_const', dest='out_type', const='terms', help= 'Parse a file and print out the stream of terms, before interpretation' ) g.add_argument('-j', '--json', default=False, action='store_const', dest='out_type', const='json', help='Parse a file and print out a JSON representation') g.add_argument('-y', '--yaml', default=False, action='store_const', dest='out_type', const='yaml', help='Parse a file and print out a YAML representation') g.add_argument( '-l', '--line', default=False, action='store_const', dest='out_type', const='line', help='Parse a file and print out a Metatab Line representation') g.add_argument( '-c', '--csv', default=False, action='store_const', dest='out_type', const='csv', help='Parse a file and print out a Metatab Line representation') g.add_argument('-p', '--prety', default=False, action='store_const', dest='out_type', const='prety', help='Pretty print the python Dict representation ') parser.add_argument( '-W', '--write-in-place', help= 'When outputting as yaml, json, csv or line, write the file instead of printing it, ' 'to a file with same base name and appropriate extension ', action='store_true') parser.set_defaults(out_type='csv') parser.add_argument( '-f', '--find-first', help='Find and print the first value for a fully qualified term name') parser.add_argument( '-d', '--show-declaration', default=False, action='store_true', help= 'Parse a declaration file and print out declaration dict. Use -j or -y for the format' ) parser.add_argument('file', nargs='?', default=DEFAULT_METATAB_FILE, help='Path to a Metatab file') cli_init() args = parser.parse_args(sys.argv[1:]) # Specing a fragment screws up setting the default metadata file name if args.file.startswith('#'): args.file = DEFAULT_METATAB_FILE + args.file if args.create is not False: if new_metatab_file(args.file, args.create): prt("Created ", args.file) else: warn("File", args.file, 'already exists.') exit(0) metadata_url = parse_app_url(args.file, proto='metatab') try: doc = MetatabDoc(metadata_url, cache=cache) except IOError as e: err("Failed to open '{}': {}".format(metadata_url, e)) def write_or_print(t): from pathlib import Path if metadata_url.scheme != 'file': err("Can only use -w with local files") return ext = 'txt' if args.out_type == 'line' else args.out_type if args.write_in_place: with metadata_url.fspath.with_suffix('.' + ext).open('w') as f: f.write(t) else: print(t) if args.show_declaration: decl_doc = MetatabDoc('', cache=cache, decl=metadata_url.path) d = {'terms': decl_doc.decl_terms, 'sections': decl_doc.decl_sections} if args.out_type == 'json': print(json.dumps(d, indent=4)) elif args.out_type == 'yaml': import yaml print(yaml.safe_dump(d, default_flow_style=False, indent=4)) elif args.find_first: t = doc.find_first(args.find_first) print(t.value) elif args.out_type == 'terms': for t in doc._term_parser: print(t) elif args.out_type == 'json': write_or_print(json.dumps(doc.as_dict(), indent=4)) elif args.out_type == 'yaml': import yaml from collections import OrderedDict def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): class OrderedDumper(Dumper): pass def _dict_representer(dumper, data): return dumper.represent_mapping( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) OrderedDumper.add_representer(OrderedDict, _dict_representer) return yaml.dump(data, stream, OrderedDumper, **kwds) write_or_print( ordered_dump(doc.as_dict(), default_flow_style=False, indent=4, Dumper=yaml.SafeDumper)) elif args.out_type == 'line': write_or_print(doc.as_lines()) elif args.out_type == 'csv': write_or_print(doc.as_csv()) elif args.out_type == 'prety': from pprint import pprint pprint(doc.as_dict()) exit(0)