def test_parent_dump_all(self): dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': None, '--gzip': False, '--all': True, '--processes': '1', '--get-request': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'datasets', '--worker', 'value-here-to-make-docopt-happy']) self.assertEqual(self.worker_processes, 1) self.assertEqual(self.worker_jobs, [(0, b'"12"\n'), (1, b'"34"\n'), (2, b'"dp"\n')])
def test_parent_maintain_order(self): dump_things(self.ckan, 'organizations', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': None, '--gzip': False, '--all': False, 'ID_OR_NAME': ['P', 'Q', 'R', 'S'], '--processes': '1', '--get-request': False, '--datastore-fields': False, }, worker_pool=self._mock_worker_pool_reversed, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'organizations', '--worker', 'value-here-to-make-docopt-happy' ]) self.assertEqual(self.worker_processes, 1) self.assertEqual( self.stdout.getvalue(), b'{"id":"P"}\n' b'{"id":"Q"}\n' b'{"id":"R"}\n' b'{"id":"S"}\n')
def test_parent_dump_all(self): dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': None, '--gzip': False, '--all': True, '--processes': '1', '--get-request': False, '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'datasets', '--worker', 'value-here-to-make-docopt-happy' ]) self.assertEqual(self.worker_processes, 1) self.assertEqual(self.worker_jobs, [(0, b'"12"\n'), (1, b'"34"\n'), (2, b'"dp"\n')])
def test_parent_parallel_limit(self): self.ckan.parallel_limit = 2 dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': None, '--gzip': False, '--all': False, 'ID_OR_NAME': ['12'], '--processes': '5', '--get-request': False, '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'datasets', '--worker', 'value-here-to-make-docopt-happy' ]) self.assertEqual(self.worker_processes, 2)
def test_parent_datapackages(self): target = tempfile.mkdtemp() try: dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': target, '--gzip': False, '--all': True, '--processes': '1', '--get-request': False, }, worker_pool=self._mock_worker_pool_with_data, stdout=self.stdout, stderr=self.stderr) assert exists(target + '/twelve/datapackage.json') assert exists(target + '/thirtyfour/datapackage.json') assert exists(target + '/dp/datapackage.json') assert exists(target + '/dp/data/test-file') finally: shutil.rmtree(target)
def test_parent_maintain_order(self): dump_things(self.ckan, 'organizations', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--gzip': False, '--all': False, 'ID_OR_NAME': ['P', 'Q', 'R', 'S'], '--processes': '1', '--get-request': False, }, worker_pool=self._mock_worker_pool_reversed, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'organizations', '--worker', 'value-here-to-make-docopt-happy']) self.assertEqual(self.worker_processes, 1) self.assertEqual(self.stdout.getvalue(), b'{"id":"P"}\n' b'{"id":"Q"}\n' b'{"id":"R"}\n' b'{"id":"S"}\n')
def test_parent_id_argument(self): dump_things(self.ckan, 'groups', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--dp-output': None, '--gzip': False, '--all': False, 'ID_OR_NAME': ['ab'], '--processes': '1', '--get-request': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'groups', '--worker', 'value-here-to-make-docopt-happy']) self.assertEqual(self.worker_processes, 1) self.assertEqual(self.worker_jobs, [(0, b'"ab"\n')])
def test_parent_parallel_limit(self): self.ckan.parallel_limit = 2 dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--dp-output': None, '--gzip': False, '--all': False, 'ID_OR_NAME': ['12'], '--processes': '5', '--get-request': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'datasets', '--worker', 'value-here-to-make-docopt-happy']) self.assertEqual(self.worker_processes, 2)
def test_parent_id_argument(self): dump_things(self.ckan, 'groups', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': None, '--gzip': False, '--all': False, 'ID_OR_NAME': ['ab'], '--processes': '1', '--get-request': False, '--datastore-fields': False, }, worker_pool=self._mock_worker_pool, stdout=self.stdout, stderr=self.stderr) self.assertEqual(self.worker_cmd, [ 'ckanapi', 'dump', 'groups', '--worker', 'value-here-to-make-docopt-happy' ]) self.assertEqual(self.worker_processes, 1) self.assertEqual(self.worker_jobs, [(0, b'"ab"\n')])
def test_parent_datapackages(self): target = tempfile.mkdtemp() try: dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': target, '--gzip': False, '--all': True, '--processes': '1', '--get-request': False, '--datastore-fields': False, }, worker_pool=self._worker_pool_with_data, stdout=self.stdout, stderr=self.stderr) assert exists(target + '/twelve/datapackage.json') assert exists(target + '/thirtyfour/datapackage.json') assert exists(target + '/dp/datapackage.json') assert exists(target + '/dp/data/resource1.csv') with open(target + '/dp/datapackage.json') as dpf: dp = json.load(dpf) self.assertEqual( dp, { 'name': 'dp', 'title': 'Test for datapackage', 'resources': [{ 'name': 'resource1', 'format': 'csv', 'path': 'data/resource1.csv', 'title': 'resource1', 'schema': { 'fields': [{ 'name': 'col1', 'title': 'Column One', 'description': 'Description One', 'type': 'string', }], } }] }) finally: shutil.rmtree(target)
def test_parent_datapackages(self): target = tempfile.mkdtemp() try: dump_things(self.ckan, 'datasets', { '--quiet': False, '--ckan-user': None, '--config': None, '--remote': None, '--apikey': None, '--worker': False, '--log': None, '--output': None, '--datapackages': target, '--gzip': False, '--all': True, '--processes': '1', '--get-request': False, '--datastore-fields': False, }, worker_pool=self._worker_pool_with_data, stdout=self.stdout, stderr=self.stderr) assert exists(target + '/twelve/datapackage.json') assert exists(target + '/thirtyfour/datapackage.json') assert exists(target + '/dp/datapackage.json') assert exists(target + '/dp/data/resource1.csv') with open(target + '/dp/datapackage.json') as dpf: dp = json.load(dpf) self.assertEqual(dp, { 'name': 'dp', 'title': 'Test for datapackage', 'resources': [{ 'name': 'resource1', 'format': 'csv', 'path': 'data/resource1.csv', 'title': 'resource1', 'schema': { 'fields': [{ 'name': 'col1', 'title': 'Column One', 'description': 'Description One', 'type': 'string', }], } }] }) finally: shutil.rmtree(target)
def main(running_with_paster=False): """ ckanapi command line entry point """ arguments = parse_arguments() if not running_with_paster and not arguments['--remote']: return _switch_to_paster(arguments) if arguments['--remote']: ckan = RemoteCKAN(arguments['--remote'], apikey=arguments['--apikey'], user_agent="ckanapi-cli/{version} (+{url})".format( version=__version__, url='https://github.com/open-data/ckanapi'), get_only=arguments['--get-request'], ) else: ckan = LocalCKAN(username=arguments['--ckan-user']) if arguments['action']: for r in action(ckan, arguments): sys.stdout.write(r) return things = ['datasets', 'groups', 'organizations'] thing = [x for x in things if arguments[x]] if arguments['load']: assert len(thing) == 1, thing return load_things(ckan, thing[0], arguments) if arguments['dump']: assert len(thing) == 1, thing return dump_things(ckan, thing[0], arguments) assert 0, arguments # we shouldn't be here
def main(running_with_paster=False): """ ckanapi command line entry point """ arguments = parse_arguments() if not running_with_paster and not arguments['--remote']: return _switch_to_paster(arguments) if arguments['--remote']: ckan = RemoteCKAN( arguments['--remote'], apikey=arguments['--apikey'], user_agent="ckanapi-cli/{version} (+{url})".format( version=__version__, url='https://github.com/open-data/ckanapi'), get_only=arguments['--get-request'], ) else: ckan = LocalCKAN(username=arguments['--ckan-user']) stdout = getattr(sys.stdout, 'buffer', sys.stdout) if arguments['action']: try: for r in action(ckan, arguments): stdout.write(r) return except CLIError as e: sys.stderr.write(e.args[0] + '\n') return 1 things = ['datasets', 'groups', 'organizations', 'users', 'related'] thing = [x for x in things if arguments[x]] if (arguments['load'] or arguments['dump'] or arguments['delete'] ) and arguments['--processes'] != '1' and os.name == 'nt': sys.stderr.write( "multiple worker processes are not supported on windows\n") arguments['--processes'] = '1' if arguments['load']: return load_things(ckan, thing[0], arguments) if arguments['dump']: return dump_things(ckan, thing[0], arguments) if arguments['delete']: return delete_things(ckan, thing[0], arguments) if arguments['search']: return search_datasets(ckan, arguments) assert 0, arguments # we shouldn't be here
def main(running_with_paster=False): """ ckanapi command line entry point """ arguments = parse_arguments() if not running_with_paster and not arguments['--remote']: return _switch_to_paster(arguments) if arguments['--remote']: ckan = RemoteCKAN(arguments['--remote'], apikey=arguments['--apikey'], user_agent="ckanapi-cli/{version} (+{url})".format( version=__version__, url='https://github.com/open-data/ckanapi'), get_only=arguments['--get-request'], ) else: ckan = LocalCKAN(username=arguments['--ckan-user']) stdout = getattr(sys.stdout, 'buffer', sys.stdout) if arguments['action']: try: for r in action(ckan, arguments): stdout.write(r) return except CLIError as e: sys.stderr.write(e.args[0] + '\n') return 1 things = ['datasets', 'groups', 'organizations', 'users', 'related'] thing = [x for x in things if arguments[x]] if (arguments['load'] or arguments['dump'] or arguments['delete'] ) and arguments['--processes'] != '1' and os.name == 'nt': sys.stderr.write( "multiple worker processes are not supported on windows\n") arguments['--processes'] = '1' if arguments['load']: return load_things(ckan, thing[0], arguments) if arguments['dump']: return dump_things(ckan, thing[0], arguments) if arguments['delete']: return delete_things(ckan, thing[0], arguments) assert 0, arguments # we shouldn't be here
def main(running_with_paster=False): """ ckanapi command line entry point """ arguments = parse_arguments() if not running_with_paster and not arguments["--remote"]: return _switch_to_paster(arguments) if arguments["--remote"]: ckan = RemoteCKAN( arguments["--remote"], apikey=arguments["--apikey"], user_agent="ckanapi-cli/{version} (+{url})".format( version=__version__, url="https://github.com/open-data/ckanapi" ), get_only=arguments["--get-request"], ) else: ckan = LocalCKAN(username=arguments["--ckan-user"]) if arguments["action"]: for r in action(ckan, arguments): sys.stdout.write(r) return things = ["datasets", "groups", "organizations"] thing = [x for x in things if arguments[x]] if (arguments["load"] or arguments["dump"]) and arguments["--processes"] != "1" and os.name == "nt": sys.stderr.write("multiple worker processes are not supported on windows\n") arguments["--processes"] = "1" if arguments["load"]: assert len(thing) == 1, thing return load_things(ckan, thing[0], arguments) if arguments["dump"]: assert len(thing) == 1, thing return dump_things(ckan, thing[0], arguments) assert 0, arguments # we shouldn't be here
return except CLIError, e: sys.stderr.write(e.args[0] + '\n') return 1 things = ['datasets', 'groups', 'organizations', 'users', 'related'] thing = [x for x in things if arguments[x]] if (arguments['load'] or arguments['dump'] ) and arguments['--processes'] != '1' and os.name == 'nt': sys.stderr.write( "multiple worker processes are not supported on windows\n") arguments['--processes'] = '1' if arguments['load']: assert len(thing) == 1, thing return load_things(ckan, thing[0], arguments) if arguments['dump']: assert len(thing) == 1, thing return dump_things(ckan, thing[0], arguments) assert 0, arguments # we shouldn't be here def _switch_to_paster(arguments): """ With --config we switch to the paster command version of the cli """ sys.argv[1:1] = ["--plugin=ckanapi", "ckanapi"] sys.exit(load_entry_point('PasteScript', 'console_scripts', 'paster')())
return 1 things = ["datasets", "groups", "organizations", "users", "related"] thing = [x for x in things if arguments[x]] if ( (arguments["load"] or arguments["dump"] or arguments["delete"]) and arguments["--processes"] != "1" and os.name == "nt" ): sys.stderr.write("multiple worker processes are not supported on windows\n") arguments["--processes"] = "1" if arguments["load"]: return load_things(ckan, thing[0], arguments) if arguments["dump"]: return dump_things(ckan, thing[0], arguments) if arguments["delete"]: return delete_things(ckan, thing[0], arguments) assert 0, arguments # we shouldn't be here def _switch_to_paster(arguments): """ With --config we switch to the paster command version of the cli """ sys.argv[1:1] = ["--plugin=ckanapi", "ckanapi"] sys.exit(load_entry_point("PasteScript", "console_scripts", "paster")())