def test_parent_parallel_limit(self):
self.ckan.parallel_limit = 2
load_things(self.ckan, 'datasets', {
'--quiet': False,
'--ckan-user': None,
'--config': None,
'--remote': None,
'--apikey': None,
'--worker': False,
'--log': None,
'--gzip': False,
'--processes': '5',
'--input': None,
'--create-only': False,
'--update-only': False,
'--start-record': '1',
'--max-records': None,
},
worker_pool=self._mock_worker_pool,
stdin=BytesIO(
b'{"name": "cd", "title": "Go"}\n'
b'{"name": "ef", "title": "Play"}\n'
),
stdout=self.stdout,
stderr=self.stderr)
self.assertEqual(self.worker_cmd, [
'ckanapi', 'load', 'datasets', '--worker'])
self.assertEqual(self.worker_processes, 2)
def test_parent_load_two(self):
load_things(self.ckan,
'datasets', {
'--quiet': False,
'--ckan-user': None,
'--config': None,
'--remote': None,
'--apikey': None,
'--worker': False,
'--log': None,
'--gzip': False,
'--processes': '1',
'--input': None,
'--create-only': False,
'--update-only': False,
'--start-record': '1',
'--max-records': None,
'--upload-resources': False,
'--upload-logo': False
},
worker_pool=self._mock_worker_pool,
stdin=BytesIO(b'{"name": "cd", "title": "Go"}\n'
b'{"name": "ef", "title": "Play"}\n'),
stdout=self.stdout,
stderr=self.stderr)
self.assertEqual(self.worker_cmd,
['ckanapi', 'load', 'datasets', '--worker'])
self.assertEqual(self.worker_processes, 1)
self.assertEqual(self.worker_jobs, [
(1, b'{"name": "cd", "title": "Go"}\n'),
(2, b'{"name": "ef", "title": "Play"}\n'),
])
def test_parent_load_start_max(self):
load_things(self.ckan, 'groups', {
'--quiet': False,
'--ckan-user': None,
'--config': None,
'--remote': None,
'--apikey': None,
'--worker': False,
'--log': None,
'--gzip': False,
'--processes': '1',
'--input': None,
'--create-only': False,
'--update-only': False,
'--start-record': '2',
'--max-records': '2',
},
worker_pool=self._mock_worker_pool,
stdin=BytesIO(
b'{"name": "cd", "title": "Go"}\n'
b'{"name": "ef", "title": "Play"}\n'
b'{"name": "gh", "title": "Hotel"}\n'
b'{"name": "ij", "title": "Ambient"}\n'
),
stdout=self.stdout,
stderr=self.stderr)
self.assertEqual(self.worker_cmd, [
'ckanapi', 'load', 'groups', '--worker'])
self.assertEqual(self.worker_processes, 1)
self.assertEqual(self.worker_jobs, [
(2, b'{"name": "ef", "title": "Play"}\n'),
(3, b'{"name": "gh", "title": "Hotel"}\n'),
])
def main(running_with_paster=False):
"""
ckanapi command line entry point
"""
arguments = parse_arguments()
if not running_with_paster and not arguments['--remote']:
return _switch_to_paster(arguments)
if arguments['--remote']:
ckan = RemoteCKAN(arguments['--remote'],
apikey=arguments['--apikey'],
user_agent="ckanapi-cli/{version} (+{url})".format(
version=__version__,
url='https://github.com/open-data/ckanapi'),
get_only=arguments['--get-request'],
)
else:
ckan = LocalCKAN(username=arguments['--ckan-user'])
if arguments['action']:
for r in action(ckan, arguments):
sys.stdout.write(r)
return
things = ['datasets', 'groups', 'organizations']
thing = [x for x in things if arguments[x]]
if arguments['load']:
assert len(thing) == 1, thing
return load_things(ckan, thing[0], arguments)
if arguments['dump']:
assert len(thing) == 1, thing
return dump_things(ckan, thing[0], arguments)
assert 0, arguments # we shouldn't be here
def main(running_with_paster=False):
"""
ckanapi command line entry point
"""
arguments = parse_arguments()
if not running_with_paster and not arguments['--remote']:
return _switch_to_paster(arguments)
if arguments['--remote']:
ckan = RemoteCKAN(
arguments['--remote'],
apikey=arguments['--apikey'],
user_agent="ckanapi-cli/{version} (+{url})".format(
version=__version__,
url='https://github.com/open-data/ckanapi'),
get_only=arguments['--get-request'],
)
else:
ckan = LocalCKAN(username=arguments['--ckan-user'])
stdout = getattr(sys.stdout, 'buffer', sys.stdout)
if arguments['action']:
try:
for r in action(ckan, arguments):
stdout.write(r)
return
except CLIError as e:
sys.stderr.write(e.args[0] + '\n')
return 1
things = ['datasets', 'groups', 'organizations', 'users', 'related']
thing = [x for x in things if arguments[x]]
if (arguments['load'] or arguments['dump'] or arguments['delete']
) and arguments['--processes'] != '1' and os.name == 'nt':
sys.stderr.write(
"multiple worker processes are not supported on windows\n")
arguments['--processes'] = '1'
if arguments['load']:
return load_things(ckan, thing[0], arguments)
if arguments['dump']:
return dump_things(ckan, thing[0], arguments)
if arguments['delete']:
return delete_things(ckan, thing[0], arguments)
if arguments['search']:
return search_datasets(ckan, arguments)
assert 0, arguments # we shouldn't be here
def main(running_with_paster=False):
"""
ckanapi command line entry point
"""
arguments = parse_arguments()
if not running_with_paster and not arguments['--remote']:
return _switch_to_paster(arguments)
if arguments['--remote']:
ckan = RemoteCKAN(arguments['--remote'],
apikey=arguments['--apikey'],
user_agent="ckanapi-cli/{version} (+{url})".format(
version=__version__,
url='https://github.com/open-data/ckanapi'),
get_only=arguments['--get-request'],
)
else:
ckan = LocalCKAN(username=arguments['--ckan-user'])
stdout = getattr(sys.stdout, 'buffer', sys.stdout)
if arguments['action']:
try:
for r in action(ckan, arguments):
stdout.write(r)
return
except CLIError as e:
sys.stderr.write(e.args[0] + '\n')
return 1
things = ['datasets', 'groups', 'organizations', 'users', 'related']
thing = [x for x in things if arguments[x]]
if (arguments['load'] or arguments['dump'] or arguments['delete']
) and arguments['--processes'] != '1' and os.name == 'nt':
sys.stderr.write(
"multiple worker processes are not supported on windows\n")
arguments['--processes'] = '1'
if arguments['load']:
return load_things(ckan, thing[0], arguments)
if arguments['dump']:
return dump_things(ckan, thing[0], arguments)
if arguments['delete']:
return delete_things(ckan, thing[0], arguments)
assert 0, arguments # we shouldn't be here
def main(running_with_paster=False):
"""
ckanapi command line entry point
"""
arguments = parse_arguments()
if not running_with_paster and not arguments["--remote"]:
return _switch_to_paster(arguments)
if arguments["--remote"]:
ckan = RemoteCKAN(
arguments["--remote"],
apikey=arguments["--apikey"],
user_agent="ckanapi-cli/{version} (+{url})".format(
version=__version__, url="https://github.com/open-data/ckanapi"
),
get_only=arguments["--get-request"],
)
else:
ckan = LocalCKAN(username=arguments["--ckan-user"])
if arguments["action"]:
for r in action(ckan, arguments):
sys.stdout.write(r)
return
things = ["datasets", "groups", "organizations"]
thing = [x for x in things if arguments[x]]
if (arguments["load"] or arguments["dump"]) and arguments["--processes"] != "1" and os.name == "nt":
sys.stderr.write("multiple worker processes are not supported on windows\n")
arguments["--processes"] = "1"
if arguments["load"]:
assert len(thing) == 1, thing
return load_things(ckan, thing[0], arguments)
if arguments["dump"]:
assert len(thing) == 1, thing
return dump_things(ckan, thing[0], arguments)
assert 0, arguments # we shouldn't be here
return
except CLIError, e:
sys.stderr.write(e.args[0] + '\n')
return 1
things = ['datasets', 'groups', 'organizations', 'users', 'related']
thing = [x for x in things if arguments[x]]
if (arguments['load'] or arguments['dump']
) and arguments['--processes'] != '1' and os.name == 'nt':
sys.stderr.write(
"multiple worker processes are not supported on windows\n")
arguments['--processes'] = '1'
if arguments['load']:
assert len(thing) == 1, thing
return load_things(ckan, thing[0], arguments)
if arguments['dump']:
assert len(thing) == 1, thing
return dump_things(ckan, thing[0], arguments)
assert 0, arguments # we shouldn't be here
def _switch_to_paster(arguments):
"""
With --config we switch to the paster command version of the cli
"""
sys.argv[1:1] = ["--plugin=ckanapi", "ckanapi"]
sys.exit(load_entry_point('PasteScript', 'console_scripts', 'paster')())
except CLIError, e:
sys.stderr.write(e.args[0] + "\n")
return 1
things = ["datasets", "groups", "organizations", "users", "related"]
thing = [x for x in things if arguments[x]]
if (
(arguments["load"] or arguments["dump"] or arguments["delete"])
and arguments["--processes"] != "1"
and os.name == "nt"
):
sys.stderr.write("multiple worker processes are not supported on windows\n")
arguments["--processes"] = "1"
if arguments["load"]:
return load_things(ckan, thing[0], arguments)
if arguments["dump"]:
return dump_things(ckan, thing[0], arguments)
if arguments["delete"]:
return delete_things(ckan, thing[0], arguments)
assert 0, arguments # we shouldn't be here
def _switch_to_paster(arguments):
"""
With --config we switch to the paster command version of the cli
"""
sys.argv[1:1] = ["--plugin=ckanapi", "ckanapi"]