def test_parent_dump_all(self):
     dump_things(self.ckan, 'datasets', {
             '--quiet': False,
             '--ckan-user': None,
             '--config': None,
             '--remote': None,
             '--apikey': None,
             '--worker': False,
             '--log': None,
             '--output': None,
             '--datapackages': None,
             '--gzip': False,
             '--all': True,
             '--processes': '1',
             '--get-request': False,
         },
         worker_pool=self._mock_worker_pool,
         stdout=self.stdout,
         stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'datasets', '--worker',
         'value-here-to-make-docopt-happy'])
     self.assertEqual(self.worker_processes, 1)
     self.assertEqual(self.worker_jobs,
         [(0, b'"12"\n'), (1, b'"34"\n'), (2, b'"dp"\n')])
Beispiel #2
0
 def test_parent_maintain_order(self):
     dump_things(self.ckan,
                 'organizations', {
                     '--quiet': False,
                     '--ckan-user': None,
                     '--config': None,
                     '--remote': None,
                     '--apikey': None,
                     '--worker': False,
                     '--log': None,
                     '--output': None,
                     '--datapackages': None,
                     '--gzip': False,
                     '--all': False,
                     'ID_OR_NAME': ['P', 'Q', 'R', 'S'],
                     '--processes': '1',
                     '--get-request': False,
                     '--datastore-fields': False,
                 },
                 worker_pool=self._mock_worker_pool_reversed,
                 stdout=self.stdout,
                 stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'organizations', '--worker',
         'value-here-to-make-docopt-happy'
     ])
     self.assertEqual(self.worker_processes, 1)
     self.assertEqual(
         self.stdout.getvalue(), b'{"id":"P"}\n'
         b'{"id":"Q"}\n'
         b'{"id":"R"}\n'
         b'{"id":"S"}\n')
Beispiel #3
0
 def test_parent_dump_all(self):
     dump_things(self.ckan,
                 'datasets', {
                     '--quiet': False,
                     '--ckan-user': None,
                     '--config': None,
                     '--remote': None,
                     '--apikey': None,
                     '--worker': False,
                     '--log': None,
                     '--output': None,
                     '--datapackages': None,
                     '--gzip': False,
                     '--all': True,
                     '--processes': '1',
                     '--get-request': False,
                     '--datastore-fields': False,
                 },
                 worker_pool=self._mock_worker_pool,
                 stdout=self.stdout,
                 stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'datasets', '--worker',
         'value-here-to-make-docopt-happy'
     ])
     self.assertEqual(self.worker_processes, 1)
     self.assertEqual(self.worker_jobs, [(0, b'"12"\n'), (1, b'"34"\n'),
                                         (2, b'"dp"\n')])
Beispiel #4
0
 def test_parent_parallel_limit(self):
     self.ckan.parallel_limit = 2
     dump_things(self.ckan,
                 'datasets', {
                     '--quiet': False,
                     '--ckan-user': None,
                     '--config': None,
                     '--remote': None,
                     '--apikey': None,
                     '--worker': False,
                     '--log': None,
                     '--output': None,
                     '--datapackages': None,
                     '--gzip': False,
                     '--all': False,
                     'ID_OR_NAME': ['12'],
                     '--processes': '5',
                     '--get-request': False,
                     '--datastore-fields': False,
                 },
                 worker_pool=self._mock_worker_pool,
                 stdout=self.stdout,
                 stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'datasets', '--worker',
         'value-here-to-make-docopt-happy'
     ])
     self.assertEqual(self.worker_processes, 2)
Beispiel #5
0
 def test_parent_datapackages(self):
     target = tempfile.mkdtemp()
     try:
         dump_things(self.ckan,
                     'datasets', {
                         '--quiet': False,
                         '--ckan-user': None,
                         '--config': None,
                         '--remote': None,
                         '--apikey': None,
                         '--worker': False,
                         '--log': None,
                         '--output': None,
                         '--datapackages': target,
                         '--gzip': False,
                         '--all': True,
                         '--processes': '1',
                         '--get-request': False,
                     },
                     worker_pool=self._mock_worker_pool_with_data,
                     stdout=self.stdout,
                     stderr=self.stderr)
         assert exists(target + '/twelve/datapackage.json')
         assert exists(target + '/thirtyfour/datapackage.json')
         assert exists(target + '/dp/datapackage.json')
         assert exists(target + '/dp/data/test-file')
     finally:
         shutil.rmtree(target)
Beispiel #6
0
 def test_parent_maintain_order(self):
     dump_things(self.ckan, 'organizations', {
             '--quiet': False,
             '--ckan-user': None,
             '--config': None,
             '--remote': None,
             '--apikey': None,
             '--worker': False,
             '--log': None,
             '--output': None,
             '--gzip': False,
             '--all': False,
             'ID_OR_NAME': ['P', 'Q', 'R', 'S'],
             '--processes': '1',
             '--get-request': False,
         },
         worker_pool=self._mock_worker_pool_reversed,
         stdout=self.stdout,
         stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'organizations', '--worker',
         'value-here-to-make-docopt-happy'])
     self.assertEqual(self.worker_processes, 1)
     self.assertEqual(self.stdout.getvalue(),
         b'{"id":"P"}\n'
         b'{"id":"Q"}\n'
         b'{"id":"R"}\n'
         b'{"id":"S"}\n')
Beispiel #7
0
    def test_parent_id_argument(self):
        dump_things(self.ckan, 'groups', {
                '--quiet': False,
                '--ckan-user': None,
                '--config': None,
                '--remote': None,
                '--apikey': None,
                '--worker': False,
                '--log': None,
                '--output': None,
                '--dp-output': None,
                '--gzip': False,
                '--all': False,
                'ID_OR_NAME': ['ab'],
                '--processes': '1',
                '--get-request': False,
            },

            worker_pool=self._mock_worker_pool,
            stdout=self.stdout,
            stderr=self.stderr)
        self.assertEqual(self.worker_cmd, [
            'ckanapi', 'dump', 'groups', '--worker',
            'value-here-to-make-docopt-happy'])
        self.assertEqual(self.worker_processes, 1)
        self.assertEqual(self.worker_jobs, [(0, b'"ab"\n')])
Beispiel #8
0
 def test_parent_parallel_limit(self):
     self.ckan.parallel_limit = 2
     dump_things(self.ckan, 'datasets', {
             '--quiet': False,
             '--ckan-user': None,
             '--config': None,
             '--remote': None,
             '--apikey': None,
             '--worker': False,
             '--log': None,
             '--output': None,
             '--dp-output': None,
             '--gzip': False,
             '--all': False,
             'ID_OR_NAME': ['12'],
             '--processes': '5',
             '--get-request': False,
         },
         worker_pool=self._mock_worker_pool,
         stdout=self.stdout,
         stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'datasets', '--worker',
         'value-here-to-make-docopt-happy'])
     self.assertEqual(self.worker_processes, 2)
 def test_parent_datapackages(self):
     target = tempfile.mkdtemp()
     try:
         dump_things(self.ckan, 'datasets', {
                 '--quiet': False,
                 '--ckan-user': None,
                 '--config': None,
                 '--remote': None,
                 '--apikey': None,
                 '--worker': False,
                 '--log': None,
                 '--output': None,
                 '--datapackages': target,
                 '--gzip': False,
                 '--all': True,
                 '--processes': '1',
                 '--get-request': False,
             },
             worker_pool=self._mock_worker_pool_with_data,
             stdout=self.stdout,
             stderr=self.stderr)
         assert exists(target + '/twelve/datapackage.json')
         assert exists(target + '/thirtyfour/datapackage.json')
         assert exists(target + '/dp/datapackage.json')
         assert exists(target + '/dp/data/test-file')
     finally:
         shutil.rmtree(target)
Beispiel #10
0
 def test_parent_id_argument(self):
     dump_things(self.ckan,
                 'groups', {
                     '--quiet': False,
                     '--ckan-user': None,
                     '--config': None,
                     '--remote': None,
                     '--apikey': None,
                     '--worker': False,
                     '--log': None,
                     '--output': None,
                     '--datapackages': None,
                     '--gzip': False,
                     '--all': False,
                     'ID_OR_NAME': ['ab'],
                     '--processes': '1',
                     '--get-request': False,
                     '--datastore-fields': False,
                 },
                 worker_pool=self._mock_worker_pool,
                 stdout=self.stdout,
                 stderr=self.stderr)
     self.assertEqual(self.worker_cmd, [
         'ckanapi', 'dump', 'groups', '--worker',
         'value-here-to-make-docopt-happy'
     ])
     self.assertEqual(self.worker_processes, 1)
     self.assertEqual(self.worker_jobs, [(0, b'"ab"\n')])
Beispiel #11
0
 def test_parent_datapackages(self):
     target = tempfile.mkdtemp()
     try:
         dump_things(self.ckan,
                     'datasets', {
                         '--quiet': False,
                         '--ckan-user': None,
                         '--config': None,
                         '--remote': None,
                         '--apikey': None,
                         '--worker': False,
                         '--log': None,
                         '--output': None,
                         '--datapackages': target,
                         '--gzip': False,
                         '--all': True,
                         '--processes': '1',
                         '--get-request': False,
                         '--datastore-fields': False,
                     },
                     worker_pool=self._worker_pool_with_data,
                     stdout=self.stdout,
                     stderr=self.stderr)
         assert exists(target + '/twelve/datapackage.json')
         assert exists(target + '/thirtyfour/datapackage.json')
         assert exists(target + '/dp/datapackage.json')
         assert exists(target + '/dp/data/resource1.csv')
         with open(target + '/dp/datapackage.json') as dpf:
             dp = json.load(dpf)
         self.assertEqual(
             dp, {
                 'name':
                 'dp',
                 'title':
                 'Test for datapackage',
                 'resources': [{
                     'name': 'resource1',
                     'format': 'csv',
                     'path': 'data/resource1.csv',
                     'title': 'resource1',
                     'schema': {
                         'fields': [{
                             'name': 'col1',
                             'title': 'Column One',
                             'description': 'Description One',
                             'type': 'string',
                         }],
                     }
                 }]
             })
     finally:
         shutil.rmtree(target)
Beispiel #12
0
 def test_parent_datapackages(self):
     target = tempfile.mkdtemp()
     try:
         dump_things(self.ckan, 'datasets', {
                 '--quiet': False,
                 '--ckan-user': None,
                 '--config': None,
                 '--remote': None,
                 '--apikey': None,
                 '--worker': False,
                 '--log': None,
                 '--output': None,
                 '--datapackages': target,
                 '--gzip': False,
                 '--all': True,
                 '--processes': '1',
                 '--get-request': False,
                 '--datastore-fields': False,
             },
             worker_pool=self._worker_pool_with_data,
             stdout=self.stdout,
             stderr=self.stderr)
         assert exists(target + '/twelve/datapackage.json')
         assert exists(target + '/thirtyfour/datapackage.json')
         assert exists(target + '/dp/datapackage.json')
         assert exists(target + '/dp/data/resource1.csv')
         with open(target + '/dp/datapackage.json') as dpf:
             dp = json.load(dpf)
         self.assertEqual(dp, {
             'name': 'dp',
             'title': 'Test for datapackage',
             'resources': [{
                 'name': 'resource1',
                 'format': 'csv',
                 'path': 'data/resource1.csv',
                 'title': 'resource1',
                 'schema': {
                     'fields': [{
                         'name': 'col1',
                         'title': 'Column One',
                         'description': 'Description One',
                         'type': 'string',
                     }],
                 }
             }]
         })
     finally:
         shutil.rmtree(target)
Beispiel #13
0
def main(running_with_paster=False):
    """
    ckanapi command line entry point
    """
    arguments = parse_arguments()
    if not running_with_paster and not arguments['--remote']:
        return _switch_to_paster(arguments)

    if arguments['--remote']:
        ckan = RemoteCKAN(arguments['--remote'],
            apikey=arguments['--apikey'],
            user_agent="ckanapi-cli/{version} (+{url})".format(
                version=__version__,
                url='https://github.com/open-data/ckanapi'),
            get_only=arguments['--get-request'],
            )
    else:
        ckan = LocalCKAN(username=arguments['--ckan-user'])

    if arguments['action']:
        for r in action(ckan, arguments):
            sys.stdout.write(r)
        return

    things = ['datasets', 'groups', 'organizations']
    thing = [x for x in things if arguments[x]]
    if arguments['load']:
        assert len(thing) == 1, thing
        return load_things(ckan, thing[0], arguments)

    if arguments['dump']:
        assert len(thing) == 1, thing
        return dump_things(ckan, thing[0], arguments)

    assert 0, arguments # we shouldn't be here
Beispiel #14
0
def main(running_with_paster=False):
    """
    ckanapi command line entry point
    """
    arguments = parse_arguments()

    if not running_with_paster and not arguments['--remote']:
        return _switch_to_paster(arguments)

    if arguments['--remote']:
        ckan = RemoteCKAN(
            arguments['--remote'],
            apikey=arguments['--apikey'],
            user_agent="ckanapi-cli/{version} (+{url})".format(
                version=__version__,
                url='https://github.com/open-data/ckanapi'),
            get_only=arguments['--get-request'],
        )
    else:
        ckan = LocalCKAN(username=arguments['--ckan-user'])

    stdout = getattr(sys.stdout, 'buffer', sys.stdout)
    if arguments['action']:
        try:
            for r in action(ckan, arguments):
                stdout.write(r)
            return
        except CLIError as e:
            sys.stderr.write(e.args[0] + '\n')
            return 1

    things = ['datasets', 'groups', 'organizations', 'users', 'related']
    thing = [x for x in things if arguments[x]]
    if (arguments['load'] or arguments['dump'] or arguments['delete']
        ) and arguments['--processes'] != '1' and os.name == 'nt':
        sys.stderr.write(
            "multiple worker processes are not supported on windows\n")
        arguments['--processes'] = '1'

    if arguments['load']:
        return load_things(ckan, thing[0], arguments)

    if arguments['dump']:
        return dump_things(ckan, thing[0], arguments)

    if arguments['delete']:
        return delete_things(ckan, thing[0], arguments)

    if arguments['search']:
        return search_datasets(ckan, arguments)

    assert 0, arguments  # we shouldn't be here
Beispiel #15
0
def main(running_with_paster=False):
    """
    ckanapi command line entry point
    """
    arguments = parse_arguments()

    if not running_with_paster and not arguments['--remote']:
        return _switch_to_paster(arguments)

    if arguments['--remote']:
        ckan = RemoteCKAN(arguments['--remote'],
            apikey=arguments['--apikey'],
            user_agent="ckanapi-cli/{version} (+{url})".format(
                version=__version__,
                url='https://github.com/open-data/ckanapi'),
            get_only=arguments['--get-request'],
            )
    else:
        ckan = LocalCKAN(username=arguments['--ckan-user'])

    stdout = getattr(sys.stdout, 'buffer', sys.stdout)
    if arguments['action']:
        try:
            for r in action(ckan, arguments):
                stdout.write(r)
            return
        except CLIError as e:
            sys.stderr.write(e.args[0] + '\n')
            return 1

    things = ['datasets', 'groups', 'organizations', 'users', 'related']
    thing = [x for x in things if arguments[x]]
    if (arguments['load'] or arguments['dump'] or arguments['delete']
            ) and arguments['--processes'] != '1' and os.name == 'nt':
        sys.stderr.write(
            "multiple worker processes are not supported on windows\n")
        arguments['--processes'] = '1'

    if arguments['load']:
        return load_things(ckan, thing[0], arguments)

    if arguments['dump']:
        return dump_things(ckan, thing[0], arguments)

    if arguments['delete']:
        return delete_things(ckan, thing[0], arguments)

    assert 0, arguments # we shouldn't be here
Beispiel #16
0
def main(running_with_paster=False):
    """
    ckanapi command line entry point
    """
    arguments = parse_arguments()

    if not running_with_paster and not arguments["--remote"]:
        return _switch_to_paster(arguments)

    if arguments["--remote"]:
        ckan = RemoteCKAN(
            arguments["--remote"],
            apikey=arguments["--apikey"],
            user_agent="ckanapi-cli/{version} (+{url})".format(
                version=__version__, url="https://github.com/open-data/ckanapi"
            ),
            get_only=arguments["--get-request"],
        )
    else:
        ckan = LocalCKAN(username=arguments["--ckan-user"])

    if arguments["action"]:
        for r in action(ckan, arguments):
            sys.stdout.write(r)
        return

    things = ["datasets", "groups", "organizations"]
    thing = [x for x in things if arguments[x]]
    if (arguments["load"] or arguments["dump"]) and arguments["--processes"] != "1" and os.name == "nt":
        sys.stderr.write("multiple worker processes are not supported on windows\n")
        arguments["--processes"] = "1"

    if arguments["load"]:
        assert len(thing) == 1, thing
        return load_things(ckan, thing[0], arguments)

    if arguments["dump"]:
        assert len(thing) == 1, thing
        return dump_things(ckan, thing[0], arguments)

    assert 0, arguments  # we shouldn't be here
Beispiel #17
0
            return
        except CLIError, e:
            sys.stderr.write(e.args[0] + '\n')
            return 1

    things = ['datasets', 'groups', 'organizations', 'users', 'related']
    thing = [x for x in things if arguments[x]]
    if (arguments['load'] or arguments['dump']
        ) and arguments['--processes'] != '1' and os.name == 'nt':
        sys.stderr.write(
            "multiple worker processes are not supported on windows\n")
        arguments['--processes'] = '1'

    if arguments['load']:
        assert len(thing) == 1, thing
        return load_things(ckan, thing[0], arguments)

    if arguments['dump']:
        assert len(thing) == 1, thing
        return dump_things(ckan, thing[0], arguments)

    assert 0, arguments  # we shouldn't be here


def _switch_to_paster(arguments):
    """
    With --config we switch to the paster command version of the cli
    """
    sys.argv[1:1] = ["--plugin=ckanapi", "ckanapi"]
    sys.exit(load_entry_point('PasteScript', 'console_scripts', 'paster')())
Beispiel #18
0
            return 1

    things = ["datasets", "groups", "organizations", "users", "related"]
    thing = [x for x in things if arguments[x]]
    if (
        (arguments["load"] or arguments["dump"] or arguments["delete"])
        and arguments["--processes"] != "1"
        and os.name == "nt"
    ):
        sys.stderr.write("multiple worker processes are not supported on windows\n")
        arguments["--processes"] = "1"

    if arguments["load"]:
        return load_things(ckan, thing[0], arguments)

    if arguments["dump"]:
        return dump_things(ckan, thing[0], arguments)

    if arguments["delete"]:
        return delete_things(ckan, thing[0], arguments)

    assert 0, arguments  # we shouldn't be here


def _switch_to_paster(arguments):
    """
    With --config we switch to the paster command version of the cli
    """
    sys.argv[1:1] = ["--plugin=ckanapi", "ckanapi"]
    sys.exit(load_entry_point("PasteScript", "console_scripts", "paster")())