예제 #1
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    headers = get_args_dict(args['--header'])
    if args['--size-hint']:
        headers['x-archive-size-hint'] = args['--size-hint']

    # Upload keyword arguments.
    upload_kwargs = dict(
        metadata=get_args_dict(args['--metadata']),
        headers=headers,
        debug=args['--debug'],
        queue_derive=True if args['--no-derive'] is False else False,
        ignore_preexisting_bucket=args['--ignore-bucket'],
        checksum=args['--checksum'],
        verbose=True if args['--quiet'] is False else False,
        retries=int(args['--retries']) if args['--retries'] else 0,
        retries_sleep=int(args['--sleep']),
        delete=args['--delete'],
    )

    if args['<file>'] == ['-'] and not args['-']:
        sys.stderr.write(
            '--remote-name is required when uploading from stdin.\n')
        call(['ia', 'upload', '--help'])
        sys.exit(1)

    # Upload from stdin.
    if args['-']:
        local_file = TemporaryFile()
        local_file.write(sys.stdin.read())
        local_file.seek(0)
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)

    # Bulk upload using spreadsheet.
    elif args['--spreadsheet']:
        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        prev_identifier = None
        for row in spreadsheet:
            local_file = row['file']
            identifier = row['identifier']
            del row['file']
            del row['identifier']
            if (not identifier) and (prev_identifier):
                identifier = prev_identifier
            # TODO: Clean up how indexed metadata items are coerced
            # into metadata.
            md_args = [
                '{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v
            ]
            metadata = get_args_dict(md_args)
            upload_kwargs['metadata'].update(metadata)
            _upload_files(args, identifier, local_file, upload_kwargs,
                          prev_identifier)
            prev_identifier = identifier

    # Upload files.
    else:
        local_file = args['<file>']
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)
예제 #2
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    headers = get_args_dict(args['--header'])
    if args['--size-hint']:
        headers['x-archive-size-hint'] = args['--size-hint']

    # Upload keyword arguments.
    upload_kwargs = dict(
        metadata=get_args_dict(args['--metadata']),
        headers=headers,
        debug=args['--debug'],
        queue_derive=True if args['--no-derive'] is False else False,
        ignore_preexisting_bucket=args['--ignore-bucket'],
        checksum=args['--checksum'],
        verbose=True if args['--quiet'] is False else False,
        retries=int(args['--retries']) if args['--retries'] else 0,
        retries_sleep=int(args['--sleep']),
        delete=args['--delete'],
    )

    if args['<file>'] == ['-'] and not args['-']:
        sys.stderr.write('--remote-name is required when uploading from stdin.\n')
        call(['ia', 'upload', '--help'])
        sys.exit(1)

    # Upload from stdin.
    if args['-']:
        local_file = TemporaryFile()
        local_file.write(sys.stdin.read())
        local_file.seek(0)
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)

    # Bulk upload using spreadsheet.
    elif args['--spreadsheet']:
        # Use the same session for each upload request.
        session = ArchiveSession()

        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        prev_identifier = None
        for row in spreadsheet:
            local_file = row['file']
            identifier = row['identifier']
            del row['file']
            del row['identifier']
            if (not identifier) and (prev_identifier):
                identifier = prev_identifier
            # TODO: Clean up how indexed metadata items are coerced
            # into metadata.
            md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
            metadata = get_args_dict(md_args)
            upload_kwargs['metadata'].update(metadata)
            _upload_files(args, identifier, local_file, upload_kwargs, prev_identifier,
                          session)
            prev_identifier = identifier

    # Upload files.
    else:
        local_file = args['<file>']
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)
예제 #3
0
def main(argv):
    args = docopt(__doc__, argv=argv)
    verbose = True if args['--quiet'] is False else False

    if verbose is not False:
        sys.stdout.write('getting item: {0}\n'.format(args['<identifier>']))

    headers = get_args_dict(args['--header'])
    if args['--size-hint']:
        headers['x-archive-size-hint'] = args['--size-hint']

    upload_kwargs = dict(
        metadata=get_args_dict(args['--metadata']),
        headers=headers,
        debug=args['--debug'],
        queue_derive=True if args['--no-derive'] is False else False,
        ignore_preexisting_bucket=args['--ignore-bucket'],
        verbose=verbose,
        delete=args['--delete'])

    # Upload stdin.
    if args['<file>'] == ['-'] and not args['-']:
        sys.stderr.write('--remote-name is required when uploading from stdin.\n')
        call(['ia', 'upload', '--help'])
        sys.exit(1)
    if args['-']:
        local_file = TemporaryFile()
        local_file.write(sys.stdin.read())
        local_file.seek(0)
        upload_kwargs['key'] = args['--remote-name']
    # Upload files.
    else:
        local_file = args['<file>']

    config = {} if not args['--log'] else {'logging': {'level': 'INFO'}}
    item = get_item(args['<identifier>'], config=config)
    response = item.upload(local_file, **upload_kwargs)

    if args['--debug']:
        for i, r in enumerate(response):
            if i != 0:
                sys.stdout.write('---\n')
            headers = '\n'.join([' {0}: {1}'.format(k, v) for (k, v) in r.headers.items()])
            sys.stdout.write('Endpoint:\n {0}\n\n'.format(r.url))
            sys.stdout.write('HTTP Headers:\n{0}\n'.format(headers))
    else:
        for resp in response:
            if resp.status_code == 200:
                continue
            error = parseString(resp.content)
            code = get_xml_text(error.getElementsByTagName('Code'))
            msg = get_xml_text(error.getElementsByTagName('Message'))
            sys.stderr.write('error "{0}" ({1}): {2}\n'.format(code, resp.status_code, msg))
            sys.exit(1)
예제 #4
0
def main(argv):
    args = docopt(__doc__, argv=argv)
    item = get_item(args['<identifier>'])

    # Check existence of item.
    if args['--exists']:
        if item.exists:
            sys.stdout.write('{0} exists\n'.format(item.identifier))
            sys.exit(0)
        else:
            sys.stderr.write('{0} does not exist\n'.format(item.identifier))
            sys.exit(1)

    # Modify metadata.
    elif args['--modify'] or args['--append']:
        append = True if args['--append'] else False
        metadata_args = args['--modify'] if args['--modify'] else args['--append']
        metadata = get_args_dict(metadata_args)
        response = modify_metadata(args['<identifier>'], metadata, append=append)
        if not response.json()['success']:
            error_msg = response.json()['error']
            sys.stderr.write('error: {0} ({1})\n'.format(error_msg, response.status_code))
            sys.exit(1)
        sys.stdout.write('success: {0}\n'.format(response.json()['log']))

    # Get metadata.
    elif args['--formats']:
        formats = set([f.format for f in item.iter_files()])
        sys.stdout.write('\n'.join(formats) + '\n')
    else:
        metadata = dumps(item.metadata)
        sys.stdout.write(metadata + '\n')
    sys.exit(0)
예제 #5
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    headers = get_args_dict(args['--header'])
    if args['--size-hint']:
        headers['x-archive-size-hint'] = args['--size-hint']

    # Upload keyword arguments.
    upload_kwargs = dict(
        metadata=get_args_dict(args['--metadata']),
        headers=headers,
        debug=args['--debug'],
        queue_derive=True if args['--no-derive'] is False else False,
        ignore_preexisting_bucket=args['--ignore-bucket'],
        checksum=args['--checksum'],
        verbose=True if args['--quiet'] is False else False,
        delete=args['--delete'],
    )

    if args['<file>'] == ['-'] and not args['-']:
        sys.stderr.write('--remote-name is required when uploading from stdin.\n')
        call(['ia', 'upload', '--help'])
        sys.exit(1)

    # Upload from stdin.
    if args['-']:
        local_file = TemporaryFile()
        local_file.write(sys.stdin.read())
        local_file.seek(0)
        upload_kwargs['key'] = args['--remote-name']
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)

    # Bulk upload using spreadsheet.
    elif args['--spreadsheet']:
        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        for row in spreadsheet:
            local_file = row['file']
            del row['file']
            metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
            upload_kwargs['metadata'].update(metadata)
            _upload_files(args, row['item'], local_file, upload_kwargs)

    # Upload files.
    else:
        local_file = args['<file>']
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)
예제 #6
0
def main(argv):
    args = docopt(__doc__, argv=argv)
    params = get_args_dict(args['--parameter'])

    row_types = {
        -1: 'done',
        0: 'green',
        1: 'blue',
        2: 'red',
        9: 'brown',
    }

    task_type = None
    if args['--green-rows']:
        task_type = 'green'
    elif args['--blue-rows']:
        task_type = 'blue'
    elif args['--red-rows']:
        task_type = 'red'

    try:
        if args['<identifier>']:
            tasks = get_tasks(identifier=args['<identifier>'],
                              task_type=task_type,
                              params=params)
        elif args['--get-task-log']:
            task = get_tasks(task_ids=args['--get-task-log'], params=params)
            if task:
                log = task[0].task_log()
                sys.stdout.write(log)
            else:
                sys.stderr.write('error retrieving task-log for {0}\n'.format(
                    args['--get-task-log']))
                sys.exit(1)
            sys.exit(0)
        elif args['--task']:
            tasks = get_tasks(task_ids=args['--task'], params=params)
        else:
            tasks = get_tasks(task_type=task_type, params=params)

        for t in tasks:
            task_info = [
                t.identifier,
                t.task_id,
                t.server,
                t.time,
                t.command,
                row_types[t.row_type],
            ]
            if args['--verbose']:
                # parse task args and append to task_info list.
                targs = '\t'.join(
                    ['{0}={1}'.format(k, v) for (k, v) in t.args.items()])
                task_info += [t.submitter, targs]
            sys.stdout.write('\t'.join([str(x) for x in task_info]) + '\n')
    except NameError as exc:
        sys.stderr.write('error: {0}'.format(exc.message))
        sys.exit(1)
예제 #7
0
def main(argv):
    args = docopt(__doc__, argv=argv)
    params = get_args_dict(args['--parameter'])

    row_types = {
        -1: 'done',
        0: 'green',
        1: 'blue',
        2: 'red',
        9: 'brown',
    }

    task_type = None
    if args['--green-rows']:
        task_type = 'green'
    elif args['--blue-rows']:
        task_type = 'blue'
    elif args['--red-rows']:
        task_type = 'red'

    try:
        if args['<identifier>']:
            tasks = get_tasks(identifier=args['<identifier>'], task_type=task_type, params=params)
        elif args['--get-task-log']:
            task = get_tasks(task_ids=args['--get-task-log'], params=params)
            if task:
                log = task[0].task_log()
                sys.stdout.write(log)
            else:
                sys.stderr.write(
                    'error retrieving task-log for {0}\n'.format(args['--get-task-log'])
                )
                sys.exit(1)
            sys.exit(0)
        elif args['--task']:
            tasks = get_tasks(task_ids=args['--task'], params=params)
        else:
            tasks = get_tasks(task_type=task_type, params=params)

        for t in tasks:
            task_info = [
                t.identifier, t.task_id, t.server, t.time, t.command, 
                row_types[t.row_type],
            ]
            if args['--verbose']:
                # parse task args and append to task_info list.
                targs = '\t'.join(['{0}={1}'.format(k, v) for (k, v) in t.args.items()])
                task_info += [t.submitter, targs]
            sys.stdout.write('\t'.join([str(x) for x in task_info]) + '\n')
    except NameError as exc:
        sys.stderr.write('error: {0}'.format(exc.message))
        sys.exit(1)
예제 #8
0
def test_get_args_dict():
    test_input = [
            'collection:test_collection',
            "description: Attention: multiple colon's",
            'unicode_test:தமிழ்',
            'subject:subject1',
            'subject:subject2',
    ]
    test_output = {
            'collection': 'test_collection', 
            'description': " Attention: multiple colon's", 
            'unicode_test': '\xe0\xae\xa4\xe0\xae\xae\xe0\xae\xbf\xe0\xae\xb4\xe0\xaf\x8d', 
            'subject': ['subject1', 'subject2']
    }
    args_dict = get_args_dict(test_input)
    for key, value in args_dict.items():
        assert test_output[key] == value
예제 #9
0
def test_get_args_dict():
    test_input = [
        'collection:test_collection',
        "description: Attention: multiple colon's",
        'unicode_test:தமிழ்',
        'subject:subject1',
        'subject:subject2',
    ]
    test_output = {
        'collection': 'test_collection',
        'description': " Attention: multiple colon's",
        'unicode_test':
        '\xe0\xae\xa4\xe0\xae\xae\xe0\xae\xbf\xe0\xae\xb4\xe0\xaf\x8d',
        'subject': ['subject1', 'subject2']
    }
    args_dict = get_args_dict(test_input)
    for key, value in args_dict.items():
        assert test_output[key] == value
예제 #10
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    formats = set()
    responses = []

    for i, _item in enumerate(args['<identifier>']):
        item = get_item(_item)

        # Check existence of item.
        if args['--exists']:
            if item.exists:
                responses.append(True)
                sys.stdout.write('{0} exists\n'.format(item.identifier))
            else:
                responses.append(False)
                sys.stderr.write('{0} does not exist\n'.format(item.identifier))
            if (i + 1) == len(args['<identifier>']):
                if all(r is True for r in responses):
                    sys.exit(0)
                else:
                    sys.exit(1)

        # Modify metadata.
        elif args['--modify'] or args['--append']:
            metadata_args = args['--modify'] if args['--modify'] else args['--append']
            metadata = get_args_dict(metadata_args)
            responses.append(modify_metadata(item, metadata, args))
            if (i + 1) == len(args['<identifier>']):
                if all(r.status_code == 200 for r in responses):
                    sys.exit(0)
                else:
                    sys.exit(1)

        # Get metadata.
        elif args['--formats']:
            for f in item.iter_files():
                formats.add(f.format)
            if (i + 1) == len(args['<identifier>']):
                sys.stdout.write('\n'.join(formats) + '\n')

        # Dump JSON to stdout.
        else:
            metadata = dumps(item.metadata)
            sys.stdout.write(metadata + '\n')

    # Edit metadata for items in bulk, using a spreadsheet as input.
    if args['--spreadsheet']:
        if not args['--priority']:
            args['--priority'] = -5
        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        responses = []
        for row in spreadsheet:
            item = get_item(row['item'])
            if row.get('file'):
                del row['file']
            metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
            responses.append(modify_metadata(item, metadata, args))

        if all(r.status_code == 200 for r in responses):
            sys.exit(0)
        else:
            sys.exit(1)

    sys.exit(0)
예제 #11
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    formats = set()
    responses = []

    for i, _item in enumerate(args['<identifier>']):
        item = get_item(_item)

        # Check existence of item.
        if args['--exists']:
            if item.exists:
                responses.append(True)
                sys.stdout.write('{0} exists\n'.format(item.identifier))
            else:
                responses.append(False)
                sys.stderr.write('{0} does not exist\n'.format(
                    item.identifier))
            if (i + 1) == len(args['<identifier>']):
                if all(r is True for r in responses):
                    sys.exit(0)
                else:
                    sys.exit(1)

        # Modify metadata.
        elif args['--modify'] or args['--append']:
            metadata_args = args['--modify'] if args['--modify'] else args[
                '--append']
            metadata = get_args_dict(metadata_args)
            responses.append(modify_metadata(item, metadata, args))
            if (i + 1) == len(args['<identifier>']):
                if all(r.status_code == 200 for r in responses):
                    sys.exit(0)
                else:
                    sys.exit(1)

        # Get metadata.
        elif args['--formats']:
            for f in item.iter_files():
                formats.add(f.format)
            if (i + 1) == len(args['<identifier>']):
                sys.stdout.write('\n'.join(formats) + '\n')

        # Dump JSON to stdout.
        else:
            metadata = json.dumps(item._json)
            sys.stdout.write(metadata + '\n')

    # Edit metadata for items in bulk, using a spreadsheet as input.
    if args['--spreadsheet']:
        if not args['--priority']:
            args['--priority'] = -5
        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        responses = []
        for row in spreadsheet:
            if not row['identifier']:
                continue
            item = get_item(row['identifier'])
            if row.get('file'):
                del row['file']
            metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
            responses.append(modify_metadata(item, metadata, args))

        if all(r.status_code == 200 for r in responses):
            sys.exit(0)
        else:
            sys.exit(1)

    sys.exit(0)
예제 #12
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    metadata = get_args_dict(args['--metadata'])
    # Make sure the collection being uploaded to exists.
    collection_ids = metadata.get('collection', [])
    if isinstance(collection_ids, six.string_types):
        collection_ids = [collection_ids]
    for cid in collection_ids:
        if args['--no-collection-check'] or args['--status-check']:
            break
        collection = get_item(cid) 
        if not collection.exists:
            sys.stderr.write(
                    'You must upload to a collection that exists. '
                    '"{0}" does not exist.\n{1}\n'.format(collection.identifier,
                                                          printable_usage(__doc__)))
            sys.exit(1)

    headers = get_args_dict(args['--header'])
    if args['--size-hint']:
        headers['x-archive-size-hint'] = args['--size-hint']

    # Upload keyword arguments.
    upload_kwargs = dict(
        metadata=metadata,
        headers=headers,
        debug=args['--debug'],
        queue_derive=True if args['--no-derive'] is False else False,
        ignore_preexisting_bucket=args['--ignore-bucket'],
        checksum=args['--checksum'],
        verbose=True if args['--quiet'] is False else False,
        retries=int(args['--retries']) if args['--retries'] else 0,
        retries_sleep=int(args['--sleep']),
        delete=args['--delete'],
    )

    if args['<file>'] == ['-'] and not args['-']:
        sys.stderr.write('--remote-name is required when uploading from stdin.\n')
        call(['ia', 'upload', '--help'])
        sys.exit(1)

    # Upload from stdin.
    if args['-']:
        local_file = TemporaryFile()
        local_file.write(sys.stdin.read())
        local_file.seek(0)
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)

    # Bulk upload using spreadsheet.
    elif args['--spreadsheet']:
        # Use the same session for each upload request.
        session = ArchiveSession()

        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        prev_identifier = None
        for row in spreadsheet:
            local_file = row['file']
            identifier = row['identifier'].strip()
            del row['file']
            del row['identifier']
            if (not identifier) and (prev_identifier):
                identifier = prev_identifier
            # TODO: Clean up how indexed metadata items are coerced
            # into metadata.
            md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
            metadata = get_args_dict(md_args)
            upload_kwargs['metadata'].update(metadata)
            _upload_files(args, identifier, local_file, upload_kwargs, prev_identifier,
                          session)
            prev_identifier = identifier

    # Upload files.
    else:
        local_file = args['<file>']
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)
예제 #13
0
def main(argv):
    args = docopt(__doc__, argv=argv)

    metadata = get_args_dict(args['--metadata'])
    # Make sure the collection being uploaded to exists.
    collection_id = metadata.get('collection')
    if collection_id and not args['--no-collection-check'] and not args['--status-check']:
        collection = get_item(collection_id) 
        if not collection.exists:
            sys.stderr.write(
                    'You must upload to a collection that exists. '
                    '"{0}" does not exist.\n{1}\n'.format(collection_id,
                                                          printable_usage(__doc__)))
            sys.exit(1)

    headers = get_args_dict(args['--header'])
    if args['--size-hint']:
        headers['x-archive-size-hint'] = args['--size-hint']

    # Upload keyword arguments.
    upload_kwargs = dict(
        metadata=metadata,
        headers=headers,
        debug=args['--debug'],
        queue_derive=True if args['--no-derive'] is False else False,
        ignore_preexisting_bucket=args['--ignore-bucket'],
        checksum=args['--checksum'],
        verbose=True if args['--quiet'] is False else False,
        retries=int(args['--retries']) if args['--retries'] else 0,
        retries_sleep=int(args['--sleep']),
        delete=args['--delete'],
    )

    if args['<file>'] == ['-'] and not args['-']:
        sys.stderr.write('--remote-name is required when uploading from stdin.\n')
        call(['ia', 'upload', '--help'])
        sys.exit(1)

    # Upload from stdin.
    if args['-']:
        local_file = TemporaryFile()
        local_file.write(sys.stdin.read())
        local_file.seek(0)
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)

    # Bulk upload using spreadsheet.
    elif args['--spreadsheet']:
        # Use the same session for each upload request.
        session = ArchiveSession()

        spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
        prev_identifier = None
        for row in spreadsheet:
            local_file = row['file']
            identifier = row['identifier']
            del row['file']
            del row['identifier']
            if (not identifier) and (prev_identifier):
                identifier = prev_identifier
            # TODO: Clean up how indexed metadata items are coerced
            # into metadata.
            md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
            metadata = get_args_dict(md_args)
            upload_kwargs['metadata'].update(metadata)
            _upload_files(args, identifier, local_file, upload_kwargs, prev_identifier,
                          session)
            prev_identifier = identifier

    # Upload files.
    else:
        local_file = args['<file>']
        _upload_files(args, args['<identifier>'], local_file, upload_kwargs)