def main(argv):
args = docopt(__doc__, argv=argv)
headers = get_args_dict(args['--header'])
if args['--size-hint']:
headers['x-archive-size-hint'] = args['--size-hint']
# Upload keyword arguments.
upload_kwargs = dict(
metadata=get_args_dict(args['--metadata']),
headers=headers,
debug=args['--debug'],
queue_derive=True if args['--no-derive'] is False else False,
ignore_preexisting_bucket=args['--ignore-bucket'],
checksum=args['--checksum'],
verbose=True if args['--quiet'] is False else False,
retries=int(args['--retries']) if args['--retries'] else 0,
retries_sleep=int(args['--sleep']),
delete=args['--delete'],
)
if args['<file>'] == ['-'] and not args['-']:
sys.stderr.write(
'--remote-name is required when uploading from stdin.\n')
call(['ia', 'upload', '--help'])
sys.exit(1)
# Upload from stdin.
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
# Bulk upload using spreadsheet.
elif args['--spreadsheet']:
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
prev_identifier = None
for row in spreadsheet:
local_file = row['file']
identifier = row['identifier']
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = [
'{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v
]
metadata = get_args_dict(md_args)
upload_kwargs['metadata'].update(metadata)
_upload_files(args, identifier, local_file, upload_kwargs,
prev_identifier)
prev_identifier = identifier
# Upload files.
else:
local_file = args['<file>']
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
def main(argv):
args = docopt(__doc__, argv=argv)
headers = get_args_dict(args['--header'])
if args['--size-hint']:
headers['x-archive-size-hint'] = args['--size-hint']
# Upload keyword arguments.
upload_kwargs = dict(
metadata=get_args_dict(args['--metadata']),
headers=headers,
debug=args['--debug'],
queue_derive=True if args['--no-derive'] is False else False,
ignore_preexisting_bucket=args['--ignore-bucket'],
checksum=args['--checksum'],
verbose=True if args['--quiet'] is False else False,
retries=int(args['--retries']) if args['--retries'] else 0,
retries_sleep=int(args['--sleep']),
delete=args['--delete'],
)
if args['<file>'] == ['-'] and not args['-']:
sys.stderr.write('--remote-name is required when uploading from stdin.\n')
call(['ia', 'upload', '--help'])
sys.exit(1)
# Upload from stdin.
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
# Bulk upload using spreadsheet.
elif args['--spreadsheet']:
# Use the same session for each upload request.
session = ArchiveSession()
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
prev_identifier = None
for row in spreadsheet:
local_file = row['file']
identifier = row['identifier']
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
metadata = get_args_dict(md_args)
upload_kwargs['metadata'].update(metadata)
_upload_files(args, identifier, local_file, upload_kwargs, prev_identifier,
session)
prev_identifier = identifier
# Upload files.
else:
local_file = args['<file>']
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
def main(argv):
args = docopt(__doc__, argv=argv)
verbose = True if args['--quiet'] is False else False
if verbose is not False:
sys.stdout.write('getting item: {0}\n'.format(args['<identifier>']))
headers = get_args_dict(args['--header'])
if args['--size-hint']:
headers['x-archive-size-hint'] = args['--size-hint']
upload_kwargs = dict(
metadata=get_args_dict(args['--metadata']),
headers=headers,
debug=args['--debug'],
queue_derive=True if args['--no-derive'] is False else False,
ignore_preexisting_bucket=args['--ignore-bucket'],
verbose=verbose,
delete=args['--delete'])
# Upload stdin.
if args['<file>'] == ['-'] and not args['-']:
sys.stderr.write('--remote-name is required when uploading from stdin.\n')
call(['ia', 'upload', '--help'])
sys.exit(1)
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
upload_kwargs['key'] = args['--remote-name']
# Upload files.
else:
local_file = args['<file>']
config = {} if not args['--log'] else {'logging': {'level': 'INFO'}}
item = get_item(args['<identifier>'], config=config)
response = item.upload(local_file, **upload_kwargs)
if args['--debug']:
for i, r in enumerate(response):
if i != 0:
sys.stdout.write('---\n')
headers = '\n'.join([' {0}: {1}'.format(k, v) for (k, v) in r.headers.items()])
sys.stdout.write('Endpoint:\n {0}\n\n'.format(r.url))
sys.stdout.write('HTTP Headers:\n{0}\n'.format(headers))
else:
for resp in response:
if resp.status_code == 200:
continue
error = parseString(resp.content)
code = get_xml_text(error.getElementsByTagName('Code'))
msg = get_xml_text(error.getElementsByTagName('Message'))
sys.stderr.write('error "{0}" ({1}): {2}\n'.format(code, resp.status_code, msg))
sys.exit(1)
def main(argv):
args = docopt(__doc__, argv=argv)
item = get_item(args['<identifier>'])
# Check existence of item.
if args['--exists']:
if item.exists:
sys.stdout.write('{0} exists\n'.format(item.identifier))
sys.exit(0)
else:
sys.stderr.write('{0} does not exist\n'.format(item.identifier))
sys.exit(1)
# Modify metadata.
elif args['--modify'] or args['--append']:
append = True if args['--append'] else False
metadata_args = args['--modify'] if args['--modify'] else args['--append']
metadata = get_args_dict(metadata_args)
response = modify_metadata(args['<identifier>'], metadata, append=append)
if not response.json()['success']:
error_msg = response.json()['error']
sys.stderr.write('error: {0} ({1})\n'.format(error_msg, response.status_code))
sys.exit(1)
sys.stdout.write('success: {0}\n'.format(response.json()['log']))
# Get metadata.
elif args['--formats']:
formats = set([f.format for f in item.iter_files()])
sys.stdout.write('\n'.join(formats) + '\n')
else:
metadata = dumps(item.metadata)
sys.stdout.write(metadata + '\n')
sys.exit(0)
def main(argv):
args = docopt(__doc__, argv=argv)
headers = get_args_dict(args['--header'])
if args['--size-hint']:
headers['x-archive-size-hint'] = args['--size-hint']
# Upload keyword arguments.
upload_kwargs = dict(
metadata=get_args_dict(args['--metadata']),
headers=headers,
debug=args['--debug'],
queue_derive=True if args['--no-derive'] is False else False,
ignore_preexisting_bucket=args['--ignore-bucket'],
checksum=args['--checksum'],
verbose=True if args['--quiet'] is False else False,
delete=args['--delete'],
)
if args['<file>'] == ['-'] and not args['-']:
sys.stderr.write('--remote-name is required when uploading from stdin.\n')
call(['ia', 'upload', '--help'])
sys.exit(1)
# Upload from stdin.
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
upload_kwargs['key'] = args['--remote-name']
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
# Bulk upload using spreadsheet.
elif args['--spreadsheet']:
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
for row in spreadsheet:
local_file = row['file']
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
upload_kwargs['metadata'].update(metadata)
_upload_files(args, row['item'], local_file, upload_kwargs)
# Upload files.
else:
local_file = args['<file>']
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
def main(argv):
args = docopt(__doc__, argv=argv)
params = get_args_dict(args['--parameter'])
row_types = {
-1: 'done',
0: 'green',
1: 'blue',
2: 'red',
9: 'brown',
}
task_type = None
if args['--green-rows']:
task_type = 'green'
elif args['--blue-rows']:
task_type = 'blue'
elif args['--red-rows']:
task_type = 'red'
try:
if args['<identifier>']:
tasks = get_tasks(identifier=args['<identifier>'],
task_type=task_type,
params=params)
elif args['--get-task-log']:
task = get_tasks(task_ids=args['--get-task-log'], params=params)
if task:
log = task[0].task_log()
sys.stdout.write(log)
else:
sys.stderr.write('error retrieving task-log for {0}\n'.format(
args['--get-task-log']))
sys.exit(1)
sys.exit(0)
elif args['--task']:
tasks = get_tasks(task_ids=args['--task'], params=params)
else:
tasks = get_tasks(task_type=task_type, params=params)
for t in tasks:
task_info = [
t.identifier,
t.task_id,
t.server,
t.time,
t.command,
row_types[t.row_type],
]
if args['--verbose']:
# parse task args and append to task_info list.
targs = '\t'.join(
['{0}={1}'.format(k, v) for (k, v) in t.args.items()])
task_info += [t.submitter, targs]
sys.stdout.write('\t'.join([str(x) for x in task_info]) + '\n')
except NameError as exc:
sys.stderr.write('error: {0}'.format(exc.message))
sys.exit(1)
def main(argv):
args = docopt(__doc__, argv=argv)
params = get_args_dict(args['--parameter'])
row_types = {
-1: 'done',
0: 'green',
1: 'blue',
2: 'red',
9: 'brown',
}
task_type = None
if args['--green-rows']:
task_type = 'green'
elif args['--blue-rows']:
task_type = 'blue'
elif args['--red-rows']:
task_type = 'red'
try:
if args['<identifier>']:
tasks = get_tasks(identifier=args['<identifier>'], task_type=task_type, params=params)
elif args['--get-task-log']:
task = get_tasks(task_ids=args['--get-task-log'], params=params)
if task:
log = task[0].task_log()
sys.stdout.write(log)
else:
sys.stderr.write(
'error retrieving task-log for {0}\n'.format(args['--get-task-log'])
)
sys.exit(1)
sys.exit(0)
elif args['--task']:
tasks = get_tasks(task_ids=args['--task'], params=params)
else:
tasks = get_tasks(task_type=task_type, params=params)
for t in tasks:
task_info = [
t.identifier, t.task_id, t.server, t.time, t.command,
row_types[t.row_type],
]
if args['--verbose']:
# parse task args and append to task_info list.
targs = '\t'.join(['{0}={1}'.format(k, v) for (k, v) in t.args.items()])
task_info += [t.submitter, targs]
sys.stdout.write('\t'.join([str(x) for x in task_info]) + '\n')
except NameError as exc:
sys.stderr.write('error: {0}'.format(exc.message))
sys.exit(1)
def test_get_args_dict():
test_input = [
'collection:test_collection',
"description: Attention: multiple colon's",
'unicode_test:தமிழ்',
'subject:subject1',
'subject:subject2',
]
test_output = {
'collection': 'test_collection',
'description': " Attention: multiple colon's",
'unicode_test': '\xe0\xae\xa4\xe0\xae\xae\xe0\xae\xbf\xe0\xae\xb4\xe0\xaf\x8d',
'subject': ['subject1', 'subject2']
}
args_dict = get_args_dict(test_input)
for key, value in args_dict.items():
assert test_output[key] == value
def test_get_args_dict():
test_input = [
'collection:test_collection',
"description: Attention: multiple colon's",
'unicode_test:தமிழ்',
'subject:subject1',
'subject:subject2',
]
test_output = {
'collection': 'test_collection',
'description': " Attention: multiple colon's",
'unicode_test':
'\xe0\xae\xa4\xe0\xae\xae\xe0\xae\xbf\xe0\xae\xb4\xe0\xaf\x8d',
'subject': ['subject1', 'subject2']
}
args_dict = get_args_dict(test_input)
for key, value in args_dict.items():
assert test_output[key] == value
def main(argv):
args = docopt(__doc__, argv=argv)
formats = set()
responses = []
for i, _item in enumerate(args['<identifier>']):
item = get_item(_item)
# Check existence of item.
if args['--exists']:
if item.exists:
responses.append(True)
sys.stdout.write('{0} exists\n'.format(item.identifier))
else:
responses.append(False)
sys.stderr.write('{0} does not exist\n'.format(item.identifier))
if (i + 1) == len(args['<identifier>']):
if all(r is True for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Modify metadata.
elif args['--modify'] or args['--append']:
metadata_args = args['--modify'] if args['--modify'] else args['--append']
metadata = get_args_dict(metadata_args)
responses.append(modify_metadata(item, metadata, args))
if (i + 1) == len(args['<identifier>']):
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Get metadata.
elif args['--formats']:
for f in item.iter_files():
formats.add(f.format)
if (i + 1) == len(args['<identifier>']):
sys.stdout.write('\n'.join(formats) + '\n')
# Dump JSON to stdout.
else:
metadata = dumps(item.metadata)
sys.stdout.write(metadata + '\n')
# Edit metadata for items in bulk, using a spreadsheet as input.
if args['--spreadsheet']:
if not args['--priority']:
args['--priority'] = -5
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
responses = []
for row in spreadsheet:
item = get_item(row['item'])
if row.get('file'):
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
responses.append(modify_metadata(item, metadata, args))
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
sys.exit(1)
sys.exit(0)
def main(argv):
args = docopt(__doc__, argv=argv)
formats = set()
responses = []
for i, _item in enumerate(args['<identifier>']):
item = get_item(_item)
# Check existence of item.
if args['--exists']:
if item.exists:
responses.append(True)
sys.stdout.write('{0} exists\n'.format(item.identifier))
else:
responses.append(False)
sys.stderr.write('{0} does not exist\n'.format(
item.identifier))
if (i + 1) == len(args['<identifier>']):
if all(r is True for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Modify metadata.
elif args['--modify'] or args['--append']:
metadata_args = args['--modify'] if args['--modify'] else args[
'--append']
metadata = get_args_dict(metadata_args)
responses.append(modify_metadata(item, metadata, args))
if (i + 1) == len(args['<identifier>']):
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Get metadata.
elif args['--formats']:
for f in item.iter_files():
formats.add(f.format)
if (i + 1) == len(args['<identifier>']):
sys.stdout.write('\n'.join(formats) + '\n')
# Dump JSON to stdout.
else:
metadata = json.dumps(item._json)
sys.stdout.write(metadata + '\n')
# Edit metadata for items in bulk, using a spreadsheet as input.
if args['--spreadsheet']:
if not args['--priority']:
args['--priority'] = -5
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
responses = []
for row in spreadsheet:
if not row['identifier']:
continue
item = get_item(row['identifier'])
if row.get('file'):
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
responses.append(modify_metadata(item, metadata, args))
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
sys.exit(1)
sys.exit(0)
def main(argv):
args = docopt(__doc__, argv=argv)
metadata = get_args_dict(args['--metadata'])
# Make sure the collection being uploaded to exists.
collection_ids = metadata.get('collection', [])
if isinstance(collection_ids, six.string_types):
collection_ids = [collection_ids]
for cid in collection_ids:
if args['--no-collection-check'] or args['--status-check']:
break
collection = get_item(cid)
if not collection.exists:
sys.stderr.write(
'You must upload to a collection that exists. '
'"{0}" does not exist.\n{1}\n'.format(collection.identifier,
printable_usage(__doc__)))
sys.exit(1)
headers = get_args_dict(args['--header'])
if args['--size-hint']:
headers['x-archive-size-hint'] = args['--size-hint']
# Upload keyword arguments.
upload_kwargs = dict(
metadata=metadata,
headers=headers,
debug=args['--debug'],
queue_derive=True if args['--no-derive'] is False else False,
ignore_preexisting_bucket=args['--ignore-bucket'],
checksum=args['--checksum'],
verbose=True if args['--quiet'] is False else False,
retries=int(args['--retries']) if args['--retries'] else 0,
retries_sleep=int(args['--sleep']),
delete=args['--delete'],
)
if args['<file>'] == ['-'] and not args['-']:
sys.stderr.write('--remote-name is required when uploading from stdin.\n')
call(['ia', 'upload', '--help'])
sys.exit(1)
# Upload from stdin.
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
# Bulk upload using spreadsheet.
elif args['--spreadsheet']:
# Use the same session for each upload request.
session = ArchiveSession()
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
prev_identifier = None
for row in spreadsheet:
local_file = row['file']
identifier = row['identifier'].strip()
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
metadata = get_args_dict(md_args)
upload_kwargs['metadata'].update(metadata)
_upload_files(args, identifier, local_file, upload_kwargs, prev_identifier,
session)
prev_identifier = identifier
# Upload files.
else:
local_file = args['<file>']
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
def main(argv):
args = docopt(__doc__, argv=argv)
metadata = get_args_dict(args['--metadata'])
# Make sure the collection being uploaded to exists.
collection_id = metadata.get('collection')
if collection_id and not args['--no-collection-check'] and not args['--status-check']:
collection = get_item(collection_id)
if not collection.exists:
sys.stderr.write(
'You must upload to a collection that exists. '
'"{0}" does not exist.\n{1}\n'.format(collection_id,
printable_usage(__doc__)))
sys.exit(1)
headers = get_args_dict(args['--header'])
if args['--size-hint']:
headers['x-archive-size-hint'] = args['--size-hint']
# Upload keyword arguments.
upload_kwargs = dict(
metadata=metadata,
headers=headers,
debug=args['--debug'],
queue_derive=True if args['--no-derive'] is False else False,
ignore_preexisting_bucket=args['--ignore-bucket'],
checksum=args['--checksum'],
verbose=True if args['--quiet'] is False else False,
retries=int(args['--retries']) if args['--retries'] else 0,
retries_sleep=int(args['--sleep']),
delete=args['--delete'],
)
if args['<file>'] == ['-'] and not args['-']:
sys.stderr.write('--remote-name is required when uploading from stdin.\n')
call(['ia', 'upload', '--help'])
sys.exit(1)
# Upload from stdin.
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
# Bulk upload using spreadsheet.
elif args['--spreadsheet']:
# Use the same session for each upload request.
session = ArchiveSession()
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
prev_identifier = None
for row in spreadsheet:
local_file = row['file']
identifier = row['identifier']
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
metadata = get_args_dict(md_args)
upload_kwargs['metadata'].update(metadata)
_upload_files(args, identifier, local_file, upload_kwargs, prev_identifier,
session)
prev_identifier = identifier
# Upload files.
else:
local_file = args['<file>']
_upload_files(args, args['<identifier>'], local_file, upload_kwargs)
