def _upload_files(item, files, upload_kwargs, prev_identifier=None, archive_session=None,
responses=None):
"""Helper function for calling :meth:`Item.upload`"""
responses = [] if not responses else responses
if (upload_kwargs['verbose']) and (prev_identifier != item.identifier):
print('{0}:'.format(item.identifier))
try:
response = item.upload(files, **upload_kwargs)
responses += response
except HTTPError as exc:
responses += [exc.response]
finally:
# Debug mode.
if upload_kwargs['debug']:
for i, r in enumerate(responses):
if i != 0:
print('---')
headers = '\n'.join(
[' {0}:{1}'.format(k, v) for (k, v) in r.headers.items()]
)
print('Endpoint:\n {0}\n'.format(r.url))
print('HTTP Headers:\n{0}'.format(headers))
return
# Format error message for any non 200 responses that
# we haven't caught yet,and write to stderr.
if responses and responses[-1] and responses[-1].status_code != 200:
filename = responses[-1].request.url.split('/')[-1]
msg = get_xml_text(responses[-1].content)
print(' * error uploading '
'{0} ({1}): {2}'.format(filename, responses[-1].status_code, msg),
file=sys.stderr)
return responses
def test_get_xml_text():
xml_str = ('<Error><Code>NoSuchBucket</Code>'
'<Message>The specified bucket does not exist.</Message>'
'<Resource>'
'does-not-exist-! not found by Metadata::get_obj()[server]'
'</Resource>'
'<RequestId>d56bdc63-169b-4b4f-8c47-0fac6de39040</RequestId></Error>')
assert get_xml_text(xml_str) == 'The specified bucket does not exist.'
def test_get_xml_text():
xml_str = (
'<Error><Code>NoSuchBucket</Code>'
'<Message>The specified bucket does not exist.</Message>'
'<Resource>'
'does-not-exist-! not found by Metadata::get_obj()[server]'
'</Resource>'
'<RequestId>d56bdc63-169b-4b4f-8c47-0fac6de39040</RequestId></Error>')
assert get_xml_text(xml_str) == 'The specified bucket does not exist.'
def _upload_files(item,
files,
upload_kwargs,
prev_identifier=None,
archive_session=None):
"""Helper function for calling :meth:`Item.upload`"""
responses = []
if (upload_kwargs['verbose']) and (prev_identifier != item.identifier):
print('{0}:'.format(item.identifier))
try:
response = item.upload(files, **upload_kwargs)
responses += response
except HTTPError as exc:
responses += [exc.response]
finally:
# Debug mode.
if upload_kwargs['debug']:
for i, r in enumerate(responses):
if i != 0:
print('---')
headers = '\n'.join(
[' {0}:{1}'.format(k, v) for (k, v) in r.headers.items()])
print('Endpoint:\n {0}\n'.format(r.url))
print('HTTP Headers:\n{0}'.format(headers))
return responses
# Format error message for any non 200 responses that
# we haven't caught yet,and write to stderr.
if responses and responses[-1] and responses[-1].status_code != 200:
filename = responses[-1].request.url.split('/')[-1]
msg = get_xml_text(responses[-1].content)
print(' * error uploading '
'{0} ({1}): {2}'.format(filename, responses[-1].status_code,
msg),
file=sys.stderr)
return responses
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
invalid_id_msg = (
'<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )')
# Validate args.
s = Schema({
text_type:
Use(lambda x: bool(x)),
'<file>':
list,
'--format':
list,
'--glob':
list,
'delete':
bool,
'<identifier>':
Or(None, And(str, validate_ia_identifier, error=invalid_id_msg)),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
verbose = True if not args['--quiet'] else False
item = session.get_item(args['<identifier>'])
if not item.exists:
print('{0}: skipping, item does\'t exist.')
# Files that cannot be deleted via S3.
no_delete = ['_meta.xml', '_files.xml', '_meta.sqlite']
if verbose:
sys.stdout.write('Deleting files from {0}\n'.format(item.identifier))
if args['--all']:
files = [f for f in item.iter_files()]
args['--cacade'] = True
elif args['--glob']:
files = item.get_files(glob_pattern=args['--glob'])
elif args['--format']:
files = item.get_files(formats=args['--format'])
else:
fnames = []
if args['<file>'] == ['-']:
fnames = [f.strip().decode('utf-8') for f in sys.stdin]
else:
fnames = [f.strip().decode('utf-8') for f in args['<file>']]
files = [f for f in [item.get_file(f) for f in fnames] if f]
if not files:
sys.stderr.write(' warning: no files found, nothing deleted.\n')
sys.exit(1)
for f in files:
if not f:
if verbose:
sys.stderr.write(' error: "{0}" does not exist\n'.format(
f.name))
sys.exit(1)
if any(f.name.endswith(s) for s in no_delete):
continue
if args['--dry-run']:
sys.stdout.write(' will delete: {0}/{1}\n'.format(
item.identifier, f.name.encode('utf-8')))
continue
resp = f.delete(verbose=verbose, cascade_delete=args['--cascade'])
if resp.status_code != 204:
msg = get_xml_text(resp.content)
sys.stderr.write(' error: {0} ({1})\n'.format(
msg, resp.status_code))
sys.exit(1)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
invalid_id_msg = ('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )')
# Validate args.
s = Schema({
text_type: Use(lambda x: bool(x)),
'<file>': list,
'--format': list,
'--glob': list,
'delete': bool,
'<identifier>': Or(None, And(str, validate_ia_identifier, error=invalid_id_msg)),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
verbose = True if not args['--quiet'] else False
item = session.get_item(args['<identifier>'])
if not item.exists:
print('{0}: skipping, item does\'t exist.')
# Files that cannot be deleted via S3.
no_delete = ['_meta.xml', '_files.xml', '_meta.sqlite']
if verbose:
sys.stdout.write('Deleting files from {0}\n'.format(item.identifier))
if args['--all']:
files = [f for f in item.iter_files()]
args['--cacade'] = True
elif args['--glob']:
files = item.get_files(glob_pattern=args['--glob'])
elif args['--format']:
files = item.get_files(formats=args['--format'])
else:
fnames = []
if args['<file>'] == ['-']:
fnames = [f.strip().decode('utf-8') for f in sys.stdin]
else:
fnames = [f.strip().decode('utf-8') for f in args['<file>']]
files = [f for f in [item.get_file(f) for f in fnames] if f]
if not files:
sys.stderr.write(' warning: no files found, nothing deleted.\n')
sys.exit(1)
for f in files:
if not f:
if verbose:
sys.stderr.write(' error: "{0}" does not exist\n'.format(f.name))
sys.exit(1)
if any(f.name.endswith(s) for s in no_delete):
continue
if args['--dry-run']:
sys.stdout.write(' will delete: {0}/{1}\n'.format(item.identifier,
f.name.encode('utf-8')))
continue
resp = f.delete(verbose=verbose, cascade_delete=args['--cascade'])
if resp.status_code != 204:
msg = get_xml_text(resp.content)
sys.stderr.write(' error: {0} ({1})\n'.format(msg, resp.status_code))
sys.exit(1)