def test_printable_and_formal_usage():
doc = """
Usage: prog [-hv] ARG
prog N M
prog is a program."""
assert printable_usage(doc) == "Usage: prog [-hv] ARG\n prog N M"
assert formal_usage(printable_usage(doc)) == "[-hv] ARG | N M"
assert printable_usage('uSaGe: prog ARG\n\t \t\n bla') == "uSaGe: prog ARG"
def test_printable_and_formal_usage():
doc = """
Usage: prog [-hv] ARG
prog N M
prog is a program."""
assert printable_usage(doc) == "Usage: prog [-hv] ARG\n prog N M"
assert formal_usage(printable_usage(doc)) == "[-hv] ARG | N M"
assert printable_usage('uSaGe: prog ARG\n\t \t\n bla') == "uSaGe: prog ARG"
def docopt(doc, argv=None, help=True, version=None, options_first=False): # @ReservedAssignment help
"""Re-implementation of docopt.docopt() function to parse ANYTHING at
the end (for proxying django options)."""
if argv is None:
argv = sys.argv[1:]
DocoptExit.usage = printable_usage(doc)
options = parse_defaults(doc)
pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
argv = parse_argv(TokenStream(argv, DocoptExit), list(options), options_first)
pattern_options = set(pattern.flat(Option))
for ao in pattern.flat(AnyOptions):
doc_options = parse_defaults(doc)
ao.children = list(set(doc_options) - pattern_options)
extras(help, version, argv, doc)
__matched, __left, collected = pattern.fix().match(argv)
# if matched and left == []: # better error message if left?
if collected: # better error message if left?
result = Dict((a.name, a.value) for a in (pattern.flat() + collected))
collected_django_options = len(result.get("DJANGO_OPTIONS", []))
result["DJANGO_OPTIONS"] = (
result.get("DJANGO_OPTIONS", []) + sys.argv[len(collected) + (collected_django_options or 1) :]
)
# If any of the collected arguments are also in the DJANGO_OPTIONS,
# then exit because we don't want users to have put options for kalite
# at the end of the command
if any(map(lambda x: x.name in map(lambda x: x.split("=")[0], result["DJANGO_OPTIONS"]), collected)):
sys.stderr.write(
"Cannot mix django manage command options with kalite options. "
"Always put django management options last.\n\n"
)
raise DocoptExit()
return result
raise DocoptExit()
def run_subcommand(self, subcommand, args, kwds=None, options_first=False):
"""
Launches subcommands with given args.
kwds - is dict of keywords that are passed directly to command call and
are not parsed with docopt.
"""
if not kwds:
kwds = {}
sub_cmd_definition = self._find_subcommand(subcommand)
is_class = inspect.isclass(sub_cmd_definition)
if is_class:
sub_cmd = sub_cmd_definition()
sub_cmd_doc = sub_cmd.help()
accepts_help = 'help' in inspect.getargspec(sub_cmd.__call__).args
else:
sub_cmd = sub_cmd_definition
sub_cmd_doc = sub_cmd.__doc__
spec_args = inspect.getargspec(sub_cmd_definition).args
accepts_help = 'help' in spec_args
if 'self' in spec_args:
kwds['self'] = self
try:
kwds.update(parse_command_line(args, sub_cmd_doc, options_first=options_first))
except (DocoptExit, DocoptLanguageError), e:
# TODO: think about showing whole help not just usage
usage = printable_usage(sub_cmd_doc)
msg = '%s: invalid call.' % subcommand
raise InvalidCall(msg, subcommand, usage)
def run_subcommand(self, subcommand, args, kwds=None, options_first=False):
"""
Launches subcommands with given args.
kwds - is dict of keywords that are passed directly to command call and
are not parsed with docopt.
"""
if not kwds:
kwds = {}
sub_cmd_definition = self._find_subcommand(subcommand)
is_class = inspect.isclass(sub_cmd_definition)
if is_class:
sub_cmd = sub_cmd_definition()
sub_cmd_doc = sub_cmd.help()
accepts_help = 'help' in inspect.getargspec(sub_cmd.__call__).args
else:
sub_cmd = sub_cmd_definition
sub_cmd_doc = sub_cmd.__doc__
spec_args = inspect.getargspec(sub_cmd_definition).args
accepts_help = 'help' in spec_args
if 'self' in spec_args:
kwds['self'] = self
try:
kwds.update(
parse_command_line(args,
sub_cmd_doc,
options_first=options_first))
except (DocoptExit, DocoptLanguageError), e:
# TODO: think about showing whole help not just usage
usage = printable_usage(sub_cmd_doc)
msg = '%s: invalid call.' % subcommand
raise InvalidCall(msg, subcommand, usage)
def main():
"""This is the CLI driver for ia-wrapper."""
args = docopt(__doc__, version=__version__, options_first=True)
# Validate args.
s = Schema({
six.text_type:
bool,
'--config-file':
Or(None,
lambda f: os.path.exists(f),
error='--config-file should be a readable file.'),
'<args>':
list,
'<command>':
Or(str, lambda _: 'help'),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Get subcommand.
cmd = args['<command>']
if cmd in cmd_aliases:
cmd = cmd_aliases[cmd]
if (cmd == 'help') or (not cmd):
if not args['<args>']:
sys.exit(print(__doc__.strip(), file=sys.stderr))
else:
ia_module = load_ia_module(args['<args>'][0])
sys.exit(print(ia_module.__doc__.strip(), file=sys.stderr))
argv = [cmd] + args['<args>']
config = dict()
if args['--log']:
config['logging'] = {'level': 'INFO'}
elif args['--debug']:
config['logging'] = {'level': 'DEBUG'}
if args['--insecure']:
config['general'] = dict(secure=False)
session = get_session(config_file=args['--config-file'],
config=config,
debug=args['--debug'])
ia_module = load_ia_module(cmd)
try:
sys.exit(ia_module.main(argv, session))
except IOError as e:
# Handle Broken Pipe errors.
if e.errno == errno.EPIPE:
sys.exit(0)
else:
raise
def get_usage(self, program_name=None):
"""Get the commandline usage string for this experiment."""
program_name = program_name or sys.argv[0]
commands = OrderedDict(self.gather_commands())
options = gather_command_line_options()
long_usage = format_usage(program_name, self.doc, commands, options)
short_usage = printable_usage(long_usage)
return short_usage, long_usage
def get_usage(self, program_name=None):
"""Get the commandline usage string for this experiment."""
program_name = program_name or sys.argv[0]
commands = OrderedDict(self.gather_commands())
options = gather_command_line_options()
long_usage = format_usage(program_name, self.doc, commands, options)
short_usage = printable_usage(long_usage)
return short_usage, long_usage
def _get_module(cmd):
if cmd is None:
print(printable_usage(__doc__), file=sys.stderr)
sys.exit(1)
elif cmd not in __commands__:
print("Command not found. Available commands: " + ", ".join(__commands__), file=sys.stderr)
sys.exit(1)
else:
return import_module(__package__ + "." + cmd)
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: Use(bool),
'<query>': Use(lambda x: ' '.join(x)),
'--parameters': Use(lambda x: get_args_dict(x, query_string=True)),
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--sort': list,
'--field': list,
'--timeout': Use(lambda x: float(x[0]),
error='--timeout must be integer or float.')
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Support comma separated values.
fields = list(chain.from_iterable([x.split(',') for x in args['--field']]))
sorts = list(chain.from_iterable([x.split(',') for x in args['--sort']]))
r_kwargs = dict(
headers=args['--header'],
timeout=args['--timeout'],
)
search = session.search_items(args['<query>'],
fields=fields,
sorts=sorts,
params=args['--parameters'],
request_kwargs=r_kwargs)
try:
if args['--num-found']:
print('{0}'.format(search.num_found))
sys.exit(0)
for result in search:
if args['--itemlist']:
print(result.get('identifier', ''))
else:
j = json.dumps(result)
print(j)
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
except ConnectTimeout as exc:
print('error: Request timed out. Increase the --timeout and try again.',
file=sys.stderr)
sys.exit(1)
except AuthenticationError as exc:
print('error: {}'.format(exc), file=sys.stderr)
sys.exit(1)
def parse_params(cmd, given_usage=None):
# This creates a parameter tree (CommandParams object) for the target docopt tool.
# Also returns a second parameter, a dict of:
# option->option-help-string
from docopt import parse_defaults, parse_pattern, formal_usage, printable_usage
usage = get_usage(cmd) if given_usage is None else given_usage
options = parse_defaults(usage)
pattern = parse_pattern(formal_usage(printable_usage(usage)), options)
param_tree = CommandParams()
build_command_tree(pattern, param_tree)
return param_tree, dict(list(get_options_descriptions(usage)))
def parse_params(cmd):
# This creates a parameter tree (CommandParams object) for the target docopt tool.
# Also returns a second parameter, a dict of:
# option->option-help-string
from docopt import parse_defaults, parse_pattern, formal_usage, printable_usage
usage = get_usage(cmd)
options = parse_defaults(usage)
pattern = parse_pattern(formal_usage(printable_usage(usage)), options)
param_tree = CommandParams()
build_command_tree(pattern, param_tree)
return param_tree, dict(list(get_options_descriptions(usage)))
def parse_params(cmd):
# this function creates a parameter tree for the target docopt tool.
# a parameter tree is a CommandParams instance, see the documentation of the class
# this function also returns a second parameter, which is a dictionary of option->option help string
from docopt import parse_defaults, parse_pattern, formal_usage, printable_usage
usage = get_usage(cmd)
options = parse_defaults(usage)
pattern = parse_pattern(formal_usage(printable_usage(usage)), options)
param_tree = CommandParams()
build_command_tree(pattern, param_tree)
return param_tree, dict(list(get_options_descriptions(usage)))
def main(argv=None, session=None):
# Parse the list of args passed in from `ia`.
args = docopt(__doc__, argv=argv)
del args['simplelists']
# Validate and prepare args.
s = Schema({
str: Use(bool),
'<identifier>': str,
'--parent': Use(lambda p: p[0]),
'--list': Use(lambda l: l[0]),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
item = session.get_item(args['<identifier>'])
# Prepare patch and POST data.
patch = json.dumps({
'op': 'set',
'list': args['--list'],
'parent': args['--parent'],
})
data = {
'-patch': patch,
'-target': 'simplelists',
'priority': '-5',
}
# Build request.
request = Request(
method='POST',
url=item.urls.metadata,
data=data,
auth=S3PostAuth(session.access_key, session.secret_key),
)
prepared_request = request.prepare()
# TODO: probably ready to uncomment this, and give it a try?
#r = session.send(prepared_request)
#if r.status_code == 200 or 'no changes' in r.text:
# print('success: {}'.format(item.urls.history))
# sys.exit(0)
#else:
# print('error: {} - {} - {}'.format(item.identifier, r.status_code, r.content))
# sys.exit(1)
# TODO: delete
print('args:\n\n{}\n'.format(args))
print('item:\n\n{}\n'.format(item))
print('POST data:\n\n{}'.format(request.data))
def _get_module(cmd):
if cmd is None:
print(printable_usage(__doc__), file=sys.stderr)
sys.exit(1)
elif cmd not in __commands__:
print("Command not found. Available commands: " +
", ".join(__commands__),
file=sys.stderr)
sys.exit(1)
else:
return import_module(__package__ + "." + cmd)
def main():
"""This is the CLI driver for ia-wrapper."""
args = docopt(__doc__, version=__version__, options_first=True)
# Validate args.
s = Schema({
six.text_type: bool,
'--config-file': Or(None, lambda f: os.path.exists(f),
error='--config-file should be a readable file.'),
'<args>': list,
'<command>': Or(str, lambda _: 'help'),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Get subcommand.
cmd = args['<command>']
if cmd in cmd_aliases:
cmd = cmd_aliases[cmd]
if (cmd == 'help') or (not cmd):
if not args['<args>']:
sys.exit(print(__doc__.strip(), file=sys.stderr))
else:
ia_module = load_ia_module(args['<args>'][0])
sys.exit(print(ia_module.__doc__.strip(), file=sys.stderr))
argv = [cmd] + args['<args>']
config = dict()
if args['--log']:
config['logging'] = {'level': 'INFO'}
elif args['--debug']:
config['logging'] = {'level': 'DEBUG'}
if args['--insecure']:
config['secure'] = False
session = get_session(config_file=args['--config-file'],
config=config,
debug=args['--debug'])
ia_module = load_ia_module(cmd)
try:
sys.exit(ia_module.main(argv, session))
except IOError as e:
# Handle Broken Pipe errors.
if e.errno == errno.EPIPE:
sys.exit(0)
else:
raise
def main():
"""This is the CLI driver for ia-wrapper."""
args = docopt(__doc__, version=__version__, options_first=True)
# Validate args.
s = Schema(
{
six.text_type: bool,
"--config-file": Or(None, lambda f: os.path.exists(f), error="--config-file should be a readable file."),
"<args>": list,
"<command>": Or(str, lambda _: "help"),
}
)
try:
args = s.validate(args)
except SchemaError as exc:
print("{0}\n{1}".format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Get subcommand.
cmd = args["<command>"]
if cmd in cmd_aliases:
cmd = cmd_aliases[cmd]
if (cmd == "help") or (not cmd):
if not args["<args>"]:
sys.exit(print(__doc__.strip(), file=sys.stderr))
else:
ia_module = load_ia_module(args["<args>"][0])
sys.exit(print(ia_module.__doc__.strip(), file=sys.stderr))
argv = [cmd] + args["<args>"]
config = dict()
if args["--log"]:
config["logging"] = {"level": "INFO"}
elif args["--debug"]:
config["logging"] = {"level": "DEBUG"}
if args["--insecure"]:
config["secure"] = False
session = get_session(config_file=args["--config-file"], config=config, debug=args["--debug"])
ia_module = load_ia_module(cmd)
try:
sys.exit(ia_module.main(argv, session))
except IOError as e:
# Handle Broken Pipe errors.
if e.errno == errno.EPIPE:
sys.exit(0)
else:
raise
def get_usage(self, program_name=None):
"""Get the commandline usage string for this experiment."""
program_name = os.path.relpath(program_name or sys.argv[0] or 'Dummy',
self.base_dir)
commands = OrderedDict(self.gather_commands())
options = gather_command_line_options()
long_usage = format_usage(program_name, self.doc, commands, options)
# internal usage is a workaround because docopt cannot handle spaces
# in program names. So for parsing we use 'dummy' as the program name.
# for printing help etc. we want to use the actual program name.
internal_usage = format_usage('dummy', self.doc, commands, options)
short_usage = printable_usage(long_usage)
return short_usage, long_usage, internal_usage
def get_usage(self, program_name=None):
"""Get the commandline usage string for this experiment."""
program_name = os.path.relpath(program_name or sys.argv[0] or 'Dummy',
self.base_dir)
commands = OrderedDict(self.gather_commands())
options = gather_command_line_options()
long_usage = format_usage(program_name, self.doc, commands, options)
# internal usage is a workaround because docopt cannot handle spaces
# in program names. So for parsing we use 'dummy' as the program name.
# for printing help etc. we want to use the actual program name.
internal_usage = format_usage('dummy', self.doc, commands, options)
short_usage = printable_usage(long_usage)
return short_usage, long_usage, internal_usage
def start_interactive():
while True:
command = input('> ')
if command in ('exit', 'quit', '\q'):
return
try:
arguments = docopt(__doc__,
argv=command.split(),
version=pkg_resources.get_distribution(
'uspherum-esc-cli').version)
dispatch_command(arguments, is_interactive=True)
except DocoptExit:
print(printable_usage(__doc__))
def main():
_setup_logging(file=False)
# TODO access centralized version information
args = docopt(__doc__, options_first=True, version="instantshare version 0.1")
if args["<command>"] == "help":
if len(args["<args>"]) == 0:
print(printable_usage(__doc__), file=sys.stderr)
sys.exit(1)
cmd = args["<args>"][0]
_execute_command([cmd, "--help"])
else:
argv = [args["<command>"]] + args["<args>"]
_execute_command(argv)
def cli():
"""Command-line interface"""
global options
options = docopt(__doc__)
try:
main()
except RuntimeWarning as w:
print(" Warning: %s" % w, file=sys.stderr)
sys.exit(1)
except RuntimeError as w:
print("%s" % w, file=sys.stderr)
print(docpt.printable_usage(__doc__))
sys.exit(1)
def _find_subcommand(self, subcommand):
"""
Searches subcommand in self.subcommands.
If no subcommand found - raises exception.
"""
for sub_cmd in self.subcommands:
name = get_command_name(sub_cmd)
is_alias = hasattr(sub_cmd, 'aliases') and subcommand in sub_cmd.aliases
if subcommand == name or is_alias:
return sub_cmd
usage = printable_usage(self.help())
raise UnknownCommand('%s: unknown subcommand %s' % (get_command_name(self), subcommand),
subcommand,
usage)
def __init__(self, argv: List[str], strict_mode=True):
arguments = docopt(__doc__, argv[1:])
self.name = arguments['--name']
self.team = self.int_or_none(arguments['--team'])
self.player_index = self.int_or_none(arguments['--player-index'])
self.spawn_id = self.int_or_none(arguments['--spawn-id'])
self.config_file = arguments['--config-file']
self.matchcomms_url = self.url_or_none(arguments['--matchcomms-url'])
self.is_missing_args = False
if self.name is None or self.team is None or self.player_index is None or self.spawn_id is None \
or self.config_file is None or self.matchcomms_url is None:
print('Standalone bot is missing required arguments!')
print(printable_usage(__doc__))
self.is_missing_args = True
def _find_subcommand(self, subcommand):
"""
Searches subcommand in self.subcommands.
If no subcommand found - raises exception.
"""
for sub_cmd in self.subcommands:
name = get_command_name(sub_cmd)
is_alias = hasattr(sub_cmd,
'aliases') and subcommand in sub_cmd.aliases
if subcommand == name or is_alias:
return sub_cmd
usage = printable_usage(self.help())
raise UnknownCommand(
'%s: unknown subcommand %s' % (get_command_name(self), subcommand),
subcommand, usage)
def main(argv, session):
args = docopt(__doc__, argv=argv)
src_path = args['<src-identifier>/<src-file>']
dest_path = args['<dest-identifier>/<dest-file>']
# Validate args.
s = Schema({
str:
Use(bool),
'--metadata':
list,
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'<src-identifier>/<src-file>':
And(str,
lambda x: '/' in x,
error='Source not formatted correctly. See usage example.'),
'<dest-identifier>/<dest-file>':
And(str,
lambda x: '/' in x,
error='Destiantion not formatted correctly. See usage example.'),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
# First we use ia_copy, prep argv for ia_copy.
argv.pop(0)
argv = ['copy'] + argv
# Call ia_copy.
r, src_file = ia_copy.main(argv, session, cmd='move')
dr = src_file.delete(headers=args['--header'], cascade_delete=True)
if dr.status_code == 204:
print('success: moved {} to {}'.format(src_path, dest_path))
sys.exit(0)
print('error: {}'.format(dr.content))
def main():
_setup_config()
_setup_logging(file=False)
# TODO access centralized version information
args = docopt(__doc__,
options_first=True,
version="instantshare version 0.2")
if args["<command>"] == "help":
if len(args["<args>"]) == 0:
print(printable_usage(__doc__), file=sys.stderr)
sys.exit(1)
cmd = args["<args>"][0]
_execute_command([cmd, "--help"])
else:
argv = [args['<command>']] + args['<args>']
_execute_command(argv)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
str: Use(bool),
'--comment': str,
'<identifier>': list,
})
# Filenames should be unicode literals. Support PY2 and PY3.
if six.PY2:
args['<file>'] = [f.decode('utf-8') for f in args['<file>']]
try:
args = s.validate(args)
except SchemaError as exc:
sys.stderr.write('{0}\n{1}\n'.format(str(exc),
printable_usage(__doc__)))
sys.exit(1)
errors = False
if args['<identifier>'] == ['-']:
args['<identifier>'] = sys.stdin
for identifier in args['<identifier>']:
item = session.get_item(identifier)
if not item.exists:
print('warning: item does not exist - {}'.format(item.identifier),
file=sys.stderr)
r = item.undark(args['--comment'])
if r is None:
print(
'warning: no task submitted, item already undark - {}'.format(
item.identifier),
file=sys.stderr)
elif r.status_code != 200:
print('error: task failed, item not undark - {}'.format(
item.identifier),
file=sys.stderr)
errors = True
else:
print('success: item undarked - {}'.format(item.identifier))
if errors:
print('\n*** Not all items were undarked! ***\n')
sys.exit(1)
def dispatch_command(arguments, is_interactive=False):
commands = OrderedDict({
'config.list': config_list,
'config.set': config_set,
'll.list': ll_list,
'll.status': ll_status,
'll.show': ll_show,
'll.cmd': ll_cmd,
'llgr.list': ll_groups_list,
'show': show_last
})
command = determine_command(commands, arguments)
if command is not None:
command(arguments)
else:
print('Unknown command requested!', printable_usage(__doc__))
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type:
Use(bool),
'<query>':
Use(lambda x: ' '.join(x)),
'--parameters':
Use(lambda x: get_args_dict(x, query_string=True)),
'--sort':
list,
'--field':
Use(lambda x: ['identifier'] if not x and args['--itemlist'] else x),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Support comma separated values.
fields = list(chain.from_iterable([x.split(',') for x in args['--field']]))
sorts = list(chain.from_iterable([x.split(',') for x in args['--sort']]))
search = search_items(args['<query>'],
fields=fields,
sorts=sorts,
params=args['--parameters'])
if args['--num-found']:
print('{0}'.format(search.num_found))
sys.exit(0)
try:
for result in search:
if args['--itemlist']:
print(result.get('identifier', ''))
else:
j = json.dumps(result)
print(j)
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
def docopt(doc,
argv=None,
help=True,
version=None,
options_first=False): # @ReservedAssignment help
"""Re-implementation of docopt.docopt() function to parse ANYTHING at
the end (for proxying django options)."""
if argv is None:
argv = sys.argv[1:]
DocoptExit.usage = printable_usage(doc)
options = parse_defaults(doc)
pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
argv = parse_argv(TokenStream(argv, DocoptExit), list(options),
options_first)
pattern_options = set(pattern.flat(Option))
for ao in pattern.flat(AnyOptions):
doc_options = parse_defaults(doc)
ao.children = list(set(doc_options) - pattern_options)
extras(help, version, argv, doc)
__matched, __left, collected = pattern.fix().match(argv)
# if matched and left == []: # better error message if left?
if collected: # better error message if left?
result = Dict((a.name, a.value) for a in (pattern.flat() + collected))
collected_django_options = len(result.get('DJANGO_OPTIONS', []))
result['DJANGO_OPTIONS'] = (result.get('DJANGO_OPTIONS', []) +
sys.argv[len(collected) +
(collected_django_options or 1):])
# If any of the collected arguments are also in the DJANGO_OPTIONS,
# then exit because we don't want users to have put options for kalite
# at the end of the command
if any(
map(
lambda x: x.name in map(lambda x: x.split("=")[0], result[
'DJANGO_OPTIONS']), collected)):
sys.stderr.write(
"Cannot mix django manage command options with kalite options. "
"Always put django management options last.\n\n")
raise DocoptExit()
return result
raise DocoptExit()
def isbg_sa_unwrap():
"""Run when this module is called from the command line."""
try:
opts = docopt(__isbg_sa_unwrap_opts__.__doc__,
version="isbg_sa_unwrap v" + isbg.__version__ +
", from: " + os.path.abspath(__file__) + "\n\n" +
isbg.__license__)
except DocoptExit:
sys.stderr.write('Error with options!!!\n')
raise
if opts.get("--usage"):
sys.stdout.write(
"{}\n".format(printable_usage(__isbg_sa_unwrap_opts__.__doc__)))
return
if opts.get("--from"):
file_in = open(opts["--from"], 'rb')
else:
file_in = sys.stdin
if hasattr(file_in, 'buffer'):
# select byte streams if they exist (on python 3)
file_in = file_in.buffer # pylint: disable=no-member
spams = unwrap(file_in.read())
file_in.close()
if spams is None:
sys.stderr.write("No spam into the mail detected.\n")
return
if opts.get("--to"):
file_out = open(opts["--to"], 'wb')
else:
file_out = sys.stdout
for spam in spams:
file_out.write(spam.as_string())
if file_out != sys.stdout:
file_out.close()
def isbg_sa_unwrap():
"""Run when this module is called from the command line."""
try:
opts = docopt(__isbg_sa_unwrap_opts__.__doc__,
version="isbg_sa_unwrap v" + isbg.__version__ +
", from: " + os.path.abspath(__file__) + "\n\n" +
isbg.__license__)
except DocoptExit:
sys.stderr.write('Error with options!!!\n')
raise
if opts.get("--usage"):
sys.stdout.write(
"{}\n".format(printable_usage(__isbg_sa_unwrap_opts__.__doc__)))
return
if opts.get("--from"):
file_in = open(opts["--from"], 'rb')
else:
file_in = sys.stdin
if hasattr(file_in, 'buffer'):
# select byte streams if they exist (on python 3)
file_in = file_in.buffer # pylint: disable=no-member
spams = unwrap(file_in.read())
file_in.close()
if spams is None:
sys.stderr.write("No spam into the mail detected.\n")
return
if opts.get("--to"):
file_out = open(opts["--to"], 'wb')
else:
file_out = sys.stdout
for spam in spams:
file_out.write(spam.as_string())
if file_out != sys.stdout:
file_out.close()
async def help(self, extra_msg: str = "", usage_only: bool = True) -> None:
body = ""
if extra_msg:
body += f"{extra_msg}\n\n"
# Get rid of extra indentation
doc = getattr(self, "__doc__", "").replace("\n ", "\n").strip()
body += "```\n"
if usage_only:
body += docopt.printable_usage(doc)
else:
body += doc
body += "\n```"
message = Notice(body)
message.formatted_body = markdown(body)
message.format = "org.matrix.custom.html"
await self.room.timeline.send(message)
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: Use(bool),
'<query>': Use(lambda x: ' '.join(x)),
'--parameters': Use(lambda x: get_args_dict(x, query_string=True)),
'--sort': list,
'--field': Use(lambda x: ['identifier'] if not x and args['--itemlist'] else x),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Support comma separated values.
fields = list(chain.from_iterable([x.split(',') for x in args['--field']]))
sorts = list(chain.from_iterable([x.split(',') for x in args['--sort']]))
search = search_items(args['<query>'],
fields=fields,
sorts=sorts,
params=args['--parameters'])
if args['--num-found']:
print('{0}'.format(search.num_found))
sys.exit(0)
try:
for result in search:
if args['--itemlist']:
print(result.get('identifier', ''))
else:
j = json.dumps(result)
print(j)
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
def docopt_cmd_completion(func, **kwargs):
options = parse_defaults(func.__doc__)
pattern = parse_pattern(formal_usage(printable_usage(func.__doc__)),
options).children[0]
def get_state(it, pattern):
try:
value = next(it)
except StopIteration:
return pattern
res = []
for x in pattern:
if ((type(x[0]) == list and value in flatten(x[0])) or
value == x[0]):
res.append(x[1:])
if res:
return get_state(it, res)
return []
def wrapper(self, text, line, begidx, endidx):
argv = shlex.split(line[:endidx])[1:]
if not line[endidx - 1].isspace():
target = argv[-1]
argv = argv[:-1]
else:
target = ''
state = get_state(iter(argv), pattern.noflat())
res = []
for x in state:
if type(x[0]) == list:
res.extend(flatten(x[0]))
else:
res.append(x[0])
return list(set(x for x in res if x.startswith(target)))
wrapper.__name__ = str('complete_' + func.__name__[3:])
wrapper.__module__ = func.__module__
wrapper.__doc__ = func.__doc__
return wrapper
def docopt_cmd_completion(func, **kwargs):
options = parse_defaults(func.__doc__)
pattern = parse_pattern(formal_usage(printable_usage(func.__doc__)),
options).children[0]
def get_state(it, pattern):
try:
value = next(it)
except StopIteration:
return pattern
res = []
for x in pattern:
if ((type(x[0]) == list and value in flatten(x[0]))
or value == x[0]):
res.append(x[1:])
if res:
return get_state(it, res)
return []
def wrapper(self, text, line, begidx, endidx):
argv = shlex.split(line[:endidx])[1:]
if not line[endidx - 1].isspace():
target = argv[-1]
argv = argv[:-1]
else:
target = ''
state = get_state(iter(argv), pattern.noflat())
res = []
for x in state:
if type(x[0]) == list:
res.extend(flatten(x[0]))
else:
res.append(x[0])
return list(set(x for x in res if x.startswith(target)))
wrapper.__name__ = str('complete_' + func.__name__[3:])
wrapper.__module__ = func.__module__
wrapper.__doc__ = func.__doc__
return wrapper
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema(
{
six.text_type: Use(bool),
"<query>": Use(lambda x: " ".join(x)),
"--parameters": Use(lambda x: get_args_dict(x, query_string=True)),
"--sort": list,
"--field": list,
}
)
try:
args = s.validate(args)
except SchemaError as exc:
print("{0}\n{1}".format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Support comma separated values.
fields = list(chain.from_iterable([x.split(",") for x in args["--field"]]))
sorts = list(chain.from_iterable([x.split(",") for x in args["--sort"]]))
search = session.search_items(args["<query>"], fields=fields, sorts=sorts, params=args["--parameters"])
try:
if args["--num-found"]:
print("{0}".format(search.num_found))
sys.exit(0)
for result in search:
if args["--itemlist"]:
print(result.get("identifier", ""))
else:
j = json.dumps(result)
print(j)
except ValueError as e:
print("error: {0}".format(e), file=sys.stderr)
def main(argv, session):
args = docopt(__doc__, argv=argv)
src_path = args['<src-identifier>/<src-file>']
dest_path = args['<dest-identifier>/<dest-file>']
# Validate args.
s = Schema({
str: Use(bool),
'--metadata': list,
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'<src-identifier>/<src-file>': And(str, lambda x: '/' in x,
error='Source not formatted correctly. See usage example.'),
'<dest-identifier>/<dest-file>': And(str, lambda x: '/' in x,
error='Destination not formatted correctly. See usage example.'),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
# First we use ia_copy, prep argv for ia_copy.
argv.pop(0)
argv = ['copy'] + argv
# Call ia_copy.
r, src_file = ia_copy.main(argv, session, cmd='move')
dr = src_file.delete(headers=args['--header'], cascade_delete=True)
if dr.status_code == 204:
print('success: moved {} to {}'.format(src_path, dest_path))
sys.exit(0)
print('error: {}'.format(dr.content))
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: Use(bool),
'<query>': Use(lambda x: ' '.join(x)),
'--parameters': Use(lambda x: get_args_dict(x)),
'--sort': list,
'--field': Use(lambda x: ['identifier'] if not x and args['--itemlist'] else x),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Format sort paramaters.
for i, field in enumerate(args['--sort']):
key = 'sort[{0}]'.format(i)
args['--parameters'][key] = field.strip().replace(':', ' ')
search = search_items(args['<query>'],
fields=args['--field'],
params=args['--parameters'])
if args['--num-found']:
print('{0}'.format(search.num_found))
sys.exit(0)
for result in search:
if args['--itemlist']:
print(result.get('identifier', ''))
else:
j = json.dumps(result)
print(j)
def main(argv, session, cmd='copy'):
args = docopt(__doc__, argv=argv)
src_path = args['<src-identifier>/<src-file>']
dest_path = args['<dest-identifier>/<dest-file>']
# If src == dest, file get's deleted!
try:
assert src_path != dest_path
except AssertionError:
print('error: The source and destination files cannot be the same!',
file=sys.stderr)
sys.exit(1)
global SRC_ITEM
SRC_ITEM = session.get_item(src_path.split('/')[0])
# Validate args.
s = Schema({
str:
Use(bool),
'<src-identifier>/<src-file>':
And(
str,
And(And(
str,
lambda x: '/' in x,
error='Destiantion not formatted correctly. See usage example.'
),
assert_src_file_exists,
error=('https://archive.org/download/{} does not exist. '
'Please check the identifier and filepath and retry.'.
format(src_path)))),
'<dest-identifier>/<dest-file>':
And(str,
lambda x: '/' in x,
error='Destiantion not formatted correctly. See usage example.'),
'--metadata':
Or(None,
And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
})
try:
args = s.validate(args)
except SchemaError as exc:
# This module is sometimes called by other modules.
# Replace references to 'ia copy' in ___doc__ to 'ia {cmd}' for clarity.
usage = printable_usage(__doc__.replace('ia copy',
'ia {}'.format(cmd)))
print('{0}\n{1}'.format(str(exc), usage), file=sys.stderr)
sys.exit(1)
args['--header']['x-amz-copy-source'] = '/{}'.format(src_path)
args['--header']['x-amz-metadata-directive'] = 'COPY'
args['--header']
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
url = '{}//s3.us.archive.org/{}'.format(session.protocol, dest_path)
req = ia.iarequest.S3Request(url=url,
method='PUT',
metadata=args['--metadata'],
headers=args['--header'],
access_key=session.access_key,
secret_key=session.secret_key)
p = req.prepare()
r = session.send(p)
if r.status_code != 200:
try:
msg = get_s3_xml_text(r.text)
except Exception as e:
msg = r.text
print('error: failed to {} "{}" to "{}" - {}'.format(
cmd, src_path, dest_path, msg))
sys.exit(1)
elif cmd == 'copy':
print('success: copied "{}" to "{}".'.format(src_path, dest_path))
else:
return (r, SRC_FILE)
p.join()
# merge the results
# 0- clear the terminal
os.system('cls' if os.name == 'nt' else 'clear')
# 1- the FeatureCollection part of the geojson
with open(outfile, 'a') as dst:
dst.write('{ "type": "FeatureCollection",\n"features": [\n')
dst.close()
# 2- the subparts of the converted source file
merge_results(list_out_files, outfile)
#3- closing the geojson FeatureCollection property
with open(outfile, 'a') as dst:
dst.write('\n]\n}\n')
dst.close()
#4- deleting the sub files
for f in list_out_files:
os.remove(f)
except IOError as e:
print "Error when handling the files. Details: "+str(e)
if __name__ == '__main__':
arguments = docopt(__doc__, version='csv2geojson utility v0.1, Author: Sofiane Imadali <*****@*****.**>')
if arguments["--input"] is not None:
print "Converting "+arguments["--input"]+" to geojson in the file: "+arguments["--output"]
whole_convert(arguments["--input"], arguments["--output"])
print "All done! Your output is in: "+arguments["--output"]
else:
print "\nYou must give an input file to convert. Try -h or --help option for help.\n"
print(printable_usage(__doc__))
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
destdir_msg = '--destdir must be a valid path to a directory.'
itemlist_msg = '--itemlist must be a valid path to an existing file.'
# Validate args.
s = Schema({
str: Use(bool),
'--destdir': Or([], And(Use(lambda d: d[0]), dir_exists), error=destdir_msg),
'--format': list,
'--glob': Use(lambda l: l[0] if l else None),
'<file>': list,
'--search': Or(str, None),
'--itemlist': Or(None, And(lambda f: os.path.isfile(f)), error=itemlist_msg),
'<identifier>': Or(str, None),
'--retries': Use(lambda x: x[0]),
'--search-parameters': Use(lambda x: get_args_dict(x, query_string=True)),
'--on-the-fly': Use(bool),
'--no-change-timestamp': Use(bool),
'--parameters': Use(lambda x: get_args_dict(x, query_string=True)),
})
# Filenames should be unicode literals. Support PY2 and PY3.
if six.PY2:
args['<file>'] = [f.decode('utf-8') for f in args['<file>']]
try:
args = s.validate(args)
if args['--glob']:
if args['--format']:
raise(SchemaError(None, '--glob and --format cannot be used together.'))
except SchemaError as exc:
sys.stderr.write('{0}\n{1}\n'.format(
str(exc), printable_usage(__doc__)))
sys.exit(1)
retries = int(args['--retries'])
if args['--itemlist']:
with open(args['--itemlist']) as fp:
ids = [x.strip() for x in fp]
total_ids = len(ids)
elif args['--search']:
try:
_search = session.search_items(args['--search'],
params=args['--search-parameters'])
total_ids = _search.num_found
if total_ids == 0:
print('error: the query "{0}" '
'returned no results'.format(args['--search']), file=sys.stderr)
sys.exit(1)
ids = _search
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
sys.exit(1)
# Download specific files.
if args['<identifier>'] and args['<identifier>'] != '-':
if '/' in args['<identifier>']:
identifier = args['<identifier>'].split('/')[0]
files = ['/'.join(args['<identifier>'].split('/')[1:])]
else:
identifier = args['<identifier>']
files = args['<file>']
total_ids = 1
ids = [identifier]
elif args['<identifier>'] == '-':
total_ids = 1
ids = sys.stdin
files = None
else:
files = None
errors = list()
for i, identifier in enumerate(ids):
if args['--stdout']:
item = session.get_item(identifier)
f = list(item.get_files(args['<file>']))
try:
assert len(f) == 1
except AssertionError:
sys.stderr.write('error: {0}/{1} does not exist!\n'.format(
identifier, args['<file>'][0]))
sys.exit(1)
if six.PY2:
stdout_buf = sys.stdout
else:
stdout_buf = sys.stdout.buffer
f[0].download(retries=args['--retries'],
fileobj=stdout_buf,
params=args['--parameters'])
sys.exit(0)
try:
identifier = identifier.strip()
except AttributeError:
identifier = identifier.get('identifier')
if total_ids > 1:
item_index = '{0}/{1}'.format((i + 1), total_ids)
else:
item_index = None
try:
item = session.get_item(identifier)
except Exception as exc:
print('{0}: failed to retrieve item metadata - errors'.format(identifier),
file=sys.stderr)
if 'You are attempting to make an HTTPS' in str(exc):
print('\n{0}'.format(exc), file=sys.stderr)
sys.exit(1)
else:
continue
# Otherwise, download the entire item.
_errors = item.download(
files=files,
formats=args['--format'],
glob_pattern=args['--glob'],
dry_run=args['--dry-run'],
verbose=args['--verbose'],
silent=args['--silent'],
ignore_existing=args['--ignore-existing'],
checksum=args['--checksum'],
destdir=args['--destdir'],
no_directory=args['--no-directories'],
retries=retries,
item_index=item_index,
ignore_errors=True,
on_the_fly=args['--on-the-fly'],
no_change_timestamp=args['--no-change-timestamp'],
params=args['--parameters']
)
if _errors:
errors.append(_errors)
if errors:
# TODO: add option for a summary/report.
sys.exit(1)
else:
sys.exit(0)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: bool,
'<identifier>': list,
'--modify': list,
'--append': list,
'--spreadsheet': Or(None, And(lambda f: os.path.exists(f),
error='<file> should be a readable file or directory.')),
'--target': Or(None, str),
'--priority': Or(None, Use(int, error='<priority> should be an integer.')),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
formats = set()
responses = []
for i, identifier in enumerate(args['<identifier>']):
item = session.get_item(identifier)
# Check existence of item.
if args['--exists']:
if item.exists:
responses.append(True)
print('{0} exists'.format(identifier))
else:
responses.append(False)
print('{0} does not exist'.format(identifier), file=sys.stderr)
if (i + 1) == len(args['<identifier>']):
if all(r is True for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Modify metadata.
elif args['--modify'] or args['--append']:
metadata_args = args['--modify'] if args['--modify'] else args['--append']
metadata = get_args_dict(metadata_args)
responses.append(modify_metadata(item, metadata, args))
if (i + 1) == len(args['<identifier>']):
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Get metadata.
elif args['--formats']:
for f in item.get_files():
formats.add(f.format)
if (i + 1) == len(args['<identifier>']):
print('\n'.join(formats))
# Dump JSON to stdout.
else:
metadata = json.dumps(item.item_metadata)
print(metadata)
# Edit metadata for items in bulk, using a spreadsheet as input.
if args['--spreadsheet']:
if not args['--priority']:
args['--priority'] = -5
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
responses = []
for row in spreadsheet:
if not row['identifier']:
continue
item = session.get_item(row['identifier'])
if row.get('file'):
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
responses.append(modify_metadata(item, metadata, args))
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
sys.exit(1)
sys.exit(0)
def main(argv, session):
args = docopt(__doc__, argv=argv)
ERRORS = False
# Validate args.
s = Schema({
six.text_type:
Use(bool),
'<identifier>':
Or(
None,
And(str,
validate_ia_identifier,
error=
('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )'))),
'<file>':
And(
And(lambda f: all(os.path.exists(x) for x in f if x != '-'),
error='<file> should be a readable file or directory.'),
And(lambda f: False
if f == ['-'] and not args['--remote-name'] else True,
error=
'--remote-name must be provided when uploading from stdin.')),
'--remote-name':
Or(None, And(str)),
'--spreadsheet':
Or(None,
os.path.isfile,
error='--spreadsheet should be a readable file.'),
'--metadata':
Or(None,
And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--retries':
Use(lambda x: int(x[0]) if x else 0),
'--sleep':
Use(lambda l: int(l[0]), error='--sleep value must be an integer.'),
'--size-hint':
Or(Use(lambda l: int(l[0]) if l else None),
int,
None,
error='--size-hint value must be an integer.'),
'--status-check':
bool,
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Status check.
if args['--status-check']:
if session.s3_is_overloaded():
print('warning: {0} is over limit, and not accepting requests. '
'Expect 503 SlowDown errors.'.format(args['<identifier>']),
file=sys.stderr)
sys.exit(1)
else:
print('success: {0} is accepting requests.'.format(
args['<identifier>']))
sys.exit()
elif args['<identifier>']:
item = session.get_item(args['<identifier>'])
# Upload keyword arguments.
if args['--size-hint']:
args['--header']['x-archive-size-hint'] = args['--size-hint']
queue_derive = True if args['--no-derive'] is False else False
verbose = True if args['--quiet'] is False else False
upload_kwargs = dict(
metadata=args['--metadata'],
headers=args['--header'],
debug=args['--debug'],
queue_derive=queue_derive,
checksum=args['--checksum'],
verbose=verbose,
retries=args['--retries'],
retries_sleep=args['--sleep'],
delete=args['--delete'],
)
# Upload files.
if not args['--spreadsheet']:
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
else:
local_file = args['<file>']
if isinstance(local_file,
(list, tuple, set)) and args['--remote-name']:
local_file = local_file[0]
if args['--remote-name']:
files = {args['--remote-name']: local_file}
else:
files = local_file
for _r in _upload_files(item, files, upload_kwargs):
if args['--debug']:
break
if (not _r) or (not _r.ok):
ERRORS = True
# Bulk upload using spreadsheet.
else:
# Use the same session for each upload request.
session = ArchiveSession()
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
prev_identifier = None
for row in spreadsheet:
local_file = row['file']
identifier = row['identifier']
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
item = session.get_item(identifier)
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = [
'{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v
]
metadata = get_args_dict(md_args)
upload_kwargs['metadata'].update(metadata)
r = _upload_files(item, local_file, upload_kwargs, prev_identifier,
session)
for _r in r:
if args['--debug']:
break
if (not _r) or (not _r.ok):
ERRORS = True
prev_identifier = identifier
if ERRORS:
sys.exit(1)
def parse_args(sbg):
"""Argument processing of the command line.
:param sbg: the `isbg.ISBG` instance which would be updated with the
parameters.
:type sbg: isbg.ISBG
:return: `None`
:Example: You can run it using:
>>> sbg = isbg.ISBG()
>>> parse_args(sbg)
"""
try:
opts = docopt(__cmd_opts__.__doc__, version="isbg_v" +
isbg.__version__ + ", from: " +
os.path.abspath(__file__) + "\n\n" + isbg.__license__)
opts = dict([(k, v) for k, v in opts.items()
if v is not None])
except DocoptExit as exc:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Option processing failed - " + str(exc))
if opts.get("--usage"):
print(printable_usage(__cmd_opts__.__doc__))
return 1
if opts.get("--deletehigherthan") is not None:
try:
sbg.deletehigherthan = float(opts["--deletehigherthan"])
except Exception: # pylint: disable=broad-except
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Unrecognized score - " +
opts["--deletehigherthan"])
if sbg.deletehigherthan < 1:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Score " + repr(sbg.deletehigherthan) +
" is too small")
if opts["--flag"] is True:
sbg.spamflags.append("\\Flagged")
sbg.imapsets.host = opts.get('--imaphost', sbg.imapsets.host)
sbg.imapsets.passwd = opts.get('--imappasswd',
sbg.imapsets.passwd)
sbg.imapsets.port = opts.get('--imapport', sbg.imapsets.port)
sbg.imapsets.user = opts.get('--imapuser', sbg.imapsets.user)
sbg.imapsets.inbox = opts.get('--imapinbox', sbg.imapsets.inbox)
sbg.imapsets.spaminbox = opts.get('--spaminbox', sbg.imapsets.spaminbox)
sbg.imapsets.learnspambox = opts.get('--learnspambox')
sbg.imapsets.learnhambox = opts.get('--learnhambox')
sbg.imapsets.nossl = opts.get('--nossl', sbg.imapsets.nossl)
sbg.lockfilegrace = float(opts.get('--lockfilegrace', sbg.lockfilegrace))
sbg.nostats = opts.get('--nostats', False)
sbg.dryrun = opts.get('--dryrun', False)
sbg.delete = opts.get('--delete', False)
sbg.gmail = opts.get('--gmail', False)
if opts.get("--maxsize") is not None:
try:
sbg.maxsize = int(opts["--maxsize"])
except (TypeError, ValueError):
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Unrecognised size - " + opts["--maxsize"])
if sbg.maxsize < 1:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Size " + repr(sbg.maxsize) + " is too small")
sbg.movehamto = opts.get('--movehamto')
if opts["--noninteractive"] is True:
sbg.interactive = 0
sbg.noreport = opts.get('--noreport', sbg.noreport)
sbg.lockfilename = opts.get('--lockfilename', sbg.lockfilename)
sbg.trackfile = opts.get('--trackfile', sbg.trackfile)
sbg.partialrun = opts.get('--partialrun', sbg.partialrun)
try:
sbg.partialrun = int(opts["--partialrun"])
except ValueError:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"partialrun \'{}\' must be a integer".format(
repr(sbg.partialrun)))
if sbg.partialrun < 0:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
("Partial run \'{}\' number must be equal to " +
"0 or higher").format(repr(sbg.partialrun)))
elif sbg.partialrun == 0:
sbg.partialrun = None
sbg.verbose = opts.get('--verbose', sbg.verbose)
sbg.verbose_mails = opts.get('--verbose-mails', sbg.verbose_mails)
sbg.ignorelockfile = opts.get("--ignorelockfile", sbg.ignorelockfile)
sbg.savepw = opts.get('--savepw', sbg.savepw)
sbg.passwdfilename = opts.get('--passwdfilename', sbg.passwdfilename)
sbg.imaplist = opts.get('--imaplist', sbg.imaplist)
sbg.learnunflagged = opts.get('--learnunflagged', sbg.learnunflagged)
sbg.learnflagged = opts.get('--learnflagged', sbg.learnflagged)
sbg.learnthendestroy = opts.get('--learnthendestroy', sbg.learnthendestroy)
sbg.learnthenflag = opts.get('--learnthendestroy', sbg.learnthenflag)
sbg.expunge = opts.get('--expunge', sbg.expunge)
sbg.teachonly = opts.get('--teachonly', sbg.teachonly)
sbg.spamc = opts.get('--spamc', sbg.spamc)
sbg.exitcodes = opts.get('--exitcodes', sbg.exitcodes)
# fixup any arguments
if opts.get("--imapport") is None:
if opts["--nossl"] is True:
sbg.imapsets.port = 143
else:
sbg.imapsets.port = 993
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
invalid_id_msg = ('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )')
# Validate args.
s = Schema({
six.text_type: Use(lambda x: bool(x)),
'<file>': And(list, Use(
lambda x: convert_str_list_to_unicode(x) if six.PY2 else x)),
'--format': list,
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--glob': list,
'delete': bool,
'--retries': Use(lambda i: int(i[0])),
'<identifier>': str,
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
verbose = True if not args['--quiet'] else False
item = session.get_item(args['<identifier>'])
if not item.exists:
print('{0}: skipping, item does\'t exist.')
# Files that cannot be deleted via S3.
no_delete = ['_meta.xml', '_files.xml', '_meta.sqlite']
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
if verbose:
sys.stdout.write('Deleting files from {0}\n'.format(item.identifier))
if args['--all']:
files = [f for f in item.get_files()]
args['--cacade'] = True
elif args['--glob']:
files = item.get_files(glob_pattern=args['--glob'])
elif args['--format']:
files = item.get_files(formats=args['--format'])
else:
fnames = []
if args['<file>'] == ['-']:
if six.PY2:
fnames = convert_str_list_to_unicode([f.strip() for f in sys.stdin])
else:
fnames = [f.strip() for f in sys.stdin]
else:
fnames = [f.strip() for f in args['<file>']]
files = list(item.get_files(fnames))
if not files:
sys.stderr.write(' warning: no files found, nothing deleted.\n')
sys.exit(1)
errors = False
for f in files:
if not f:
if verbose:
sys.stderr.write(' error: "{0}" does not exist\n'.format(f.name))
errors = True
if any(f.name.endswith(s) for s in no_delete):
continue
if args['--dry-run']:
sys.stdout.write(' will delete: {0}/{1}\n'.format(item.identifier,
f.name.encode('utf-8')))
continue
try:
resp = f.delete(verbose=verbose,
cascade_delete=args['--cascade'],
headers=args['--header'],
retries=args['--retries'])
except requests.exceptions.RetryError as e:
print(' error: max retries exceeded for {0}'.format(f.name), file=sys.stderr)
errors = True
continue
if resp.status_code != 204:
errors = True
msg = get_s3_xml_text(resp.content)
print(' error: {0} ({1})'.format(msg, resp.status_code), file=sys.stderr)
continue
if errors is True:
sys.exit(1)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
destdir_msg = '--destdir must be a valid path to a directory.'
# Validate args.
s = Schema({
str: Use(bool),
'--destdir': Or([], And(Use(lambda d: d[0]), dir_exists), error=destdir_msg),
'--format': list,
'--glob': Use(lambda l: l[0] if l else None),
'<file>': list,
'--search': Or(str, None),
'--itemlist': Or(str, None),
'<identifier>': Or(str, None),
'--retries': Use(lambda x: x[0]),
})
# Filenames should be unicode literals. Support PY2 and PY3.
if six.PY2:
args['<file>'] = [f.decode('utf-8') for f in args['<file>']]
try:
args = s.validate(args)
except SchemaError as exc:
sys.stderr.write('{0}\n{1}\n'.format(
str(exc), printable_usage(__doc__)))
sys.exit(1)
retries = int(args['--retries'])
if args['--itemlist']:
ids = [x.strip() for x in open(args['--itemlist'])]
total_ids = len(ids)
elif args['--search']:
_search = search_items(args['--search'])
total_ids = _search.num_found
ids = search_ids(args['--search'])
# Download specific files.
if args['<identifier>']:
if '/' in args['<identifier>']:
identifier = args['<identifier>'].split('/')[0]
files = ['/'.join(args['<identifier>'].split('/')[1:])]
else:
identifier = args['<identifier>']
files = args['<file>']
total_ids = 1
ids = [identifier]
else:
files = None
errors = list()
for i, identifier in enumerate(ids):
if total_ids > 1:
item_index = '{0}/{1}'.format((i + 1), total_ids)
else:
item_index = None
try:
item = session.get_item(identifier)
except Exception as exc:
print('{0}: failed to retrieve item metadata - errors'.format(identifier))
continue
# Otherwise, download the entire item.
_errors = item.download(
files=files,
formats=args['--format'],
glob_pattern=args['--glob'],
dry_run=args['--dry-run'],
verbose=args['--verbose'],
silent=args['--silent'],
ignore_existing=args['--ignore-existing'],
checksum=args['--checksum'],
destdir=args['--destdir'],
no_directory=args['--no-directories'],
retries=retries,
item_index=item_index,
ignore_errors=True
)
if _errors:
errors.append(_errors)
if errors:
# TODO: add option for a summary/report.
sys.exit(1)
else:
sys.exit(0)
def main(argv, session):
if six.PY2:
args = docopt(__doc__.encode('utf-8'), argv=argv)
else:
args = docopt(__doc__, argv=argv)
ERRORS = False
# Validate args.
s = Schema({
str: Use(bool),
'<identifier>': Or(None, And(str, validate_ia_identifier,
error=('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, periods ".", '
'underscores "_", or dashes "-". However, <identifier> cannot begin '
'with periods, underscores, or dashes.'))),
'<file>': And(
Use(lambda l: l if not six.PY2 else convert_str_list_to_unicode(l)),
And(lambda f: all(os.path.exists(x) for x in f if x != '-'),
error='<file> should be a readable file or directory.'),
And(lambda f: False if f == ['-'] and not args['--remote-name'] else True,
error='--remote-name must be provided when uploading from stdin.')),
'--remote-name': Or(None,
Use(lambda x: x.decode(sys.getfilesystemencoding()) if six.PY2 else x)),
'--spreadsheet': Or(None, os.path.isfile,
error='--spreadsheet should be a readable file.'),
'--metadata': Or(None, And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--retries': Use(lambda x: int(x[0]) if x else 0),
'--sleep': Use(lambda l: int(l[0]), error='--sleep value must be an integer.'),
'--size-hint': Or(Use(lambda l: str(l[0]) if l else None), int, None,
error='--size-hint value must be an integer.'),
'--status-check': bool,
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Make sure the collection being uploaded to exists.
collection_id = args['--metadata'].get('collection')
if collection_id and not args['--no-collection-check'] and not args['--status-check']:
if isinstance(collection_id, list):
collection_id = collection_id[0]
collection = session.get_item(collection_id)
if not collection.exists:
sys.stderr.write(
'You must upload to a collection that exists. '
'"{0}" does not exist.\n{1}\n'.format(collection_id,
printable_usage(__doc__)))
sys.exit(1)
# Status check.
if args['--status-check']:
if session.s3_is_overloaded():
print('warning: {0} is over limit, and not accepting requests. '
'Expect 503 SlowDown errors.'.format(args['<identifier>']),
file=sys.stderr)
sys.exit(1)
else:
print('success: {0} is accepting requests.'.format(args['<identifier>']))
sys.exit()
elif args['<identifier>']:
item = session.get_item(args['<identifier>'])
# Upload keyword arguments.
if args['--size-hint']:
args['--header']['x-archive-size-hint'] = args['--size-hint']
# Upload with backups turned on by default.
if not args['--header'].get('x-archive-keep-old-version'):
args['--header']['x-archive-keep-old-version'] = '1'
queue_derive = True if args['--no-derive'] is False else False
verbose = True if args['--quiet'] is False else False
upload_kwargs = dict(
metadata=args['--metadata'],
headers=args['--header'],
debug=args['--debug'],
queue_derive=queue_derive,
verbose=verbose,
verify=args['--verify'],
checksum=args['--checksum'],
retries=args['--retries'],
retries_sleep=args['--sleep'],
delete=args['--delete'],
)
# Upload files.
if not args['--spreadsheet']:
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
else:
local_file = args['<file>']
if isinstance(local_file, (list, tuple, set)) and args['--remote-name']:
local_file = local_file[0]
if args['--remote-name']:
files = {args['--remote-name']: local_file}
else:
files = local_file
for _r in _upload_files(item, files, upload_kwargs):
if args['--debug']:
break
if (not _r.status_code) or (not _r.ok):
ERRORS = True
# Bulk upload using spreadsheet.
else:
# Use the same session for each upload request.
with io.open(args['--spreadsheet'], 'rU', newline='', encoding='utf-8') as csvfp:
spreadsheet = csv.DictReader(csvfp)
prev_identifier = None
for row in spreadsheet:
upload_kwargs_copy = deepcopy(upload_kwargs)
local_file = row['file']
identifier = row['identifier']
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
item = session.get_item(identifier)
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
metadata = get_args_dict(md_args)
upload_kwargs_copy['metadata'].update(metadata)
r = _upload_files(item, local_file, upload_kwargs_copy, prev_identifier,
session)
for _r in r:
if args['--debug']:
break
if (not _r) or (not _r.ok):
ERRORS = True
prev_identifier = identifier
if ERRORS:
sys.exit(1)
def command_not_avialable(args=None):
usage = docopt.printable_usage(main.__doc__)
print(usage, file=sys.stderr)
exit(-1)
def main(cliargs=None):
"""Entry point for the application script
:param list(str) cliargs: Arguments to parse or None (=use ``sys.argv``)
:return: return codes from :func:`molden_modifier.common.errorcode`
:rtype: int
"""
try:
args = parsecli(cliargs)
LOG.info('%s version: %s', __package__, __version__)
LOG.debug('Python version: %s', sys.version.split()[0])
LOG.debug("CLI result: %s", args)
checkargs(args)
data = ""
with open(args['<molden_file>'], 'r') as infile:
for line in infile.readlines():
data += re.sub(r" *$", "", line)
if data is None:
Log.error("Empty file")
sys.exit(1)
molecules = parse(data)
if molecules is None:
LOG.info("No molecules found!")
sys.exit(1)
LOG.info("Found %s molecules in %s", len(molecules),
args['<molden_file>'])
results = None
cvs = False
if args["mirror"]:
results = mirror(molecules, args["--compare"])
elif args["filter"]:
with open(args["-f"], 'r') as filter_file:
filter_data = filter_file.read()
molecules_filter = parse(filter_data)
results = filter(molecules_filter, molecules)
elif args["info"]:
info(molecules)
elif args["sort"]:
results = sort(molecules)
elif args["shortest_distance"]:
results = shortest_distance(molecules, args["-s"])
cvs = True
else:
raise RuntimeError("Unknown command")
output(results, args["--output"], cvs)
LOG.info("Done.")
return 0
except DocoptExit as error:
LOG.fatal("Need a molden file.")
printable_usage(__doc__)
return errorcode(error)
except FileNotFoundError as error:
LOG.fatal("File not found '%s'", error)
return errorcode(error)
except RuntimeError as error:
LOG.fatal("Something failed '%s'", error)
return 1
except KeyboardInterrupt as error:
return errorcode(error)
def main(argv, session, cmd='copy'):
args = docopt(__doc__, argv=argv)
src_path = args['<src-identifier>/<src-file>']
dest_path = args['<dest-identifier>/<dest-file>']
# If src == dest, file get's deleted!
try:
assert src_path != dest_path
except AssertionError:
print('error: The source and destination files cannot be the same!',
file=sys.stderr)
sys.exit(1)
global SRC_ITEM
SRC_ITEM = session.get_item(src_path.split('/')[0])
# Validate args.
s = Schema({
str: Use(bool),
'<src-identifier>/<src-file>': And(str, And(And(str, lambda x: '/' in x,
error='Destination not formatted correctly. See usage example.'),
assert_src_file_exists, error=(
'https://archive.org/download/{} does not exist. '
'Please check the identifier and filepath and retry.'.format(src_path)))),
'<dest-identifier>/<dest-file>': And(str, lambda x: '/' in x,
error='Destination not formatted correctly. See usage example.'),
'--metadata': Or(None, And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
})
try:
args = s.validate(args)
except SchemaError as exc:
# This module is sometimes called by other modules.
# Replace references to 'ia copy' in ___doc__ to 'ia {cmd}' for clarity.
usage = printable_usage(__doc__.replace('ia copy', 'ia {}'.format(cmd)))
print('{0}\n{1}'.format(str(exc), usage), file=sys.stderr)
sys.exit(1)
args['--header']['x-amz-copy-source'] = '/{}'.format(src_path)
args['--header']['x-amz-metadata-directive'] = 'COPY'
args['--header']
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
url = '{}//s3.us.archive.org/{}'.format(session.protocol, dest_path)
req = ia.iarequest.S3Request(url=url,
method='PUT',
metadata=args['--metadata'],
headers=args['--header'],
access_key=session.access_key,
secret_key=session.secret_key)
p = req.prepare()
r = session.send(p)
if r.status_code != 200:
try:
msg = get_s3_xml_text(r.text)
except Exception as e:
msg = r.text
print('error: failed to {} "{}" to "{}" - {}'.format(
cmd, src_path, dest_path, msg))
sys.exit(1)
elif cmd == 'copy':
print('success: copied "{}" to "{}".'.format(src_path, dest_path))
else:
return (r, SRC_FILE)
def parse_args(sbg):
"""Argument processing of the command line.
:param sbg: the `isbg.ISBG` instance which would be updated with the
parameters.
:type sbg: isbg.ISBG
:return: `None`
:Example: You can run it using:
>>> sbg = isbg.ISBG()
>>> parse_args(sbg)
"""
try:
opts = docopt(__cmd_opts__.__doc__, version="isbg_v" +
isbg.__version__ + ", from: " +
os.path.abspath(__file__) + "\n\n" + isbg.__license__)
opts = dict([(k, v) for k, v in opts.items()
if v is not None])
except DocoptExit as exc:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Option processing failed - " + str(exc))
if opts.get("--usage"):
print(printable_usage(__cmd_opts__.__doc__))
return 1
if opts.get("--deletehigherthan") is not None:
try:
sbg.deletehigherthan = float(opts["--deletehigherthan"])
except Exception: # pylint: disable=broad-except
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Unrecognized score - " +
opts["--deletehigherthan"])
if sbg.deletehigherthan < 1:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Score " + repr(sbg.deletehigherthan) +
" is too small")
if opts["--flag"] is True:
sbg.spamflags.append("\\Flagged")
sbg.imapsets.host = opts.get('--imaphost', sbg.imapsets.host)
sbg.imapsets.passwd = opts.get('--imappasswd',
sbg.imapsets.passwd)
sbg.imapsets.port = opts.get('--imapport', sbg.imapsets.port)
sbg.imapsets.user = opts.get('--imapuser', sbg.imapsets.user)
sbg.imapsets.inbox = opts.get('--imapinbox', sbg.imapsets.inbox)
sbg.imapsets.spaminbox = opts.get('--spaminbox', sbg.imapsets.spaminbox)
sbg.imapsets.learnspambox = opts.get('--learnspambox')
sbg.imapsets.learnhambox = opts.get('--learnhambox')
sbg.imapsets.nossl = opts.get('--nossl', sbg.imapsets.nossl)
sbg.lockfilegrace = float(opts.get('--lockfilegrace', sbg.lockfilegrace))
sbg.nostats = opts.get('--nostats', False)
sbg.dryrun = opts.get('--dryrun', False)
sbg.delete = opts.get('--delete', False)
sbg.gmail = opts.get('--gmail', False)
if opts.get("--maxsize") is not None:
try:
sbg.maxsize = int(opts["--maxsize"])
except (TypeError, ValueError):
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Unrecognised size - " + opts["--maxsize"])
if sbg.maxsize < 1:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"Size " + repr(sbg.maxsize) + " is too small")
sbg.movehamto = opts.get('--movehamto')
if opts["--noninteractive"] is True:
sbg.interactive = 0
sbg.noreport = opts.get('--noreport', sbg.noreport)
sbg.lockfilename = opts.get('--lockfilename', sbg.lockfilename)
sbg.trackfile = opts.get('--trackfile', sbg.trackfile)
#: ..v2.0: partialrun now has a default value of 50, use 0 if not
#: partialrun shoud be used.
sbg.partialrun = opts.get('--partialrun', sbg.partialrun)
try:
sbg.partialrun = int(opts["--partialrun"])
except ValueError:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
"partialrun \'{}\' must be a integer".format(
repr(sbg.partialrun)))
if sbg.partialrun < 0:
raise isbg.ISBGError(isbg.__exitcodes__['flags'],
("Partial run \'{}\' number must be equal to " +
"0 or higher").format(repr(sbg.partialrun)))
elif sbg.partialrun == 0:
sbg.partialrun = None
sbg.verbose = opts.get('--verbose', sbg.verbose)
sbg.verbose_mails = opts.get('--verbose-mails', sbg.verbose_mails)
sbg.ignorelockfile = opts.get("--ignorelockfile", sbg.ignorelockfile)
sbg.savepw = opts.get('--savepw', sbg.savepw)
sbg.passwdfilename = opts.get('--passwdfilename', sbg.passwdfilename)
sbg.imaplist = opts.get('--imaplist', sbg.imaplist)
sbg.learnunflagged = opts.get('--learnunflagged', sbg.learnunflagged)
sbg.learnflagged = opts.get('--learnflagged', sbg.learnflagged)
sbg.learnthendestroy = opts.get('--learnthendestroy', sbg.learnthendestroy)
sbg.learnthenflag = opts.get('--learnthendestroy', sbg.learnthenflag)
sbg.expunge = opts.get('--expunge', sbg.expunge)
sbg.teachonly = opts.get('--teachonly', sbg.teachonly)
sbg.spamc = opts.get('--spamc', sbg.spamc)
sbg.exitcodes = opts.get('--exitcodes', sbg.exitcodes)
# fixup any arguments
if opts.get("--imapport") is None:
if opts["--nossl"] is True:
sbg.imapsets.port = 143
else:
sbg.imapsets.port = 993
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: Use(bool),
'<identifier>': Or(None, And(str, validate_ia_identifier,
error=('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )'))),
'<file>': And(
And(lambda f: all(os.path.exists(x) for x in f if x != '-'),
error='<file> should be a readable file or directory.'),
And(lambda f: False if f == ['-'] and not args['--remote-name'] else True,
error='--remote-name must be provided when uploading from stdin.')),
'--remote-name': Or(None, And(str)),
'--spreadsheet': Or(None, os.path.isfile,
error='--spreadsheet should be a readable file.'),
'--metadata': Or(None, And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--retries': Use(lambda x: int(x[0]) if x else 0),
'--sleep': Use(lambda l: int(l[0]), error='--sleep value must be an integer.'),
'--size-hint': Or(Use(lambda l: int(l[0]) if l else None), int, None,
error='--size-hint value must be an integer.'),
'--status-check': bool,
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
# Status check.
if args['--status-check']:
if session.s3_is_overloaded():
print('warning: {0} is over limit, and not accepting requests. '
'Expect 503 SlowDown errors.'.format(args['<identifier>']),
file=sys.stderr)
sys.exit(1)
else:
print('success: {0} is accepting requests.'.format(args['<identifier>']))
sys.exit()
elif args['<identifier>']:
item = session.get_item(args['<identifier>'])
# Upload keyword arguments.
if args['--size-hint']:
args['--header']['x-archive-size-hint'] = args['--size-hint']
queue_derive = True if args['--no-derive'] is False else False
verbose = True if args['--quiet'] is False else False
upload_kwargs = dict(
metadata=args['--metadata'],
headers=args['--header'],
debug=args['--debug'],
queue_derive=queue_derive,
checksum=args['--checksum'],
verbose=verbose,
retries=args['--retries'],
retries_sleep=args['--sleep'],
delete=args['--delete'],
)
# Upload files.
if not args['--spreadsheet']:
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
else:
local_file = args['<file>']
if isinstance(local_file, (list, tuple, set)) and args['--remote-name']:
local_file = local_file[0]
if args['--remote-name']:
files = {args['--remote-name']: local_file}
else:
files = local_file
responses = _upload_files(item, files, upload_kwargs)
# Bulk upload using spreadsheet.
else:
# Use the same session for each upload request.
session = ArchiveSession()
spreadsheet = csv.DictReader(open(args['--spreadsheet'], 'rU'))
prev_identifier = None
responses = []
for row in spreadsheet:
local_file = row['file']
identifier = row['identifier']
del row['file']
del row['identifier']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
item = session.get_item(identifier)
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = ['{0}:{1}'.format(k.lower(), v) for (k, v) in row.items() if v]
metadata = get_args_dict(md_args)
upload_kwargs['metadata'].update(metadata)
r = _upload_files(item, local_file, upload_kwargs, prev_identifier, session,
responses)
responses += r
prev_identifier = identifier
if responses and not all(r and r.ok for r in responses):
sys.exit(1)
from multiprocessing.pool import Pool
from tempfile import NamedTemporaryFile
from codecs import EncodedFile
from authorate.text_features import text_to_vector
import fileinput
import sys
import os
import re
import subprocess
import random
import warnings
VERSION = "0.1.0-SNAPSHOT"
VERBOSE = False
USAGE_TEXT = printable_usage(__doc__)
# Regexes
BOOK_REGEX = re.compile('^.*\.(mobi|txt|epub)$')
TITLE_REGEX = re.compile('^(.*) - .*$')
DEFAULT_SNIPPETS_COUNT = 100
MIN_SNIPPET_SIZE = 128
engine = None
def display_error(e):
"""Display the given message as an error to the user and print the usage
string.
def _help(_stdin, stdout, _stderr, **_kwargs):
print >> stdout, docopt.printable_usage(__doc__)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
invalid_id_msg = ('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )')
# Validate args.
s = Schema({
six.text_type: Use(lambda x: bool(x)),
'<file>': And(list, Use(
lambda x: convert_str_list_to_unicode(x) if six.PY2 else x)),
'--format': list,
'--header': Or(None, And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--glob': list,
'delete': bool,
'--retries': Use(lambda i: int(i[0])),
'<identifier>': str,
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
verbose = True if not args['--quiet'] else False
item = session.get_item(args['<identifier>'])
if not item.exists:
print('{0}: skipping, item does\'t exist.')
# Files that cannot be deleted via S3.
no_delete = ['_meta.xml', '_files.xml', '_meta.sqlite']
# Add keep-old-version by default.
if 'x-archive-keep-old-version' not in args['--header']:
args['--header']['x-archive-keep-old-version'] = '1'
if verbose:
sys.stdout.write('Deleting files from {0}\n'.format(item.identifier))
if args['--all']:
files = [f for f in item.get_files()]
args['--cacade'] = True
elif args['--glob']:
files = item.get_files(glob_pattern=args['--glob'])
elif args['--format']:
files = item.get_files(formats=args['--format'])
else:
fnames = []
if args['<file>'] == ['-']:
if six.PY2:
fnames = convert_str_list_to_unicode([f.strip() for f in sys.stdin])
else:
fnames = [f.strip() for f in sys.stdin]
else:
fnames = [f.strip() for f in args['<file>']]
files = list(item.get_files(fnames))
if not files:
sys.stderr.write(' warning: no files found, nothing deleted.\n')
sys.exit(1)
errors = False
for f in files:
if not f:
if verbose:
sys.stderr.write(' error: "{0}" does not exist\n'.format(f.name))
errors = True
if any(f.name.endswith(s) for s in no_delete):
continue
if args['--dry-run']:
sys.stdout.write(' will delete: {0}/{1}\n'.format(item.identifier,
f.name.encode('utf-8')))
continue
try:
resp = f.delete(verbose=verbose,
cascade_delete=args['--cascade'],
headers=args['--headers'],
retries=args['--retries'])
except requests.exceptions.RetryError as e:
print(' error: max retries exceeded for {0}'.format(f.name), file=sys.stderr)
errors = True
continue
if resp.status_code != 204:
errors = True
msg = get_s3_xml_text(resp.content)
print(' error: {0} ({1})'.format(msg, resp.status_code), file=sys.stderr)
continue
if errors is True:
sys.exit(1)
def _parse_docopt(doc):
options = docopt.parse_defaults(doc)
pure_doc = docopt.formal_usage(docopt.printable_usage(doc))
pattern = docopt.parse_pattern(pure_doc, options)
return pattern
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
destdir_msg = '--destdir must be a valid path to a directory.'
itemlist_msg = '--itemlist must be a valid path to an existing file.'
# Validate args.
s = Schema({
str:
Use(bool),
'--destdir':
Or([], And(Use(lambda d: d[0]), dir_exists), error=destdir_msg),
'--format':
list,
'--glob':
Use(lambda l: l[0] if l else None),
'<file>':
list,
'--search':
Or(str, None),
'--itemlist':
Or(None, And(lambda f: os.path.isfile(f)), error=itemlist_msg),
'<identifier>':
Or(str, None),
'--retries':
Use(lambda x: x[0]),
'--search-parameters':
Use(lambda x: get_args_dict(x, query_string=True)),
'--on-the-fly':
Use(bool),
'--no-change-timestamp':
Use(bool)
})
# Filenames should be unicode literals. Support PY2 and PY3.
if six.PY2:
args['<file>'] = [f.decode('utf-8') for f in args['<file>']]
try:
args = s.validate(args)
if args['--glob']:
if args['--format']:
raise (SchemaError(
None, '--glob and --format cannot be used together.'))
except SchemaError as exc:
sys.stderr.write('{0}\n{1}\n'.format(str(exc),
printable_usage(__doc__)))
sys.exit(1)
retries = int(args['--retries'])
if args['--itemlist']:
with open(args['--itemlist']) as fp:
ids = [x.strip() for x in fp]
total_ids = len(ids)
elif args['--search']:
try:
_search = session.search_items(args['--search'],
params=args['--search-parameters'])
total_ids = _search.num_found
if total_ids == 0:
print('error: the query "{0}" '
'returned no results'.format(args['--search']),
file=sys.stderr)
sys.exit(1)
ids = _search
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
sys.exit(1)
# Download specific files.
if args['<identifier>'] and args['<identifier>'] != '-':
if '/' in args['<identifier>']:
identifier = args['<identifier>'].split('/')[0]
files = ['/'.join(args['<identifier>'].split('/')[1:])]
else:
identifier = args['<identifier>']
files = args['<file>']
total_ids = 1
ids = [identifier]
elif args['<identifier>'] == '-':
total_ids = 1
ids = sys.stdin
files = None
else:
files = None
errors = list()
for i, identifier in enumerate(ids):
if args['--stdout']:
item = session.get_item(identifier)
f = list(item.get_files(args['<file>']))
try:
assert len(f) == 1
except AssertionError:
sys.stderr.write('error: {0}/{1} does not exist!\n'.format(
identifier, args['<file>'][0]))
sys.exit(1)
if six.PY2:
stdout_buf = sys.stdout
else:
stdout_buf = sys.stdout.buffer
f[0].download(retries=args['--retries'], fileobj=stdout_buf)
sys.exit(0)
try:
identifier = identifier.strip()
except AttributeError:
identifier = identifier.get('identifier')
if total_ids > 1:
item_index = '{0}/{1}'.format((i + 1), total_ids)
else:
item_index = None
try:
item = session.get_item(identifier)
except Exception as exc:
print('{0}: failed to retrieve item metadata - errors'.format(
identifier),
file=sys.stderr)
if 'You are attempting to make an HTTPS' in str(exc):
print('\n{0}'.format(exc), file=sys.stderr)
sys.exit(1)
else:
continue
# Otherwise, download the entire item.
_errors = item.download(
files=files,
formats=args['--format'],
glob_pattern=args['--glob'],
dry_run=args['--dry-run'],
verbose=args['--verbose'],
silent=args['--silent'],
ignore_existing=args['--ignore-existing'],
checksum=args['--checksum'],
destdir=args['--destdir'],
no_directory=args['--no-directories'],
retries=retries,
item_index=item_index,
ignore_errors=True,
on_the_fly=args['--on-the-fly'],
no_change_timestamp=args['--no-change-timestamp'])
if _errors:
errors.append(_errors)
if errors:
# TODO: add option for a summary/report.
sys.exit(1)
else:
sys.exit(0)
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type: bool,
'<identifier>': list,
'--modify': list,
'--append': list,
'--append-list': list,
'--remove': list,
'--spreadsheet': Or(None, And(lambda f: os.path.exists(f),
error='<file> should be a readable file or directory.')),
'--target': Or(None, str),
'--priority': Or(None, Use(int, error='<priority> should be an integer.')),
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr)
sys.exit(1)
formats = set()
responses = []
for i, identifier in enumerate(args['<identifier>']):
item = session.get_item(identifier)
# Check existence of item.
if args['--exists']:
if item.exists:
responses.append(True)
print('{0} exists'.format(identifier))
else:
responses.append(False)
print('{0} does not exist'.format(identifier), file=sys.stderr)
if (i + 1) == len(args['<identifier>']):
if all(r is True for r in responses):
sys.exit(0)
else:
sys.exit(1)
# Modify metadata.
elif args['--modify'] or args['--append'] or args['--append-list'] \
or args['--remove']:
if args['--modify']:
metadata_args = args['--modify']
elif args['--append']:
metadata_args = args['--append']
elif args['--append-list']:
metadata_args = args['--append-list']
if args['--remove']:
metadata_args = args['--remove']
try:
metadata = get_args_dict(metadata_args)
if any('/' in k for k in metadata):
metadata = get_args_dict_many_write(metadata)
except ValueError:
print("error: The value of --modify, --remove, --append or --append-list "
"is invalid. It must be formatted as: --modify=key:value",
file=sys.stderr)
sys.exit(1)
if args['--remove']:
responses.append(remove_metadata(item, metadata, args))
else:
responses.append(modify_metadata(item, metadata, args))
if (i + 1) == len(args['<identifier>']):
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
for r in responses:
if r.status_code == 200:
continue
# We still want to exit 0 if the non-200 is a
# "no changes to xml" error.
elif 'no changes' in r.content.decode('utf-8'):
continue
else:
sys.exit(1)
# Get metadata.
elif args['--formats']:
for f in item.get_files():
formats.add(f.format)
if (i + 1) == len(args['<identifier>']):
print('\n'.join(formats))
# Dump JSON to stdout.
else:
metadata = json.dumps(item.item_metadata)
print(metadata)
# Edit metadata for items in bulk, using a spreadsheet as input.
if args['--spreadsheet']:
if not args['--priority']:
args['--priority'] = -5
with io.open(args['--spreadsheet'], 'rU', newline='', encoding='utf-8') as csvfp:
spreadsheet = csv.DictReader(csvfp)
responses = []
for row in spreadsheet:
if not row['identifier']:
continue
item = session.get_item(row['identifier'])
if row.get('file'):
del row['file']
metadata = dict((k.lower(), v) for (k, v) in row.items() if v)
responses.append(modify_metadata(item, metadata, args))
if all(r.status_code == 200 for r in responses):
sys.exit(0)
else:
for r in responses:
if r.status_code == 200:
continue
# We still want to exit 0 if the non-200 is a
# "no changes to xml" error.
elif 'no changes' in r.content.decode('utf-8'):
continue
else:
sys.exit(1)
