def test_sync_error_debug(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None,
conflicts=None,
dry_run=False,
log_level="DEBUG")
config = {
"targets": {
"bar": {
"local_folder": "/home/mike/barmil",
"s3_uri": "s3://foobar/barrel",
"aws_access_key_id": "3223",
"aws_secret_access_key": "23#eWEa@423#@",
"region_name": "us-west-2",
}
}
}
SyncWorker.side_effect = ValueError("something bad happened")
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert SyncWorker.call_count == 1
assert out == ""
assert err.split("\n")[:2] == [
"something bad happened",
"Traceback (most recent call last):",
]
def test_keyboard_interrupt(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None, conflicts=None, dry_run=False)
config = {
"targets": {
"foo": {
"local_folder": "/home/mike/docs",
"s3_uri": "s3://foobar/docs",
"aws_access_key_id": "3223323",
"aws_secret_access_key": "23#@423#@",
"region_name": "us-east-1",
},
"bar": {
"local_folder": "/home/mike/barmil",
"s3_uri": "s3://foobar/barrel",
"aws_access_key_id": "3223",
"aws_secret_access_key": "23#eWEa@423#@",
"region_name": "us-west-2",
},
}
}
SyncWorker.side_effect = KeyboardInterrupt
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert SyncWorker.call_count == 1
assert out == ""
assert err == ("Quitting due to Keyboard Interrupt...\n")
def test_all_targets(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None, conflicts=None, dry_run=False)
config = {
'targets': {
'foo': {
'local_folder': '/home/mike/docs',
's3_uri': 's3://foobar/docs',
'aws_access_key_id': '3223323',
'aws_secret_access_key': '23#@423#@',
'region_name': 'us-east-1',
},
'bar': {
'local_folder': '/home/mike/barmil',
's3_uri': 's3://foobar/barrel',
'aws_access_key_id': '3223',
'aws_secret_access_key': '23#eWEa@423#@',
'region_name': 'us-west-2',
}
}
}
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert out == ''
assert err == (
'Syncing bar [/home/mike/barmil/ <=> s3://foobar/barrel/]\n'
'Syncing foo [/home/mike/docs/ <=> s3://foobar/docs/]\n')
assert SyncWorker.call_count == 2
def test_sync_error(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None,
conflicts=None,
dry_run=False,
log_level="INFO")
config = {
"targets": {
"foo": {
"local_folder": "/home/mike/docs",
"s3_uri": "s3://foobar/docs",
"aws_access_key_id": "3223323",
"aws_secret_access_key": "23#@423#@",
"region_name": "us-east-1",
},
"bar": {
"local_folder": "/home/mike/barmil",
"s3_uri": "s3://foobar/barrel",
"aws_access_key_id": "3223",
"aws_secret_access_key": "23#eWEa@423#@",
"region_name": "us-west-2",
},
}
}
SyncWorker.side_effect = ValueError("something bad happened")
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert SyncWorker.call_count == 2
assert out == ""
assert err == (
"There was an error syncing 'bar': something bad happened\n"
"There was an error syncing 'foo': something bad happened\n")
def test_keyboard_interrupt(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None, conflicts=None, dry_run=False)
config = {
'targets': {
'foo': {
'local_folder': '/home/mike/docs',
's3_uri': 's3://foobar/docs',
'aws_access_key_id': '3223323',
'aws_secret_access_key': '23#@423#@',
'region_name': 'us-east-1',
},
'bar': {
'local_folder': '/home/mike/barmil',
's3_uri': 's3://foobar/barrel',
'aws_access_key_id': '3223',
'aws_secret_access_key': '23#eWEa@423#@',
'region_name': 'us-west-2',
}
}
}
SyncWorker.side_effect = KeyboardInterrupt
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert SyncWorker.call_count == 1
assert out == ''
assert err == ('Quitting due to Keyboard Interrupt...\n')
def test_sync_error_debug(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None,
conflicts=None,
dry_run=False,
log_level="DEBUG")
config = {
'targets': {
'bar': {
'local_folder': '/home/mike/barmil',
's3_uri': 's3://foobar/barrel',
'aws_access_key_id': '3223',
'aws_secret_access_key': '23#eWEa@423#@',
'region_name': 'us-west-2',
}
}
}
SyncWorker.side_effect = ValueError('something bad happened')
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert SyncWorker.call_count == 1
assert out == ''
assert err.split('\n')[:2] == [
"something bad happened",
"Traceback (most recent call last):",
]
def test_sync_error(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None,
conflicts=None,
dry_run=False,
log_level="INFO")
config = {
'targets': {
'foo': {
'local_folder': '/home/mike/docs',
's3_uri': 's3://foobar/docs',
'aws_access_key_id': '3223323',
'aws_secret_access_key': '23#@423#@',
'region_name': 'us-east-1',
},
'bar': {
'local_folder': '/home/mike/barmil',
's3_uri': 's3://foobar/barrel',
'aws_access_key_id': '3223',
'aws_secret_access_key': '23#eWEa@423#@',
'region_name': 'us-west-2',
}
}
}
SyncWorker.side_effect = ValueError('something bad happened')
command = SyncCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert SyncWorker.call_count == 2
assert out == ''
assert err == (
"There was an error syncing 'bar': something bad happened\n"
"There was an error syncing 'foo': something bad happened\n")
def test_no_targets(self, SyncWorker, capsys):
args = argparse.Namespace(targets=None, conflicts=None, dry_run=False)
command = SyncCommand(args, {"targets": {}}, create_logger())
command.run()
out, err = capsys.readouterr()
assert out == err == ""
assert SyncWorker.call_count == 0
def test_wrong_target(self, SyncWorker, capsys):
args = argparse.Namespace(targets=["foo", "bar"],
conflicts=None,
dry_run=False)
command = SyncCommand(args, {"targets": {"baz": {}}}, create_logger())
command.run()
out, err = capsys.readouterr()
assert out == ""
assert err == ("\"bar\" is an unknown target. Choices are: ['baz']\n"
"\"foo\" is an unknown target. Choices are: ['baz']\n")
assert SyncWorker.call_count == 0
def test_wrong_target(self, SyncWorker, capsys):
args = argparse.Namespace(targets=['foo', 'bar'],
conflicts=None,
dry_run=False)
command = SyncCommand(args, {'targets': {'baz': {}}}, create_logger())
command.run()
out, err = capsys.readouterr()
assert out == ''
assert err == ('"bar" is an unknown target. Choices are: [\'baz\']\n'
'"foo" is an unknown target. Choices are: [\'baz\']\n')
assert SyncWorker.call_count == 0
def main(arguments):
parser = argparse.ArgumentParser(
prog='s4',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=('Fast and cheap synchronisation of files with Amazon S3\n'
'\n'
'Version: {}\n'
'\n'
'To start off, add a Target with the "add" command\n'
).format(VERSION),
)
parser.add_argument(
'--log-level',
default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
)
parser.add_argument(
'--no-colors',
action='store_true',
help='Display without colors',
)
parser.add_argument(
'--timestamps',
action='store_true',
help='Display timestamps for each log message',
)
subparsers = parser.add_subparsers(dest='command')
daemon_parser = subparsers.add_parser('daemon',
help="Run S4 sync continiously")
daemon_parser.add_argument('targets', nargs='*')
daemon_parser.add_argument('--read-delay', default=1000, type=int)
daemon_parser.add_argument('--conflicts',
default='ignore',
choices=['1', '2', 'ignore'])
add_parser = subparsers.add_parser('add',
help="Add a new Target to synchronise")
add_parser.add_argument(
'--copy-target-credentials',
'-C',
help=
"Copy credentials from an existing target instead of typing them in again"
)
sync_parser = subparsers.add_parser('sync',
help="Synchronise Targets with S3")
sync_parser.add_argument('targets', nargs='*')
sync_parser.add_argument('--conflicts',
default=None,
choices=['1', '2', 'ignore'])
sync_parser.add_argument('--dry-run', action='store_true')
edit_parser = subparsers.add_parser('edit', help="Edit Target details")
edit_parser.add_argument('target')
subparsers.add_parser('targets', help="Print available Targets")
subparsers.add_parser('version', help="Print S4 Version")
ls_parser = subparsers.add_parser(
'ls', help="Display list of files for a Target")
ls_parser.add_argument('target')
ls_parser.add_argument('--sort-by',
'-s',
choices=['key', 'local', 's3'],
default='key')
ls_parser.add_argument('--descending', '-d', action='store_true')
ls_parser.add_argument(
'--all',
'-A',
dest='show_all',
action='store_true',
help='show deleted files',
)
remove_parser = subparsers.add_parser('rm', help="Remove a Target")
remove_parser.add_argument('target')
args = parser.parse_args(arguments)
if args.log_level == 'DEBUG':
log_format = '%(levelname)s:%(module)s:%(lineno)s %(message)s'
else:
log_format = '%(message)s'
if args.timestamps:
log_format = '%(asctime)s: ' + log_format
logging.basicConfig(format=log_format, level=args.log_level)
# shut boto up
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)
logging.getLogger('nose').setLevel(logging.CRITICAL)
logging.getLogger('s3transfer').setLevel(logging.CRITICAL)
logging.getLogger('filelock').setLevel(logging.CRITICAL)
logger = logging.getLogger(__name__)
logger.setLevel(args.log_level)
config = utils.get_config()
try:
command = None
if args.command == 'version':
print(VERSION)
return
elif args.command == 'sync':
command = SyncCommand(args, config, logger)
elif args.command == 'targets':
command = TargetsCommand(args, config, logger)
elif args.command == 'add':
command = AddCommand(args, config, logger)
elif args.command == 'edit':
command = EditCommand(args, config, logger)
elif args.command == 'ls':
command = LsCommand(args, config, logger)
elif args.command == 'rm':
command = RmCommand(args, config, logger)
elif args.command == 'daemon':
command = DaemonCommand(args, config, logger)
if command:
try:
command.run()
except Exception as e:
logger.error('An unhandled error has occurred: %s', e)
# Only display a scary stack trace to the user if in DEBUG mode
if args.log_level == 'DEBUG':
raise e
else:
parser.print_help()
except KeyboardInterrupt:
pass
def main(arguments):
parser = argparse.ArgumentParser(
prog="s4",
formatter_class=argparse.RawDescriptionHelpFormatter,
description=("Fast and cheap synchronisation of files with Amazon S3\n"
"\n"
"Version: {}\n"
"\n"
'To start off, add a Target with the "add" command\n'
).format(VERSION),
)
parser.add_argument("--log-level",
default="INFO",
choices=["DEBUG", "INFO", "WARNING", "ERROR"])
parser.add_argument("--no-colors",
action="store_true",
help="Display without colors")
parser.add_argument(
"--timestamps",
action="store_true",
help="Display timestamps for each log message",
)
subparsers = parser.add_subparsers(dest="command")
daemon_parser = subparsers.add_parser("daemon",
help="Run S4 sync continiously",
aliases=["d"])
daemon_parser.add_argument("targets", nargs="*")
daemon_parser.add_argument("--read-delay", default=1000, type=int)
daemon_parser.add_argument("--conflicts",
default="ignore",
choices=["1", "2", "ignore"])
add_parser = subparsers.add_parser("add",
help="Add a new Target to synchronise",
aliases=["a"])
add_parser.add_argument(
"--copy-target-credentials",
"-C",
help=
"Copy credentials from an existing target instead of typing them in again",
)
sync_parser = subparsers.add_parser("sync",
help="Synchronise Targets with S3",
aliases=["s"])
sync_parser.add_argument("targets", nargs="*")
sync_parser.add_argument("--conflicts",
default=None,
choices=["1", "2", "ignore"])
sync_parser.add_argument("--dry-run", action="store_true")
edit_parser = subparsers.add_parser("edit",
help="Edit Target details",
aliases=["e"])
edit_parser.add_argument("target")
subparsers.add_parser("targets",
help="Print available Targets",
aliases=["t"])
subparsers.add_parser("version", help="Print S4 Version", aliases=["v"])
ls_parser = subparsers.add_parser(
"ls", help="Display list of files for a Target")
ls_parser.add_argument("target")
ls_parser.add_argument("--sort-by",
"-s",
choices=["key", "local", "s3"],
default="key")
ls_parser.add_argument("--descending", "-d", action="store_true")
ls_parser.add_argument("--all",
"-A",
dest="show_all",
action="store_true",
help="show deleted files")
remove_parser = subparsers.add_parser("rm", help="Remove a Target")
remove_parser.add_argument("target")
args = parser.parse_args(arguments)
if args.log_level == "DEBUG":
log_format = "%(levelname)s:%(module)s:%(lineno)s %(message)s"
else:
log_format = "%(message)s"
if args.timestamps:
log_format = "%(asctime)s: " + log_format
logging.basicConfig(format=log_format, level=args.log_level)
# shut boto up
logging.getLogger("boto3").setLevel(logging.CRITICAL)
logging.getLogger("botocore").setLevel(logging.CRITICAL)
logging.getLogger("nose").setLevel(logging.CRITICAL)
logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
logging.getLogger("filelock").setLevel(logging.CRITICAL)
logger = logging.getLogger(__name__)
logger.setLevel(args.log_level)
config = utils.get_config()
try:
command = None
if args.command in ("version", "v"):
print(VERSION)
return
elif args.command in ("sync", "s"):
command = SyncCommand(args, config, logger)
elif args.command in ("targets", "t"):
command = TargetsCommand(args, config, logger)
elif args.command in ("add", "a"):
command = AddCommand(args, config, logger)
elif args.command in ("edit", "e"):
command = EditCommand(args, config, logger)
elif args.command == "ls":
command = LsCommand(args, config, logger)
elif args.command == "rm":
command = RmCommand(args, config, logger)
elif args.command in ("daemon", "d"):
command = DaemonCommand(args, config, logger)
if command:
try:
command.run()
except Exception as e:
logger.error("An unhandled error has occurred: %s", e)
# Only display a scary stack trace to the user if in DEBUG mode
if args.log_level == "DEBUG":
raise e
else:
parser.print_help()
except KeyboardInterrupt:
pass
