acd_cli.py

#!/usr/bin/env python3
import sys
import os
import json
import argparse
import logging
import signal
from datetime import datetime, timedelta
import time
import re
import appdirs
from functools import partial
from collections import namedtuple

from pkgutil import walk_packages
from pkg_resources import iter_entry_points

from acdcli.api import account, common, content, metadata, trash
from acdcli.api.common import RequestError
from acdcli.cache import *
from acdcli.utils import hashing, progress
from acdcli.utils.threading import QueuedLoader

# load local plugin modules (default ones, for developers)
from acdcli import plugins

for importer, modname, ispkg in walk_packages(path=plugins.__path__, prefix=plugins.__name__ + '.',
                                              onerror=lambda x: None):
    if not ispkg:
        __import__(modname)

# load additional plugins from entry point
for plug_mod in iter_entry_points(group='acdcli.plugins', name=None):
    __import__(plug_mod.module_name)

__version__ = '0.3.0a5'
_app_name = 'acd_cli'

logger = logging.getLogger(_app_name)

# noinspection PyBroadException
# monkey patch the user agent
try:
    import requests.utils

    if 'old_dau' not in dir(requests.utils):
        requests.utils.old_dau = requests.utils.default_user_agent

        def new_dau():
            return _app_name + '/' + __version__ + ' ' + requests.utils.old_dau()

        requests.utils.default_user_agent = new_dau
except:
    pass


# monkey patch for suppressing overlong http.client's HTTPConnection send debug prints
def ellipses_print(*args, **kwargs):
    for a in args:
        if len(a) > 2000:
            print('[...]', **kwargs)
            return

    print(*args, **kwargs)

import http.client as cl
cl.print = ellipses_print


# path settings

cp = os.environ.get('ACD_CLI_CACHE_PATH')
sp = os.environ.get('ACD_CLI_SETTINGS_PATH')

CACHE_PATH = cp if cp else appdirs.user_cache_dir(_app_name)
SETTINGS_PATH = sp if sp else appdirs.user_config_dir(_app_name)

if not os.path.isdir(CACHE_PATH):
    try:
        os.makedirs(CACHE_PATH, mode=0o0700)  # private data
    except OSError:
        logger.critical('Error creating cache directory "%s"' % CACHE_PATH)
        sys.exit(1)

# return values

ERROR_RETVAL = 1
INVALID_ARG_RETVAL = 2  # doubles as flag
INIT_FAILED_RETVAL = 3
KEYB_INTERR_RETVAL = 4

# additional retval flags
UL_DL_FAILED = 8
UL_TIMEOUT = 16
HASH_MISMATCH = 32
ERR_CR_FOLDER = 64
SIZE_MISMATCH = 128
CACHE_ASYNC = 256
DUPLICATE = 512
DUPLICATE_DIR = 1024
NAME_COLLISION = 2048


def signal_handler(signal_, frame):
    # if db.Session:
    #     db.Session.rollback()
    sys.exit(KEYB_INTERR_RETVAL)


signal.signal(signal.SIGINT, signal_handler)
if hasattr(signal, 'SIGPIPE'):
    signal.signal(signal.SIGPIPE, signal_handler)


def pprint(d: dict):
    print(json.dumps(d, indent=4, sort_keys=True))


#
# Glue functions (API, cache)
#


class CacheConsts(object):
    CHECKPOINT_KEY = 'checkpoint'
    LAST_SYNC_KEY = 'last_sync'
    MAX_AGE = 30


def sync_node_list(full=False):
    cp = db.KeyValueStorage.get(CacheConsts.CHECKPOINT_KEY) if not full else None

    try:
        nodes, purged, ncp, full = metadata.get_changes(checkpoint=cp, include_purged=not not cp)
    except RequestError as e:
        logger.critical('Sync failed.')
        print(e)
        return ERROR_RETVAL

    if full:
        db.drop_all()
        db.init(CACHE_PATH)
    else:
        sync.remove_purged(purged)

    if len(nodes) > 0:
        sync.insert_nodes(nodes, partial=not full)
    db.KeyValueStorage.update({CacheConsts.LAST_SYNC_KEY: time.time()})

    if len(nodes) > 0 or len(purged) > 0:
        db.KeyValueStorage.update({CacheConsts.CHECKPOINT_KEY: ncp})


def old_sync():
    db.drop_all()
    db.init(CACHE_PATH)
    try:
        folders = metadata.get_folder_list()
        folders.extend(metadata.get_trashed_folders())
        files = metadata.get_file_list()
        files.extend(metadata.get_trashed_files())
    except RequestError as e:
        logger.critical('Sync failed.')
        print(e)
        return ERROR_RETVAL

    sync.insert_nodes(files + folders, partial=False)
    db.KeyValueStorage['sync_date'] = time.time()


#
# File transfer
#

RetryRetVal = namedtuple('RetryRetVal', ['ret_val', 'retry'])
STD_RETRY_RETVALS = [UL_DL_FAILED]


def retry_on(ret_vals: list):
    """Retry decorator that sets the wrapped function's progress handler argument according to its
    return value and wraps the return value in RetryRetVal object.
    :param ret_vals: list of retry values on which execution should be repeated"""

    def wrap(f):
        def wrapped(*args, **kwargs) -> RetryRetVal:
            ret_val = ERROR_RETVAL
            try:
                ret_val = f(*args, **kwargs)
            except Exception:
                import traceback
                logger.error(traceback.format_exc())
            h = kwargs.get('pg_handler')
            h.status = ret_val
            retry = ret_val in ret_vals
            if retry:
                h.reset()
            return RetryRetVal(ret_val, ret_val in ret_vals)

        return wrapped

    return wrap


def compare_hashes(hash1: str, hash2: str, file_name: str) -> int:
    if hash1 != hash2:
        logger.error('Hash mismatch between local and remote file for "%s".' % file_name)
        return HASH_MISMATCH

    logger.debug('Local and remote hashes match for "%s".' % file_name)
    return 0


def create_upload_jobs(dirs: list, path: str, parent_id: str, overwr: bool, force: bool,
                       dedup: bool, exclude: list, exclude_paths: list, jobs: list) -> int:
    """Creates upload job if passed path is a file, delegates directory traversal otherwise.
    Detects soft links that link to an already queued directory.
    :param dirs: List of directories' inodes traversed so far"""

    if os.path.realpath(path) in [os.path.realpath(p) for p in exclude_paths]:
        logger.info('Skipping upload of path "%s".' % path)
        return 0

    if not os.access(path, os.R_OK):
        logger.error('Path "%s" is not accessible.' % path)
        return INVALID_ARG_RETVAL

    if os.path.isdir(path):
        ino = os.stat(path).st_ino
        if ino in dirs:
            logger.warning('Duplicate directory detected: "%s".' % path)
            return DUPLICATE_DIR
        dirs.append(ino)
        return traverse_ul_dir(dirs, path, parent_id, overwr, force, dedup,
                               exclude, exclude_paths, jobs)
    elif os.path.isfile(path):
        short_nm = os.path.basename(path)
        for reg in exclude:
            if re.match(reg, short_nm):
                logger.info('Skipping upload of "%s" because of exclusion pattern.' % short_nm)
                return 0

        prog = progress.FileProgress(os.path.getsize(path))
        fo = partial(upload_file, path, parent_id, overwr, force, dedup, pg_handler=prog)
        jobs.append(fo)
        return 0


def traverse_ul_dir(dirs: list, directory: str, parent_id: str, overwr: bool, force: bool,
                    dedup: bool, exclude: list, exclude_paths: list, jobs: list) -> int:
    """Duplicates local directory structure."""

    if parent_id is None:
        parent_id = query.get_root_id()
    parent = query.get_node(parent_id)

    real_path = os.path.realpath(directory)
    short_nm = os.path.basename(real_path)

    curr_node = parent.get_child(short_nm)
    if not curr_node or not curr_node.is_available() or not parent.is_available():
        try:
            r = content.create_folder(short_nm, parent_id)
            logger.info('Created folder "%s"' % (parent.full_path() + short_nm))
            sync.insert_node(r)
            curr_node = query.get_node(r['id'])
        except RequestError as e:
            if e.status_code == 409:
                logger.error('Folder "%s" already exists. Please sync.' % short_nm)
            else:
                logger.error('Error creating remote folder "%s": %s.' % (short_nm, e))
            return ERR_CR_FOLDER
    elif curr_node.is_file():
        logger.error('Cannot create remote folder "%s", '
                     'because a file of the same name already exists.' % short_nm)
        return ERR_CR_FOLDER

    try:
        entries = sorted(os.listdir(directory))
    except OSError as e:
        logger.error('Skipping directory %s because of an error.' % directory)
        logger.info(e)
        return ERROR_RETVAL

    ret_val = 0
    for entry in entries:
        full_path = os.path.join(real_path, entry)
        ret_val |= create_upload_jobs(dirs, full_path, curr_node.id,
                                      overwr, force, dedup, exclude, exclude_paths, jobs)

    return ret_val


@retry_on(STD_RETRY_RETVALS)
def upload_file(path: str, parent_id: str, overwr: bool, force: bool, dedup: bool,
                pg_handler: progress.FileProgress=None) -> RetryRetVal:
    short_nm = os.path.basename(path)

    if dedup and query.file_size_exists(os.path.getsize(path)):
        nodes = query.find_md5(hashing.hash_file(path))
        nodes = [n for n in format.PathFormatter(nodes)]
        if len(nodes) > 0:
            # print('Skipping upload of duplicate file "%s".' % short_nm)
            logger.info('Location of duplicates: %s' % nodes)
            pg_handler.done()
            return DUPLICATE

    conflicting_node = query.conflicting_node(short_nm, parent_id)
    file_id = None
    if conflicting_node:
        if conflicting_node.is_folder():
            logger.error('Name collision with existing folder '
                         'in the same location: "%".' % short_nm)
            return NAME_COLLISION

        file_id = conflicting_node.id

    if not file_id:
        logger.info('Uploading %s' % path)
        hasher = hashing.IncrementalHasher()
        try:
            r = content.upload_file(path, parent_id,
                                    read_callbacks=[hasher.update, pg_handler.update],
                                    deduplication=dedup)
        except RequestError as e:
            if e.status_code == 409:  # might happen if cache is outdated
                if not dedup:
                    logger.error('Uploading "%s" failed. Name collision with non-cached file. '
                                 'If you want to overwrite, please sync and try again.' % short_nm)
                else:
                    logger.error(
                        'Uploading "%s" failed. '
                        'Name or hash collision with non-cached file.' % short_nm)
                    logger.info(e)
                # colliding node ID is returned in error message -> could be used to continue
                return CACHE_ASYNC
            elif e.status_code == 504 or e.status_code == 408:  # proxy timeout / request timeout
                logger.warning('Timeout while uploading "%s".' % short_nm)
                # TODO: wait; request parent folder's children
                return UL_TIMEOUT
            else:
                logger.error(
                    'Uploading "%s" failed. Code: %s, msg: %s' % (short_nm, e.status_code, e.msg))
                return UL_DL_FAILED
        else:
            sync.insert_node(r)
            file_id = r['id']
            md5 = query.get_node(file_id).md5
            return compare_hashes(hasher.get_result(), md5, short_nm)

    # else: file exists
    if not overwr and not force:
        logger.info('Skipping upload of existing file "%s".' % short_nm)
        pg_handler.done()
        return 0

    rmod = (conflicting_node.modified - datetime(1970, 1, 1)) / timedelta(seconds=1)
    rmod = datetime.utcfromtimestamp(rmod)
    lmod = datetime.utcfromtimestamp(os.path.getmtime(path))
    lcre = datetime.utcfromtimestamp(os.path.getctime(path))

    logger.debug('Remote mtime: %s, local mtime: %s, local ctime: %s' % (rmod, lmod, lcre))

    # ctime is checked because files can be overwritten by files with older mtime
    if rmod < lmod or (rmod < lcre and conflicting_node.size != os.path.getsize(path)) \
            or force:
        return overwrite(file_id, path, dedup=dedup, pg_handler=pg_handler).ret_val
    elif not force:
        logger.info('Skipping upload of "%s" because of mtime or ctime and size.' % short_nm)
        pg_handler.done()
        return 0


@retry_on(STD_RETRY_RETVALS)
def overwrite(node_id, local_file, dedup=False,
              pg_handler: progress.FileProgress=None) -> RetryRetVal:
    hasher = hashing.IncrementalHasher()
    try:
        r = content.overwrite_file(node_id, local_file,
                                   read_callbacks=[hasher.update, pg_handler.update],
                                   deduplication=dedup)
        sync.insert_node(r)
        node = query.get_node(r['id'])
        md5 = node.md5

        return compare_hashes(md5, hasher.get_result(), local_file)
    except RequestError as e:
        logger.error('Error overwriting file. Code: %s, msg: %s' % (e.status_code, e.msg))
        return UL_DL_FAILED


@retry_on([])
def upload_stream(stream, file_name, parent_id, dedup=False,
                  pg_handler: progress.FileProgress=None) -> RetryRetVal:
    hasher = hashing.IncrementalHasher()
    try:
        r = content.upload_stream(stream, file_name, parent_id,
                                  read_callbacks=[hasher.update, pg_handler.update],
                                  deduplication=dedup)
        sync.insert_node(r)
        node = query.get_node(r['id'])
        return compare_hashes(node.md5, hasher.get_result(), 'stream')
    except RequestError as e:
        logger.error('Error uploading stream. Code: %s, msg: %s' % (e.status_code, e.msg))
        return UL_DL_FAILED


def create_dl_jobs(node_id: str, local_path: str, exclude: list, jobs: list) -> int:
    """Populates passed jobs list with download partials."""
    local_path = local_path if local_path else ''

    node = query.get_node(node_id)
    if not node.is_available():
        return 0

    if node.is_folder():
        return traverse_dl_folder(node_id, local_path, exclude, jobs)

    loc_name = node.name

    for reg in exclude:
        if re.match(reg, loc_name):
            logger.info('Skipping download of "%s" because of exclusion pattern.' % loc_name)
            return 0

    flp = os.path.join(local_path, loc_name)
    if os.path.isfile(flp):
        logger.info('Skipping download of existing file "%s"' % loc_name)
        if os.path.getsize(flp) != node.size:
            logger.info('Skipped file "%s" has different size than local file.' % loc_name)
            return SIZE_MISMATCH
        return 0

    prog = progress.FileProgress(node.size)
    fo = partial(download_file, node_id, local_path, pg_handler=prog)
    jobs.append(fo)

    return 0


def traverse_dl_folder(node_id: str, local_path: str, exclude: list, jobs: list) -> int:
    """Duplicates remote folder structure."""

    if not local_path:
        local_path = os.getcwd()

    node = query.get_node(node_id)

    if node.name is None:
        curr_path = os.path.join(local_path, 'acd')
    else:
        curr_path = os.path.join(local_path, node.name)

    try:
        os.makedirs(curr_path, exist_ok=True)
    except OSError:
        logger.error('Error creating directory "%s".' % curr_path)
        return ERR_CR_FOLDER

    ret_val = 0
    children = sorted(node.children)
    for child in children:
        ret_val |= create_dl_jobs(child.id, curr_path, exclude, jobs)
    return ret_val


@retry_on(STD_RETRY_RETVALS)
def download_file(node_id: str, local_path: str,
                  pg_handler: progress.FileProgress=None) -> RetryRetVal:
    node = query.get_node(node_id)
    name, md5, size = node.name, node.md5, node.size
    # db.Session.remove()  # otherwise, sqlalchemy will complain if thread crashes

    logger.info('Downloading "%s"' % name)

    hasher = hashing.IncrementalHasher()
    try:
        content.download_file(node_id, name, local_path, length=size,
                              write_callbacks=[hasher.update, pg_handler.update])
    except RequestError as e:
        logger.error('Downloading "%s" failed. Code: %s, msg: %s' % (name, e.status_code, e.msg))
        return UL_DL_FAILED
    else:
        return compare_hashes(hasher.get_result(), md5, name)


#
# Subparser actions. Return value [typeof(None), int] will be used as sys exit status.
#

# decorators

nocache_actions = []
offline_actions = []
no_autores_trash_actions = []


def nocache_action(func):
    """Decorator for actions that do not need to read from cache or autoresolve."""
    nocache_actions.append(func)
    return func


def offline_action(func):
    """Decorator for actions that can be performed without API calls."""
    offline_actions.append(func)
    return func


def no_autores_trash_action(func):
    """Decorator for actions that should not have trash paths auto-resolved."""
    no_autores_trash_actions.append(func)
    return func


# actual actions

def sync_action(args: argparse.Namespace):
    print('Syncing...')
    r = sync_node_list(full=args.full)
    print('Done.')
    return r


def old_sync_action(args: argparse.Namespace):
    print('Syncing...')
    r = old_sync()
    print('Done.')
    return r


@nocache_action
@offline_action
def delete_everything_action(args: argparse.Namespace):
    from distutils.util import strtobool
    from shutil import rmtree

    a = input('Do you really want to delete %s? [y/n] ' % CACHE_PATH)
    try:
        if strtobool(a):
            rmtree(CACHE_PATH)
    except ValueError:
        pass
    except OSError as e:
        print(e)
        print('Deleting directory failed.')


@nocache_action
@offline_action
def clear_action(args: argparse.Namespace):
    if not db.remove_db_file(CACHE_PATH):
        return ERROR_RETVAL


@nocache_action
@offline_action
def print_version_action(args: argparse.Namespace):
    print(' '.join([_app_name, __version__]))


@offline_action
def tree_action(args: argparse.Namespace):
    tree = query.tree(args.node, args.include_trash)
    if tree is None:
        logger.critical('Invalid folder.')
        return INVALID_ARG_RETVAL

    tree = format.TreeFormatter(tree)
    for node in tree:
        print(node)


@nocache_action
def usage_action(args: argparse.Namespace):
    r = account.get_account_usage()
    print(r, end='')


@nocache_action
def quota_action(args: argparse.Namespace):
    r = account.get_quota()
    pprint(r)


def regex_helper(args: argparse.Namespace) -> list:
    """Pre-compiles regexes from strings in args namespace."""
    excl_re = []
    for re_ in args.exclude_re:
        try:
            excl_re.append(re.compile(re_, flags=re.IGNORECASE))
        except re.error as e:
            logger.critical('Invalid regular expression: %s. %s' % (re_, e))
            sys.exit(INVALID_ARG_RETVAL)

    for ending in args.exclude_fe:
        excl_re.append(re.compile('^.*\.' + re.escape(ending) + '$', flags=re.IGNORECASE))

    return excl_re


@no_autores_trash_action
def upload_action(args: argparse.Namespace) -> int:
    if not query.get_node(args.parent):
        logger.critical('Invalid upload folder.')
        return INVALID_ARG_RETVAL

    excl_re = regex_helper(args)

    jobs = []
    ret_val = 0
    for path in args.path:
        if not os.path.exists(path):
            logger.error('Path "%s" does not exist.' % path)
            ret_val |= INVALID_ARG_RETVAL
            continue

        ret_val |= create_upload_jobs([], path, args.parent, args.overwrite, args.force,
                                      args.deduplicate, excl_re, args.exclude_path, jobs)

    ql = QueuedLoader(args.max_connections, max_retries=args.max_retries)
    ql.add_jobs(jobs)

    return ret_val | ql.start()


@no_autores_trash_action
def upload_stream_action(args: argparse.Namespace) -> int:
    if not query.get_node(args.parent):
        logger.critical('Invalid upload folder')
        return INVALID_ARG_RETVAL

    prog = progress.FileProgress(0)
    ql = QueuedLoader(max_retries=0)
    job = partial(upload_stream,
                  sys.stdin.buffer, args.name, args.parent, args.deduplicate, pg_handler=prog)
    ql.add_jobs([job])

    return ql.start()


def overwrite_action(args: argparse.Namespace) -> int:
    if not os.path.isfile(args.file):
        logger.error('Invalid file.')
        return INVALID_ARG_RETVAL

    prog = progress.FileProgress(os.path.getsize(args.file))
    ql = QueuedLoader(max_retries=args.max_retries)
    job = partial(overwrite, args.node, args.file, pg_handler=prog)
    ql.add_jobs([job])

    return ql.start()


def download_action(args: argparse.Namespace) -> int:
    excl_re = regex_helper(args)

    jobs = []
    ret_val = 0
    ret_val |= create_dl_jobs(args.node, args.path, excl_re, jobs)

    ql = QueuedLoader(args.max_connections, max_retries=args.max_retries)
    ql.add_jobs(jobs)

    return ret_val | ql.start()


def cat_action(args: argparse.Namespace) -> int:
    if not query.get_node(args.node):
        return INVALID_ARG_RETVAL

    try:
        content.chunked_download(args.node, sys.stdout.buffer)
    except RequestError as e:
        logger.error('Downloading failed. Code: %s, msg: %s' % (e.status_code, e.msg))
        return UL_DL_FAILED

    return 0


def create_action(args: argparse.Namespace) -> int:
    parent, folder = os.path.split(args.new_folder)
    # no trailing slash
    if not folder:
        parent, folder = os.path.split(parent)

    if not folder:
        logger.error('Cannot create folder with empty name.')
        return INVALID_ARG_RETVAL

    if parent[-1:] == '' or parent[0] != '/':
        parent = '/' + parent
    p_id = query.resolve_path(parent)
    if not p_id:
        logger.error('Invalid parent path "%s".' % parent)
        return INVALID_ARG_RETVAL

    try:
        r = content.create_folder(folder, p_id)
    except RequestError as e:
        logger.debug(str(e.status_code) + e.msg)
        if e.status_code == 409:
            logger.warning('Folder "%s" already exists.' % folder)
        else:
            logger.error('Error creating folder "%s".' % folder)
            return ERR_CR_FOLDER
    else:
        sync.insert_node(r)


@no_autores_trash_action
@offline_action
def list_trash_action(args: argparse.Namespace):
    t_list = query.list_trash(args.recursive)
    for node in format.ListFormatter(t_list, recursive=args.recursive):
        print(node)


def trash_action(args: argparse.Namespace) -> int:
    try:
        r = trash.move_to_trash(args.node)
        sync.insert_node(r)
    except RequestError as e:
        print(e)
        return ERROR_RETVAL


def restore_action(args: argparse.Namespace) -> int:
    try:
        r = trash.restore(args.node)
    except RequestError as e:
        logger.error('Error restoring "%s": %s' % (args.node, e))
        return ERROR_RETVAL
    sync.insert_node(r)


@offline_action
def resolve_action(args: argparse.Namespace) -> int:
    node = query.resolve_path(args.path)
    if node:
        print(node)
    else:
        return INVALID_ARG_RETVAL


@offline_action
def find_action(args: argparse.Namespace):
    r = query.find(args.name)
    r = format.LongIDFormatter(r)
    for node in r:
        print(node)
    if not r:
        return INVALID_ARG_RETVAL


@offline_action
def find_md5_action(args: argparse.Namespace) -> int:
    if len(args.md5) != 32:
        logger.critical('Invalid MD5 specified')
        return INVALID_ARG_RETVAL
    nodes = query.find_md5(args.md5)
    for node in format.LongIDFormatter(nodes):
        print(node)


@offline_action
def find_regex_action(args: argparse.Namespace) -> int:
    try:
        re.compile(args.regex)
    except re.error as e:
        logger.critical('Invalid regular expression specified')
        return INVALID_ARG_RETVAL
    r = query.find_regex(args.regex)
    r = format.LongIDFormatter(r)
    for node in r:
        print(node)
    return 0


@offline_action
def children_action(args: argparse.Namespace) -> int:
    nodes = query.list_children(args.node, args.recursive, args.include_trash)
    for entry in format.ListFormatter(nodes, recursive=args.recursive, long=args.long):
        print(entry)


def move_action(args: argparse.Namespace) -> int:
    node = query.get_node(args.child)
    if not node:
        return INVALID_ARG_RETVAL
    if len(node.parents) > 1:
        logger.error('Cannot move node with multiple parents.')
        return ERROR_RETVAL

    try:
        r = metadata.move_node(args.child, node.parents[0].id, args.parent)
        sync.insert_node(r)
    except RequestError as e:
        print(e)
        return ERROR_RETVAL


def rename_action(args: argparse.Namespace) -> int:
    try:
        r = metadata.rename_node(args.node, args.name)
        sync.insert_node(r)
    except RequestError as e:
        print(e)
        return ERROR_RETVAL


def add_child_action(args: argparse.Namespace) -> int:
    try:
        r = metadata.add_child(args.parent, args.child)
        sync.insert_node(r)
    except RequestError as e:
        print(e)
        return ERROR_RETVAL


def remove_child_action(args: argparse.Namespace) -> int:
    try:
        r = metadata.remove_child(args.parent, args.child)
        sync.insert_node(r)
    except RequestError as e:
        print(e)
        return ERROR_RETVAL


def metadata_action(args: argparse.Namespace) -> int:
    try:
        r = metadata.get_metadata(args.node, args.assets)
        pprint(r)
    except RequestError as e:
        print(e)
        return INVALID_ARG_RETVAL


@offline_action
@nocache_action
def dump_sql_action(args: argparse.Namespace):
    db.dump_table_sql()


def mount_action(args: argparse.Namespace):
    import acdcli.fuse
    acdcli.fuse.mount(args.path, dict(nlinks=args.nlinks, interval=args.interval),
                      ro=args.ro, foreground=args.foreground, nothreads=args.single_threaded,
                      nonempty=args.nonempty,
                      allow_root=args.allow_root, allow_other=args.allow_other)


@offline_action
@nocache_action
def unmount_action(args: argparse.Namespace):
    import acdcli.fuse
    return acdcli.fuse.unmount(args.path, args.lazy)


def init_action(args: argparse.Namespace):
    from importlib import reload


#
# helper methods
#


def resolve_remote_path_args(args: argparse.Namespace, attrs: list, incl_trash: bool=True):
    """In-place replaces certain attributes in Namespace by resolved node ID.
    :param attrs: list of attributes that may be given in absolute path form
    :param incl_trash: whether to resolve trashed files
    """
    for id_attr in attrs:
        if hasattr(args, id_attr):
            val = getattr(args, id_attr)
            if not val:
                continue
            if '/' in val:
                v = query.resolve_path(val, trash=incl_trash)
                if not v:
                    logger.critical('Could not resolve path "%s".' % val)
                    sys.exit(INVALID_ARG_RETVAL)
                logger.info('Resolved "%s" to "%s"' % (val, v))
                setattr(args, id_attr, v)
                setattr(args, id_attr + '_path', val)
            elif not common.is_valid_id(val):
                logger.critical('Invalid ID format: "%s".' % val)
                sys.exit(INVALID_ARG_RETVAL)


def set_log_level(args: argparse.Namespace):
    format_ = '%(asctime)s.%(msecs).03d [%(levelname)s] [%(name)s] - %(message)s\x1b[K'
    time_fmt = '%y-%m-%d %H:%M:%S'

    if not args.verbose and not args.debug:
        logging.basicConfig(level=logging.WARNING, format=format_, datefmt=time_fmt)

    if args.verbose:
        logging.basicConfig(level=logging.INFO, format=format_, datefmt=time_fmt)
        if args.verbose > 1:
            logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
            logging.getLogger('sqlalchemy.orm').setLevel(logging.INFO)
    elif args.debug:
        logging.basicConfig(level=logging.DEBUG, format=format_, datefmt=time_fmt)

        # these debug messages (prints) will not show up in log file
        import http.client

        http.client.HTTPConnection.debuglevel = 1

        if args.debug > 1:
            logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
            logging.getLogger('sqlalchemy.orm').setLevel(logging.DEBUG)


def set_encoding(force_utf: bool=False):
    """Sets the default encoding to UTF-8 if none is set.
    :param force_utf: force UTF-8 output
    """
    enc = str.lower(sys.stdout.encoding)
    utf_flag = False

    if not enc or force_utf:
        import io

        sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8')
        sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding='utf-8')
        utf_flag = True

        logger.info('Stdout/stderr encoding changed to UTF-8.')

    return utf_flag


def check_cache():
    """Checks for existence of root node and cache age."""

    if not query.get_root_node():
        logger.critical('Root node not found. Please sync.')
        return False

    last_sync = db.KeyValueStorage.get(CacheConsts.LAST_SYNC_KEY)
    if not last_sync:
        return True
    last_sync = datetime.utcfromtimestamp(float(last_sync))
    age = (datetime.utcnow() - last_sync) / timedelta(days=1)
    if age > CacheConsts.MAX_AGE:
        logger.warning('Cache data may be outdated. Please sync.')
    else:
        logger.info('Last sync at %s.' % last_sync)
    return True


def check_py_version():
    if sys.version_info[:3] in [(3, 2, 3), (3, 3, 0), (3, 3, 1)]:
        logger.warning('Your Python version is known to cause issues. Uploading might not work.')


# noinspection PyProtectedMember
class Argument(object):
    """Simple argparse argument container"""

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

    def attach(self, subparser: argparse._ActionsContainer):
        subparser.add_argument(*self.args, **self.kwargs)


def get_parser() -> tuple:
    max_ret = Argument('--max-retries', '-r', action='store', type=int, default=0,
                       help='set the maximum number of retries [default: 0]')

    opt_parser = argparse.ArgumentParser(
        prog=_app_name, formatter_class=argparse.RawTextHelpFormatter,
        epilog='Hints: \n'
               '  * Remote locations may be specified as path in most cases, '
               'e.g. "/folder/file", or via ID \n'
               '  * If you need to enter a node ID that contains a leading dash (minus) sign, '
               'precede it by two dashes and a space, e.g. \'-- -xfH...\'\n'
               '  * actions marked with [+] have optional arguments'
               '')
    log_group = opt_parser.add_mutually_exclusive_group()
    log_group.add_argument('-v', '--verbose', action='count',
                           help='prints some info messages to stderr; '
                                 'use "-vv" to also get sqlalchemy info')
    log_group.add_argument('-d', '--debug', action='count',
                           help='prints info and debug to stderr; '
                                 'use "-dd" to also get sqlalchemy debug messages')
    opt_parser.add_argument('-c', '--color', default=format.ColorMode['never'],
                            choices=format.ColorMode.keys(),
                            help='"never" [default] turns coloring off, '
                                 '"always" turns coloring on '
                                 'and "auto" colors listings when stdout is a tty '
                                 '[uses the Linux-style LS_COLORS environment variable]')
    opt_parser.add_argument('-i', '--check', default=db.IntegrityCheckType['full'],
                            choices=db.IntegrityCheckType.keys(),
                            help='select database integrity check type [default: full]')
    opt_parser.add_argument('-u', '--utf', action='store_true',
                            help='force utf output')
    opt_parser.add_argument('-nw', '--no-wait', action='store_true', help=argparse.SUPPRESS)

    subparsers = opt_parser.add_subparsers(title='action', dest='action')
    subparsers.required = True

    vers_sp = subparsers.add_parser('version', aliases=['v'], help='print version and exit\n')
    vers_sp.set_defaults(func=print_version_action)

    sync_sp = subparsers.add_parser('sync', aliases=['s'],
                                    help='[+] refresh node list cache; necessary for many actions')
    sync_sp.add_argument('--full', '-f', action='store_true', help='force a full sync')
    sync_sp.set_defaults(func=sync_action)

    old_sync_sp = subparsers.add_parser('old-sync', add_help=False)
    old_sync_sp.set_defaults(func=old_sync_action)

    clear_sp = subparsers.add_parser('clear-cache', aliases=['cc'],
                                     help='clear node cache [offline operation]\n\n')
    clear_sp.set_defaults(func=clear_action)

    tree_sp = subparsers.add_parser('tree', aliases=['t'],
                                    help='[+] print directory tree [offline operation]')
    tree_sp.add_argument('--include-trash', '-t', action='store_true')
    tree_sp.add_argument('node', nargs='?', default=None, help='root folder for the tree')
    tree_sp.set_defaults(func=tree_action)

    list_c_sp = subparsers.add_parser('children', aliases=['ls', 'dir'],
                                      help='[+] list folder\'s children [offline operation]\n\n')
    list_c_sp.add_argument('--long', '-l', action='store_true', help='long listing format')
    list_c_sp.add_argument('--include-trash', '-t', action='store_true')
    list_c_sp.add_argument('--recursive', '-r', action='store_true')
    list_c_sp.add_argument('node', nargs='?', default='/',
                           help='folder to display contents of [optional]')
    list_c_sp.set_defaults(func=children_action)

    find_sp = subparsers.add_parser('find', aliases=['f'], help=
    'find nodes by name [offline operation] [case insensitive]')
    find_sp.add_argument('name')
    find_sp.set_defaults(func=find_action)

    find_hash_sp = subparsers.add_parser('find-md5', aliases=['fh'],
                                         help='find files by MD5 hash [offline operation]')
    find_hash_sp.add_argument('md5')
    find_hash_sp.set_defaults(func=find_md5_action)

    find_regex_sp = subparsers.add_parser('find-regex', aliases=['fr'],
                                          help='find nodes by regular expression '
                                               '[offline operation] [case insensitive]\n\n')
    find_regex_sp.add_argument('regex')
    find_regex_sp.set_defaults(func=find_regex_action)

    dummy_p = argparse.ArgumentParser().add_subparsers()
    re_dummy_sp = dummy_p.add_parser('', add_help=False)
    re_dummy_sp.add_argument('--max-connections', '-x', action='store', type=int, default=1,
                             help='set the maximum concurrent connections [default: 1, '
                                  'maximum: %i' % QueuedLoader.MAX_NUM_WORKERS + ']')
    max_ret.attach(re_dummy_sp)
    re_dummy_sp.add_argument('--exclude-ending', '-xe', action='append', dest='exclude_fe',
                             default=[], help='exclude files whose endings match the given string,'
                                              ' e.g. "bak" [case insensitive]')
    re_dummy_sp.add_argument('--exclude-regex', '-xr', action='append', dest='exclude_re',
                             default=[],
                             help='exclude files whose names match the given regular expression,'
                                  ' e.g. "^thumbs\.db$" [case insensitive]')

    upload_sp = subparsers.add_parser('upload', aliases=['ul'], parents=[re_dummy_sp],
                                      help='[+] file and directory upload to a remote destination')
    upload_sp.add_argument('--exclude-path', '-xp', action='append', dest='exclude_path',
                           default=[], help='exclude file or directory '
                                            'that match the given string')
    upload_sp.add_argument('--overwrite', '-o', action='store_true',
                           help='overwrite if local modification time is higher '
                                'or local ctime is higher than remote modification time '
                                'and local/remote file sizes do not match.')
    upload_sp.add_argument('--force', '-f', action='store_true', help='force overwrite')
    upload_sp.add_argument('--deduplicate', '-d', action='store_true',
                           help='exclude duplicate files from upload')
    upload_sp.add_argument('path', nargs='+', help='a path to a local file or directory')
    upload_sp.add_argument('parent', help='remote parent folder')
    upload_sp.set_defaults(func=upload_action)

    overwrite_sp = subparsers.add_parser('overwrite', aliases=['ov'], help=
    'overwrite file A [remote] with content of file B [local]')
    max_ret.attach(overwrite_sp)
    overwrite_sp.add_argument('node')
    overwrite_sp.add_argument('file')
    overwrite_sp.set_defaults(func=overwrite_action)

    stream_sp = subparsers.add_parser('stream', aliases=['st'],
                                      help='[+] upload the standard input stream to a file')
    stream_sp.add_argument('--deduplicate', '-d', action='store_true',
                           help='prevent duplicates from getting stored after upload')
    stream_sp.add_argument('name', help='the remote file name')
    stream_sp.add_argument('parent', help='remote parent folder')
    stream_sp.set_defaults(func=upload_stream_action)

    download_sp = subparsers.add_parser('download', aliases=['dl'], parents=[re_dummy_sp],
                                        help='download a remote folder or file; '
                                             'will skip existing local files')
    download_sp.add_argument('node')
    download_sp.add_argument('path', nargs='?', default=None, help='local download path [optional]')
    download_sp.set_defaults(func=download_action)

    cat_sp = subparsers.add_parser('cat', help='output a file to the standard output stream\n\n')
    cat_sp.add_argument('node')
    cat_sp.set_defaults(func=cat_action)

    cr_fo_sp = subparsers.add_parser('create', aliases=['c', 'mkdir'],
                                     help='create folder using an absolute path\n\n')
    cr_fo_sp.add_argument('new_folder', help='an absolute folder path, '
                                             'e.g. "/my/dir/"; trailing slash is optional')
    cr_fo_sp.set_defaults(func=create_action)

    trash_sp = subparsers.add_parser('list-trash', aliases=['lt'],
                                     help='[+] list trashed nodes [offline operation]')
    trash_sp.add_argument('--recursive', '-r', action='store_true')
    trash_sp.set_defaults(func=list_trash_action)

    m_trash_sp = subparsers.add_parser('trash', aliases=['rm'], help='move node to trash')
    m_trash_sp.add_argument('node')
    m_trash_sp.set_defaults(func=trash_action)

    rest_sp = subparsers.add_parser('restore', aliases=['re'], help='restore node from trash\n\n')
    rest_sp.add_argument('node', help='ID of the node')
    rest_sp.set_defaults(func=restore_action)

    move_sp = subparsers.add_parser('move', aliases=['mv'], help='move node A into folder B')
    move_sp.add_argument('child')
    move_sp.add_argument('parent')
    move_sp.set_defaults(func=move_action)

    rename_sp = subparsers.add_parser('rename', aliases=['rn'], help='rename a node\n\n')
    rename_sp.add_argument('node')
    rename_sp.add_argument('name')
    rename_sp.set_defaults(func=rename_action)

    res_sp = subparsers.add_parser('resolve', aliases=['rs'],
                                   help='resolve a path to a node ID [offline operation]\n\n')
    res_sp.add_argument('path')
    res_sp.set_defaults(func=resolve_action)

    # maybe the child operations should not be exposed
    # they can be used for creating hardlinks
    add_c_sp = subparsers.add_parser('add-child', aliases=['ac'],
                                     help='add a node to a parent folder')
    add_c_sp.add_argument('parent')
    add_c_sp.add_argument('child')
    add_c_sp.set_defaults(func=add_child_action)

    rem_c_sp = subparsers.add_parser('remove-child', aliases=['rc'],
                                     help='remove a node from a parent folder\n\n')
    rem_c_sp.add_argument('parent')
    rem_c_sp.add_argument('child')
    rem_c_sp.set_defaults(func=remove_child_action)

    usage_sp = subparsers.add_parser('usage', aliases=['u'], help='show drive usage data')
    usage_sp.set_defaults(func=usage_action)

    quota_sp = subparsers.add_parser('quota', aliases=['q'], help='show drive quota [raw JSON]')
    quota_sp.set_defaults(func=quota_action)

    meta_sp = subparsers.add_parser('metadata', aliases=['m'],
                                    help='print a node\'s metadata [raw JSON]\n\n')
    meta_sp.add_argument('--assets', '-a', action='store_true')
    meta_sp.add_argument('node')
    meta_sp.set_defaults(func=metadata_action)

    fuse_sp = subparsers.add_parser('mount', help='[+] mount the cloud drive at a local directory')
    fuse_sp.add_argument('--ro', '-ro', action='store_true', help='mount read-only')
    fuse_sp.add_argument('--foreground', '-fg', action='store_true', help='do not detach')
    fuse_sp.add_argument('--single-threaded', '-st', action='store_true')
    fuse_sp.add_argument('--nonempty', '-ne', action='store_true',
                         help='allow mounting over a non-empty directory')
    fuse_sp.add_argument('--allow-root', '-ar', action='store_true',
                         help='allow access to root user')
    fuse_sp.add_argument('--allow-other', '-ao', action='store_true',
                         help='allow access to other users')
    fuse_sp.add_argument('--nlinks', '-n', action='store_true', help='calculate nlinks')
    fuse_sp.add_argument('--interval', '-i', type=int, default=60,
                         help='sync every x seconds [default: 60, off: 0]')
    fuse_sp.add_argument('path')
    fuse_sp.set_defaults(func=mount_action)

    umount_sp = subparsers.add_parser('umount', help='[+] unmount cloud drive(s)')
    umount_sp.add_argument('--lazy', '-l', '-z', action='store_true')
    umount_sp.add_argument('path', nargs='?', default=None, help='local path to unmount [optional]')
    umount_sp.set_defaults(func=unmount_action)

    # undocumented actions

    de_sp = subparsers.add_parser('delete-everything', add_help=False)
    de_sp.set_defaults(func=delete_everything_action)

    # useful for interactive mode
    dn_sp = subparsers.add_parser('init', aliases=['i'], add_help=False)
    dn_sp.set_defaults(func=init_action)

    # dump sql database creation sequence to stdout
    dmp_sp = subparsers.add_parser('dumpsql', add_help=False)
    dmp_sp.set_defaults(func=dump_sql_action)

    return opt_parser, subparsers


def main():
    opt_parser, subparsers = get_parser()

    # plugins

    plugin_log = [str(plugins.Plugin)]
    for plugin in plugins.Plugin:
        if plugin.check_version(__version__):
            log = []
            plugin.attach(subparsers, log)
            plugin_log.extend(log)
        else:
            plugin_log.append('Script version is not compatible with "%s".' % plugin)

    args = opt_parser.parse_args()

    set_log_level(args)
    for msg in plugin_log:
        logger.info(msg)

    set_encoding(force_utf=args.utf)
    check_py_version()

    if args.func not in offline_actions:
        if not common.init(CACHE_PATH):
            sys.exit(INIT_FAILED_RETVAL)

    if args.func not in nocache_actions:
        if not db.init(CACHE_PATH, args.check):
            sys.exit(INIT_FAILED_RETVAL)
        if args.func != sync_action:
            if not check_cache():
                sys.exit(INIT_FAILED_RETVAL)

    format.init(args.color)

    if args.no_wait:
        common.BackOffRequest._wait = lambda: None

    autoresolve_attrs = ['child', 'parent', 'node']
    resolve_remote_path_args(args, autoresolve_attrs,
                             incl_trash=args.action not in no_autores_trash_actions)

    # call appropriate sub-parser action
    if args.func:
        sys.exit(args.func(args))


if __name__ == "__main__":
    main()