Example #1
0
def processArgs():
    parser = argparse.ArgumentParser(description='Set a token/password')
    parser.add_argument('--database', '-D', dest='database', default=Defaults.getDefault('TARDIS_DB'),      help="Database to use.  Default: %(default)s")
    parser.add_argument('--client', '-C',   dest='client',   default=Defaults.getDefault('TARDIS_CLIENT'),  help="Client to list on.  Default: %(default)s")
    parser.add_argument('--dbname',         dest='dbname',   default=Defaults.getDefault('TARDIS_DBNAME'),  help="Name of the database file. Default: %(default)s")

    passgroup= parser.add_argument_group("Password/Encryption specification options")
    pwgroup = passgroup.add_mutually_exclusive_group()
    pwgroup.add_argument('--password', '-p',dest='password', default=None, nargs='?', const=True,       help='Encrypt files with this password')
    pwgroup.add_argument('--password-file', dest='passwordfile', default=None,                          help='Read password from file')
    pwgroup.add_argument('--password-url',  dest='passwordurl', default=None,                           help='Retrieve password from the specified URL')
    pwgroup.add_argument('--password-prog', dest='passwordprog', default=None,                          help='Use the specified command to generate the password on stdout')

    return parser.parse_args()
Example #2
0
def processArgs():
    parser = argparse.ArgumentParser(description='Set a token/password')
    parser.add_argument('--database', '-D', dest='database', default=Defaults.getDefault('TARDIS_DB'),      help="Database to use.  Default: %(default)s")
    parser.add_argument('--client', '-C',   dest='client',   default=Defaults.getDefault('TARDIS_CLIENT'),  help="Client to list on.  Default: %(default)s")
    parser.add_argument('--dbname',         dest='dbname',   default=Defaults.getDefault('TARDIS_DBNAME'),  help="Name of the database file. Default: %(default)s")

    passgroup= parser.add_argument_group("Password/Encryption specification options")
    pwgroup = passgroup.add_mutually_exclusive_group()
    pwgroup.add_argument('--password', '-p',dest='password', default=None, nargs='?', const=True,       help='Encrypt files with this password')
    pwgroup.add_argument('--password-file', dest='passwordfile', default=None,                          help='Read password from file')
    pwgroup.add_argument('--password-url',  dest='passwordurl', default=None,                           help='Retrieve password from the specified URL')
    pwgroup.add_argument('--password-prog', dest='passwordprog', default=None,                          help='Use the specified command to generate the password on stdout')

    return parser.parse_args()
Example #3
0
def getDB(crypt, password, new=False, allowRemote=True, allowUpgrade=False):
    loc = urllib.parse.urlparse(args.database)
    # This is basically the same code as in Util.setupDataConnection().  Should consider moving to it.
    if (loc.scheme == 'http') or (loc.scheme == 'https'):
        if not allowRemote:
            raise Exception("This command cannot be executed remotely.  You must execute it on the server directly.")
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
            dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        else:
            dbLoc = args.database
        tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
        cache = tardisdb
    else:
        basedir = os.path.join(args.database, args.client)
        if not args.dbdir:
            dbdir = os.path.join(args.database, args.client)
        else:
            dbdir = os.path.join(args.dbdir, args.client)
        dbfile = os.path.join(dbdir, args.dbname)
        if new and os.path.exists(dbfile):
            raise Exception("Database for client %s already exists." % (args.client))

        cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
        schema = args.schema if new else None
        tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, allow_upgrade=allowUpgrade)

    if tardisdb.needsAuthentication():
        if password is None:
            password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=False, confirm=False)
            crypt = TardisCrypto.TardisCrypto(password, args.client)
        Util.authenticate(tardisdb, args.client, password)

    return (tardisdb, cache, crypt)
Example #4
0
def getDB(crypt, password, new=False, allowRemote=True, allowUpgrade=False):
    loc = urlparse.urlparse(args.database)
    # This is basically the same code as in Util.setupDataConnection().  Should consider moving to it.
    if (loc.scheme == 'http') or (loc.scheme == 'https'):
        if not allowRemote:
            raise Exception("This command cannot be executed remotely.  You must execute it on the server directly.")
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
            dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        else:
            dbLoc = args.database
        tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
        cache = tardisdb
    else:
        basedir = os.path.join(args.database, args.client)
        if not args.dbdir:
            dbdir = os.path.join(args.database, args.client)
        else:
            dbdir = os.path.join(args.dbdir, args.client)
        dbfile = os.path.join(dbdir, args.dbname)
        if new and os.path.exists(dbfile):
            raise Exception("Database for client %s already exists." % (args.client))

        cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
        schema = args.schema if new else None
        tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, allow_upgrade=allowUpgrade)

    if tardisdb.needsAuthentication():
        if password is None:
            password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=False, confirm=False)
        Util.authenticate(tardisdb, args.client, password)

    return (tardisdb, cache)
Example #5
0
def getBackupSet(db, bset):
    bsetInfo = None
    # First, try as an integer
    try:
        bset = int(bset)
        bsetInfo = db.getBackupSetInfoById(bset)
    except ValueError:
        # Else, let's look it up based on name
        if bset == Defaults.getDefault('TARDIS_RECENT_SET') or bset == '' or bset == None:
            bsetInfo = db.lastBackupSet()
        else:
            bsetInfo = db.getBackupSetInfo(bset)
        if not bsetInfo:
            # still nothing, hm, let's try a date format
            cal = parsedatetime.Calendar()
            (then, success) = cal.parse(bset)
            if success:
                timestamp = time.mktime(then)
                logger.debug("Using time: %s", time.asctime(then))
                bsetInfo = db.getBackupSetInfoForTime(timestamp)
                if bsetInfo and bsetInfo['backupset'] != 1:
                    bset = bsetInfo['backupset']
                    logger.debug("Using backupset: %s %d for %s", bsetInfo['name'], bsetInfo['backupset'], bset)
                else:
                    # Weed out the ".Initial" set
                    logger.critical("No backupset at date: %s (%s)", bset, time.asctime(then))
                    bsetInfo = None
            else:
                logger.critical("Could not parse string: %s", bset)
    return bsetInfo
Example #6
0
def getBackupSet(db, bset):
    bsetInfo = None
    # First, try as an integer
    try:
        bset = int(bset)
        bsetInfo = db.getBackupSetInfoById(bset)
    except ValueError:
        # Else, let's look it up based on name
        if bset == Defaults.getDefault('TARDIS_RECENT_SET') or bset == '' or bset == None:
            bsetInfo = db.lastBackupSet()
        else:
            bsetInfo = db.getBackupSetInfo(bset)
        if not bsetInfo:
            # still nothing, hm, let's try a date format
            cal = parsedatetime.Calendar()
            (then, success) = cal.parse(bset)
            if success:
                timestamp = time.mktime(then)
                logger.debug("Using time: %s", time.asctime(then))
                bsetInfo = db.getBackupSetInfoForTime(timestamp)
                if bsetInfo and bsetInfo['backupset'] != 1:
                    bset = bsetInfo['backupset']
                    logger.debug("Using backupset: %s %d for %s", bsetInfo['name'], bsetInfo['backupset'], bset)
                else:
                    # Weed out the ".Initial" set
                    logger.critical("No backupset at date: %s (%s)", bset, time.asctime(then))
                    bsetInfo = None
            else:
                logger.critical("Could not parse string: %s", bset)
    return bsetInfo
Example #7
0
def getDB(crypt, new=False, allowRemote=True):
    token = crypt.createToken() if crypt else None
    loc = urlparse.urlparse(args.database)
    # This is basically the same code as in Util.setupDataConnection().  Should consider moving to it.
    if (loc.scheme == "http") or (loc.scheme == "https"):
        if not allowRemote:
            raise Exception("This command cannot be executed remotely.  You must execute it on the server directly.")
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault("TARDIS_REMOTE_PORT")
            dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        else:
            dbLoc = args.database
        tardisdb = RemoteDB.RemoteDB(dbLoc, args.client, token=token)
        cache = tardisdb
    else:
        basedir = os.path.join(args.database, args.client)
        if not args.dbdir:
            dbdir = os.path.join(args.database, args.client)
        else:
            dbdir = os.path.join(args.dbdir, args.client)
        dbfile = os.path.join(dbdir, args.dbname)
        if new and os.path.exists(dbfile):
            raise Exception("Database for client %s already exists." % (args.client))

        cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
        schema = args.schema if new else None
        tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, token=token)

    return (tardisdb, cache)
Example #8
0
def main():
    global args, logger
    tardis = None
    try:
        args = processArgs()
        logger = Util.setupLogging(args.verbose)

        setColors(Defaults.getDefault('TARDIS_LS_COLORS'))

        # Load any password info
        password = Util.getPassword(args.password,
                                    args.passwordfile,
                                    args.passwordprog,
                                    prompt="Password for %s: " % (args.client))
        args.password = None

        (tardis, _, crypt) = Util.setupDataConnection(args.database,
                                                      args.client, password,
                                                      args.keys, args.dbname,
                                                      args.dbdir)

        setupDisplay(tardis)

        if args.headers:
            doprint("Client: %s    DB: %s" % (args.client, args.database),
                    color=colors['name'],
                    eol=True)

        if args.glob:
            directories = []
            for d in args.directories:
                if not Util.isMagic(d):
                    directories.append(d)
                else:
                    directories += globPath(os.path.abspath(d), tardis, crypt)
        else:
            directories = args.directories

        for d in directories:
            d = unicode(os.path.abspath(d).decode(fsEncoding))
            if args.realpath:
                d = os.path.realpath(d)
            fInfos = collectFileInfo(d, tardis, crypt)
            recurse = args.maxdepth if args.recurse else 0
            processFile(d,
                        fInfos,
                        tardis,
                        crypt,
                        printContents=(not args.dirinfo),
                        recurse=recurse)
    except KeyboardInterrupt:
        pass
    except Exception as e:
        logger.error("Caught exception: %s", str(e))
        if args.exceptions:
            logger.exception(e)
    finally:
        if tardis:
            tardis.close()
Example #9
0
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
    """ Setup a data connection to a client.   Determines the correct way to connect, either via direct filesystem, 
    or via TardisRemote (http).
    Returns a 3-tuple, the TardisDB object, the CacheDir object, and the appropriate crypto object
    """
    logger.debug("Connection requested for %s under %s", client, dataLoc)
    crypt = None

    loc = urllib.parse.urlparse(dataLoc)
    if (loc.scheme == 'http') or (loc.scheme == 'https'):
        logger.debug("Creating remote connection to %s", dataLoc)
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
            dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        else:
            dbLoc = dataLoc
        # get the RemoteURL object
        logger.debug("==> %s %s", dbLoc, client)
        tardis = RemoteDB.RemoteDB(dbLoc, client)
        cache = tardis
    else:
        logger.debug("Creating direct connection to %s", dataLoc)
        cacheDir = os.path.join(loc.path, client)
        cache = CacheDir.CacheDir(cacheDir, create=False)
        if not dbLoc:
            dbDir = cacheDir
        else:
            dbDir = os.path.join(dbLoc, client)
        dbPath = os.path.join(dbDir, dbName)
        tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)

    needsAuth = tardis.needsAuthentication()
    if needsAuth and password is None:
        password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)

    if needsAuth:
        authenticate(tardis, client, password)
    elif password:
        raise TardisDB.AuthenticationFailed()

    # Password specified, so create the crypto unit
    #cryptoScheme = tardis.getConfigValue('CryptoScheme', '1')
    cryptoScheme = tardis.getCryptoScheme()

    crypt = TardisCrypto.getCrypto(cryptoScheme, password, client)
    if keyFile:
        (f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
    else:
        (f, c) = tardis.getKeys()
    crypt.setKeys(f, c)

    if retpassword:
        return (tardis, cache, crypt, password)
    else:
        return (tardis, cache, crypt)
Example #10
0
def checkPasswordStrength(password):
    pwStrMin     = float(Defaults.getDefault('TARDIS_PW_STRENGTH'))
    strength, improvements = passwordmeter.test(password)
    if strength < pwStrMin:
        logger.error("Password too weak: %f (%f required)", strength, pwStrMin)
        for i in improvements:
            logger.error("    %s", improvements[i])
        return False
    else:
        return True
Example #11
0
def checkPasswordStrength(password):
    pwStrMin     = float(Defaults.getDefault('TARDIS_PW_STRENGTH'))
    strength, improvements = passwordmeter.test(password)
    if strength < pwStrMin:
        logger.error("Password too weak: %f (%f required)", strength, pwStrMin)
        for i in improvements:
            logger.error("    %s", improvements[i])
        return False
    else:
        return True
Example #12
0
    def __init__(self, password, client=None):
        self._random = Cryptodome.Random.new()
        if client is None:
            client = Defaults.getDefault('TARDIS_CLIENT')

        self.client = client
        self.salt = hashlib.sha256(client).digest()
        keys = PBKDF2(password, self.salt, count=20000, dkLen=self._keysize * 2)      # 2x256 bit keys
        self._keyKey     = keys[0:self._keysize]                                      # First 256 bit key
        self._tokenKey   = keys[self._keysize:]                                       # And the other one
Example #13
0
def setupDataConnection(dataLoc,
                        client,
                        password,
                        keyFile,
                        dbName,
                        dbLoc=None,
                        allow_upgrade=False):
    logger.debug("Connection requested for %s under %s", client, dataLoc)
    crypt = None

    loc = urlparse.urlparse(dataLoc)
    if (loc.scheme == 'http') or (loc.scheme == 'https'):
        logger.debug("Creating remote connection to %s", dataLoc)
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault(
                'TARDIS_REMOTE_PORT')
            dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path,
                                         loc.params, loc.query, loc.fragment))
        else:
            dbLoc = dataLoc
        # get the RemoteURL object
        logger.debug("==> %s %s", dbLoc, client)
        tardis = RemoteDB.RemoteDB(dbLoc, client)
        cache = tardis
    else:
        logger.debug("Creating direct connection to %s", dataLoc)
        cacheDir = os.path.join(loc.path, client)
        cache = CacheDir.CacheDir(cacheDir, create=False)
        if not dbLoc:
            dbDir = cacheDir
        else:
            dbDir = os.path.join(dbLoc, client)
        dbPath = os.path.join(dbDir, dbName)
        tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)

    needsAuth = tardis.needsAuthentication()
    if needsAuth and password is None:
        password = getPassword(True, None, None, "Password for %s: " % client)

    if password:
        if needsAuth:
            authenticate(tardis, client, password)
        else:
            raise TardisDB.AuthenticationFailed()

        # Password specified, so create the crypto unit
        crypt = TardisCrypto.TardisCrypto(password, client)
        if keyFile:
            (f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
        else:
            (f, c) = tardis.getKeys()
        crypt.setKeys(f, c)

    return (tardis, cache, crypt)
Example #14
0
    def __init__(self, password, client=None, fsencoding=sys.getfilesystemencoding()):
        self._random = Cryptodome.Random.new()
        if client is None:
            client = Defaults.getDefault('TARDIS_CLIENT')

        self.client = bytes(client, 'utf8')
        self.salt = hashlib.sha256(self.client).digest()
        keys = self.genKeyKey(password)
        self._keyKey     = keys[0:self._keysize]                                      # First 256 bit key

        self._fsEncoding = fsencoding
Example #15
0
    def __init__(self, password, client=None, fsencoding=sys.getfilesystemencoding()):
        self._random = Cryptodome.Random.new()
        if client is None:
            client = Defaults.getDefault('TARDIS_CLIENT')

        self.client = bytes(client, 'utf8')
        self.salt = hashlib.sha256(self.client).digest()
        keys = PBKDF2(password, self.salt, count=20000, dkLen=self._keysize * 2)      # 2x256 bit keys
        self._keyKey     = keys[0:self._keysize]                                      # First 256 bit key
        self._tokenKey   = keys[self._keysize:]                                       # And the other one

        self._fsEncoding = fsencoding
Example #16
0
def main():
    global args, logger
    tardis = None
    try:
        args = processArgs()
        logger = Util.setupLogging(args.verbose)

        setColors(Defaults.getDefault('TARDIS_LS_COLORS'))

        # Load any password info
        password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client))
        args.password = None

        (tardis, _, crypt) = Util.setupDataConnection(args.database, args.client, password, args.keys, args.dbname, args.dbdir)

        setupDisplay(tardis)

        if args.headers:
            doprint("Client: %s    DB: %s" %(args.client, args.database), color=colors['name'], eol=True)

        if args.glob:
            directories = []
            for d in args.directories:
                if not Util.isMagic(d):
                    directories.append(d)
                else:
                    directories += globPath(os.path.abspath(d), tardis, crypt)
        else:
            directories = args.directories

        for d in directories:
            d = os.path.abspath(d)
            if args.realpath:
                d = os.path.realpath(d)
            fInfos = collectFileInfo(d, tardis, crypt)
            recurse = args.maxdepth if args.recurse else 0
            processFile(d, fInfos, tardis, crypt, printContents=(not args.dirinfo), recurse=recurse)
    except KeyboardInterrupt:
        pass
    except TardisDB.AuthenticationException as e:
        logger.error("Authentication failed.  Bad password")
        if args.exceptions:
            logger.exception(e)
    except Exception as e:
        logger.error("Caught exception: %s", str(e))
        if args.exceptions:
            logger.exception(e)
    finally:
        if tardis:
            tardis.close()
Example #17
0
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
    logger.debug("Connection requested for %s under %s", client, dataLoc)
    crypt = None

    loc = urllib.parse.urlparse(dataLoc)
    if (loc.scheme == 'http') or (loc.scheme == 'https'):
        logger.debug("Creating remote connection to %s", dataLoc)
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
            dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        else:
            dbLoc = dataLoc
        # get the RemoteURL object
        logger.debug("==> %s %s", dbLoc, client)
        tardis = RemoteDB.RemoteDB(dbLoc, client)
        cache = tardis
    else:
        logger.debug("Creating direct connection to %s", dataLoc)
        cacheDir = os.path.join(loc.path, client)
        cache = CacheDir.CacheDir(cacheDir, create=False)
        if not dbLoc:
            dbDir = cacheDir
        else:
            dbDir = os.path.join(dbLoc, client)
        dbPath = os.path.join(dbDir, dbName)
        tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)

    needsAuth = tardis.needsAuthentication()
    if needsAuth and password is None:
        password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)

    if password:
        if needsAuth:
            authenticate(tardis, client, password)
        else:
            raise TardisDB.AuthenticationFailed()

        # Password specified, so create the crypto unit
        crypt = TardisCrypto.TardisCrypto(password, client)
        if keyFile:
            (f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
        else:
            (f, c) = tardis.getKeys()
        crypt.setKeys(f, c)

    if retpassword:
        return (tardis, cache, crypt, password)
    else:
        return (tardis, cache, crypt)
Example #18
0
    def __init__(self,
                 password,
                 client=None,
                 fsencoding=sys.getfilesystemencoding()):
        self._random = Cryptodome.Random.new()
        if client is None:
            client = Defaults.getDefault('TARDIS_CLIENT')

        self.client = client
        self.salt = hashlib.sha256(client).digest()
        keys = PBKDF2(password,
                      self.salt,
                      count=20000,
                      dkLen=self._keysize * 2)  # 2x256 bit keys
        self._keyKey = keys[0:self._keysize]  # First 256 bit key
        self._tokenKey = keys[self._keysize:]  # And the other one

        self._fsEncoding = fsencoding
Example #19
0
def main():
    global args, logger
    try:
        FORMAT = "%(levelname)s : %(message)s"
        logging.basicConfig(stream=sys.stderr, format=FORMAT, level=logging.INFO)
        logger = logging.getLogger("")

        args = processArgs()

        setColors(Defaults.getDefault('TARDIS_LS_COLORS'))

        # Load any password info
        password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client))
        args.password = None

        (tardis, _, crypt) = Util.setupDataConnection(args.database, args.client, password, args.keys, args.dbname, args.dbdir)

        setupDisplay(tardis)

        if args.headers:
            doprint("Client: %s    DB: %s" %(args.client, args.database), color=colors['name'], eol=True)

        if args.glob:
            directories = []
            for d in args.directories:
                if not Util.isMagic(d):
                    directories.append(d)
                else:
                    directories += globPath(os.path.abspath(d), tardis, crypt)
        else:
            directories = args.directories

        for d in directories:
            d = os.path.abspath(d)
            if args.realpath:
                d = os.path.realpath(d)
            fInfos = collectFileInfo(d, tardis, crypt)
            recurse = args.maxdepth if args.recurse else 0
            processFile(d, fInfos, tardis, crypt, printContents=(not args.dirinfo), recurse=recurse)
    except KeyboardInterrupt:
        pass
    except Exception as e:
        logger.error("Caught exception: %s", str(e))
        logger.exception(e)
Example #20
0
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False):
    crypt = None
    if password:
        crypt = TardisCrypto.TardisCrypto(password, client)
    password = None
    token = None
    if crypt:
        token = crypt.createToken()

    loc = urlparse.urlparse(dataLoc)
    if (loc.scheme == 'http') or (loc.scheme == 'https'):
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
            dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        else:
            dbLoc = dataLoc
        # get the RemoteURL object
        tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
        cache = tardis
    else:
        cacheDir = os.path.join(loc.path, client)
        cache = CacheDir.CacheDir(cacheDir, create=False)
        if not dbLoc:
            dbDir = cacheDir
        else:
            dbDir = os.path.join(dbLoc, client)
        dbPath = os.path.join(dbDir, dbName)
        tardis = TardisDB.TardisDB(dbPath, token=token, allow_upgrade=allow_upgrade)

    if crypt:
        if keyFile:
            (f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
        else:
            (f, c) = tardis.getKeys()
        crypt.setKeys(f, c)

    return (tardis, cache, crypt)
Example #21
0
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None):
    crypt = None
    if password:
        crypt = TardisCrypto.TardisCrypto(password, client)
    password = None
    token = None
    if crypt:
        token = crypt.createToken()

    loc = urlparse.urlparse(dataLoc)
    if (loc.scheme == "http") or (loc.scheme == "https"):
        # If no port specified, insert the port
        if loc.port is None:
            netloc = loc.netloc + ":" + Defaults.getDefault("TARDIS_REMOTE_PORT")
            dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
        # get the RemoteURL object
        tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
        cache = tardis
    else:
        cacheDir = os.path.join(loc.path, client)
        cache = CacheDir.CacheDir(cacheDir, create=False)
        if not dbLoc:
            dbDir = cacheDir
        else:
            dbDir = os.path.join(dbLoc, client)
        dbPath = os.path.join(dbDir, dbName)
        tardis = TardisDB.TardisDB(dbPath, token=token)

    if crypt:
        if keyFile:
            (f, c) = loadKeys(keyFile, tardis.getConfigValue("ClientID"))
        else:
            (f, c) = tardis.getKeys()
        crypt.setKeys(f, c)

    return (tardis, cache, crypt)
Example #22
0
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import configparser

import Tardis.Defaults as Defaults
import Tardis.Util as Util
import Tardis.TardisCrypto as TardisCrypto

configDefaults = {
    'Database': Defaults.getDefault('TARDIS_DB'),
    'Client': Defaults.getDefault('TARDIS_CLIENT'),
    'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
    'DBName': Defaults.getDefault('TARDIS_DBNAME'),
    'Password': None,
    'PasswordFile': Defaults.getDefault('TARDIS_PWFILE'),
    'PasswordProg': None,
    'Crypt': str(True),
    'KeyFile': Defaults.getDefault('TARDIS_KEYFILE'),
    'LogFiles': None,
    'Verbosity': str(0),
    'Schema': Defaults.getDefault('TARDIS_SCHEMA')
}

config = configparser.ConfigParser(configDefaults, allow_no_value=True)
job = None
Example #23
0
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import ConfigParser

import Tardis.Defaults as Defaults
import Tardis.Util as Util

configDefaults = {
    'Database': Defaults.getDefault('TARDIS_DB'),
    'Client': Defaults.getDefault('TARDIS_CLIENT'),
    'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
    'DBName': Defaults.getDefault('TARDIS_DBNAME'),
    'Password': None,
    'PasswordFile': None,
    'PasswordProg': None,
    'Crypt': str(True),
    'KeyFile': None,
    'LogFiles': None,
    'Verbosity': str(0),
    'Schema': Defaults.getDefault('TARDIS_SCHEMA')
}

config = ConfigParser.ConfigParser(configDefaults)
job = None
Example #24
0
import pprint
import urlparse

import parsedatetime
import passwordmeter

import Tardis
import Tardis.Util as Util
import Tardis.Defaults as Defaults
import Tardis.TardisDB as TardisDB
import Tardis.TardisCrypto as TardisCrypto
import Tardis.CacheDir as CacheDir
import Tardis.RemoteDB as RemoteDB
import Tardis.Config as Config

current      = Defaults.getDefault('TARDIS_RECENT_SET')
pwStrMin     = Defaults.getDefault('TARDIS_PW_STRENGTH')

# Config keys which can be gotten or set.
configKeys = ['Formats', 'Priorities', 'KeepDays', 'ForceFull', 'SaveFull', 'MaxDeltaChain', 'MaxChangePercent', 'VacuumInterval', 'AutoPurge', 'Disabled']
# Extra keys that we print when everything is requested
sysKeys    = ['ClientID', 'SchemaVersion', 'FilenameKey', 'ContentKey']

minPwStrength = 0
logger = None
args = None

def getDB(crypt, new=False, allowRemote=True):
    token = crypt.createToken() if crypt else None
    loc = urlparse.urlparse(args.database)
    # This is basically the same code as in Util.setupDataConnection().  Should consider moving to it.
Example #25
0
                            d = f.read(128 * 1024)
                        res = m.hexdigest()
                        if res != checksum:
                            print("Checksums don't match.  Expected: %s, result %s" % (checksum, res))
                            checked[checksum] = 0
                            output.write(checksum + '\n')
                            output.flush()
                        else:
                            checked[checksum] = 1
                            valid.write(checksum + "\n")
                except Exception as e:
                    print("Caught exception processing %s: %s" % (checksum, str(e)))
                    output.write(checksum + '\n')
                    output.flush()

            row = cur.fetchone()
        except sqlite3.OperationalError as e:
            print("Caught operational error.  DB is probably locked.  Sleeping for a bit")
            time.sleep(90)
    pbar.finish()

if __name__ == "__main__":
    root   = Defaults.getDefault('TARDIS_DB')
    client = Defaults.getDefault('TARDIS_CLIENT')
    dbname = Defaults.getDefault('TARDIS_DBNAME')
    password = None # 'PassWord'

    logging.basicConfig(level=logging.INFO)

    validate(root, client, dbname, password)
Example #26
0
import time

import termcolor
import parsedatetime
import binaryornot.check

import Tardis
import Tardis.Util as Util
import Tardis.Regenerator as Regenerator
import Tardis.Defaults as Defaults
import Tardis.Config as Config

logger = None
args = None

current = Defaults.getDefault("TARDIS_RECENT_SET")


def parseArgs():
    isatty = os.isatty(sys.stdout.fileno())
    global args

    parser = argparse.ArgumentParser(
        description="Diff files between current and a Tardis backup, or multiple Tardis versions",
        fromfile_prefix_chars="@",
        formatter_class=Util.HelpFormatter,
        add_help=False,
    )
    (args, remaining) = Config.parseConfigOptions(parser)

    Config.addCommonOptions(parser)
Example #27
0
import logging
import time

import termcolor
import parsedatetime

import Tardis
import Tardis.Util as Util
import Tardis.Regenerator as Regenerator
import Tardis.Defaults as Defaults
import Tardis.Config as Config

logger = None
args = None

current = Defaults.getDefault('TARDIS_RECENT_SET')

def parseArgs():
    isatty = os.isatty(sys.stdout.fileno())
    global args

    parser = argparse.ArgumentParser(description='Diff files between current and a Tardis backup, or multiple Tardis versions', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatter, add_help=False)
    (args, remaining) = Config.parseConfigOptions(parser)

    Config.addCommonOptions(parser)
    Config.addPasswordOptions(parser)

    parser.add_argument("--backup", '-b',   nargs='+', dest='backup', default=[current], help="Backup set(s) to use (Default: %(default)s)")

    parser.add_argument('--color',                  dest='color',   default=isatty, action=Util.StoreBoolean,   help='Use colors')
Example #28
0
import termcolor
import parsedatetime
import binaryornot.check

import Tardis
import Tardis.Util as Util
import Tardis.Regenerator as Regenerator
import Tardis.Defaults as Defaults
import Tardis.Config as Config
import Tardis.TardisDB as TardisDB

logger = None
exceptionLogger = None
args = None

current = Defaults.getDefault('TARDIS_RECENT_SET')

def parseArgs():
    isatty = os.isatty(sys.stdout.fileno())
    global args

    parser = argparse.ArgumentParser(description='Diff files between current and a Tardis backup, or multiple Tardis versions', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatter, add_help=False)
    (args, remaining) = Config.parseConfigOptions(parser)

    Config.addCommonOptions(parser)
    Config.addPasswordOptions(parser)

    parser.add_argument("--backup", '-b',   nargs='+', dest='backup', default=[current], help="Backup set(s) to use (Default: %(default)s)")

    parser.add_argument('--color',                  dest='color',   default=isatty, action=Util.StoreBoolean,   help='Use colors')
Example #29
0
import pprint
import urlparse

import parsedatetime
import passwordmeter

import Tardis
import Tardis.Util as Util
import Tardis.Defaults as Defaults
import Tardis.TardisDB as TardisDB
import Tardis.TardisCrypto as TardisCrypto
import Tardis.CacheDir as CacheDir
import Tardis.RemoteDB as RemoteDB
import Tardis.Config as Config

current = Defaults.getDefault("TARDIS_RECENT_SET")
pwStrMin = Defaults.getDefault("TARDIS_PW_STRENGTH")

# Config keys which can be gotten or set.
configKeys = [
    "Formats",
    "Priorities",
    "KeepDays",
    "ForceFull",
    "SaveFull",
    "MaxDeltaChain",
    "MaxChangePercent",
    "VacuumInterval",
    "AutoPurge",
    "Disabled",
]
Example #30
0
import pprint
import urlparse

import parsedatetime
import passwordmeter

import Tardis
import Tardis.Util as Util
import Tardis.Defaults as Defaults
import Tardis.TardisDB as TardisDB
import Tardis.TardisCrypto as TardisCrypto
import Tardis.CacheDir as CacheDir
import Tardis.RemoteDB as RemoteDB
import Tardis.Config as Config

current = Defaults.getDefault('TARDIS_RECENT_SET')
pwStrMin = Defaults.getDefault('TARDIS_PW_STRENGTH')

# Config keys which can be gotten or set.
configKeys = [
    'Formats', 'Priorities', 'KeepDays', 'ForceFull', 'SaveFull',
    'MaxDeltaChain', 'MaxChangePercent', 'VacuumInterval', 'AutoPurge',
    'Disabled', 'SaveConfig'
]
# Extra keys that we print when everything is requested
sysKeys = ['ClientID', 'SchemaVersion', 'FilenameKey', 'ContentKey']

minPwStrength = 0
logger = None
args = None
Example #31
0
def processArgs():
    parser = argparse.ArgumentParser(description='Encrypt the database')
    parser.add_argument('--database',
                        '-D',
                        dest='database',
                        default=Defaults.getDefault('TARDIS_DB'),
                        help="Database to use.  Default: %(default)s")
    parser.add_argument('--client',
                        '-C',
                        dest='client',
                        default=Defaults.getDefault('TARDIS_CLIENT'),
                        help="Client to list on.  Default: %(default)s")
    parser.add_argument('--dbname',
                        dest='dbname',
                        default=Defaults.getDefault('TARDIS_DBNAME'),
                        help="Name of the database file. Default: %(default)s")

    parser.add_argument('--filenames',
                        dest='filenames',
                        action='store_true',
                        default=False,
                        help='Encrypt filenames. Default=%(default)s')
    parser.add_argument('--files',
                        dest='files',
                        action='store_true',
                        default=False,
                        help='Encrypt files. Default=%(default)s')
    parser.add_argument('--dirhashes',
                        dest='dirhash',
                        action='store_true',
                        default=False,
                        help='Generate directory hashes.  Default=%(default)s')
    parser.add_argument('--meta',
                        dest='meta',
                        action='store_true',
                        default=False,
                        help='Generate metadata files.  Default=%(default)s')
    #parser.add_argument('--signatures',     dest='sigs',      action='store_true', default=False,       help='Generate signatures. Default=%(default)s')

    passgroup = parser.add_argument_group(
        "Password/Encryption specification options")
    pwgroup = passgroup.add_mutually_exclusive_group(required=True)
    pwgroup.add_argument('--password',
                         '-P',
                         dest='password',
                         default=None,
                         nargs='?',
                         const=True,
                         help='Encrypt files with this password')
    pwgroup.add_argument('--password-file',
                         dest='passwordfile',
                         default=None,
                         help='Read password from file')
    pwgroup.add_argument('--password-url',
                         dest='passwordurl',
                         default=None,
                         help='Retrieve password from the specified URL')
    pwgroup.add_argument(
        '--password-prog',
        dest='passwordprog',
        default=None,
        help='Use the specified command to generate the password on stdout')

    return parser.parse_args()
Example #32
0
def getPassword(password, pwurl, pwprog, prompt='Password: '******'TARDIS_PWTIMEOUT')):
    methods = 0
    if password: methods += 1
    if pwurl:    methods += 1
    if pwprog:   methods += 1

    if methods > 1:
        raise Exception("Cannot specify more than one password retrieval mechanism")

    if methods == 0 and not allowNone:
        # Nothing specified, and it wants a value.  Set password to True to fetch
        password = True

    if password == True or password == '':
        password = _readWithTimeout(prompt, int(timeout))
        password = password.rstrip()       # Delete trailing characters
        if confirm:
            pw2 = _readWithTimeout("Confirm password:"******"Passwords don't match")

    if pwurl:
        loc = urllib.parse.urlunparse(urllib.parse.urlparse(pwurl, scheme='file'))
        pwf = urllib.request.urlopen(loc)
        password = pwf.readline().rstrip()
        pwf.close()

    if pwprog:
        a = shlex.split(pwprog)
        output = subprocess.check_output(a)
        password = output.split('\n')[0].rstrip()

    if not allowNone and not password:
        raise Exception("Password required")

    if strength and password:
        if not checkPasswordStrength(password):
            raise Exception("Password not strong enough")

    return password
Example #33
0
def parseArgs():
    parser = argparse.ArgumentParser(description='Recover Backed Up Files', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatter, add_help=False)

    (_, remaining) = Config.parseConfigOptions(parser)
    Config.addCommonOptions(parser)
    Config.addPasswordOptions(parser)

    parser.add_argument("--output", "-o",   dest="output", help="Output file", default=None)
    parser.add_argument("--checksum", "-c", help="Use checksum instead of filename", dest='cksum', action='store_true', default=False)

    bsetgroup = parser.add_mutually_exclusive_group()
    bsetgroup.add_argument("--backup", "-b", help="Backup set to use.  Default: %(default)s", dest='backup', default=Defaults.getDefault('TARDIS_RECENT_SET'))
    bsetgroup.add_argument("--date", "-d",   help="Regenerate as of date", dest='date', default=None)
    bsetgroup.add_argument("--last", "-l",   dest='last', default=False, action='store_true', help="Regenerate the most recent version of the file")

    parser.add_argument('--recurse',        dest='recurse', default=True, action=Util.StoreBoolean, help='Recurse directory trees.  Default: %(default)s')
    parser.add_argument('--recovername',    dest='recovername', default=False, action=Util.StoreBoolean,    help='Recover the name when recovering a checksum.  Default: %(default)s')

    parser.add_argument('--authenticate',    dest='auth', default=True, action=Util.StoreBoolean,    help='Authenticate files while regenerating them.  Default: %(default)s')
    parser.add_argument('--authfail-action', dest='authfailaction', default='rename', choices=['keep', 'rename', 'delete'], help='Action to take for files that do not authenticate.  Default: %(default)s')

    parser.add_argument('--reduce-path', '-R',  dest='reduce',  default=0, const=sys.maxsize, type=int, nargs='?',   metavar='N',
                        help='Reduce path by N directories.  No value for "smart" reduction')
    parser.add_argument('--set-times', dest='settime', default=True, action=Util.StoreBoolean,      help='Set file times to match original file. Default: %(default)s')
    parser.add_argument('--set-perms', dest='setperm', default=True, action=Util.StoreBoolean,      help='Set file owner and permisions to match original file. Default: %(default)s')
    parser.add_argument('--set-attrs', dest='setattrs', default=True, action=Util.StoreBoolean,     help='Set file extended attributes to match original file.  May only set attributes in user space. Default: %(default)s')
    parser.add_argument('--set-acl',   dest='setacl', default=True, action=Util.StoreBoolean,       help='Set file access control lists to match the original file. Default: %(default)s')
    parser.add_argument('--overwrite', '-O', dest='overwrite', default=owModeDefault, const='always', nargs='?',
                        choices=['always', 'newer', 'older', 'never', 'ask'],
                        help='Mode for handling existing files. Default: %(default)s')

    parser.add_argument('--hardlinks',  dest='hardlinks',   default=True,   action=Util.StoreBoolean,   help='Create hardlinks of multiple copies of same inode created. Default: %(default)s')

    parser.add_argument('--exceptions',         default=False, action=Util.StoreBoolean, dest='exceptions', help="Log full exception data");
    parser.add_argument('--verbose', '-v',      action='count', default=0, dest='verbose', help='Increase the verbosity')
    parser.add_argument('--version',            action='version', version='%(prog)s ' + Tardis.__versionstring__,    help='Show the version')
    parser.add_argument('--help', '-h',         action='help')

    parser.add_argument('files', nargs='+', default=None, help="List of files to regenerate")

    Util.addGenCompletions(parser)

    return parser.parse_args(remaining)
Example #34
0
                            d = f.read(128 * 1024)
                        res = m.hexdigest()
                        if res != checksum:
                            print("Checksums don't match.  Expected: %s, result %s" % (checksum, res))
                            checked[checksum] = 0
                            output.write(checksum + '\n')
                            output.flush()
                        else:
                            checked[checksum] = 1
                            valid.write(checksum + "\n")
                except Exception as e:
                    print("Caught exception processing %s: %s" % (checksum, str(e)))
                    output.write(checksum + '\n')
                    output.flush()

            row = cur.fetchone()
        except sqlite3.OperationalError as e:
            print("Caught operational error.  DB is probably locked.  Sleeping for a bit")
            time.sleep(90)
    pbar.finish()

if __name__ == "__main__":
    root   = Defaults.getDefault('TARDIS_DB')
    client = Defaults.getDefault('TARDIS_CLIENT')
    dbname = Defaults.getDefault('TARDIS_DBNAME')
    password = None # 'PassWord'

    logging.basicConfig(level=logging.INFO)

    validate(root, client, dbname, password)
Example #35
0
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import ConfigParser

import Tardis.Defaults as Defaults
import Tardis.Util as Util


configDefaults = {
    'Database':             Defaults.getDefault('TARDIS_DB'),
    'Client':               Defaults.getDefault('TARDIS_CLIENT'),
    'DBDir':		        Defaults.getDefault('TARDIS_DBDIR'),
    'DBName':		        Defaults.getDefault('TARDIS_DBNAME'),
    'Password':             None,
    'PasswordFile':         None,
    'PasswordProg':         None,
    'Crypt':                str(True),
    'KeyFile':              None,
    'LogFiles':             None,
    'Verbosity':            str(0),
    'Schema':				Defaults.getDefault('TARDIS_SCHEMA')
}

config = ConfigParser.ConfigParser(configDefaults)
job = None
Example #36
0
    def __init__(self, *args, **kw):
        super(TardisFS, self).__init__(*args, **kw)

        try:
            client = Defaults.getDefault('TARDIS_CLIENT')
            database = Defaults.getDefault('TARDIS_DB')
            dbdir = Defaults.getDefault('TARDIS_DBDIR') % {
                'TARDIS_DB': database
            }  # HACK
            dbname = Defaults.getDefault('TARDIS_DBNAME')
            current = Defaults.getDefault('TARDIS_RECENT_SET')

            # Parameters
            self.database = database
            self.client = client
            self.repoint = False
            self.password = None
            self.pwfile = None
            self.pwprog = None
            self.keys = None
            self.dbname = dbname
            self.dbdir = dbdir
            self.cachetime = 60
            self.nocrypt = False
            self.noauth = False
            self.current = current
            self.authenticate = True

            self.crypt = None

            logging.basicConfig(level=logging.WARNING)
            self.log = logging.getLogger("TardisFS")

            self.parser.add_option(
                "--database",
                '-D',
                help="Path to the Tardis database directory")
            self.parser.add_option("--client",
                                   '-C',
                                   help="Client to load database for")
            self.parser.add_option(
                "--password",
                '-P',
                help=
                "Password for this archive (use '-o password='******'-o password='******'nocrypt',
                                   help="Disable encryption")
            self.parser.add_option(mountopt='noauth',
                                   help="Disable authentication")
            self.parser.add_option(
                mountopt='current',
                help="Name to use for most recent complete backup")

            res = self.parse(values=self, errex=1)

            self.mountpoint = res.mountpoint

            self.name = "TardisFS:<{}/{}>".format(self.database, self.client)

            password = Util.getPassword(self.password,
                                        self.pwfile,
                                        self.pwprog,
                                        prompt="Password for %s: " %
                                        (self.client))
            self.password = None

            self.cache = Cache.Cache(0, float(self.cachetime))
            self.fileCache = Cache.Cache(0, float(self.cachetime), 'FileCache')

            #if password:
            #    self.crypt = TardisCrypto.TardisCrypto(password, self.client)
            (self.tardis, self.cacheDir,
             self.crypt) = Util.setupDataConnection(self.database, self.client,
                                                    password, self.keys,
                                                    self.dbname, self.dbdir)
            password = None

            # Remove the crypto object if not encyrpting files.
            if self.nocrypt or self.nocrypt is None:
                self.crypt = None

            if self.noauth or self.noauth is None:
                self.authenticate = False

            # Create a regenerator.
            self.regenerator = Regenerator.Regenerator(self.cacheDir,
                                                       self.tardis,
                                                       crypt=self.crypt)
            self.files = {}

            # Fuse variables
            self.flags = 0
            self.multithreaded = 0

        except Exception as e:
            self.log.exception(e)
            sys.exit(2)
Example #37
0
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import configparser

import Tardis.Defaults as Defaults
import Tardis.Util as Util


configDefaults = {
    'Database':             Defaults.getDefault('TARDIS_DB'),
    'Client':               Defaults.getDefault('TARDIS_CLIENT'),
    'DBDir':		    Defaults.getDefault('TARDIS_DBDIR'),
    'DBName':		    Defaults.getDefault('TARDIS_DBNAME'),
    'Password':             None,
    'PasswordFile':         Defaults.getDefault('TARDIS_PWFILE'),
    'PasswordProg':         None,
    'Crypt':                str(True),
    'KeyFile':              Defaults.getDefault('TARDIS_KEYFILE'),
    'LogFiles':             None,
    'Verbosity':            str(0),
    'Schema':				Defaults.getDefault('TARDIS_SCHEMA')
}

config = configparser.ConfigParser(configDefaults, allow_no_value=True)
job = None
Example #38
0
import zlib
import daemonize
import base64

from flask import Flask, Response, session, request, url_for, abort, redirect, make_response
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

import Tardis
import Tardis.TardisDB as TardisDB
import Tardis.Util as Util
import Tardis.CacheDir as CacheDir
import Tardis.Defaults as Defaults

basedir     = Defaults.getDefault('TARDIS_DB')
dbname      = Defaults.getDefault('TARDIS_DBNAME')
port        = Defaults.getDefault('TARDIS_REMOTE_PORT')
configName  = Defaults.getDefault('TARDIS_REMOTE_CONFIG')
pidFile     = Defaults.getDefault('TARDIS_REMOTE_PIDFILE')

configDefaults = {
    'Port'              : port,
    'Database'          : basedir,
    'DBName'            : dbname,
    'LogFile'           : None,
    'LogExceptions'     : str(False),
    'Verbose'           : '0',
    'Daemon'            : str(False),
    'User'              : None,
    'Group'             : None,
Example #39
0
import zlib
import daemonize
import base64

from flask import Flask, Response, session, request, url_for, abort, redirect, make_response
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

import Tardis
import Tardis.TardisDB as TardisDB
import Tardis.Util as Util
import Tardis.CacheDir as CacheDir
import Tardis.Defaults as Defaults

basedir     = Defaults.getDefault('TARDIS_DB')
dbname      = Defaults.getDefault('TARDIS_DBNAME')
port        = Defaults.getDefault('TARDIS_REMOTE_PORT')
configName  = Defaults.getDefault('TARDIS_REMOTE_CONFIG')
pidFile     = Defaults.getDefault('TARDIS_REMOTE_PIDFILE')

configDefaults = {
    'Port'              : port,
    'Database'          : basedir,
    'DBName'            : dbname,
    'LogFile'           : '',
    'LogExceptions'     : str(False),
    'Verbose'           : '0',
    'Daemon'            : str(False),
    'User'              : '',
    'Group'             : '',
Example #40
0
    def __init__(self, *args, **kw):
        super(TardisFS, self).__init__(*args, **kw)

        try:
            client   = Defaults.getDefault('TARDIS_CLIENT')
            database = Defaults.getDefault('TARDIS_DB')
            dbdir    = Defaults.getDefault('TARDIS_DBDIR') % { 'TARDIS_DB': database }          # HACK
            dbname   = Defaults.getDefault('TARDIS_DBNAME')
            current  = Defaults.getDefault('TARDIS_RECENT_SET')

            # Parameters
            self.database       = database
            self.client         = client
            self.repoint        = False
            self.password       = None
            self.pwfile         = None
            self.pwprog         = None
            self.keys           = None
            self.dbname         = dbname
            self.dbdir          = dbdir
            self.cachetime      = 60
            self.nocrypt        = False
            self.noauth         = False
            self.current        = current
            self.authenticate   = True

            self.crypt          = None

            logging.basicConfig(level=logging.WARNING)
            self.log = logging.getLogger("TardisFS")

            self.parser.add_option("--database", '-D',      help="Path to the Tardis database directory")
            self.parser.add_option("--client", '-C',        help="Client to load database for")
            self.parser.add_option("--password", '-P',      help="Password for this archive (use '-o password='******'-o password='******'nocrypt',      help="Disable encryption")
            self.parser.add_option(mountopt='noauth',       help="Disable authentication")
            self.parser.add_option(mountopt='current',      help="Name to use for most recent complete backup")

            res = self.parse(values=self, errex=1)

            self.mountpoint = res.mountpoint

            self.name = "TardisFS:<{}/{}>".format(self.database, self.client)

            password = Util.getPassword(self.password, self.pwfile, self.pwprog, prompt="Password for %s: " % (self.client))
            self.password = None

            self.cache      = Cache.Cache(0, float(self.cachetime))
            self.fileCache  = Cache.Cache(0, float(self.cachetime), 'FileCache')

            #if password:
            #    self.crypt = TardisCrypto.TardisCrypto(password, self.client)
            (self.tardis, self.cacheDir, self.crypt) = Util.setupDataConnection(self.database, self.client, password, self.keys, self.dbname, self.dbdir)
            password = None

            # Remove the crypto object if not encyrpting files.
            if self.nocrypt or self.nocrypt is None:
                self.crypt = None

            if self.noauth or self.noauth is None:
                self.authenticate = False

            # Create a regenerator.
            self.regenerator = Regenerator.Regenerator(self.cacheDir, self.tardis, crypt=self.crypt)
            self.files = {}

            # Fuse variables
            self.flags = 0
            self.multithreaded = 0

        except Exception as e:
            self.log.exception(e)
            sys.exit(2)
Example #41
0
class TardisFS(LoggingMixIn, Operations):
    """
    FUSE filesystem to read data from a Tardis Backup Database
    """
    # Disable pylint complaints about "could me a function" and "unused argument" as lots of required FUSE functions
    # just return "read-only FS" status
    # pragma pylint: disable=no-self-use,unused-argument
    backupsets = {}
    dirInfo = {}
    fsencoding = sys.getfilesystemencoding()
    name = "TardisFS"

    client   = Defaults.getDefault('TARDIS_CLIENT')
    database = Defaults.getDefault('TARDIS_DB')
    dbdir    = Defaults.getDefault('TARDIS_DBDIR') % { 'TARDIS_DB': database }          # HACK
    dbname   = Defaults.getDefault('TARDIS_DBNAME')
    current  = Defaults.getDefault('TARDIS_RECENT_SET')


    def __init__(self, db, cache, crypto, args):
        self.cacheDir = cache
        self.crypt = crypto
        self.tardis = db

        # Create a regenerator.
        self.regenerator = Regenerator.Regenerator(self.cacheDir, self.tardis, crypt=self.crypt)
        self.files = {}

        # Set up some caches.
        self.cachetime  = 60

        self.cache      = Cache.Cache(0, float(self.cachetime))
        self.fileCache  = Cache.Cache(0, float(self.cachetime), 'FileCache')

        self.authenticate = True


    def __del__(self):
        if self.tardis:
            self.tardis.close()

    def __repr__(self):
        return self.name

    def fsEncodeName(self, name):
        return name

    def getBackupSetInfo(self, b):
        key = (_BackupSetInfo, b)
        info = self.cache.retrieve(key)
        if info:
            return info
        info = self.tardis.getBackupSetInfo(b)
        self.cache.insert(key, info)
        return info

    def lastBackupSet(self, completed):
        key = (_LastBackupSet, completed)
        backupset = self.cache.retrieve(key)
        if backupset:
            return backupset
        backupset = self.tardis.lastBackupSet(completed=completed)
        self.cache.insert(key, backupset)
        return backupset

    def getDirInfo(self, path):
        """ Return the inode and backupset of a directory """
        #self.log.info("getDirInfo: %s", path)
        key = (_DirInfo, path)
        info = self.cache.retrieve(key)
        if info:
            return info

        #self.log.debug("No cache info available for %s", path)
        parts = getParts(path)
        bsInfo = self.getBackupSetInfo(parts[0])
        if len(parts) == 2:
            subpath = parts[1]
            if self.crypt:
                subpath = self.crypt.encryptPath(subpath)
            #fInfo = self.getFileInfoByPath(subpath, bsInfo['backupset'])
            fInfo = self.getFileInfoByPath(path)
            #self.log.info("fInfo %s %s %s", parts[1], "**", str(fInfo))
            info = (bsInfo, fInfo)
        else:
            fInfo = {'inode': 0, 'device': 0, 'dir': 1}
            info = (bsInfo, fInfo)

        if info:
            self.cache.insert(key, info)
        return info

    def getFileInfoByPath(self, path):
        #self.log.info("getFileInfoByPath: %s", path)

        # First, check the cache
        f = self.fileCache.retrieve(path)
        if f:
            #self.log.debug("getFileInfoByPath: %s found in cache", path)
            return f

        # Not in the cache, look things up
        #self.log.debug("File info for %s not in cache", path)
        (head, tail) = os.path.split(path)
        data = self.getDirInfo(head)
        if data:
            bsInfo, dInfo = data
        else:
            return None

        if bsInfo:
            if self.crypt:
                tail = self.crypt.encryptPath(tail)
            #self.log.debug(str(dInfo))
            f = self.tardis.getFileInfoByName(tail, (dInfo['inode'], dInfo['device']), bsInfo['backupset'])
        else:
            parts = getParts(path)
            b = self.getBackupSetInfo(parts[0])
            subpath = parts[1]
            if self.crypt:
                subpath = self.crypt.encryptPath(subpath)
            #self.log.debug("getFileInfoByPath: %s=>%s", parts[1], subpath)
            f = self.tardis.getFileInfoByPath(subpath, b['backupset'])
        # Cache it.
        self.fileCache.insert(path, f)
        # Return it
        return f

    #@tracer
    def getattr(self, path, fh=None):
        """
        - st_mode (protection bits)
        - st_ino (inode number)
        - st_dev (device)
        - st_nlink (number of hard links)
        - st_uid (user ID of owner)
        - st_gid (group ID of owner)
        - st_size (size of file, in bytes)
        - st_atime (time of most recent access)
        - st_mtime (time of most recent content modification)
        - st_ctime (platform dependent; time of most recent metadata change on Unix,
                    or the time of creation on Windows).
        """

        #self.log.info("CALL getattr: %s",  path)
        path = self.fsEncodeName(path)

        depth = getDepth(path) # depth of path, zero-based from root
        if depth == 0:
            # Fake the root
            target = self.lastBackupSet(False)
            timestamp = float(target['starttime'])
            st = {
                'st_mode': stat.S_IFDIR | 0o555,
                'st_ino': 0,
                'st_dev': 0,
                'st_nlink': 32,
                'st_uid': 0,
                'st_gid': 0,
                'st_size': 4096,
                'st_atime': timestamp,
                'st_mtime': timestamp,
                'st_ctime': timestamp,
            }
            return st
        elif depth == 1:
            # Root directory contents
            lead = getParts(path)
            if lead[0] == self.current:
                target = self.lastBackupSet(True)
                timestamp = float(target['endtime'])
                st = {
                    'st_mode': stat.S_IFLNK | 0o755,
                    'st_ino': 1,
                    'st_dev': 0,
                    'st_nlink': 1,
                    'st_uid': 0,
                    'st_gid': 0,
                    'st_size': 4096,
                    'st_atime': timestamp,
                    'st_mtime': timestamp,
                    'st_ctime': timestamp
                }
                return st
            else:
                f = self.getBackupSetInfo(lead[0])
                #self.log.debug("Got backupset info for %s: %s", lead[0], str(f))
                if f:
                    timestamp = float(f['starttime'])
                    st = {
                        'st_mode': stat.S_IFDIR | 0o555,
                        'st_ino': int(float(f['starttime'])),
                        'st_dev': 0,
                        'st_nlink': 2,
                        'st_uid': 0,
                        'st_gid': 0,
                        'st_size': 4096,
                        'st_atime': timestamp,
                        'st_mtime': timestamp,
                        'st_ctime': timestamp
                    }
                    return st
        else:
            f = self.getFileInfoByPath(path)
            if f:
                st = {
                    'st_mode': f["mode"],
                    'st_ino': f["inode"],
                    'st_dev': 0,
                    'st_nlink': f["nlinks"],
                    'st_uid': f["uid"],
                    'st_gid': f["gid"],
                    'st_atime': f["mtime"],
                    'st_mtime': f["mtime"],
                    'st_ctime': f["ctime"]
                }
                if f["size"] is not None:
                    st['st_size'] = int(f["size"])
                elif f["dir"]:
                    st['st_size'] = 4096       # Arbitrary number
                else:
                    st['st_size'] = 0
                return st
        logger.debug("File not found: %s", path)
        raise FuseOSError(errno.ENOENT)

    #@tracer
    #def getdir(self, _, fh):
        #"""
        #return: [[('file1', 0), ('file2', 0), ... ]]
        #"""
        ##self.log.info('CALL getdir {}'.format(path))
        #raise FuseOSError(errno.ENOSYS)

    #@tracer
    def readdir(self, path, offset):
        #self.log.info("CALL readdir %s Offset: %d", path, offset)
        parent = None

        path = self.fsEncodeName(path)

        key = (_DirContents, path)
        dirents = self.cache.retrieve(key)
        if not dirents:
            dirents = ['.', '..']
            depth = getDepth(path)
            if depth == 0:
                dirents.append(self.current)
                entries = self.tardis.listBackupSets()
                dirents.extend([y['name'] for y in entries])
            else:
                parts = getParts(path)
                if depth == 1:
                    b = self.getBackupSetInfo(parts[0])
                    entries = self.tardis.readDirectory((0, 0), b['backupset'])
                else:
                    (b, parent) = self.getDirInfo(path)
                    entries = self.tardis.readDirectory((parent["inode"], parent["device"]), b['backupset'])
                #if self.crypt:
                    #entries = self.decryptNames(entries)

                # For each entry, cache it, so a later getattr() call can use it.
                # Get attr will typically be called promptly after a call to
                now = time.time()
                for e in entries:
                    name  = e['name']
                    if self.crypt:
                        name = self.crypt.decryptFilename(name)
                    name = self.fsEncodeName(name)
                    p = os.path.join(path, name)
                    self.fileCache.insert(p, e, now=now)
                    dirents.append(name)
            self.cache.insert(key, dirents)

        #self.log.debug("Direntries: %s", str(dirents))

        # Now, return each entry in the list.
        for e in dirents:
            name = e
            #self.log.debug("readdir %s yielding dir entry for %s.  Mode: %s. Type: %s ", path, e, mode, type(mode))
            yield name

    #@tracer
    def mythread ( self ):
        #self.log.info('mythread')
        raise FuseOSError(errno.ENOSYS)

    #@tracer
    def chmod ( self, path, mode ):
        #self.log.info('CALL chmod {} {}'.format(path, oct(mode)))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def chown ( self, path, uid, gid ):
        #self.log.info( 'CALL chown {} {} {}'.format(path, uid, gid))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def fsync ( self, path, isFsyncFile ):
        #self.log.info( 'CALL fsync {} {}'.format(path, isFsyncFile))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def link ( self, targetPath, linkPath ):
        #self.log.info( 'CALL link {} {}'.format(targetPath, linkPath))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def mkdir ( self, path, mode ):
        #self.log.info( 'CALL mkdir {} {}'.format(path, oct(mode)))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def mknod ( self, path, mode, dev ):
        #self.log.info( 'CALL mknod {} {} {}'.format(path, oct(mode), dev))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def open ( self, path, flags ):
        #self.log.info('CALL open {} {})'.format(path, flags))
        path = self.fsEncodeName(path)

        depth = getDepth(path) # depth of path, zero-based from root

        if depth < 2:
            raise FuseOSError(errno.ENOENT)

        # TODO: Lock this
        if path in self.files:
            self.files[path]["opens"] += 1
            return 0

        parts = getParts(path)
        b = self.getBackupSetInfo(parts[0])
        if b:
            subpath = parts[1]
            if self.crypt:
                subpath = self.crypt.encryptPath(subpath)
            f = self.regenerator.recoverFile(subpath, b['backupset'], nameEncrypted=True, authenticate=self.authenticate)
            if f:
                logger.debug("Opened file %s", path)
                try:
                    f.flush()
                    f.seek(0)
                except (AttributeError, IOError) as e:
                    logger.exception(e)
                    bytesCopied = 0
                    logger.debug("Copying file to tempfile")
                    temp = tempfile.TemporaryFile()
                    chunk = f.read(65536)
                    while chunk:
                        bytesCopied = bytesCopied + len(chunk)
                        temp.write(chunk)
                        chunk = f.read(65536)
                    f.close()
                    logger.debug("Copied %d bytes to tempfile", bytesCopied)
                    temp.flush()
                    temp.seek(0)
                    f = temp

                self.files[path] = {"file": f, "opens": 1}
                logger.debug("Set files[%s] => %s", path, str(self.files[path]))
                return 0
        # Otherwise.....
        raise FuseOSError(errno.ENOENT)


    #@tracer
    def read ( self, path, length, offset, fh ):
        #self.log.info('CALL read {} {} {}'.format(path, length, offset))
        path = self.fsEncodeName(path)
        f = self.files[path]["file"]
        if f:
            f.seek(offset)
            data = f.read(length)
            logger.debug("Actually read %d bytes of %s", len(data), type(data))
            return data
        logger.warning("No file for path %s", path)
        raise FuseOSError(errno.EINVAL)

    #@tracer
    def readlink ( self, path ):
        #self.log.info('CALL readlink {}'.format(path))
        path = self.fsEncodeName(path)

        key = (_LinkContents, path)
        link = self.cache.retrieve(key)
        if link:
            return link
        if path == '/' + self.current:
            target = self.lastBackupSet(True)
            logger.debug("Path: %s Target: %s %s", path, target['name'], target['backupset'])
            link = str(target['name'])
            self.cache.insert(key, link)
            return link
        elif getDepth(path) > 1:
            parts = getParts(path)
            b = self.getBackupSetInfo(parts[0])
            if b:
                subpath = parts[1]
                if self.crypt:
                    subpath = self.crypt.encryptPath(subpath)
                f = self.regenerator.recoverFile(subpath, b['backupset'], nameEncrypted=True, authenticate=self.authenticate)
                f.flush()
                link = f.readline()
                f.close()
                if self.repoint:
                    if os.path.isabs(link):
                        link = os.path.join(self.mountpoint, parts[0], os.path.relpath(link, "/"))
                self.cache.insert(key, link)
                return link
        raise FuseOSError(errno.ENOENT)

    #@tracer
    def release ( self, path, flags ):
        path = self.fsEncodeName(path)

        if self.files[path]:
            self.files[path]["opens"] -= 1
            if self.files[path]["opens"] == 0:
                self.files[path]["file"].close()
                del self.files[path]
            return 0
        raise FuseOSError(errno.EINVAL)

    #@tracer
    def rename ( self, oldPath, newPath ):
        #self.log.info('CALL rename {} {}'.format(oldPath, newPath))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def rmdir ( self, path ):
        #self.log.info('CALL rmdir {}'.format(path))
        raise FuseOSError(errno.EROFS)

    #@tracer
    def statfs ( self, path ):
        #self.log.info('CALL statfs: %s', path)
        if isinstance(self.cacheDir, CacheDir.CacheDir):
            fs = os.statvfs(self.cacheDir.root)

            return dict((key, getattr(fs, key)) for key in (
                'f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail',
                'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax'))
        raise FuseOSError(errno.EINVAL)

    def symlink ( self, targetPath, linkPath ):
        #self.log.info('CALL symlink {} {}'.format(path, linkPath))
        raise FuseOSError(errno.EROFS)

    def truncate ( self, path, size ):
        #self.log.info('CALL truncate {} {}'.format(path, size))
        raise FuseOSError(errno.EROFS)

    def unlink ( self, path ):
        #self.log.info('CALL unlink {}'.format(path))
        raise FuseOSError(errno.EROFS)

    def write ( self, path, buf, offset ):
        #self.log.info('CALL write {} {} {}'.format(path, offset, len(buf)))
        raise FuseOSError(errno.EROFS)

    # Map extrenal attribute names for the top level directories to backupset info names
    attrMap = {
        'user.priority' : 'priority',
        'user.complete' : 'completed',
        'user.backupset': 'backupset',
        'user.session'  : 'session'
    }

    #@tracer
    #def listxattr ( self, path, size ):
    def listxattr(self, path):
        path = self.fsEncodeName(path)
        #self.log.info('CALL listxattr %s %d', path, size)
        if getDepth(path) == 1:
            parts = getParts(path)
            b = self.getBackupSetInfo(parts[0])
            if b:
                return list(self.attrMap.keys())

        if getDepth(path) > 1:
            parts = getParts(path)
            b = self.getBackupSetInfo(parts[0])
            if b:
                subpath = parts[1]
                if self.crypt:
                    subpath = self.crypt.encryptPath(subpath)
                info = self.tardis.getFileInfoByPath(subpath, b['backupset'])
                if info:
                    attrs = ['user.tardis_checksum', 'user.tardis_since', 'user.tardis_chain']
                    logger.info("xattrs: %s", info['xattrs'])
                    if info['xattrs']:
                        f = self.regenerator.recoverChecksum(info['xattrs'], authenticate=self.authenticate)
                        xattrs = json.loads(f.read())
                        logger.debug("Xattrs: %s", str(xattrs))
                        attrs += list(map(str, list(xattrs.keys())))
                        logger.debug("Adding xattrs: %s", list(xattrs.keys()))
                        logger.info("Xattrs: %s", str(attrs))
                        logger.info("Returning: %s", str(attrs))

                    return attrs

        return None

    #@tracer
    #def getxattr (self, path, attr, size, *args):
    def getxattr(self, path, attr, position=0):
        path = self.fsEncodeName(path)
        #logger.info('CALL getxattr: %s %s', path, attr)
        attr = str(attr)

        depth = getDepth(path)
        #logger.info("Got depth of path %s -> %s", path, depth)

        if depth == 1:
            if attr in self.attrMap:
                parts = getParts(path)
                b = self.getBackupSetInfo(parts[0])
                if self.attrMap[attr] in list(b.keys()):
                    return bytes(str(b[self.attrMap[attr]]), 'utf-8')

        if depth > 1:
            parts = getParts(path)
            b = self.getBackupSetInfo(parts[0])

            subpath = parts[1]
            if self.crypt:
                subpath = self.crypt.encryptPath(subpath)
            #logger.debug("-----> Asking for attribute %s", attr)
            if attr == 'user.tardis_checksum':
                if b:
                    checksum = self.tardis.getChecksumByPath(subpath, b['backupset'])
                    #logger.debug("Got checksum {}", str(checksum))
                    if checksum:
                        return bytes(str(checksum), 'utf-8')
            elif attr == 'user.tardis_since':
                if b:
                    since = self.tardis.getFirstBackupSet(subpath, b['backupset'])
                    #self.log.debug(str(since))
                    if since:
                        return bytes(str(since), 'utf-8')
            elif attr == 'user.tardis_chain':
                info = self.tardis.getChecksumInfoByPath(subpath, b['backupset'])
                #self.log.debug(str(checksum))
                if info:
                    chain = info['chainlength']
                    return bytes(str(chain), 'utf-8')
            else:
                # Must be an imported value.  Let's generate it.
                info = self.getFileInfoByPath(path)
                if info['xattrs']:
                    f = self.regenerator.recoverChecksum(info['xattrs'], authenticate=self.authenticate)
                    xattrs = json.loads(f.read())
                    if attr in xattrs:
                        value = base64.b64decode(xattrs[attr])
                        return bytes(str(value), 'utf-8')

        #self.log.debug("Getxattr -- default return value")
        return bytes('', 'utf-8')