def processArgs():
parser = argparse.ArgumentParser(description='Set a token/password')
parser.add_argument('--database', '-D', dest='database', default=Defaults.getDefault('TARDIS_DB'), help="Database to use. Default: %(default)s")
parser.add_argument('--client', '-C', dest='client', default=Defaults.getDefault('TARDIS_CLIENT'), help="Client to list on. Default: %(default)s")
parser.add_argument('--dbname', dest='dbname', default=Defaults.getDefault('TARDIS_DBNAME'), help="Name of the database file. Default: %(default)s")
passgroup= parser.add_argument_group("Password/Encryption specification options")
pwgroup = passgroup.add_mutually_exclusive_group()
pwgroup.add_argument('--password', '-p',dest='password', default=None, nargs='?', const=True, help='Encrypt files with this password')
pwgroup.add_argument('--password-file', dest='passwordfile', default=None, help='Read password from file')
pwgroup.add_argument('--password-url', dest='passwordurl', default=None, help='Retrieve password from the specified URL')
pwgroup.add_argument('--password-prog', dest='passwordprog', default=None, help='Use the specified command to generate the password on stdout')
return parser.parse_args()
def processArgs():
parser = argparse.ArgumentParser(description='Set a token/password')
parser.add_argument('--database', '-D', dest='database', default=Defaults.getDefault('TARDIS_DB'), help="Database to use. Default: %(default)s")
parser.add_argument('--client', '-C', dest='client', default=Defaults.getDefault('TARDIS_CLIENT'), help="Client to list on. Default: %(default)s")
parser.add_argument('--dbname', dest='dbname', default=Defaults.getDefault('TARDIS_DBNAME'), help="Name of the database file. Default: %(default)s")
passgroup= parser.add_argument_group("Password/Encryption specification options")
pwgroup = passgroup.add_mutually_exclusive_group()
pwgroup.add_argument('--password', '-p',dest='password', default=None, nargs='?', const=True, help='Encrypt files with this password')
pwgroup.add_argument('--password-file', dest='passwordfile', default=None, help='Read password from file')
pwgroup.add_argument('--password-url', dest='passwordurl', default=None, help='Retrieve password from the specified URL')
pwgroup.add_argument('--password-prog', dest='passwordprog', default=None, help='Use the specified command to generate the password on stdout')
return parser.parse_args()
def getDB(crypt, password, new=False, allowRemote=True, allowUpgrade=False):
loc = urllib.parse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
if (loc.scheme == 'http') or (loc.scheme == 'https'):
if not allowRemote:
raise Exception("This command cannot be executed remotely. You must execute it on the server directly.")
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = args.database
tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
cache = tardisdb
else:
basedir = os.path.join(args.database, args.client)
if not args.dbdir:
dbdir = os.path.join(args.database, args.client)
else:
dbdir = os.path.join(args.dbdir, args.client)
dbfile = os.path.join(dbdir, args.dbname)
if new and os.path.exists(dbfile):
raise Exception("Database for client %s already exists." % (args.client))
cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
schema = args.schema if new else None
tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, allow_upgrade=allowUpgrade)
if tardisdb.needsAuthentication():
if password is None:
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=False, confirm=False)
crypt = TardisCrypto.TardisCrypto(password, args.client)
Util.authenticate(tardisdb, args.client, password)
return (tardisdb, cache, crypt)
def getDB(crypt, password, new=False, allowRemote=True, allowUpgrade=False):
loc = urlparse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
if (loc.scheme == 'http') or (loc.scheme == 'https'):
if not allowRemote:
raise Exception("This command cannot be executed remotely. You must execute it on the server directly.")
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = args.database
tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
cache = tardisdb
else:
basedir = os.path.join(args.database, args.client)
if not args.dbdir:
dbdir = os.path.join(args.database, args.client)
else:
dbdir = os.path.join(args.dbdir, args.client)
dbfile = os.path.join(dbdir, args.dbname)
if new and os.path.exists(dbfile):
raise Exception("Database for client %s already exists." % (args.client))
cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
schema = args.schema if new else None
tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, allow_upgrade=allowUpgrade)
if tardisdb.needsAuthentication():
if password is None:
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=False, confirm=False)
Util.authenticate(tardisdb, args.client, password)
return (tardisdb, cache)
def getBackupSet(db, bset):
bsetInfo = None
# First, try as an integer
try:
bset = int(bset)
bsetInfo = db.getBackupSetInfoById(bset)
except ValueError:
# Else, let's look it up based on name
if bset == Defaults.getDefault('TARDIS_RECENT_SET') or bset == '' or bset == None:
bsetInfo = db.lastBackupSet()
else:
bsetInfo = db.getBackupSetInfo(bset)
if not bsetInfo:
# still nothing, hm, let's try a date format
cal = parsedatetime.Calendar()
(then, success) = cal.parse(bset)
if success:
timestamp = time.mktime(then)
logger.debug("Using time: %s", time.asctime(then))
bsetInfo = db.getBackupSetInfoForTime(timestamp)
if bsetInfo and bsetInfo['backupset'] != 1:
bset = bsetInfo['backupset']
logger.debug("Using backupset: %s %d for %s", bsetInfo['name'], bsetInfo['backupset'], bset)
else:
# Weed out the ".Initial" set
logger.critical("No backupset at date: %s (%s)", bset, time.asctime(then))
bsetInfo = None
else:
logger.critical("Could not parse string: %s", bset)
return bsetInfo
def getBackupSet(db, bset):
bsetInfo = None
# First, try as an integer
try:
bset = int(bset)
bsetInfo = db.getBackupSetInfoById(bset)
except ValueError:
# Else, let's look it up based on name
if bset == Defaults.getDefault('TARDIS_RECENT_SET') or bset == '' or bset == None:
bsetInfo = db.lastBackupSet()
else:
bsetInfo = db.getBackupSetInfo(bset)
if not bsetInfo:
# still nothing, hm, let's try a date format
cal = parsedatetime.Calendar()
(then, success) = cal.parse(bset)
if success:
timestamp = time.mktime(then)
logger.debug("Using time: %s", time.asctime(then))
bsetInfo = db.getBackupSetInfoForTime(timestamp)
if bsetInfo and bsetInfo['backupset'] != 1:
bset = bsetInfo['backupset']
logger.debug("Using backupset: %s %d for %s", bsetInfo['name'], bsetInfo['backupset'], bset)
else:
# Weed out the ".Initial" set
logger.critical("No backupset at date: %s (%s)", bset, time.asctime(then))
bsetInfo = None
else:
logger.critical("Could not parse string: %s", bset)
return bsetInfo
def getDB(crypt, new=False, allowRemote=True):
token = crypt.createToken() if crypt else None
loc = urlparse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
if (loc.scheme == "http") or (loc.scheme == "https"):
if not allowRemote:
raise Exception("This command cannot be executed remotely. You must execute it on the server directly.")
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault("TARDIS_REMOTE_PORT")
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = args.database
tardisdb = RemoteDB.RemoteDB(dbLoc, args.client, token=token)
cache = tardisdb
else:
basedir = os.path.join(args.database, args.client)
if not args.dbdir:
dbdir = os.path.join(args.database, args.client)
else:
dbdir = os.path.join(args.dbdir, args.client)
dbfile = os.path.join(dbdir, args.dbname)
if new and os.path.exists(dbfile):
raise Exception("Database for client %s already exists." % (args.client))
cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
schema = args.schema if new else None
tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, token=token)
return (tardisdb, cache)
def main():
global args, logger
tardis = None
try:
args = processArgs()
logger = Util.setupLogging(args.verbose)
setColors(Defaults.getDefault('TARDIS_LS_COLORS'))
# Load any password info
password = Util.getPassword(args.password,
args.passwordfile,
args.passwordprog,
prompt="Password for %s: " % (args.client))
args.password = None
(tardis, _, crypt) = Util.setupDataConnection(args.database,
args.client, password,
args.keys, args.dbname,
args.dbdir)
setupDisplay(tardis)
if args.headers:
doprint("Client: %s DB: %s" % (args.client, args.database),
color=colors['name'],
eol=True)
if args.glob:
directories = []
for d in args.directories:
if not Util.isMagic(d):
directories.append(d)
else:
directories += globPath(os.path.abspath(d), tardis, crypt)
else:
directories = args.directories
for d in directories:
d = unicode(os.path.abspath(d).decode(fsEncoding))
if args.realpath:
d = os.path.realpath(d)
fInfos = collectFileInfo(d, tardis, crypt)
recurse = args.maxdepth if args.recurse else 0
processFile(d,
fInfos,
tardis,
crypt,
printContents=(not args.dirinfo),
recurse=recurse)
except KeyboardInterrupt:
pass
except Exception as e:
logger.error("Caught exception: %s", str(e))
if args.exceptions:
logger.exception(e)
finally:
if tardis:
tardis.close()
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
""" Setup a data connection to a client. Determines the correct way to connect, either via direct filesystem,
or via TardisRemote (http).
Returns a 3-tuple, the TardisDB object, the CacheDir object, and the appropriate crypto object
"""
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urllib.parse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)
if needsAuth:
authenticate(tardis, client, password)
elif password:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
#cryptoScheme = tardis.getConfigValue('CryptoScheme', '1')
cryptoScheme = tardis.getCryptoScheme()
crypt = TardisCrypto.getCrypto(cryptoScheme, password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
if retpassword:
return (tardis, cache, crypt, password)
else:
return (tardis, cache, crypt)
def checkPasswordStrength(password):
pwStrMin = float(Defaults.getDefault('TARDIS_PW_STRENGTH'))
strength, improvements = passwordmeter.test(password)
if strength < pwStrMin:
logger.error("Password too weak: %f (%f required)", strength, pwStrMin)
for i in improvements:
logger.error(" %s", improvements[i])
return False
else:
return True
def checkPasswordStrength(password):
pwStrMin = float(Defaults.getDefault('TARDIS_PW_STRENGTH'))
strength, improvements = passwordmeter.test(password)
if strength < pwStrMin:
logger.error("Password too weak: %f (%f required)", strength, pwStrMin)
for i in improvements:
logger.error(" %s", improvements[i])
return False
else:
return True
def __init__(self, password, client=None):
self._random = Cryptodome.Random.new()
if client is None:
client = Defaults.getDefault('TARDIS_CLIENT')
self.client = client
self.salt = hashlib.sha256(client).digest()
keys = PBKDF2(password, self.salt, count=20000, dkLen=self._keysize * 2) # 2x256 bit keys
self._keyKey = keys[0:self._keysize] # First 256 bit key
self._tokenKey = keys[self._keysize:] # And the other one
def setupDataConnection(dataLoc,
client,
password,
keyFile,
dbName,
dbLoc=None,
allow_upgrade=False):
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault(
'TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path,
loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client)
if password:
if needsAuth:
authenticate(tardis, client, password)
else:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
crypt = TardisCrypto.TardisCrypto(password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def __init__(self, password, client=None, fsencoding=sys.getfilesystemencoding()):
self._random = Cryptodome.Random.new()
if client is None:
client = Defaults.getDefault('TARDIS_CLIENT')
self.client = bytes(client, 'utf8')
self.salt = hashlib.sha256(self.client).digest()
keys = self.genKeyKey(password)
self._keyKey = keys[0:self._keysize] # First 256 bit key
self._fsEncoding = fsencoding
def __init__(self, password, client=None, fsencoding=sys.getfilesystemencoding()):
self._random = Cryptodome.Random.new()
if client is None:
client = Defaults.getDefault('TARDIS_CLIENT')
self.client = bytes(client, 'utf8')
self.salt = hashlib.sha256(self.client).digest()
keys = PBKDF2(password, self.salt, count=20000, dkLen=self._keysize * 2) # 2x256 bit keys
self._keyKey = keys[0:self._keysize] # First 256 bit key
self._tokenKey = keys[self._keysize:] # And the other one
self._fsEncoding = fsencoding
def main():
global args, logger
tardis = None
try:
args = processArgs()
logger = Util.setupLogging(args.verbose)
setColors(Defaults.getDefault('TARDIS_LS_COLORS'))
# Load any password info
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client))
args.password = None
(tardis, _, crypt) = Util.setupDataConnection(args.database, args.client, password, args.keys, args.dbname, args.dbdir)
setupDisplay(tardis)
if args.headers:
doprint("Client: %s DB: %s" %(args.client, args.database), color=colors['name'], eol=True)
if args.glob:
directories = []
for d in args.directories:
if not Util.isMagic(d):
directories.append(d)
else:
directories += globPath(os.path.abspath(d), tardis, crypt)
else:
directories = args.directories
for d in directories:
d = os.path.abspath(d)
if args.realpath:
d = os.path.realpath(d)
fInfos = collectFileInfo(d, tardis, crypt)
recurse = args.maxdepth if args.recurse else 0
processFile(d, fInfos, tardis, crypt, printContents=(not args.dirinfo), recurse=recurse)
except KeyboardInterrupt:
pass
except TardisDB.AuthenticationException as e:
logger.error("Authentication failed. Bad password")
if args.exceptions:
logger.exception(e)
except Exception as e:
logger.error("Caught exception: %s", str(e))
if args.exceptions:
logger.exception(e)
finally:
if tardis:
tardis.close()
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urllib.parse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)
if password:
if needsAuth:
authenticate(tardis, client, password)
else:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
crypt = TardisCrypto.TardisCrypto(password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
if retpassword:
return (tardis, cache, crypt, password)
else:
return (tardis, cache, crypt)
def __init__(self,
password,
client=None,
fsencoding=sys.getfilesystemencoding()):
self._random = Cryptodome.Random.new()
if client is None:
client = Defaults.getDefault('TARDIS_CLIENT')
self.client = client
self.salt = hashlib.sha256(client).digest()
keys = PBKDF2(password,
self.salt,
count=20000,
dkLen=self._keysize * 2) # 2x256 bit keys
self._keyKey = keys[0:self._keysize] # First 256 bit key
self._tokenKey = keys[self._keysize:] # And the other one
self._fsEncoding = fsencoding
def main():
global args, logger
try:
FORMAT = "%(levelname)s : %(message)s"
logging.basicConfig(stream=sys.stderr, format=FORMAT, level=logging.INFO)
logger = logging.getLogger("")
args = processArgs()
setColors(Defaults.getDefault('TARDIS_LS_COLORS'))
# Load any password info
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client))
args.password = None
(tardis, _, crypt) = Util.setupDataConnection(args.database, args.client, password, args.keys, args.dbname, args.dbdir)
setupDisplay(tardis)
if args.headers:
doprint("Client: %s DB: %s" %(args.client, args.database), color=colors['name'], eol=True)
if args.glob:
directories = []
for d in args.directories:
if not Util.isMagic(d):
directories.append(d)
else:
directories += globPath(os.path.abspath(d), tardis, crypt)
else:
directories = args.directories
for d in directories:
d = os.path.abspath(d)
if args.realpath:
d = os.path.realpath(d)
fInfos = collectFileInfo(d, tardis, crypt)
recurse = args.maxdepth if args.recurse else 0
processFile(d, fInfos, tardis, crypt, printContents=(not args.dirinfo), recurse=recurse)
except KeyboardInterrupt:
pass
except Exception as e:
logger.error("Caught exception: %s", str(e))
logger.exception(e)
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False):
crypt = None
if password:
crypt = TardisCrypto.TardisCrypto(password, client)
password = None
token = None
if crypt:
token = crypt.createToken()
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
cache = tardis
else:
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, token=token, allow_upgrade=allow_upgrade)
if crypt:
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None):
crypt = None
if password:
crypt = TardisCrypto.TardisCrypto(password, client)
password = None
token = None
if crypt:
token = crypt.createToken()
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == "http") or (loc.scheme == "https"):
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault("TARDIS_REMOTE_PORT")
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
# get the RemoteURL object
tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
cache = tardis
else:
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, token=token)
if crypt:
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue("ClientID"))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import sys
import configparser
import Tardis.Defaults as Defaults
import Tardis.Util as Util
import Tardis.TardisCrypto as TardisCrypto
configDefaults = {
'Database': Defaults.getDefault('TARDIS_DB'),
'Client': Defaults.getDefault('TARDIS_CLIENT'),
'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
'DBName': Defaults.getDefault('TARDIS_DBNAME'),
'Password': None,
'PasswordFile': Defaults.getDefault('TARDIS_PWFILE'),
'PasswordProg': None,
'Crypt': str(True),
'KeyFile': Defaults.getDefault('TARDIS_KEYFILE'),
'LogFiles': None,
'Verbosity': str(0),
'Schema': Defaults.getDefault('TARDIS_SCHEMA')
}
config = configparser.ConfigParser(configDefaults, allow_no_value=True)
job = None
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import sys
import ConfigParser
import Tardis.Defaults as Defaults
import Tardis.Util as Util
configDefaults = {
'Database': Defaults.getDefault('TARDIS_DB'),
'Client': Defaults.getDefault('TARDIS_CLIENT'),
'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
'DBName': Defaults.getDefault('TARDIS_DBNAME'),
'Password': None,
'PasswordFile': None,
'PasswordProg': None,
'Crypt': str(True),
'KeyFile': None,
'LogFiles': None,
'Verbosity': str(0),
'Schema': Defaults.getDefault('TARDIS_SCHEMA')
}
config = ConfigParser.ConfigParser(configDefaults)
job = None
import pprint
import urlparse
import parsedatetime
import passwordmeter
import Tardis
import Tardis.Util as Util
import Tardis.Defaults as Defaults
import Tardis.TardisDB as TardisDB
import Tardis.TardisCrypto as TardisCrypto
import Tardis.CacheDir as CacheDir
import Tardis.RemoteDB as RemoteDB
import Tardis.Config as Config
current = Defaults.getDefault('TARDIS_RECENT_SET')
pwStrMin = Defaults.getDefault('TARDIS_PW_STRENGTH')
# Config keys which can be gotten or set.
configKeys = ['Formats', 'Priorities', 'KeepDays', 'ForceFull', 'SaveFull', 'MaxDeltaChain', 'MaxChangePercent', 'VacuumInterval', 'AutoPurge', 'Disabled']
# Extra keys that we print when everything is requested
sysKeys = ['ClientID', 'SchemaVersion', 'FilenameKey', 'ContentKey']
minPwStrength = 0
logger = None
args = None
def getDB(crypt, new=False, allowRemote=True):
token = crypt.createToken() if crypt else None
loc = urlparse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
d = f.read(128 * 1024)
res = m.hexdigest()
if res != checksum:
print("Checksums don't match. Expected: %s, result %s" % (checksum, res))
checked[checksum] = 0
output.write(checksum + '\n')
output.flush()
else:
checked[checksum] = 1
valid.write(checksum + "\n")
except Exception as e:
print("Caught exception processing %s: %s" % (checksum, str(e)))
output.write(checksum + '\n')
output.flush()
row = cur.fetchone()
except sqlite3.OperationalError as e:
print("Caught operational error. DB is probably locked. Sleeping for a bit")
time.sleep(90)
pbar.finish()
if __name__ == "__main__":
root = Defaults.getDefault('TARDIS_DB')
client = Defaults.getDefault('TARDIS_CLIENT')
dbname = Defaults.getDefault('TARDIS_DBNAME')
password = None # 'PassWord'
logging.basicConfig(level=logging.INFO)
validate(root, client, dbname, password)
import time
import termcolor
import parsedatetime
import binaryornot.check
import Tardis
import Tardis.Util as Util
import Tardis.Regenerator as Regenerator
import Tardis.Defaults as Defaults
import Tardis.Config as Config
logger = None
args = None
current = Defaults.getDefault("TARDIS_RECENT_SET")
def parseArgs():
isatty = os.isatty(sys.stdout.fileno())
global args
parser = argparse.ArgumentParser(
description="Diff files between current and a Tardis backup, or multiple Tardis versions",
fromfile_prefix_chars="@",
formatter_class=Util.HelpFormatter,
add_help=False,
)
(args, remaining) = Config.parseConfigOptions(parser)
Config.addCommonOptions(parser)
import logging
import time
import termcolor
import parsedatetime
import Tardis
import Tardis.Util as Util
import Tardis.Regenerator as Regenerator
import Tardis.Defaults as Defaults
import Tardis.Config as Config
logger = None
args = None
current = Defaults.getDefault('TARDIS_RECENT_SET')
def parseArgs():
isatty = os.isatty(sys.stdout.fileno())
global args
parser = argparse.ArgumentParser(description='Diff files between current and a Tardis backup, or multiple Tardis versions', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatter, add_help=False)
(args, remaining) = Config.parseConfigOptions(parser)
Config.addCommonOptions(parser)
Config.addPasswordOptions(parser)
parser.add_argument("--backup", '-b', nargs='+', dest='backup', default=[current], help="Backup set(s) to use (Default: %(default)s)")
parser.add_argument('--color', dest='color', default=isatty, action=Util.StoreBoolean, help='Use colors')
import termcolor
import parsedatetime
import binaryornot.check
import Tardis
import Tardis.Util as Util
import Tardis.Regenerator as Regenerator
import Tardis.Defaults as Defaults
import Tardis.Config as Config
import Tardis.TardisDB as TardisDB
logger = None
exceptionLogger = None
args = None
current = Defaults.getDefault('TARDIS_RECENT_SET')
def parseArgs():
isatty = os.isatty(sys.stdout.fileno())
global args
parser = argparse.ArgumentParser(description='Diff files between current and a Tardis backup, or multiple Tardis versions', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatter, add_help=False)
(args, remaining) = Config.parseConfigOptions(parser)
Config.addCommonOptions(parser)
Config.addPasswordOptions(parser)
parser.add_argument("--backup", '-b', nargs='+', dest='backup', default=[current], help="Backup set(s) to use (Default: %(default)s)")
parser.add_argument('--color', dest='color', default=isatty, action=Util.StoreBoolean, help='Use colors')
import pprint
import urlparse
import parsedatetime
import passwordmeter
import Tardis
import Tardis.Util as Util
import Tardis.Defaults as Defaults
import Tardis.TardisDB as TardisDB
import Tardis.TardisCrypto as TardisCrypto
import Tardis.CacheDir as CacheDir
import Tardis.RemoteDB as RemoteDB
import Tardis.Config as Config
current = Defaults.getDefault("TARDIS_RECENT_SET")
pwStrMin = Defaults.getDefault("TARDIS_PW_STRENGTH")
# Config keys which can be gotten or set.
configKeys = [
"Formats",
"Priorities",
"KeepDays",
"ForceFull",
"SaveFull",
"MaxDeltaChain",
"MaxChangePercent",
"VacuumInterval",
"AutoPurge",
"Disabled",
]
import pprint
import urlparse
import parsedatetime
import passwordmeter
import Tardis
import Tardis.Util as Util
import Tardis.Defaults as Defaults
import Tardis.TardisDB as TardisDB
import Tardis.TardisCrypto as TardisCrypto
import Tardis.CacheDir as CacheDir
import Tardis.RemoteDB as RemoteDB
import Tardis.Config as Config
current = Defaults.getDefault('TARDIS_RECENT_SET')
pwStrMin = Defaults.getDefault('TARDIS_PW_STRENGTH')
# Config keys which can be gotten or set.
configKeys = [
'Formats', 'Priorities', 'KeepDays', 'ForceFull', 'SaveFull',
'MaxDeltaChain', 'MaxChangePercent', 'VacuumInterval', 'AutoPurge',
'Disabled', 'SaveConfig'
]
# Extra keys that we print when everything is requested
sysKeys = ['ClientID', 'SchemaVersion', 'FilenameKey', 'ContentKey']
minPwStrength = 0
logger = None
args = None
def processArgs():
parser = argparse.ArgumentParser(description='Encrypt the database')
parser.add_argument('--database',
'-D',
dest='database',
default=Defaults.getDefault('TARDIS_DB'),
help="Database to use. Default: %(default)s")
parser.add_argument('--client',
'-C',
dest='client',
default=Defaults.getDefault('TARDIS_CLIENT'),
help="Client to list on. Default: %(default)s")
parser.add_argument('--dbname',
dest='dbname',
default=Defaults.getDefault('TARDIS_DBNAME'),
help="Name of the database file. Default: %(default)s")
parser.add_argument('--filenames',
dest='filenames',
action='store_true',
default=False,
help='Encrypt filenames. Default=%(default)s')
parser.add_argument('--files',
dest='files',
action='store_true',
default=False,
help='Encrypt files. Default=%(default)s')
parser.add_argument('--dirhashes',
dest='dirhash',
action='store_true',
default=False,
help='Generate directory hashes. Default=%(default)s')
parser.add_argument('--meta',
dest='meta',
action='store_true',
default=False,
help='Generate metadata files. Default=%(default)s')
#parser.add_argument('--signatures', dest='sigs', action='store_true', default=False, help='Generate signatures. Default=%(default)s')
passgroup = parser.add_argument_group(
"Password/Encryption specification options")
pwgroup = passgroup.add_mutually_exclusive_group(required=True)
pwgroup.add_argument('--password',
'-P',
dest='password',
default=None,
nargs='?',
const=True,
help='Encrypt files with this password')
pwgroup.add_argument('--password-file',
dest='passwordfile',
default=None,
help='Read password from file')
pwgroup.add_argument('--password-url',
dest='passwordurl',
default=None,
help='Retrieve password from the specified URL')
pwgroup.add_argument(
'--password-prog',
dest='passwordprog',
default=None,
help='Use the specified command to generate the password on stdout')
return parser.parse_args()
def getPassword(password, pwurl, pwprog, prompt='Password: '******'TARDIS_PWTIMEOUT')):
methods = 0
if password: methods += 1
if pwurl: methods += 1
if pwprog: methods += 1
if methods > 1:
raise Exception("Cannot specify more than one password retrieval mechanism")
if methods == 0 and not allowNone:
# Nothing specified, and it wants a value. Set password to True to fetch
password = True
if password == True or password == '':
password = _readWithTimeout(prompt, int(timeout))
password = password.rstrip() # Delete trailing characters
if confirm:
pw2 = _readWithTimeout("Confirm password:"******"Passwords don't match")
if pwurl:
loc = urllib.parse.urlunparse(urllib.parse.urlparse(pwurl, scheme='file'))
pwf = urllib.request.urlopen(loc)
password = pwf.readline().rstrip()
pwf.close()
if pwprog:
a = shlex.split(pwprog)
output = subprocess.check_output(a)
password = output.split('\n')[0].rstrip()
if not allowNone and not password:
raise Exception("Password required")
if strength and password:
if not checkPasswordStrength(password):
raise Exception("Password not strong enough")
return password
def parseArgs():
parser = argparse.ArgumentParser(description='Recover Backed Up Files', fromfile_prefix_chars='@', formatter_class=Util.HelpFormatter, add_help=False)
(_, remaining) = Config.parseConfigOptions(parser)
Config.addCommonOptions(parser)
Config.addPasswordOptions(parser)
parser.add_argument("--output", "-o", dest="output", help="Output file", default=None)
parser.add_argument("--checksum", "-c", help="Use checksum instead of filename", dest='cksum', action='store_true', default=False)
bsetgroup = parser.add_mutually_exclusive_group()
bsetgroup.add_argument("--backup", "-b", help="Backup set to use. Default: %(default)s", dest='backup', default=Defaults.getDefault('TARDIS_RECENT_SET'))
bsetgroup.add_argument("--date", "-d", help="Regenerate as of date", dest='date', default=None)
bsetgroup.add_argument("--last", "-l", dest='last', default=False, action='store_true', help="Regenerate the most recent version of the file")
parser.add_argument('--recurse', dest='recurse', default=True, action=Util.StoreBoolean, help='Recurse directory trees. Default: %(default)s')
parser.add_argument('--recovername', dest='recovername', default=False, action=Util.StoreBoolean, help='Recover the name when recovering a checksum. Default: %(default)s')
parser.add_argument('--authenticate', dest='auth', default=True, action=Util.StoreBoolean, help='Authenticate files while regenerating them. Default: %(default)s')
parser.add_argument('--authfail-action', dest='authfailaction', default='rename', choices=['keep', 'rename', 'delete'], help='Action to take for files that do not authenticate. Default: %(default)s')
parser.add_argument('--reduce-path', '-R', dest='reduce', default=0, const=sys.maxsize, type=int, nargs='?', metavar='N',
help='Reduce path by N directories. No value for "smart" reduction')
parser.add_argument('--set-times', dest='settime', default=True, action=Util.StoreBoolean, help='Set file times to match original file. Default: %(default)s')
parser.add_argument('--set-perms', dest='setperm', default=True, action=Util.StoreBoolean, help='Set file owner and permisions to match original file. Default: %(default)s')
parser.add_argument('--set-attrs', dest='setattrs', default=True, action=Util.StoreBoolean, help='Set file extended attributes to match original file. May only set attributes in user space. Default: %(default)s')
parser.add_argument('--set-acl', dest='setacl', default=True, action=Util.StoreBoolean, help='Set file access control lists to match the original file. Default: %(default)s')
parser.add_argument('--overwrite', '-O', dest='overwrite', default=owModeDefault, const='always', nargs='?',
choices=['always', 'newer', 'older', 'never', 'ask'],
help='Mode for handling existing files. Default: %(default)s')
parser.add_argument('--hardlinks', dest='hardlinks', default=True, action=Util.StoreBoolean, help='Create hardlinks of multiple copies of same inode created. Default: %(default)s')
parser.add_argument('--exceptions', default=False, action=Util.StoreBoolean, dest='exceptions', help="Log full exception data");
parser.add_argument('--verbose', '-v', action='count', default=0, dest='verbose', help='Increase the verbosity')
parser.add_argument('--version', action='version', version='%(prog)s ' + Tardis.__versionstring__, help='Show the version')
parser.add_argument('--help', '-h', action='help')
parser.add_argument('files', nargs='+', default=None, help="List of files to regenerate")
Util.addGenCompletions(parser)
return parser.parse_args(remaining)
d = f.read(128 * 1024)
res = m.hexdigest()
if res != checksum:
print("Checksums don't match. Expected: %s, result %s" % (checksum, res))
checked[checksum] = 0
output.write(checksum + '\n')
output.flush()
else:
checked[checksum] = 1
valid.write(checksum + "\n")
except Exception as e:
print("Caught exception processing %s: %s" % (checksum, str(e)))
output.write(checksum + '\n')
output.flush()
row = cur.fetchone()
except sqlite3.OperationalError as e:
print("Caught operational error. DB is probably locked. Sleeping for a bit")
time.sleep(90)
pbar.finish()
if __name__ == "__main__":
root = Defaults.getDefault('TARDIS_DB')
client = Defaults.getDefault('TARDIS_CLIENT')
dbname = Defaults.getDefault('TARDIS_DBNAME')
password = None # 'PassWord'
logging.basicConfig(level=logging.INFO)
validate(root, client, dbname, password)
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import sys
import ConfigParser
import Tardis.Defaults as Defaults
import Tardis.Util as Util
configDefaults = {
'Database': Defaults.getDefault('TARDIS_DB'),
'Client': Defaults.getDefault('TARDIS_CLIENT'),
'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
'DBName': Defaults.getDefault('TARDIS_DBNAME'),
'Password': None,
'PasswordFile': None,
'PasswordProg': None,
'Crypt': str(True),
'KeyFile': None,
'LogFiles': None,
'Verbosity': str(0),
'Schema': Defaults.getDefault('TARDIS_SCHEMA')
}
config = ConfigParser.ConfigParser(configDefaults)
job = None
def __init__(self, *args, **kw):
super(TardisFS, self).__init__(*args, **kw)
try:
client = Defaults.getDefault('TARDIS_CLIENT')
database = Defaults.getDefault('TARDIS_DB')
dbdir = Defaults.getDefault('TARDIS_DBDIR') % {
'TARDIS_DB': database
} # HACK
dbname = Defaults.getDefault('TARDIS_DBNAME')
current = Defaults.getDefault('TARDIS_RECENT_SET')
# Parameters
self.database = database
self.client = client
self.repoint = False
self.password = None
self.pwfile = None
self.pwprog = None
self.keys = None
self.dbname = dbname
self.dbdir = dbdir
self.cachetime = 60
self.nocrypt = False
self.noauth = False
self.current = current
self.authenticate = True
self.crypt = None
logging.basicConfig(level=logging.WARNING)
self.log = logging.getLogger("TardisFS")
self.parser.add_option(
"--database",
'-D',
help="Path to the Tardis database directory")
self.parser.add_option("--client",
'-C',
help="Client to load database for")
self.parser.add_option(
"--password",
'-P',
help=
"Password for this archive (use '-o password='******'-o password='******'nocrypt',
help="Disable encryption")
self.parser.add_option(mountopt='noauth',
help="Disable authentication")
self.parser.add_option(
mountopt='current',
help="Name to use for most recent complete backup")
res = self.parse(values=self, errex=1)
self.mountpoint = res.mountpoint
self.name = "TardisFS:<{}/{}>".format(self.database, self.client)
password = Util.getPassword(self.password,
self.pwfile,
self.pwprog,
prompt="Password for %s: " %
(self.client))
self.password = None
self.cache = Cache.Cache(0, float(self.cachetime))
self.fileCache = Cache.Cache(0, float(self.cachetime), 'FileCache')
#if password:
# self.crypt = TardisCrypto.TardisCrypto(password, self.client)
(self.tardis, self.cacheDir,
self.crypt) = Util.setupDataConnection(self.database, self.client,
password, self.keys,
self.dbname, self.dbdir)
password = None
# Remove the crypto object if not encyrpting files.
if self.nocrypt or self.nocrypt is None:
self.crypt = None
if self.noauth or self.noauth is None:
self.authenticate = False
# Create a regenerator.
self.regenerator = Regenerator.Regenerator(self.cacheDir,
self.tardis,
crypt=self.crypt)
self.files = {}
# Fuse variables
self.flags = 0
self.multithreaded = 0
except Exception as e:
self.log.exception(e)
sys.exit(2)
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import sys
import configparser
import Tardis.Defaults as Defaults
import Tardis.Util as Util
configDefaults = {
'Database': Defaults.getDefault('TARDIS_DB'),
'Client': Defaults.getDefault('TARDIS_CLIENT'),
'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
'DBName': Defaults.getDefault('TARDIS_DBNAME'),
'Password': None,
'PasswordFile': Defaults.getDefault('TARDIS_PWFILE'),
'PasswordProg': None,
'Crypt': str(True),
'KeyFile': Defaults.getDefault('TARDIS_KEYFILE'),
'LogFiles': None,
'Verbosity': str(0),
'Schema': Defaults.getDefault('TARDIS_SCHEMA')
}
config = configparser.ConfigParser(configDefaults, allow_no_value=True)
job = None
import zlib
import daemonize
import base64
from flask import Flask, Response, session, request, url_for, abort, redirect, make_response
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
import Tardis
import Tardis.TardisDB as TardisDB
import Tardis.Util as Util
import Tardis.CacheDir as CacheDir
import Tardis.Defaults as Defaults
basedir = Defaults.getDefault('TARDIS_DB')
dbname = Defaults.getDefault('TARDIS_DBNAME')
port = Defaults.getDefault('TARDIS_REMOTE_PORT')
configName = Defaults.getDefault('TARDIS_REMOTE_CONFIG')
pidFile = Defaults.getDefault('TARDIS_REMOTE_PIDFILE')
configDefaults = {
'Port' : port,
'Database' : basedir,
'DBName' : dbname,
'LogFile' : None,
'LogExceptions' : str(False),
'Verbose' : '0',
'Daemon' : str(False),
'User' : None,
'Group' : None,
import zlib
import daemonize
import base64
from flask import Flask, Response, session, request, url_for, abort, redirect, make_response
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
import Tardis
import Tardis.TardisDB as TardisDB
import Tardis.Util as Util
import Tardis.CacheDir as CacheDir
import Tardis.Defaults as Defaults
basedir = Defaults.getDefault('TARDIS_DB')
dbname = Defaults.getDefault('TARDIS_DBNAME')
port = Defaults.getDefault('TARDIS_REMOTE_PORT')
configName = Defaults.getDefault('TARDIS_REMOTE_CONFIG')
pidFile = Defaults.getDefault('TARDIS_REMOTE_PIDFILE')
configDefaults = {
'Port' : port,
'Database' : basedir,
'DBName' : dbname,
'LogFile' : '',
'LogExceptions' : str(False),
'Verbose' : '0',
'Daemon' : str(False),
'User' : '',
'Group' : '',
def __init__(self, *args, **kw):
super(TardisFS, self).__init__(*args, **kw)
try:
client = Defaults.getDefault('TARDIS_CLIENT')
database = Defaults.getDefault('TARDIS_DB')
dbdir = Defaults.getDefault('TARDIS_DBDIR') % { 'TARDIS_DB': database } # HACK
dbname = Defaults.getDefault('TARDIS_DBNAME')
current = Defaults.getDefault('TARDIS_RECENT_SET')
# Parameters
self.database = database
self.client = client
self.repoint = False
self.password = None
self.pwfile = None
self.pwprog = None
self.keys = None
self.dbname = dbname
self.dbdir = dbdir
self.cachetime = 60
self.nocrypt = False
self.noauth = False
self.current = current
self.authenticate = True
self.crypt = None
logging.basicConfig(level=logging.WARNING)
self.log = logging.getLogger("TardisFS")
self.parser.add_option("--database", '-D', help="Path to the Tardis database directory")
self.parser.add_option("--client", '-C', help="Client to load database for")
self.parser.add_option("--password", '-P', help="Password for this archive (use '-o password='******'-o password='******'nocrypt', help="Disable encryption")
self.parser.add_option(mountopt='noauth', help="Disable authentication")
self.parser.add_option(mountopt='current', help="Name to use for most recent complete backup")
res = self.parse(values=self, errex=1)
self.mountpoint = res.mountpoint
self.name = "TardisFS:<{}/{}>".format(self.database, self.client)
password = Util.getPassword(self.password, self.pwfile, self.pwprog, prompt="Password for %s: " % (self.client))
self.password = None
self.cache = Cache.Cache(0, float(self.cachetime))
self.fileCache = Cache.Cache(0, float(self.cachetime), 'FileCache')
#if password:
# self.crypt = TardisCrypto.TardisCrypto(password, self.client)
(self.tardis, self.cacheDir, self.crypt) = Util.setupDataConnection(self.database, self.client, password, self.keys, self.dbname, self.dbdir)
password = None
# Remove the crypto object if not encyrpting files.
if self.nocrypt or self.nocrypt is None:
self.crypt = None
if self.noauth or self.noauth is None:
self.authenticate = False
# Create a regenerator.
self.regenerator = Regenerator.Regenerator(self.cacheDir, self.tardis, crypt=self.crypt)
self.files = {}
# Fuse variables
self.flags = 0
self.multithreaded = 0
except Exception as e:
self.log.exception(e)
sys.exit(2)
class TardisFS(LoggingMixIn, Operations):
"""
FUSE filesystem to read data from a Tardis Backup Database
"""
# Disable pylint complaints about "could me a function" and "unused argument" as lots of required FUSE functions
# just return "read-only FS" status
# pragma pylint: disable=no-self-use,unused-argument
backupsets = {}
dirInfo = {}
fsencoding = sys.getfilesystemencoding()
name = "TardisFS"
client = Defaults.getDefault('TARDIS_CLIENT')
database = Defaults.getDefault('TARDIS_DB')
dbdir = Defaults.getDefault('TARDIS_DBDIR') % { 'TARDIS_DB': database } # HACK
dbname = Defaults.getDefault('TARDIS_DBNAME')
current = Defaults.getDefault('TARDIS_RECENT_SET')
def __init__(self, db, cache, crypto, args):
self.cacheDir = cache
self.crypt = crypto
self.tardis = db
# Create a regenerator.
self.regenerator = Regenerator.Regenerator(self.cacheDir, self.tardis, crypt=self.crypt)
self.files = {}
# Set up some caches.
self.cachetime = 60
self.cache = Cache.Cache(0, float(self.cachetime))
self.fileCache = Cache.Cache(0, float(self.cachetime), 'FileCache')
self.authenticate = True
def __del__(self):
if self.tardis:
self.tardis.close()
def __repr__(self):
return self.name
def fsEncodeName(self, name):
return name
def getBackupSetInfo(self, b):
key = (_BackupSetInfo, b)
info = self.cache.retrieve(key)
if info:
return info
info = self.tardis.getBackupSetInfo(b)
self.cache.insert(key, info)
return info
def lastBackupSet(self, completed):
key = (_LastBackupSet, completed)
backupset = self.cache.retrieve(key)
if backupset:
return backupset
backupset = self.tardis.lastBackupSet(completed=completed)
self.cache.insert(key, backupset)
return backupset
def getDirInfo(self, path):
""" Return the inode and backupset of a directory """
#self.log.info("getDirInfo: %s", path)
key = (_DirInfo, path)
info = self.cache.retrieve(key)
if info:
return info
#self.log.debug("No cache info available for %s", path)
parts = getParts(path)
bsInfo = self.getBackupSetInfo(parts[0])
if len(parts) == 2:
subpath = parts[1]
if self.crypt:
subpath = self.crypt.encryptPath(subpath)
#fInfo = self.getFileInfoByPath(subpath, bsInfo['backupset'])
fInfo = self.getFileInfoByPath(path)
#self.log.info("fInfo %s %s %s", parts[1], "**", str(fInfo))
info = (bsInfo, fInfo)
else:
fInfo = {'inode': 0, 'device': 0, 'dir': 1}
info = (bsInfo, fInfo)
if info:
self.cache.insert(key, info)
return info
def getFileInfoByPath(self, path):
#self.log.info("getFileInfoByPath: %s", path)
# First, check the cache
f = self.fileCache.retrieve(path)
if f:
#self.log.debug("getFileInfoByPath: %s found in cache", path)
return f
# Not in the cache, look things up
#self.log.debug("File info for %s not in cache", path)
(head, tail) = os.path.split(path)
data = self.getDirInfo(head)
if data:
bsInfo, dInfo = data
else:
return None
if bsInfo:
if self.crypt:
tail = self.crypt.encryptPath(tail)
#self.log.debug(str(dInfo))
f = self.tardis.getFileInfoByName(tail, (dInfo['inode'], dInfo['device']), bsInfo['backupset'])
else:
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
subpath = parts[1]
if self.crypt:
subpath = self.crypt.encryptPath(subpath)
#self.log.debug("getFileInfoByPath: %s=>%s", parts[1], subpath)
f = self.tardis.getFileInfoByPath(subpath, b['backupset'])
# Cache it.
self.fileCache.insert(path, f)
# Return it
return f
#@tracer
def getattr(self, path, fh=None):
"""
- st_mode (protection bits)
- st_ino (inode number)
- st_dev (device)
- st_nlink (number of hard links)
- st_uid (user ID of owner)
- st_gid (group ID of owner)
- st_size (size of file, in bytes)
- st_atime (time of most recent access)
- st_mtime (time of most recent content modification)
- st_ctime (platform dependent; time of most recent metadata change on Unix,
or the time of creation on Windows).
"""
#self.log.info("CALL getattr: %s", path)
path = self.fsEncodeName(path)
depth = getDepth(path) # depth of path, zero-based from root
if depth == 0:
# Fake the root
target = self.lastBackupSet(False)
timestamp = float(target['starttime'])
st = {
'st_mode': stat.S_IFDIR | 0o555,
'st_ino': 0,
'st_dev': 0,
'st_nlink': 32,
'st_uid': 0,
'st_gid': 0,
'st_size': 4096,
'st_atime': timestamp,
'st_mtime': timestamp,
'st_ctime': timestamp,
}
return st
elif depth == 1:
# Root directory contents
lead = getParts(path)
if lead[0] == self.current:
target = self.lastBackupSet(True)
timestamp = float(target['endtime'])
st = {
'st_mode': stat.S_IFLNK | 0o755,
'st_ino': 1,
'st_dev': 0,
'st_nlink': 1,
'st_uid': 0,
'st_gid': 0,
'st_size': 4096,
'st_atime': timestamp,
'st_mtime': timestamp,
'st_ctime': timestamp
}
return st
else:
f = self.getBackupSetInfo(lead[0])
#self.log.debug("Got backupset info for %s: %s", lead[0], str(f))
if f:
timestamp = float(f['starttime'])
st = {
'st_mode': stat.S_IFDIR | 0o555,
'st_ino': int(float(f['starttime'])),
'st_dev': 0,
'st_nlink': 2,
'st_uid': 0,
'st_gid': 0,
'st_size': 4096,
'st_atime': timestamp,
'st_mtime': timestamp,
'st_ctime': timestamp
}
return st
else:
f = self.getFileInfoByPath(path)
if f:
st = {
'st_mode': f["mode"],
'st_ino': f["inode"],
'st_dev': 0,
'st_nlink': f["nlinks"],
'st_uid': f["uid"],
'st_gid': f["gid"],
'st_atime': f["mtime"],
'st_mtime': f["mtime"],
'st_ctime': f["ctime"]
}
if f["size"] is not None:
st['st_size'] = int(f["size"])
elif f["dir"]:
st['st_size'] = 4096 # Arbitrary number
else:
st['st_size'] = 0
return st
logger.debug("File not found: %s", path)
raise FuseOSError(errno.ENOENT)
#@tracer
#def getdir(self, _, fh):
#"""
#return: [[('file1', 0), ('file2', 0), ... ]]
#"""
##self.log.info('CALL getdir {}'.format(path))
#raise FuseOSError(errno.ENOSYS)
#@tracer
def readdir(self, path, offset):
#self.log.info("CALL readdir %s Offset: %d", path, offset)
parent = None
path = self.fsEncodeName(path)
key = (_DirContents, path)
dirents = self.cache.retrieve(key)
if not dirents:
dirents = ['.', '..']
depth = getDepth(path)
if depth == 0:
dirents.append(self.current)
entries = self.tardis.listBackupSets()
dirents.extend([y['name'] for y in entries])
else:
parts = getParts(path)
if depth == 1:
b = self.getBackupSetInfo(parts[0])
entries = self.tardis.readDirectory((0, 0), b['backupset'])
else:
(b, parent) = self.getDirInfo(path)
entries = self.tardis.readDirectory((parent["inode"], parent["device"]), b['backupset'])
#if self.crypt:
#entries = self.decryptNames(entries)
# For each entry, cache it, so a later getattr() call can use it.
# Get attr will typically be called promptly after a call to
now = time.time()
for e in entries:
name = e['name']
if self.crypt:
name = self.crypt.decryptFilename(name)
name = self.fsEncodeName(name)
p = os.path.join(path, name)
self.fileCache.insert(p, e, now=now)
dirents.append(name)
self.cache.insert(key, dirents)
#self.log.debug("Direntries: %s", str(dirents))
# Now, return each entry in the list.
for e in dirents:
name = e
#self.log.debug("readdir %s yielding dir entry for %s. Mode: %s. Type: %s ", path, e, mode, type(mode))
yield name
#@tracer
def mythread ( self ):
#self.log.info('mythread')
raise FuseOSError(errno.ENOSYS)
#@tracer
def chmod ( self, path, mode ):
#self.log.info('CALL chmod {} {}'.format(path, oct(mode)))
raise FuseOSError(errno.EROFS)
#@tracer
def chown ( self, path, uid, gid ):
#self.log.info( 'CALL chown {} {} {}'.format(path, uid, gid))
raise FuseOSError(errno.EROFS)
#@tracer
def fsync ( self, path, isFsyncFile ):
#self.log.info( 'CALL fsync {} {}'.format(path, isFsyncFile))
raise FuseOSError(errno.EROFS)
#@tracer
def link ( self, targetPath, linkPath ):
#self.log.info( 'CALL link {} {}'.format(targetPath, linkPath))
raise FuseOSError(errno.EROFS)
#@tracer
def mkdir ( self, path, mode ):
#self.log.info( 'CALL mkdir {} {}'.format(path, oct(mode)))
raise FuseOSError(errno.EROFS)
#@tracer
def mknod ( self, path, mode, dev ):
#self.log.info( 'CALL mknod {} {} {}'.format(path, oct(mode), dev))
raise FuseOSError(errno.EROFS)
#@tracer
def open ( self, path, flags ):
#self.log.info('CALL open {} {})'.format(path, flags))
path = self.fsEncodeName(path)
depth = getDepth(path) # depth of path, zero-based from root
if depth < 2:
raise FuseOSError(errno.ENOENT)
# TODO: Lock this
if path in self.files:
self.files[path]["opens"] += 1
return 0
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
if b:
subpath = parts[1]
if self.crypt:
subpath = self.crypt.encryptPath(subpath)
f = self.regenerator.recoverFile(subpath, b['backupset'], nameEncrypted=True, authenticate=self.authenticate)
if f:
logger.debug("Opened file %s", path)
try:
f.flush()
f.seek(0)
except (AttributeError, IOError) as e:
logger.exception(e)
bytesCopied = 0
logger.debug("Copying file to tempfile")
temp = tempfile.TemporaryFile()
chunk = f.read(65536)
while chunk:
bytesCopied = bytesCopied + len(chunk)
temp.write(chunk)
chunk = f.read(65536)
f.close()
logger.debug("Copied %d bytes to tempfile", bytesCopied)
temp.flush()
temp.seek(0)
f = temp
self.files[path] = {"file": f, "opens": 1}
logger.debug("Set files[%s] => %s", path, str(self.files[path]))
return 0
# Otherwise.....
raise FuseOSError(errno.ENOENT)
#@tracer
def read ( self, path, length, offset, fh ):
#self.log.info('CALL read {} {} {}'.format(path, length, offset))
path = self.fsEncodeName(path)
f = self.files[path]["file"]
if f:
f.seek(offset)
data = f.read(length)
logger.debug("Actually read %d bytes of %s", len(data), type(data))
return data
logger.warning("No file for path %s", path)
raise FuseOSError(errno.EINVAL)
#@tracer
def readlink ( self, path ):
#self.log.info('CALL readlink {}'.format(path))
path = self.fsEncodeName(path)
key = (_LinkContents, path)
link = self.cache.retrieve(key)
if link:
return link
if path == '/' + self.current:
target = self.lastBackupSet(True)
logger.debug("Path: %s Target: %s %s", path, target['name'], target['backupset'])
link = str(target['name'])
self.cache.insert(key, link)
return link
elif getDepth(path) > 1:
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
if b:
subpath = parts[1]
if self.crypt:
subpath = self.crypt.encryptPath(subpath)
f = self.regenerator.recoverFile(subpath, b['backupset'], nameEncrypted=True, authenticate=self.authenticate)
f.flush()
link = f.readline()
f.close()
if self.repoint:
if os.path.isabs(link):
link = os.path.join(self.mountpoint, parts[0], os.path.relpath(link, "/"))
self.cache.insert(key, link)
return link
raise FuseOSError(errno.ENOENT)
#@tracer
def release ( self, path, flags ):
path = self.fsEncodeName(path)
if self.files[path]:
self.files[path]["opens"] -= 1
if self.files[path]["opens"] == 0:
self.files[path]["file"].close()
del self.files[path]
return 0
raise FuseOSError(errno.EINVAL)
#@tracer
def rename ( self, oldPath, newPath ):
#self.log.info('CALL rename {} {}'.format(oldPath, newPath))
raise FuseOSError(errno.EROFS)
#@tracer
def rmdir ( self, path ):
#self.log.info('CALL rmdir {}'.format(path))
raise FuseOSError(errno.EROFS)
#@tracer
def statfs ( self, path ):
#self.log.info('CALL statfs: %s', path)
if isinstance(self.cacheDir, CacheDir.CacheDir):
fs = os.statvfs(self.cacheDir.root)
return dict((key, getattr(fs, key)) for key in (
'f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail',
'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax'))
raise FuseOSError(errno.EINVAL)
def symlink ( self, targetPath, linkPath ):
#self.log.info('CALL symlink {} {}'.format(path, linkPath))
raise FuseOSError(errno.EROFS)
def truncate ( self, path, size ):
#self.log.info('CALL truncate {} {}'.format(path, size))
raise FuseOSError(errno.EROFS)
def unlink ( self, path ):
#self.log.info('CALL unlink {}'.format(path))
raise FuseOSError(errno.EROFS)
def write ( self, path, buf, offset ):
#self.log.info('CALL write {} {} {}'.format(path, offset, len(buf)))
raise FuseOSError(errno.EROFS)
# Map extrenal attribute names for the top level directories to backupset info names
attrMap = {
'user.priority' : 'priority',
'user.complete' : 'completed',
'user.backupset': 'backupset',
'user.session' : 'session'
}
#@tracer
#def listxattr ( self, path, size ):
def listxattr(self, path):
path = self.fsEncodeName(path)
#self.log.info('CALL listxattr %s %d', path, size)
if getDepth(path) == 1:
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
if b:
return list(self.attrMap.keys())
if getDepth(path) > 1:
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
if b:
subpath = parts[1]
if self.crypt:
subpath = self.crypt.encryptPath(subpath)
info = self.tardis.getFileInfoByPath(subpath, b['backupset'])
if info:
attrs = ['user.tardis_checksum', 'user.tardis_since', 'user.tardis_chain']
logger.info("xattrs: %s", info['xattrs'])
if info['xattrs']:
f = self.regenerator.recoverChecksum(info['xattrs'], authenticate=self.authenticate)
xattrs = json.loads(f.read())
logger.debug("Xattrs: %s", str(xattrs))
attrs += list(map(str, list(xattrs.keys())))
logger.debug("Adding xattrs: %s", list(xattrs.keys()))
logger.info("Xattrs: %s", str(attrs))
logger.info("Returning: %s", str(attrs))
return attrs
return None
#@tracer
#def getxattr (self, path, attr, size, *args):
def getxattr(self, path, attr, position=0):
path = self.fsEncodeName(path)
#logger.info('CALL getxattr: %s %s', path, attr)
attr = str(attr)
depth = getDepth(path)
#logger.info("Got depth of path %s -> %s", path, depth)
if depth == 1:
if attr in self.attrMap:
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
if self.attrMap[attr] in list(b.keys()):
return bytes(str(b[self.attrMap[attr]]), 'utf-8')
if depth > 1:
parts = getParts(path)
b = self.getBackupSetInfo(parts[0])
subpath = parts[1]
if self.crypt:
subpath = self.crypt.encryptPath(subpath)
#logger.debug("-----> Asking for attribute %s", attr)
if attr == 'user.tardis_checksum':
if b:
checksum = self.tardis.getChecksumByPath(subpath, b['backupset'])
#logger.debug("Got checksum {}", str(checksum))
if checksum:
return bytes(str(checksum), 'utf-8')
elif attr == 'user.tardis_since':
if b:
since = self.tardis.getFirstBackupSet(subpath, b['backupset'])
#self.log.debug(str(since))
if since:
return bytes(str(since), 'utf-8')
elif attr == 'user.tardis_chain':
info = self.tardis.getChecksumInfoByPath(subpath, b['backupset'])
#self.log.debug(str(checksum))
if info:
chain = info['chainlength']
return bytes(str(chain), 'utf-8')
else:
# Must be an imported value. Let's generate it.
info = self.getFileInfoByPath(path)
if info['xattrs']:
f = self.regenerator.recoverChecksum(info['xattrs'], authenticate=self.authenticate)
xattrs = json.loads(f.read())
if attr in xattrs:
value = base64.b64decode(xattrs[attr])
return bytes(str(value), 'utf-8')
#self.log.debug("Getxattr -- default return value")
return bytes('', 'utf-8')