def getDB(password, new=False, allowRemote=True, allowUpgrade=False):
loc = urllib.parse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
if (loc.scheme == 'http') or (loc.scheme == 'https'):
if not allowRemote:
raise Exception(
"This command cannot be executed remotely. You must execute it on the server directly."
)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault(
'TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse(
(loc.scheme, netloc, loc.path, loc.params, loc.query,
loc.fragment))
else:
dbLoc = args.database
tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
cache = tardisdb
else:
basedir = os.path.join(args.database, args.client)
if not args.dbdir:
dbdir = os.path.join(args.database, args.client)
else:
dbdir = os.path.join(args.dbdir, args.client)
dbfile = os.path.join(dbdir, args.dbname)
if new and os.path.exists(dbfile):
raise Exception("Database for client %s already exists." %
(args.client))
cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
schema = args.schema if new else None
tardisdb = TardisDB.TardisDB(dbfile,
backup=False,
initialize=schema,
allow_upgrade=allowUpgrade)
if tardisdb.needsAuthentication():
if password is None:
password = Util.getPassword(args.password,
args.passwordfile,
args.passwordprog,
prompt="Password for %s: " %
(args.client),
allowNone=False,
confirm=False)
Util.authenticate(tardisdb, args.client, password)
scheme = tardisdb.getCryptoScheme()
logger.info("Using crypto scheme %d", scheme)
crypt = TardisCrypto.getCrypto(scheme, password, args.client)
else:
crypt = TardisCrypto.getCrypto(0, None, None)
return (tardisdb, cache, crypt)
def addPasswordOptions(parser, addscheme=False):
passgroup = parser.add_argument_group(
"Password/Encryption specification options")
pwgroup = passgroup.add_mutually_exclusive_group()
pwgroup.add_argument('--password',
'-P',
dest='password',
default=config.get(job, 'Password'),
nargs='?',
const=True,
help='Encrypt files with this password')
pwgroup.add_argument(
'--password-file',
'-F',
dest='passwordfile',
default=config.get(job, 'PasswordFile'),
help='Read password from file. Can be a URL (HTTP/HTTPS or FTP)')
pwgroup.add_argument(
'--password-prog',
dest='passwordprog',
default=config.get(job, 'PasswordProg'),
help='Use the specified command to generate the password on stdout')
if addscheme:
passgroup.add_argument('--scheme',
dest='scheme',
type=int,
choices=range(TardisCrypto.maxCryptoScheme + 1),
default=TardisCrypto.defaultCryptoScheme,
help='Use cryptography scheme\n' +
TardisCrypto.getCryptoNames())
passgroup.add_argument('--keys',
dest='keys',
default=config.get(job, 'KeyFile'),
help='Load keys from file.')
def main():
global logger
progressbar.streams.wrap_stderr()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('')
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
Util.authenticate(db, args.client, password)
(f, c) = db.getKeys()
crypto.setKeys(f, c)
cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
if args.names or args.all:
encryptFilenames(db, crypto)
if args.dirs or args.all:
generateDirHashes(db, crypto, cacheDir)
if args.sigs or args.all:
generateSignatures(db, crypto, cacheDir)
if args.files or args.all:
encryptFiles(db, crypto, cacheDir)
if args.meta or args.all:
generateMetadata(db, cacheDir)
def setPassword(password):
try:
(db, _, _) = getDB(None)
crypt = TardisCrypto.getCrypto(TardisCrypto.defaultCryptoScheme,
password)
crypt.genKeys()
(f, c) = crypt.getKeys()
(salt,
vkey) = srp.create_salted_verification_key(args.client, password)
if args.keys:
db.beginTransaction()
db.setSrpValues(salt, vkey)
db.setConfigValue('CryptoScheme', crypt.getCryptoScheme())
Util.saveKeys(args.keys, db.getConfigValue('ClientID'), f, c)
else:
db.setKeys(salt, vkey, f, c)
db.setConfigValue('CryptoScheme', crypt.getCryptoScheme())
return 0
except TardisDB.NotAuthenticated:
logger.error('Client %s already has a password', args.client)
if args.exceptions:
logger.exception(e)
return 1
except TardisDB.AuthenticationFailed as e:
logger.error("Authentication failed. Bad password")
if args.exceptions:
logger.exception(e)
return 1
except Exception as e:
logger.error(str(e))
if args.exceptions:
logger.exception(e)
return 1
def main():
global logger
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('')
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
token = crypto.createToken()
#logger.info("Created token: %s", token)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, token=token, backup=False)
(f, c) = db.getKeys()
crypto.setKeys(f, c)
cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
#if args.sigs:
# generateSignatures(db, cacheDir)
if args.filenames:
encryptFilenames(db, crypto)
if args.files:
encryptFiles(db, crypto, cacheDir)
if args.dirhash:
generateDirHashes(db, crypto, cacheDir)
if args.meta:
generateMetadata(db, cacheDir)
def changePassword(crypt, oldpw):
try:
(db, _, crypt) = getDB(oldpw)
# Get the new password
try:
newpw = Util.getPassword(args.newpw,
args.newpwf,
args.newpwp,
prompt="New Password for %s: " %
(args.client),
allowNone=False,
confirm=True,
strength=True)
except Exception as e:
logger.critical(str(e))
if args.exceptions:
logger.exception(e)
return -1
scheme = db.getConfigValue('CryptoScheme', 1)
crypt2 = TardisCrypto.getCrypto(scheme, newpw, args.client)
# Load the keys, and insert them into the crypt object, to decyrpt them
if args.keys:
(f, c) = Util.loadKeys(args.keys, db.getConfigValue('ClientID'))
# No need to check here, loadKeys() throws exception if nothing set.
else:
(f, c) = db.getKeys()
if f is None or c is None:
logger.critical(
"No keys loaded from database. Please specify --keys as appropriate"
)
raise Exception("No keys loaded")
crypt.setKeys(f, c)
# Grab the keys from one crypt object.
# Need to do this because getKeys/setKeys assumes they're encrypted, and we need the raw
# versions
crypt2._filenameKey = crypt._filenameKey
crypt2._contentKey = crypt._contentKey
# Now get the encrypted versions
(f, c) = crypt2.getKeys()
(salt, vkey) = srp.create_salted_verification_key(args.client, newpw)
if args.keys:
db.beginTransaction()
db.setSrpValues(salt, vkey)
Util.saveKeys(args.keys, db.getConfigValue('ClientID'), f, c)
db.commit()
else:
db.setKeys(salt, vkey, f, c)
return 0
except Exception as e:
logger.error(str(e))
if args.exceptions:
logger.exception(e)
return 1
def validate(root, client, dbname, password):
crypto = None
token = None
base = os.path.join(root, client)
cache = CacheDir.CacheDir(base)
if password:
crypto = TardisCrypto.TardisCrypto(password, client)
token = crypto.encryptFilename(client)
db = TardisDB.TardisDB(os.path.join(base, dbname), token=token, backup=False)
regen = Regenerate.Regenerator(cache, db, crypto)
conn = db.conn
cur = conn.execute("SELECT count(*) FROM CheckSums WHERE IsFile = 1")
row = cur.fetchone()
num = row[0]
print("Checksums: %d" % (num))
cur = conn.execute("SELECT Checksum FROM CheckSums WHERE IsFile = 1 ORDER BY Checksum ASC");
pbar = pb.ProgressBar(widgets=[pb.Percentage(), ' ', pb.Counter(), ' ', pb.Bar(), ' ', pb.ETA(), ' ', pb.Timer() ], maxval=num)
pbar.start()
row = cur.fetchone()
i = 1
while row is not None:
pbar.update(i)
i += 1
try:
checksum = row['Checksum']
if not checksum in checked:
try:
f = regen.recoverChecksum(checksum)
if f:
m = hashlib.md5()
d = f.read(128 * 1024)
while d:
m.update(d)
d = f.read(128 * 1024)
res = m.hexdigest()
if res != checksum:
print("Checksums don't match. Expected: %s, result %s" % (checksum, res))
checked[checksum] = 0
output.write(checksum + '\n')
output.flush()
else:
checked[checksum] = 1
valid.write(checksum + "\n")
except Exception as e:
print("Caught exception processing %s: %s" % (checksum, str(e)))
output.write(checksum + '\n')
output.flush()
row = cur.fetchone()
except sqlite3.OperationalError as e:
print("Caught operational error. DB is probably locked. Sleeping for a bit")
time.sleep(90)
pbar.finish()
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
""" Setup a data connection to a client. Determines the correct way to connect, either via direct filesystem,
or via TardisRemote (http).
Returns a 3-tuple, the TardisDB object, the CacheDir object, and the appropriate crypto object
"""
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urllib.parse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)
if needsAuth:
authenticate(tardis, client, password)
elif password:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
#cryptoScheme = tardis.getConfigValue('CryptoScheme', '1')
cryptoScheme = tardis.getCryptoScheme()
crypt = TardisCrypto.getCrypto(cryptoScheme, password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
if retpassword:
return (tardis, cache, crypt, password)
else:
return (tardis, cache, crypt)
def setupDataConnection(dataLoc,
client,
password,
keyFile,
dbName,
dbLoc=None,
allow_upgrade=False):
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault(
'TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path,
loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client)
if password:
if needsAuth:
authenticate(tardis, client, password)
else:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
crypt = TardisCrypto.TardisCrypto(password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def main():
logging.basicConfig(level=logging.DEBUG)
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
token = crypto.createToken()
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
db.setToken(token)
def main():
logging.basicConfig(level=logging.INFO)
crypto = None
token = None
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
if password:
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, token=token, backup=False)
if crypto:
(a, b) = db.getKeys()
crypto.setKeys(a, b)
conn = db.conn
dirs = conn.execute(
"SELECT Name as name, Inode AS inode, Device AS device, FirstSet as firstset, LastSet AS lastset FROM Files JOIN Names ON Files.NameId = Names.NameId WHERE Dir = 1"
)
while True:
batch = dirs.fetchmany(1000)
if not batch:
break
for d in batch:
name = d['name']
inode = d['inode']
device = d['device']
firstset = d['firstset']
lastset = d['lastset']
files = db.readDirectory((inode, device), current=lastset)
(checksum, nfiles) = Util.hashDir(crypto, files, True)
print("%-20s (%d, %d) [%d %d] -- %s %d") % (
name, inode, device, firstset, lastset, checksum, nfiles)
ckinfo = db.getChecksumInfo(checksum)
if ckinfo:
cksid = ckinfo['checksumid']
else:
cksid = db.insertChecksumFile(checksum,
size=nfiles,
isFile=False)
db.updateDirChecksum((inode, device), cksid, current=lastset)
conn.commit()
def main():
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
crypto = None
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile, args.passwordurl, args.passwordprog)
if password:
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
token = createToken(crypto, args.client)
if not checkToken(db, token):
logger.error("Password does not match")
sys.exit(1)
salt, vkey = srp.create_salted_verification_key(args.client, password)
db.setSrpValues(salt, vkey)
db._setConfigValue('Token', None)
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False):
crypt = None
if password:
crypt = TardisCrypto.TardisCrypto(password, client)
password = None
token = None
if crypt:
token = crypt.createToken()
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
cache = tardis
else:
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, token=token, allow_upgrade=allow_upgrade)
if crypt:
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def sendDataPlain(sender, data, chunksize=(16 * 1024), compress=None, stats=None):
"""
Send data, with no encryption, or calculation
"""
encrypt = TardisCrypto.NullEncryptor()
sendData(sender, data, encrypt, chunksize=chunksize, compress=compress, stats=stats)
def main():
global logger, exceptionLogger, args
parseArgs()
logger = Util.setupLogging(args.verbose)
exceptionLogger = Util.ExceptionLogger(logger, args.exceptions)
# Commands which cannot be executed on remote databases
allowRemote = args.command not in ['create', 'upgrade']
db = None
crypt = None
cache = None
try:
confirm = args.command in ['setpass', 'create']
allowNone = args.command not in ['setpass', 'chpass']
try:
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=allowNone, confirm=confirm)
except Exception as e:
logger.critical(str(e))
exceptionLogger.log(e)
return -1
if password:
crypt = TardisCrypto.TardisCrypto(password, args.client)
args.password = None
if args.command == 'create':
return createClient(crypt, password)
if args.command == 'setpass':
if not Util.checkPasswordStrength(password):
return -1
if not crypt:
logger.error("No password specified")
return -1
return setPassword(crypt, password)
if args.command == 'chpass':
return changePassword(crypt, password)
upgrade = (args.command == 'upgrade')
try:
(db, cache) = getDB(crypt, password, allowRemote=allowRemote, allowUpgrade=upgrade)
if crypt and args.command != 'keys':
if args.keys:
(f, c) = Util.loadKeys(args.keys, db.getConfigValue('ClientID'))
else:
(f, c) = db.getKeys()
crypt.setKeys(f, c)
except TardisDB.AuthenticationException as e:
logger.error("Authentication failed. Bad password")
exceptionLogger.log(e)
sys.exit(1)
except Exception as e:
logger.critical("Unable to connect to database: %s", e)
exceptionLogger.log(e)
sys.exit(1)
if args.command == 'keys':
return moveKeys(db, crypt)
elif args.command == 'list':
return listBSets(db, crypt, cache)
elif args.command == 'files':
return listFiles(db, crypt)
elif args.command == 'info':
return bsetInfo(db)
elif args.command == 'purge':
return purge(db, cache)
elif args.command == 'delete':
return deleteBsets(db, cache)
elif args.command == 'priority':
return setPriority(db)
elif args.command == 'rename':
return renameSet(db)
elif args.command == 'getconfig':
return getConfig(db)
elif args.command == 'setconfig':
return setConfig(db)
elif args.command == 'orphans':
return removeOrphans(db, cache)
elif args.command == 'upgrade':
return
except KeyboardInterrupt:
pass
except TardisDB.AuthenticationException as e:
logger.error("Authentication failed. Bad password")
sys.exit(1)
except Exception as e:
logger.error("Caught exception: %s", str(e))
exceptionLogger.log(e)
finally:
if db:
db.close()
def main():
global logger
parseArgs()
logger = Util.setupLogging(args.verbose)
# Commands which cannot be executed on remote databases
allowRemote = args.command not in ['create']
db = None
crypt = None
cache = None
try:
password = Util.getPassword(args.password,
args.passwordfile,
args.passwordprog,
prompt="Password for %s: " % (args.client),
allowNone=(args.command != 'setPass'))
if args.command in ['setpass', 'create']:
if password and not checkPasswordStrength(password):
return -1
if args.password:
pw2 = Util.getPassword(args.password,
args.passwordfile,
args.passwordprog,
prompt='Confirm Password: '******'t match")
return -1
pw2 = None
if password:
crypt = TardisCrypto.TardisCrypto(password, args.client)
password = None
args.password = None
if args.command == 'create':
return createClient(crypt)
if args.command == 'setpass':
if not crypt:
logger.error("No password specified")
return -1
return setToken(crypt)
if args.command == 'chpass':
newpw = Util.getPassword(args.newpw,
args.newpwf,
args.newpwp,
prompt="New Password for %s: " %
(args.client),
allowNone=False)
if not checkPasswordStrength(newpw):
return -1
if args.newpw is True:
newpw2 = Util.getPassword(args.newpw,
args.newpwf,
args.newpwp,
prompt="New Password for %s: " %
(args.client),
allowNone=False)
if newpw2 != newpw:
logger.error("Passwords don't match")
return -1
newpw2 = None
crypt2 = TardisCrypto.TardisCrypto(newpw, args.client)
newpw = None
args.newpw = None
return changePassword(crypt, crypt2)
try:
(db, cache) = getDB(crypt, allowRemote=allowRemote)
if crypt:
if args.keys:
(f, c) = Util.loadKeys(args.keys,
db.getConfigValue('ClientID'))
else:
(f, c) = db.getKeys()
crypt.setKeys(f, c)
except Exception as e:
logger.critical("Unable to connect to database: %s", e)
sys.exit(1)
if args.command == 'keys':
return moveKeys(db, crypt)
elif args.command == 'list':
return listBSets(db)
elif args.command == 'files':
return listFiles(db, crypt)
elif args.command == 'info':
return bsetInfo(db)
elif args.command == 'purge':
return purge(db, cache)
elif args.command == 'delete':
return deleteBsets(db, cache)
elif args.command == 'getconfig':
return getConfig(db)
elif args.command == 'setconfig':
return setConfig(db)
elif args.command == 'orphans':
return removeOrphans(db, cache)
except KeyboardInterrupt:
pass
except Exception as e:
logger.error("Caught exception: %s", str(e))
logger.exception(e)
finally:
if db:
db.close()