def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
""" Setup a data connection to a client. Determines the correct way to connect, either via direct filesystem,
or via TardisRemote (http).
Returns a 3-tuple, the TardisDB object, the CacheDir object, and the appropriate crypto object
"""
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urllib.parse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)
if needsAuth:
authenticate(tardis, client, password)
elif password:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
#cryptoScheme = tardis.getConfigValue('CryptoScheme', '1')
cryptoScheme = tardis.getCryptoScheme()
crypt = TardisCrypto.getCrypto(cryptoScheme, password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
if retpassword:
return (tardis, cache, crypt, password)
else:
return (tardis, cache, crypt)
def setupDataConnection(dataLoc,
client,
password,
keyFile,
dbName,
dbLoc=None,
allow_upgrade=False):
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault(
'TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path,
loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client)
if password:
if needsAuth:
authenticate(tardis, client, password)
else:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
crypt = TardisCrypto.TardisCrypto(password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def login():
if request.method == 'POST':
try:
#app.logger.debug(str(request))
host = request.form['host']
dbPath = os.path.join(args.database, host, dbname)
cache = CacheDir.CacheDir(os.path.join(args.database, host), create=False)
upgrade = config.getboolean('Tardis', 'AllowSchemaUpgrades')
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=upgrade)
#session['tardis'] = tardis
session['host'] = host
#app.logger.debug(str(session))
dbs[host] = tardis
caches[host] = cache
if tardis.needsAuthentication():
status = 'AUTH'
else:
status = 'OK'
return createResponse({"status": status }, compress=False, cacheable=False)
except Exception as e:
app.logger.exception(e)
abort(401)
else:
return '''
def getDB(crypt, password, new=False, allowRemote=True, allowUpgrade=False):
loc = urlparse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
if (loc.scheme == 'http') or (loc.scheme == 'https'):
if not allowRemote:
raise Exception("This command cannot be executed remotely. You must execute it on the server directly.")
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = args.database
tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
cache = tardisdb
else:
basedir = os.path.join(args.database, args.client)
if not args.dbdir:
dbdir = os.path.join(args.database, args.client)
else:
dbdir = os.path.join(args.dbdir, args.client)
dbfile = os.path.join(dbdir, args.dbname)
if new and os.path.exists(dbfile):
raise Exception("Database for client %s already exists." % (args.client))
cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
schema = args.schema if new else None
tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, allow_upgrade=allowUpgrade)
if tardisdb.needsAuthentication():
if password is None:
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=False, confirm=False)
Util.authenticate(tardisdb, args.client, password)
return (tardisdb, cache)
def main():
global logger
progressbar.streams.wrap_stderr()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('')
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
Util.authenticate(db, args.client, password)
(f, c) = db.getKeys()
crypto.setKeys(f, c)
cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
if args.names or args.all:
encryptFilenames(db, crypto)
if args.dirs or args.all:
generateDirHashes(db, crypto, cacheDir)
if args.sigs or args.all:
generateSignatures(db, crypto, cacheDir)
if args.files or args.all:
encryptFiles(db, crypto, cacheDir)
if args.meta or args.all:
generateMetadata(db, cacheDir)
def main():
global logger
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('')
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
token = crypto.createToken()
#logger.info("Created token: %s", token)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, token=token, backup=False)
(f, c) = db.getKeys()
crypto.setKeys(f, c)
cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
#if args.sigs:
# generateSignatures(db, cacheDir)
if args.filenames:
encryptFilenames(db, crypto)
if args.files:
encryptFiles(db, crypto, cacheDir)
if args.dirhash:
generateDirHashes(db, crypto, cacheDir)
if args.meta:
generateMetadata(db, cacheDir)
def authenticate(db, client, password):
usr = srp.User(client, password)
uname, A = usr.start_authentication()
s, B = db.authenticate1(uname, A)
M = usr.process_challenge(s, B)
if M is None:
raise TardisDB.AuthenticationFailed()
HAMK = db.authenticate2(M)
usr.verify_session(HAMK)
if not usr.authenticated():
raise TardisDB.AuthenticationFailed()
def validate(root, client, dbname, password):
crypto = None
token = None
base = os.path.join(root, client)
cache = CacheDir.CacheDir(base)
if password:
crypto = TardisCrypto.TardisCrypto(password, client)
token = crypto.encryptFilename(client)
db = TardisDB.TardisDB(os.path.join(base, dbname), token=token, backup=False)
regen = Regenerate.Regenerator(cache, db, crypto)
conn = db.conn
cur = conn.execute("SELECT count(*) FROM CheckSums WHERE IsFile = 1")
row = cur.fetchone()
num = row[0]
print("Checksums: %d" % (num))
cur = conn.execute("SELECT Checksum FROM CheckSums WHERE IsFile = 1 ORDER BY Checksum ASC");
pbar = pb.ProgressBar(widgets=[pb.Percentage(), ' ', pb.Counter(), ' ', pb.Bar(), ' ', pb.ETA(), ' ', pb.Timer() ], maxval=num)
pbar.start()
row = cur.fetchone()
i = 1
while row is not None:
pbar.update(i)
i += 1
try:
checksum = row['Checksum']
if not checksum in checked:
try:
f = regen.recoverChecksum(checksum)
if f:
m = hashlib.md5()
d = f.read(128 * 1024)
while d:
m.update(d)
d = f.read(128 * 1024)
res = m.hexdigest()
if res != checksum:
print("Checksums don't match. Expected: %s, result %s" % (checksum, res))
checked[checksum] = 0
output.write(checksum + '\n')
output.flush()
else:
checked[checksum] = 1
valid.write(checksum + "\n")
except Exception as e:
print("Caught exception processing %s: %s" % (checksum, str(e)))
output.write(checksum + '\n')
output.flush()
row = cur.fetchone()
except sqlite3.OperationalError as e:
print("Caught operational error. DB is probably locked. Sleeping for a bit")
time.sleep(90)
pbar.finish()
def main():
logging.basicConfig(level=logging.DEBUG)
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
token = crypto.createToken()
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
db.setToken(token)
def authenticate2(self, srpValueM):
postData = {'srpValueM': str(base64.b64encode(srpValueM), 'utf8')}
response = self.session.post(self.baseURL + 'authenticate2',
data=postData)
# Check for "not authenticated", which indicates authentication failed.
if response.status_code == 401:
raise TardisDB.AuthenticationFailed("Bad Password")
# Catch other errors.
response.raise_for_status()
data = response.json()
srpValueH = base64.b64decode(data['srpValueH'])
return srpValueH
def main():
logging.basicConfig(level=logging.INFO)
crypto = None
token = None
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
if password:
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, token=token, backup=False)
if crypto:
(a, b) = db.getKeys()
crypto.setKeys(a, b)
conn = db.conn
dirs = conn.execute(
"SELECT Name as name, Inode AS inode, Device AS device, FirstSet as firstset, LastSet AS lastset FROM Files JOIN Names ON Files.NameId = Names.NameId WHERE Dir = 1"
)
while True:
batch = dirs.fetchmany(1000)
if not batch:
break
for d in batch:
name = d['name']
inode = d['inode']
device = d['device']
firstset = d['firstset']
lastset = d['lastset']
files = db.readDirectory((inode, device), current=lastset)
(checksum, nfiles) = Util.hashDir(crypto, files, True)
print("%-20s (%d, %d) [%d %d] -- %s %d") % (
name, inode, device, firstset, lastset, checksum, nfiles)
ckinfo = db.getChecksumInfo(checksum)
if ckinfo:
cksid = ckinfo['checksumid']
else:
cksid = db.insertChecksumFile(checksum,
size=nfiles,
isFile=False)
db.updateDirChecksum((inode, device), cksid, current=lastset)
conn.commit()
def authenticate1(self, uname, srpValueA):
postData = {
'srpUname': base64.b64encode(uname),
'srpValueA': base64.b64encode(srpValueA)
}
response = self.session.post(self.baseURL + 'authenticate1',
data=postData)
# Check for "not authenticated", which indicates authentication failed.
if response.status_code == 401:
raise TardisDB.AuthenticationFailed("Bad Password")
# Catch other errors.
response.raise_for_status()
data = response.json()
srpValueS = base64.b64decode(data['srpValueS'])
srpValueB = base64.b64decode(data['srpValueB'])
return srpValueS, srpValueB
def main():
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
crypto = None
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile, args.passwordurl, args.passwordprog)
if password:
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
token = createToken(crypto, args.client)
if not checkToken(db, token):
logger.error("Password does not match")
sys.exit(1)
salt, vkey = srp.create_salted_verification_key(args.client, password)
db.setSrpValues(salt, vkey)
db._setConfigValue('Token', None)
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False):
crypt = None
if password:
crypt = TardisCrypto.TardisCrypto(password, client)
password = None
token = None
if crypt:
token = crypt.createToken()
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
cache = tardis
else:
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, token=token, allow_upgrade=allow_upgrade)
if crypt:
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def login():
if request.method == 'POST':
try:
#app.logger.debug(str(request))
host = request.form['host']
token = request.form['token'] if 'token' in request.form else None
upgrade = request.form[
'upgrade'] if 'upgrade' in request.form else False
dbPath = os.path.join(args.database, host, dbname)
cache = CacheDir.CacheDir(os.path.join(args.database, host),
create=False)
tardis = TardisDB.TardisDB(dbPath,
token=token,
allow_upgrade=upgrade)
#session['tardis'] = tardis
session['host'] = host
#app.logger.debug(str(session))
dbs[host] = tardis
caches[host] = cache
return "OK"
except Exception as e:
app.logger.exception(e)
abort(401)
return '''