def getDB(crypt, password, new=False, allowRemote=True, allowUpgrade=False):
loc = urlparse.urlparse(args.database)
# This is basically the same code as in Util.setupDataConnection(). Should consider moving to it.
if (loc.scheme == 'http') or (loc.scheme == 'https'):
if not allowRemote:
raise Exception("This command cannot be executed remotely. You must execute it on the server directly.")
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = args.database
tardisdb = RemoteDB.RemoteDB(dbLoc, args.client)
cache = tardisdb
else:
basedir = os.path.join(args.database, args.client)
if not args.dbdir:
dbdir = os.path.join(args.database, args.client)
else:
dbdir = os.path.join(args.dbdir, args.client)
dbfile = os.path.join(dbdir, args.dbname)
if new and os.path.exists(dbfile):
raise Exception("Database for client %s already exists." % (args.client))
cache = CacheDir.CacheDir(basedir, 2, 2, create=new)
schema = args.schema if new else None
tardisdb = TardisDB.TardisDB(dbfile, backup=False, initialize=schema, allow_upgrade=allowUpgrade)
if tardisdb.needsAuthentication():
if password is None:
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog, prompt="Password for %s: " % (args.client), allowNone=False, confirm=False)
Util.authenticate(tardisdb, args.client, password)
return (tardisdb, cache)
def login():
if request.method == 'POST':
try:
#app.logger.debug(str(request))
host = request.form['host']
dbPath = os.path.join(args.database, host, dbname)
cache = CacheDir.CacheDir(os.path.join(args.database, host), create=False)
upgrade = config.getboolean('Tardis', 'AllowSchemaUpgrades')
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=upgrade)
#session['tardis'] = tardis
session['host'] = host
#app.logger.debug(str(session))
dbs[host] = tardis
caches[host] = cache
if tardis.needsAuthentication():
status = 'AUTH'
else:
status = 'OK'
return createResponse({"status": status }, compress=False, cacheable=False)
except Exception as e:
app.logger.exception(e)
abort(401)
else:
return '''
def main():
global logger
progressbar.streams.wrap_stderr()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('')
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, backup=False)
Util.authenticate(db, args.client, password)
(f, c) = db.getKeys()
crypto.setKeys(f, c)
cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
if args.names or args.all:
encryptFilenames(db, crypto)
if args.dirs or args.all:
generateDirHashes(db, crypto, cacheDir)
if args.sigs or args.all:
generateSignatures(db, crypto, cacheDir)
if args.files or args.all:
encryptFiles(db, crypto, cacheDir)
if args.meta or args.all:
generateMetadata(db, cacheDir)
def main():
global logger
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('')
args = processArgs()
password = Util.getPassword(args.password, args.passwordfile,
args.passwordurl, args.passwordprog)
crypto = TardisCrypto.TardisCrypto(password, args.client)
token = crypto.createToken()
#logger.info("Created token: %s", token)
path = os.path.join(args.database, args.client, args.dbname)
db = TardisDB.TardisDB(path, token=token, backup=False)
(f, c) = db.getKeys()
crypto.setKeys(f, c)
cacheDir = CacheDir.CacheDir(os.path.join(args.database, args.client))
#if args.sigs:
# generateSignatures(db, cacheDir)
if args.filenames:
encryptFilenames(db, crypto)
if args.files:
encryptFiles(db, crypto, cacheDir)
if args.dirhash:
generateDirHashes(db, crypto, cacheDir)
if args.meta:
generateMetadata(db, cacheDir)
def validate(root, client, dbname, password):
crypto = None
token = None
base = os.path.join(root, client)
cache = CacheDir.CacheDir(base)
if password:
crypto = TardisCrypto.TardisCrypto(password, client)
token = crypto.encryptFilename(client)
db = TardisDB.TardisDB(os.path.join(base, dbname), token=token, backup=False)
regen = Regenerate.Regenerator(cache, db, crypto)
conn = db.conn
cur = conn.execute("SELECT count(*) FROM CheckSums WHERE IsFile = 1")
row = cur.fetchone()
num = row[0]
print("Checksums: %d" % (num))
cur = conn.execute("SELECT Checksum FROM CheckSums WHERE IsFile = 1 ORDER BY Checksum ASC");
pbar = pb.ProgressBar(widgets=[pb.Percentage(), ' ', pb.Counter(), ' ', pb.Bar(), ' ', pb.ETA(), ' ', pb.Timer() ], maxval=num)
pbar.start()
row = cur.fetchone()
i = 1
while row is not None:
pbar.update(i)
i += 1
try:
checksum = row['Checksum']
if not checksum in checked:
try:
f = regen.recoverChecksum(checksum)
if f:
m = hashlib.md5()
d = f.read(128 * 1024)
while d:
m.update(d)
d = f.read(128 * 1024)
res = m.hexdigest()
if res != checksum:
print("Checksums don't match. Expected: %s, result %s" % (checksum, res))
checked[checksum] = 0
output.write(checksum + '\n')
output.flush()
else:
checked[checksum] = 1
valid.write(checksum + "\n")
except Exception as e:
print("Caught exception processing %s: %s" % (checksum, str(e)))
output.write(checksum + '\n')
output.flush()
row = cur.fetchone()
except sqlite3.OperationalError as e:
print("Caught operational error. DB is probably locked. Sleeping for a bit")
time.sleep(90)
pbar.finish()
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False, retpassword=False):
""" Setup a data connection to a client. Determines the correct way to connect, either via direct filesystem,
or via TardisRemote (http).
Returns a 3-tuple, the TardisDB object, the CacheDir object, and the appropriate crypto object
"""
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urllib.parse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urllib.parse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client, allowNone=False)
if needsAuth:
authenticate(tardis, client, password)
elif password:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
#cryptoScheme = tardis.getConfigValue('CryptoScheme', '1')
cryptoScheme = tardis.getCryptoScheme()
crypt = TardisCrypto.getCrypto(cryptoScheme, password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
if retpassword:
return (tardis, cache, crypt, password)
else:
return (tardis, cache, crypt)
def setupDataConnection(dataLoc,
client,
password,
keyFile,
dbName,
dbLoc=None,
allow_upgrade=False):
logger.debug("Connection requested for %s under %s", client, dataLoc)
crypt = None
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
logger.debug("Creating remote connection to %s", dataLoc)
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault(
'TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path,
loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
logger.debug("==> %s %s", dbLoc, client)
tardis = RemoteDB.RemoteDB(dbLoc, client)
cache = tardis
else:
logger.debug("Creating direct connection to %s", dataLoc)
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, allow_upgrade=allow_upgrade)
needsAuth = tardis.needsAuthentication()
if needsAuth and password is None:
password = getPassword(True, None, None, "Password for %s: " % client)
if password:
if needsAuth:
authenticate(tardis, client, password)
else:
raise TardisDB.AuthenticationFailed()
# Password specified, so create the crypto unit
crypt = TardisCrypto.TardisCrypto(password, client)
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def setupDataConnection(dataLoc, client, password, keyFile, dbName, dbLoc=None, allow_upgrade=False):
crypt = None
if password:
crypt = TardisCrypto.TardisCrypto(password, client)
password = None
token = None
if crypt:
token = crypt.createToken()
loc = urlparse.urlparse(dataLoc)
if (loc.scheme == 'http') or (loc.scheme == 'https'):
# If no port specified, insert the port
if loc.port is None:
netloc = loc.netloc + ":" + Defaults.getDefault('TARDIS_REMOTE_PORT')
dbLoc = urlparse.urlunparse((loc.scheme, netloc, loc.path, loc.params, loc.query, loc.fragment))
else:
dbLoc = dataLoc
# get the RemoteURL object
tardis = RemoteDB.RemoteDB(dbLoc, client, token=token)
cache = tardis
else:
cacheDir = os.path.join(loc.path, client)
cache = CacheDir.CacheDir(cacheDir, create=False)
if not dbLoc:
dbDir = cacheDir
else:
dbDir = os.path.join(dbLoc, client)
dbPath = os.path.join(dbDir, dbName)
tardis = TardisDB.TardisDB(dbPath, token=token, allow_upgrade=allow_upgrade)
if crypt:
if keyFile:
(f, c) = loadKeys(keyFile, tardis.getConfigValue('ClientID'))
else:
(f, c) = tardis.getKeys()
crypt.setKeys(f, c)
return (tardis, cache, crypt)
def login():
if request.method == 'POST':
try:
#app.logger.debug(str(request))
host = request.form['host']
token = request.form['token'] if 'token' in request.form else None
upgrade = request.form[
'upgrade'] if 'upgrade' in request.form else False
dbPath = os.path.join(args.database, host, dbname)
cache = CacheDir.CacheDir(os.path.join(args.database, host),
create=False)
tardis = TardisDB.TardisDB(dbPath,
token=token,
allow_upgrade=upgrade)
#session['tardis'] = tardis
session['host'] = host
#app.logger.debug(str(session))
dbs[host] = tardis
caches[host] = cache
return "OK"
except Exception as e:
app.logger.exception(e)
abort(401)
return '''
#! /usr/bin/python
from Tardis import CacheDir, Util
import argparse
import sys
parser = argparse.ArgumentParser(
description="Generate file paths in a cache dir directory", add_help=True)
parser.add_argument('--base',
'-b',
dest='base',
default='.',
help='Base CacheDir directory')
parser.add_argument('files', nargs='*', help='List of files to print')
Util.addGenCompletions(parser)
args = parser.parse_args()
c = CacheDir.CacheDir(args.base)
for i in args.files:
print c.path(i)
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
from Tardis import CacheDir, Util
import argparse
import sys
import logging
logging.basicConfig()
parser = argparse.ArgumentParser(
description="Generate file paths in a cache dir directory", add_help=True)
parser.add_argument('--base',
'-b',
dest='base',
default='.',
help='Base CacheDir directory')
parser.add_argument('files', nargs='*', help='List of files to print')
Util.addGenCompletions(parser)
args = parser.parse_args()
c = CacheDir.CacheDir(args.base, create=False)
for i in args.files:
print(c.path(i))