def main():
cmdline = get_cmdline()
dbfilename = cmdline.dbfilename
if not os.path.exists(dbfilename) or os.path.isdir(dbfilename):
errexit(1,"file '%s' not found!" % (dbfilename))
#open database
print("opening '%s' recovery info database..." % (dbfilename))
db = RecDB(dbfilename)
#get data on all uids present
uidDataList = db.GetUIDDataList()
#get blocksizes for every supported SBx version
blocksizes = {}
for v in seqbox.supported_vers:
blocksizes[v] = seqbox.SbxBlock(ver=v).blocksize
#info/report
if cmdline.info:
report(db, uidDataList, blocksizes)
errexit(0)
#build a list of uids to recover:
uidRecoList = []
if cmdline.all:
uidRecoList = list(uidDataList)
else:
if cmdline.uid:
for hexuid in cmdline.uid:
if len(hexuid) % 2 != 0:
errexit(1, "invalid UID!")
uid = int.from_bytes(binascii.unhexlify(hexuid),
byteorder="big")
if db.GetBlocksCountFromUID(uid):
uidRecoList.append(uid)
else:
errexit(1,"no recoverable UID '%s'" % (hexuid))
if cmdline.sbx:
for sbxname in cmdline.sbx:
uid = db.GetUIDFromSbxName(sbxname)
if uid:
uidRecoList.append(uid)
else:
errexit(1,"no recoverable sbx file '%s'" % (sbxname))
if cmdline.file:
for filename in cmdline.file:
uid = db.GetUIDFromFileName(filename)
if uid:
uidRecoList.append(uid)
else:
errexit(1,"no recoverable file '%s'" % (filename))
if len(uidRecoList) == 0:
errexit(1, "nothing to recover!")
print("recovering SBX files...")
uid_list = sorted(set(uidRecoList))
#open all the sources
finlist = {}
for key, value in db.GetSourcesList():
finlist[key] = open(value, "rb")
uidcount = 0
totblocks = 0
totblockserr = 0
uiderrlist = []
for uid in uidRecoList:
uidcount += 1
sbxver = uidDataList[uid]
sbx = seqbox.SbxBlock(ver=sbxver, pswd=cmdline.password)
hexuid = binascii.hexlify(uid.to_bytes(6, byteorder="big")).decode()
print("UID %s (%i/%i)" % (hexuid, uidcount, len(uid_list)))
blocksnum = db.GetBlocksCountFromUID(uid)
print(" blocks: %i - size: %i bytes" %
(blocksnum, blocksnum * sbx.blocksize))
meta = db.GetMetaFromUID(uid)
if "sbxname" in meta:
sbxname = meta["sbxname"]
else:
#use hex uid as name if no metadata present
sbxname = (binascii.hexlify(uid.to_bytes(6, byteorder="big")).decode() +
".sbx")
if cmdline.destpath:
sbxname = os.path.join(cmdline.destpath, sbxname)
print(" to: '%s'" % sbxname)
if not cmdline.overwrite:
sbxname = uniquifyFileName(sbxname)
fout = open(sbxname, "wb", buffering = 1024*1024)
blockdatalist = db.GetBlocksListFromUID(uid)
#read 1 block to initialize the correct block parameters
#(needed for filling in missing blocks)
blockdata = blockdatalist[0]
fin = finlist[blockdata[1]]
bpos = blockdata[2]
fin.seek(bpos, 0)
try:
sbx.decode(fin.read(sbx.blocksize))
except seqbox.SbxDecodeError as err:
print(err)
errexit(1, "invalid block at offset %s file '%s'" %
(hex(bpos), fin.name))
lastblock = -1
ticks = 0
missingblocks = 0
updatetime = time() -1
maxbnum = blockdatalist[-1][0]
#loop trough the block list and recreate SBx file
for blockdata in blockdatalist:
bnum = blockdata[0]
#check for missing blocks and fill in
if bnum != lastblock +1 and bnum != 1:
for b in range(lastblock+1, bnum):
#no point in an empty block 0 with no metadata
if b > 0 and cmdline.fill:
sbx.blocknum = b
sbx.data = bytes(sbx.datasize)
buffer = sbx.encode()
fout.write(buffer)
missingblocks += 1
fin = finlist[blockdata[1]]
bpos = blockdata[2]
fin.seek(bpos, 0)
buffer = fin.read(sbx.blocksize)
fout.write(buffer)
lastblock = bnum
#some progress report
if time() > updatetime or bnum == maxbnum:
print(" %.1f%%" % (bnum*100.0/maxbnum), " ",
"(missing blocks: %i)" % missingblocks,
end="\r", flush=True)
updatetime = time() + .5
fout.close()
print()
if missingblocks > 0:
uiderrlist.append((uid, missingblocks))
totblockserr += missingblocks
print("\ndone.")
if len(uiderrlist) == 0:
print("all SBx files recovered with no errors!")
else:
print("errors detected in %i SBx file(s)!" % len(uiderrlist))
report_err(db, uiderrlist, uidDataList, blocksizes)
def main():
cmdline = get_cmdline()
filenames = []
for filename in cmdline.filename:
if os.path.exists(filename):
filenames.append(filename)
else:
errexit(1, "file '%s' not found!" % (filename))
filenames = sorted(set(filenames), key=os.path.getsize)
dbfilename = cmdline.dbfilename
if os.path.isdir(dbfilename):
dbfilename = os.path.join(dbfilename, "sbxscan.db3")
#create database tables
print("creating '%s' database..." % (dbfilename))
if os.path.exists(dbfilename):
os.remove(dbfilename)
conn = sqlite3.connect(dbfilename)
c = conn.cursor()
c.execute("CREATE TABLE sbx_source (id INTEGER, name TEXT)")
c.execute(
"CREATE TABLE sbx_meta (uid INTEGER, size INTEGER, name TEXT, sbxname TEXT, datetime INTEGER, sbxdatetime INTEGER, fileid INTEGER)"
)
c.execute("CREATE TABLE sbx_uids (uid INTEGER, ver INTEGER)")
c.execute(
"CREATE TABLE sbx_blocks (uid INTEGER, num INTEGER, fileid INTEGER, pos INTEGER )"
)
c.execute("CREATE INDEX blocks ON sbx_blocks (uid, num, pos)")
#scan all the files/devices
sbx = seqbox.SbxBlock(ver=cmdline.sbxver, pswd=cmdline.password)
offset = cmdline.offset
filenum = 0
uids = {}
magic = b'SBx' + bytes([cmdline.sbxver])
if cmdline.password:
magic = seqbox.EncDec(cmdline.password, len(magic)).xor(magic)
scanstep = cmdline.step
if scanstep == 0:
scanstep = sbx.blocksize
for filename in filenames:
filenum += 1
print("scanning file/device '%s' (%i/%i)..." %
(filename, filenum, len(filenames)))
filesize = getFileSize(filename)
c.execute("INSERT INTO sbx_source (id, name) VALUES (?, ?)",
(filenum, filename))
conn.commit()
fin = open(filename, "rb", buffering=cmdline.buffer * 1024)
blocksfound = 0
blocksmetafound = 0
updatetime = time() - 1
starttime = time()
docommit = False
for pos in range(offset, filesize, scanstep):
fin.seek(pos, 0)
buffer = fin.read(sbx.blocksize)
#check for magic
if buffer[:4] == magic:
#check for valid block
try:
sbx.decode(buffer)
#update uids table & list
if not sbx.uid in uids:
uids[sbx.uid] = True
c.execute(
"INSERT INTO sbx_uids (uid, ver) VALUES (?, ?)",
(int.from_bytes(sbx.uid,
byteorder='big'), sbx.ver))
docommit = True
#update blocks table
blocksfound += 1
c.execute(
"INSERT INTO sbx_blocks (uid, num, fileid, pos) VALUES (?, ?, ?, ?)",
(int.from_bytes(sbx.uid, byteorder='big'),
sbx.blocknum, filenum, pos))
docommit = True
#update meta table
if sbx.blocknum == 0:
blocksmetafound += 1
if not "filedatetime" in sbx.metadata:
sbx.metadata["filedatetime"] = -1
sbx.metadata["sbxdatetime"] = -1
c.execute(
"INSERT INTO sbx_meta (uid , size, name, sbxname, datetime, sbxdatetime, fileid) VALUES (?, ?, ?, ?, ?, ?, ?)",
(int.from_bytes(sbx.uid, byteorder='big'),
sbx.metadata["filesize"],
sbx.metadata["filename"], sbx.metadata["sbxname"],
sbx.metadata["filedatetime"],
sbx.metadata["sbxdatetime"], filenum))
docommit = True
except seqbox.SbxDecodeError:
pass
#status update
if (time() > updatetime) or (pos >= filesize - scanstep):
etime = (time() - starttime)
if etime == 0:
etime = 1
print("%5.1f%% blocks: %i - meta: %i - files: %i - %.2fMB/s" %
(pos * 100.0 /
(filesize - scanstep), blocksfound, blocksmetafound,
len(uids), pos / (1024 * 1024) / etime),
end="\r",
flush=True)
if docommit:
conn.commit()
docommit = False
updatetime = time() + .5
fin.close()
print()
c.close()
conn.close()
print("scan completed!")
def main():
cmdline = get_cmdline()
filename = cmdline.filename
sbxfilename = cmdline.sbxfilename
if not sbxfilename:
sbxfilename = filename + ".sbx"
elif os.path.isdir(sbxfilename):
sbxfilename = os.path.join(sbxfilename,
os.path.split(filename)[1] + ".sbx")
if os.path.exists(sbxfilename) and not cmdline.overwrite:
errexit(1, "SBX file '%s' already exists!" % (sbxfilename))
#parse eventual custom uid
uid = cmdline.uid
if uid !="r":
uid = uid[-12:]
try:
uid = int(uid, 16).to_bytes(6, byteorder='big')
except:
errexit(1, "invalid UID")
if not os.path.exists(filename):
errexit(1, "file '%s' not found" % (filename))
filesize = os.path.getsize(filename)
fout = open(sbxfilename, "wb", buffering=1024*1024)
#calc hash - before all processing, and not while reading the file,
#just to be cautious
if not cmdline.nometa:
print("hashing file '%s'..." % (filename))
sha256 = getsha256(filename)
print("SHA256",binascii.hexlify(sha256).decode())
fin = open(filename, "rb", buffering=1024*1024)
print("creating file '%s'..." % sbxfilename)
sbx = seqbox.SbxBlock(uid=uid, ver=cmdline.sbxver, pswd=cmdline.password)
#write metadata block 0
if not cmdline.nometa:
sbx.metadata = {"filesize":filesize,
"filename":os.path.split(filename)[1],
"sbxname":os.path.split(sbxfilename)[1],
"filedatetime":int(os.path.getmtime(filename)),
"sbxdatetime":int(time()),
"hash":b'\x12\x20'+sha256} #multihash
fout.write(sbx.encode())
#write all other blocks
ticks = 0
updatetime = time()
while True:
buffer = fin.read(sbx.datasize)
if len(buffer) < sbx.datasize:
if len(buffer) == 0:
break
sbx.blocknum += 1
sbx.data = buffer
fout.write(sbx.encode())
#some progress update
if time() > updatetime:
print("%.1f%%" % (fin.tell()*100.0/filesize), " ",
end="\r", flush=True)
updatetime = time() + .1
print("100% ")
fin.close()
fout.close()
totblocks = sbx.blocknum if cmdline.nometa else sbx.blocknum + 1
sbxfilesize = totblocks * sbx.blocksize
overhead = 100.0 * sbxfilesize / filesize - 100 if filesize > 0 else 0
print("SBX file size: %i - blocks: %i - overhead: %.1f%%" %
(sbxfilesize, totblocks, overhead))
def main():
cmdline = get_cmdline()
sbxfilename = cmdline.sbxfilename
filename = cmdline.filename
if not os.path.exists(sbxfilename):
errexit(1, "sbx file '%s' not found" % (sbxfilename))
sbxfilesize = os.path.getsize(sbxfilename)
print("decoding '%s'..." % (sbxfilename))
fin = open(sbxfilename, "rb", buffering=1024 * 1024)
#check magic and get version
header = fin.read(4)
fin.seek(0, 0)
if cmdline.password:
e = seqbox.EncDec(cmdline.password, len(header))
header = e.xor(header)
if header[:3] != b"SBx":
errexit(1, "not a SeqBox file!")
sbxver = header[3]
sbx = seqbox.SbxBlock(ver=sbxver, pswd=cmdline.password)
metadata = {}
trimfilesize = False
hashtype = 0
hashlen = 0
hashdigest = b""
hashcheck = False
buffer = fin.read(sbx.blocksize)
try:
sbx.decode(buffer)
except seqbox.SbxDecodeError as err:
if cmdline.cont == False:
print(err)
errexit(errlev=1, mess="invalid block at offset 0x0")
if sbx.blocknum > 1:
errexit(errlev=1, mess="blocks missing or out of order at offset 0x0")
elif sbx.blocknum == 0:
print("metadata block found!")
metadata = sbx.metadata
if "filesize" in metadata:
trimfilesize = True
if "hash" in metadata:
hashtype = metadata["hash"][0]
if hashtype == 0x12:
hashlen = metadata["hash"][1]
hashdigest = metadata["hash"][2:2 + hashlen]
hashcheck = True
else:
#first block is data, so reset from the start
print("no metadata available")
fin.seek(0, 0)
#display some info and stop
if cmdline.info:
print("\nSeqBox container info:")
print(" file size: %i bytes" % (sbxfilesize))
print(" blocks: %i" % (sbxfilesize / sbx.blocksize))
print(" version: %i" % (sbx.ver))
print(" UID: %s" % (binascii.hexlify(sbx.uid).decode()))
if metadata:
print("metadata:")
if "sbxname" in metadata:
print(" SBX name : '%s'" % (metadata["sbxname"]))
if "filename" in metadata:
print(" file name: '%s'" % (metadata["filename"]))
if "filesize" in metadata:
print(" file size: %i bytes" % (metadata["filesize"]))
if "sbxdatetime" in metadata:
print(" SBX date&time : %s" %
(time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(metadata["sbxdatetime"]))))
if "filedatetime" in metadata:
print(
" file date&time: %s" %
(time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(metadata["filedatetime"]))))
if "hash" in metadata:
if hashtype == 0x12:
print(" SHA256: %s" %
(binascii.hexlify(hashdigest).decode()))
else:
print(" hash type not recognized!")
sys.exit(0)
#evaluate target filename
if not cmdline.test:
if not filename:
if "filename" in metadata:
filename = metadata["filename"]
else:
filename = sbxfilename + ".out"
elif os.path.isdir(filename):
if "filename" in metadata:
filename = os.path.join(filename, metadata["filename"])
else:
filename = os.path.join(filename,
os.path.split(sbxfilename)[1] + ".out")
if os.path.exists(filename) and not cmdline.overwrite:
errexit(1, "target file '%s' already exists!" % (filename))
print("creating file '%s'..." % (filename))
fout = open(filename, "wb", buffering=1024 * 1024)
if hashtype == 0x12:
d = hashlib.sha256()
lastblocknum = 0
filesize = 0
blockmiss = 0
updatetime = time.time()
while True:
buffer = fin.read(sbx.blocksize)
if len(buffer) < sbx.blocksize:
break
try:
sbx.decode(buffer)
if sbx.blocknum > lastblocknum + 1:
if cmdline.cont:
blockmiss += 1
lastblocknum += 1
else:
errexit(errlev=1,
mess="block %i out of order or missing" %
(lastblocknum + 1))
lastblocknum += 1
if trimfilesize:
filesize += sbx.datasize
if filesize > metadata["filesize"]:
sbx.data = sbx.data[:-(filesize - metadata["filesize"])]
if hashcheck:
d.update(sbx.data)
if not cmdline.test:
fout.write(sbx.data)
except seqbox.SbxDecodeError as err:
if cmdline.cont:
blockmiss += 1
lastblocknum += 1
else:
print(err)
errexit(errlev=1,
mess="invalid block at offset %s" %
(hex(fin.tell() - sbx.blocksize)))
#some progress report
if time.time() > updatetime:
print(" %.1f%%" % (fin.tell() * 100.0 / sbxfilesize),
end="\r",
flush=True)
updatetime = time.time() + .1
fin.close()
if not cmdline.test:
fout.close()
if metadata:
if "filedatetime" in metadata:
os.utime(filename,
(int(time.time()), metadata["filedatetime"]))
print("SBX decoding complete")
if blockmiss:
errexit(1, "missing blocks: %i" % blockmiss)
if hashcheck:
if hashtype == 0x12:
print("SHA256", d.hexdigest())
if d.digest() == hashdigest:
print("hash match!")
else:
errexit(1, "hash mismatch! decoded file corrupted!")
else:
print("can't check integrity via hash!")
#if filesize unknown, estimate based on 0x1a padding at block's end
if not trimfilesize:
c = lastEofCount(sbx.data[-4:])
print("EOF markers at the end of last block: %i/4" % c)