def buildFile(self, fromPath, label="", catg=""):
count = self.getCountParts(fromPath)
Sys.pwlog([(" Get Hash... ", Const.CLZ_7, True)])
sha256 = hash_sha256_file(fromPath)
Sys.pwlog([(" hash : ", Const.CLZ_0), (sha256, Const.CLZ_2, True), (" Build File...", Const.CLZ_0, True)])
kg = KeyGen(128)
size = Sys.getsize(fromPath)
row = self.idxu.index.get(sha256)
if row is None:
if label == "":
label, ext = Sys.getFileExt(Sys.basename(fromPath))
else:
label, ext = Sys.getFileExt(label)
if catg == "":
catg = self.idxu.index.getAutoCatg(ext)
size = Sys.getsize(fromPath)
Sys.pwlog(
[
(" Splitting ", Const.CLZ_1),
(label, Const.CLZ_7),
(ext, Const.CLZ_7),
(" (", Const.CLZ_0),
(Sys.readableBytes(size), Const.CLZ_3),
(")", Const.CLZ_0, True),
]
)
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
km = Kirmah(kg.key)
km.DIR_OUTBOX = self.outbox
# hlst genetate with sha256
hlst = km.ck.getHashList(sha256, int(count), True)
usr = self.idxu.conf.get("name", "infos")
ownerHash = self.idxu.mb.getHashName(usr)
km.split(fromPath, hlst)
Sys.pwlog([(" done ", Const.CLZ_2, True)])
row = [kg.key, label, ext, count, catg, hlst, usr, ownerHash, sha256, size]
self.backupAddMap(row)
else:
Sys.pwlog(
[
(" File Already exist ! ", Const.CLZ_1, True),
(" id : ".rjust(10, " "), Const.CLZ_0),
(str(row[ImpraIndex.UID]), Const.CLZ_1, True),
(" label : ".rjust(10, " "), Const.CLZ_0),
(row[ImpraIndex.LABEL], Const.CLZ_3, True),
]
)
row = None
return row
def on_refresh(self, btn, data=None):
""""""
self.BLOCK_REPOPULATE = True
self.get('treestore1').clear()
if btn is not None :
Sys.cli_emit_progress(0)
self.get('checkbutton1').set_sensitive(False)
self.get('checkbutton1').set_active(False)
self.unselect_actions()
self.get('button9').set_sensitive(False)
self.launch_action(ImpraThread.TASK_REFRESH, [])
else :
self.populate_index()
def onend_cmd(self, title, stime, done, outputfile):
""""""
s = Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN
Sys.echo(s, Sys.CLZ_HEAD_LINE)
Sys.wlog([(s, Const.CLZ_HEAD_SEP)])
Sys.pstep(title, stime, done, True)
Sys.echo(s, Sys.CLZ_HEAD_LINE)
Sys.wlog([(s, Const.CLZ_HEAD_SEP)])
if done and outputfile is not None:
Sys.cli_emit_progress(100)
Sys.echo(' '*5+Sys.realpath(outputfile), Sys.Clz.fgB1, False)
Sys.echo(' ('+Sys.getFileSize(outputfile)+')', Sys.Clz.fgB3)
bdata = [(' '*5+Sys.realpath(outputfile), 'io'),(' ('+Sys.getFileSize(outputfile)+')','func')]
Sys.wlog(bdata)
Sys.wlog(Sys.dprint())
def onend_cmd(self, title, stime, done, outputfile):
""""""
s = Const.LINE_SEP_CHAR * Const.LINE_SEP_LEN
Sys.print(s, Sys.CLZ_HEAD_LINE)
Sys.wlog([(s, Const.CLZ_HEAD_SEP)])
Sys.pstep(title, stime, done, True)
Sys.print(s, Sys.CLZ_HEAD_LINE)
Sys.wlog([(s, Const.CLZ_HEAD_SEP)])
if done and outputfile is not None:
Sys.cli_emit_progress(100)
Sys.print(' ' * 5 + Sys.realpath(outputfile), Sys.Clz.fgB1, False)
Sys.print(' (' + Sys.getFileSize(outputfile) + ')', Sys.Clz.fgB3)
bdata = [(' ' * 5 + Sys.realpath(outputfile), 'io'),
(' (' + Sys.getFileSize(outputfile) + ')', 'func')]
Sys.wlog(bdata)
Sys.wlog(Sys.dprint())
def getFileParts(self, row, ids):
""""""
done = False
if len(ids) >= row[self.idxu.index.PARTS]:
for i, uid in enumerate(ids):
d = Sys.datetime.now()
self.idxu.ih.getAttachment(uid, self.inbox, True)
Sys.pwlog(
[
(" part ", Const.CLZ_0),
(str(i + 1).rjust(2, " "), Const.CLZ_2),
(" / ", Const.CLZ_0),
(str(len(ids)), Const.CLZ_3),
(" downloaded in ", Const.CLZ_0),
(Sys.getDelta(d), Const.CLZ_4, True),
]
)
Sys.cli_emit_progress(int((i + 1) * 100 / len(ids)))
Sys.sleep(0.5)
Sys.mkdir_p(self.deploy + row[self.idxu.index.CATG])
Sys.cli_emit_progress(100)
def onCommandSplit(self):
""""""
done = True
Sys.cli_emit_progress(1)
if not self.o.parts is None and not(int(self.o.parts)>=12 and int(self.o.parts) <=62) :
self.parser.error_cmd((('invalid option ',('-p, --parts', Sys.Clz.fgb3), ' value (', ('12',Sys.Clz.fgb3),' to ', ('62',Sys.Clz.fgb3),')'),))
else : self.o.parts = int(self.o.parts)
if not Sys.g.QUIET : self.parser.print_header()
if self.o.outputfile is not None :
if self.o.outputfile[-5:]!='.tark' : self.o.outputfile += '.tark'
if Io.file_exists(self.o.outputfile) and not self.o.force:
Sys.pwarn((('the file ',(self.o.outputfile, Sys.Clz.fgb3), ' already exists !'),))
done = Sys.pask('Are you sure to rewrite this file')
self.stime = Sys.datetime.now()
if done :
try :
Sys.ptask()
Sys.cli_emit_progress(2)
key = Io.get_data(self.o.keyfile)
km = Kirmah(key)
hlst = km.ck.getHashList(Sys.basename(self.a[1]), self.o.parts, True)
Sys.cli_emit_progress(3)
kcf = km.splitFile(self.a[1], hlst)
t = int(Sys.time())
times = (t,t)
p = 85
Sys.cli_emit_progress(p)
Io.touch(kcf, times)
frav = 0.24
for row in hlst['data']:
p += frav
Io.touch(row[1]+km.EXT,times)
Sys.cli_emit_progress(p)
if self.o.outputfile is not None :
d = Sys.datetime.now()
if Sys.g.DEBUG : Sys.wlog(Sys.dprint())
Sys.ptask('Preparing tark file')
hlst['data'] = sorted(hlst['data'], key=lambda lst: lst[4])
with tarfile.open(self.o.outputfile, mode='w') as tar:
tar.add(kcf, arcname=Sys.basename(kcf))
p = 90
for row in hlst['data']:
tar.add(row[1]+km.EXT, arcname=Sys.basename(row[1]+km.EXT))
p += frav
Sys.cli_emit_progress(p)
Sys.pstep('Packing destination file', d, True)
d = Sys.datetime.now()
Sys.ptask('Finalize')
for row in hlst['data']:
Io.removeFile(row[1]+km.EXT)
p += frav
Sys.cli_emit_progress(p)
Io.removeFile(kcf)
Sys.pstep('Cleaning', d, True)
except Exception as e :
done = False
if Sys.g.DEBUG :
print('split exception')
print(e)
#~ raise e
elif not Sys.g.QUIET :
Sys.pwarn((str(e),))
if not Sys.g.QUIET:
Sys.cli_emit_progress(100)
self.onend_cmd('Kirmah Split', self.stime, done, self.o.outputfile)
def onCommandSplit(self):
""""""
done = True
Sys.cli_emit_progress(1)
if not self.o.parts is None and not (int(self.o.parts) >= 12
and int(self.o.parts) <= 62):
self.parser.error_cmd(
(('invalid option ', ('-p, --parts', Sys.Clz.fgb3), ' value (',
('12', Sys.Clz.fgb3), ' to ', ('62', Sys.Clz.fgb3), ')'), ))
else:
self.o.parts = int(self.o.parts)
if not Sys.g.QUIET: self.parser.print_header()
if self.o.outputfile is not None:
if self.o.outputfile[-5:] != '.tark': self.o.outputfile += '.tark'
if Io.file_exists(self.o.outputfile) and not self.o.force:
Sys.pwarn((('the file ', (self.o.outputfile, Sys.Clz.fgb3),
' already exists !'), ))
done = Sys.pask('Are you sure to rewrite this file')
self.stime = Sys.datetime.now()
if done:
try:
Sys.ptask()
Sys.cli_emit_progress(2)
key = Io.get_data(self.o.keyfile)
km = Kirmah(key)
hlst = km.ck.getHashList(Sys.basename(self.a[1]), self.o.parts,
True)
Sys.cli_emit_progress(3)
kcf = km.splitFile(self.a[1], hlst)
t = int(Sys.time())
times = (t, t)
p = 85
Sys.cli_emit_progress(p)
Io.touch(kcf, times)
frav = 0.24
for row in hlst['data']:
p += frav
Io.touch(row[1] + km.EXT, times)
Sys.cli_emit_progress(p)
if self.o.outputfile is not None:
d = Sys.datetime.now()
if Sys.g.DEBUG: Sys.wlog(Sys.dprint())
Sys.ptask('Preparing tark file')
hlst['data'] = sorted(hlst['data'], key=lambda lst: lst[4])
with tarfile.open(self.o.outputfile, mode='w') as tar:
tar.add(kcf, arcname=Sys.basename(kcf))
p = 90
for row in hlst['data']:
tar.add(row[1] + km.EXT,
arcname=Sys.basename(row[1] + km.EXT))
p += frav
Sys.cli_emit_progress(p)
Sys.pstep('Packing destination file', d, True)
d = Sys.datetime.now()
Sys.ptask('Finalize')
for row in hlst['data']:
Io.removeFile(row[1] + km.EXT)
p += frav
Sys.cli_emit_progress(p)
Io.removeFile(kcf)
Sys.pstep('Cleaning', d, True)
except Exception as e:
done = False
if Sys.g.DEBUG:
print('split exception')
print(e)
#~ raise e
elif not Sys.g.QUIET:
Sys.pwarn((str(e), ))
if not Sys.g.QUIET:
Sys.cli_emit_progress(100)
self.onend_cmd('Kirmah Split', self.stime, done, self.o.outputfile)
def getFile(self, uid):
""""""
# ~ Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0, True)])
done = False
key = self.idxu.index.getById(uid)
row = self.idxu.index.get(key)
filePath = None
try:
if row is not None:
account = self.idxu.switchFileAccount(row[self.idxu.index.ACCOUNT])
km = Kirmah(row[self.idxu.index.KEY])
hlst = km.ck.getHashList(key, row[self.idxu.index.PARTS], True)
ids = self.idxu.ih.searchBySubject(hlst["head"][2], True)
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
Sys.pwlog(
[
(" Downloading : ", Const.CLZ_7),
(row[self.idxu.index.LABEL] + row[self.idxu.index.EXT], Const.CLZ_2),
(" (", Const.CLZ_0),
(Sys.readableBytes(row[self.idxu.index.SIZE]), Const.CLZ_3),
(")", Const.CLZ_0),
(" please wait...", Const.CLZ_7, True),
]
)
if len(ids) >= row[self.idxu.index.PARTS]:
self.getFileParts(row, ids)
Sys.pwlog([(" Merging parts...", Const.CLZ_7, True)])
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
filePath = km.merge(
hlst,
self.deploy + row[self.idxu.index.CATG] + Sys.sep + row[self.idxu.index.LABEL],
ext=row[self.idxu.index.EXT],
uid=row[self.idxu.index.UID],
dirs=self.inbox,
)
Sys.pwlog(
[
(" Deployed as ", Const.CLZ_7),
(filePath, Const.CLZ_2),
(" (", Const.CLZ_0),
(Sys.readableBytes(Sys.getsize(filePath)), Const.CLZ_3),
(") ", Const.CLZ_0, True),
(" Checking integrity...", Const.CLZ_7, True),
]
)
Sys.sleep(0.2)
sha256 = hash_sha256_file(filePath)
done = sha256 == row[ImpraIndex.HASH]
done = True
else:
print("incomplete")
except Exception as e:
print(e)
Sys.pwlog([(" done" if done else "ko", Const.CLZ_2 if done else Const.CLZ_1, True)])
return done, filePath
def sendFile(self, data, retry=False):
""""""
done = None
key = None
if data is not None:
key, label, ext, count, catg, hlst, usr, ownerHash, sha256, size = data
self.idxu.index.addUser(usr, ownerHash)
account = self.idxu.switchFileAccount()
sendIds = []
cancel = False
d = None
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
if not retry:
Sys.pwlog(
[
(" Sending... ", Const.CLZ_7),
(" (", Const.CLZ_0),
(
" ~" + Sys.readableBytes(Sys.getsize(self.outbox + hlst["data"][0][1] + Kirmah.EXT)),
Const.CLZ_3,
),
(" per msg ) ", Const.CLZ_0, True),
]
)
else:
Sys.pwlog(
[
(" Retry sending last file... ", Const.CLZ_0),
(label + ext, Const.CLZ_7),
(" (" + catg + ")", Const.CLZ_3, True),
]
)
ignore = False
for i, row in enumerate(hlst["data"]):
""""""
if retry:
if not Io.file_exists(self.outbox + row[1] + Kirmah.EXT):
continue
elif not ignore:
Sys.pwlog([(" Ignoring file 1 to " + str(i), Const.CLZ_1, True)])
ignore = True
d = Sys.datetime.now()
msg = self.idxu.mb.build(usr, "all", hlst["head"][2], self.outbox + row[1] + Kirmah.EXT)
try:
mid = self.idxu.ih.send(msg.as_string(), self.rootBox)
except Exception as e:
Sys.pwarn((("addFile : ", (str(e), Sys.CLZ_WARN_PARAM), " !"),))
Sys.echo("waiting 5 s and retry")
Sys.sleep(5)
# force reconnect
self.impst.idxu.switchFileAccount(account, True)
# retry
mid = self.idxu.ih.send(msg.as_string(), self.rootBox)
finally:
if not mid is None:
status, resp = self.idxu.ih.fetch(mid[1], "(UID BODYSTRUCTURE)", True)
if status == self.idxu.ih.OK:
sendIds.append((mid[1], row))
Sys.pwlog(
[
(" part ", Const.CLZ_0),
(str(row[0]).rjust(2, "0"), Const.CLZ_2),
(" sent as msg ", Const.CLZ_0),
(str(mid[1]).rjust(5, "0"), Const.CLZ_1),
(" (", Const.CLZ_7),
(str(int(row[4]) + 1).rjust(2, "0"), Const.CLZ_2),
("/", Const.CLZ_7),
(str(count), Const.CLZ_3),
(") in ", Const.CLZ_7),
(Sys.getDelta(d), Const.CLZ_4, True),
]
)
Sys.cli_emit_progress(int((i + 1) * 100 / len(hlst["data"])))
Sys.removeFile(self.outbox + row[1] + Kirmah.EXT)
else:
Sys.pwarn((("error occured when sending part ", (row[0], Sys.Clz.fgb3), " !"),))
diff = self.checkSendIds(sendIds, hlst["head"][2])
if len(sendIds) == count or retry:
self.idxu.get(True)
self.idxu.index.add(key, label, hlst["head"][1], ext, ownerHash, catg, sha256, size, account)
done = self.idxu.update()
Io.removeFile(self.addmapPath + Kirmah.EXT)
# resending missing parts
else:
Sys.pwarn((("TODO => must resending ", ("missing", Sys.CLZ_WARN_PARAM), " parts"),))
print(diff)
# clean
for mid, row in sendIds:
if Io.file_exists(self.outbox + row[1] + Kirmah.EXT):
Sys.removeFile(self.outbox + row[1] + Kirmah.EXT)
if cancel:
delids = [mid for mid, row in senids]
print(delids)
self.idxu.ih.delete(delids, True)
return done, key
