def buildFile(self, fromPath, label="", catg=""): count = self.getCountParts(fromPath) Sys.pwlog([(" Get Hash... ", Const.CLZ_7, True)]) sha256 = hash_sha256_file(fromPath) Sys.pwlog([(" hash : ", Const.CLZ_0), (sha256, Const.CLZ_2, True), (" Build File...", Const.CLZ_0, True)]) kg = KeyGen(128) size = Sys.getsize(fromPath) row = self.idxu.index.get(sha256) if row is None: if label == "": label, ext = Sys.getFileExt(Sys.basename(fromPath)) else: label, ext = Sys.getFileExt(label) if catg == "": catg = self.idxu.index.getAutoCatg(ext) size = Sys.getsize(fromPath) Sys.pwlog( [ (" Splitting ", Const.CLZ_1), (label, Const.CLZ_7), (ext, Const.CLZ_7), (" (", Const.CLZ_0), (Sys.readableBytes(size), Const.CLZ_3), (")", Const.CLZ_0, True), ] ) Sys.cli_emit_progress(0) Sys.sleep(0.2) km = Kirmah(kg.key) km.DIR_OUTBOX = self.outbox # hlst genetate with sha256 hlst = km.ck.getHashList(sha256, int(count), True) usr = self.idxu.conf.get("name", "infos") ownerHash = self.idxu.mb.getHashName(usr) km.split(fromPath, hlst) Sys.pwlog([(" done ", Const.CLZ_2, True)]) row = [kg.key, label, ext, count, catg, hlst, usr, ownerHash, sha256, size] self.backupAddMap(row) else: Sys.pwlog( [ (" File Already exist ! ", Const.CLZ_1, True), (" id : ".rjust(10, " "), Const.CLZ_0), (str(row[ImpraIndex.UID]), Const.CLZ_1, True), (" label : ".rjust(10, " "), Const.CLZ_0), (row[ImpraIndex.LABEL], Const.CLZ_3, True), ] ) row = None return row
def on_refresh(self, btn, data=None): """""" self.BLOCK_REPOPULATE = True self.get('treestore1').clear() if btn is not None : Sys.cli_emit_progress(0) self.get('checkbutton1').set_sensitive(False) self.get('checkbutton1').set_active(False) self.unselect_actions() self.get('button9').set_sensitive(False) self.launch_action(ImpraThread.TASK_REFRESH, []) else : self.populate_index()
def onend_cmd(self, title, stime, done, outputfile): """""" s = Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN Sys.echo(s, Sys.CLZ_HEAD_LINE) Sys.wlog([(s, Const.CLZ_HEAD_SEP)]) Sys.pstep(title, stime, done, True) Sys.echo(s, Sys.CLZ_HEAD_LINE) Sys.wlog([(s, Const.CLZ_HEAD_SEP)]) if done and outputfile is not None: Sys.cli_emit_progress(100) Sys.echo(' '*5+Sys.realpath(outputfile), Sys.Clz.fgB1, False) Sys.echo(' ('+Sys.getFileSize(outputfile)+')', Sys.Clz.fgB3) bdata = [(' '*5+Sys.realpath(outputfile), 'io'),(' ('+Sys.getFileSize(outputfile)+')','func')] Sys.wlog(bdata) Sys.wlog(Sys.dprint())
def onend_cmd(self, title, stime, done, outputfile): """""" s = Const.LINE_SEP_CHAR * Const.LINE_SEP_LEN Sys.print(s, Sys.CLZ_HEAD_LINE) Sys.wlog([(s, Const.CLZ_HEAD_SEP)]) Sys.pstep(title, stime, done, True) Sys.print(s, Sys.CLZ_HEAD_LINE) Sys.wlog([(s, Const.CLZ_HEAD_SEP)]) if done and outputfile is not None: Sys.cli_emit_progress(100) Sys.print(' ' * 5 + Sys.realpath(outputfile), Sys.Clz.fgB1, False) Sys.print(' (' + Sys.getFileSize(outputfile) + ')', Sys.Clz.fgB3) bdata = [(' ' * 5 + Sys.realpath(outputfile), 'io'), (' (' + Sys.getFileSize(outputfile) + ')', 'func')] Sys.wlog(bdata) Sys.wlog(Sys.dprint())
def getFileParts(self, row, ids): """""" done = False if len(ids) >= row[self.idxu.index.PARTS]: for i, uid in enumerate(ids): d = Sys.datetime.now() self.idxu.ih.getAttachment(uid, self.inbox, True) Sys.pwlog( [ (" part ", Const.CLZ_0), (str(i + 1).rjust(2, " "), Const.CLZ_2), (" / ", Const.CLZ_0), (str(len(ids)), Const.CLZ_3), (" downloaded in ", Const.CLZ_0), (Sys.getDelta(d), Const.CLZ_4, True), ] ) Sys.cli_emit_progress(int((i + 1) * 100 / len(ids))) Sys.sleep(0.5) Sys.mkdir_p(self.deploy + row[self.idxu.index.CATG]) Sys.cli_emit_progress(100)
def onCommandSplit(self): """""" done = True Sys.cli_emit_progress(1) if not self.o.parts is None and not(int(self.o.parts)>=12 and int(self.o.parts) <=62) : self.parser.error_cmd((('invalid option ',('-p, --parts', Sys.Clz.fgb3), ' value (', ('12',Sys.Clz.fgb3),' to ', ('62',Sys.Clz.fgb3),')'),)) else : self.o.parts = int(self.o.parts) if not Sys.g.QUIET : self.parser.print_header() if self.o.outputfile is not None : if self.o.outputfile[-5:]!='.tark' : self.o.outputfile += '.tark' if Io.file_exists(self.o.outputfile) and not self.o.force: Sys.pwarn((('the file ',(self.o.outputfile, Sys.Clz.fgb3), ' already exists !'),)) done = Sys.pask('Are you sure to rewrite this file') self.stime = Sys.datetime.now() if done : try : Sys.ptask() Sys.cli_emit_progress(2) key = Io.get_data(self.o.keyfile) km = Kirmah(key) hlst = km.ck.getHashList(Sys.basename(self.a[1]), self.o.parts, True) Sys.cli_emit_progress(3) kcf = km.splitFile(self.a[1], hlst) t = int(Sys.time()) times = (t,t) p = 85 Sys.cli_emit_progress(p) Io.touch(kcf, times) frav = 0.24 for row in hlst['data']: p += frav Io.touch(row[1]+km.EXT,times) Sys.cli_emit_progress(p) if self.o.outputfile is not None : d = Sys.datetime.now() if Sys.g.DEBUG : Sys.wlog(Sys.dprint()) Sys.ptask('Preparing tark file') hlst['data'] = sorted(hlst['data'], key=lambda lst: lst[4]) with tarfile.open(self.o.outputfile, mode='w') as tar: tar.add(kcf, arcname=Sys.basename(kcf)) p = 90 for row in hlst['data']: tar.add(row[1]+km.EXT, arcname=Sys.basename(row[1]+km.EXT)) p += frav Sys.cli_emit_progress(p) Sys.pstep('Packing destination file', d, True) d = Sys.datetime.now() Sys.ptask('Finalize') for row in hlst['data']: Io.removeFile(row[1]+km.EXT) p += frav Sys.cli_emit_progress(p) Io.removeFile(kcf) Sys.pstep('Cleaning', d, True) except Exception as e : done = False if Sys.g.DEBUG : print('split exception') print(e) #~ raise e elif not Sys.g.QUIET : Sys.pwarn((str(e),)) if not Sys.g.QUIET: Sys.cli_emit_progress(100) self.onend_cmd('Kirmah Split', self.stime, done, self.o.outputfile)
def onCommandSplit(self): """""" done = True Sys.cli_emit_progress(1) if not self.o.parts is None and not (int(self.o.parts) >= 12 and int(self.o.parts) <= 62): self.parser.error_cmd( (('invalid option ', ('-p, --parts', Sys.Clz.fgb3), ' value (', ('12', Sys.Clz.fgb3), ' to ', ('62', Sys.Clz.fgb3), ')'), )) else: self.o.parts = int(self.o.parts) if not Sys.g.QUIET: self.parser.print_header() if self.o.outputfile is not None: if self.o.outputfile[-5:] != '.tark': self.o.outputfile += '.tark' if Io.file_exists(self.o.outputfile) and not self.o.force: Sys.pwarn((('the file ', (self.o.outputfile, Sys.Clz.fgb3), ' already exists !'), )) done = Sys.pask('Are you sure to rewrite this file') self.stime = Sys.datetime.now() if done: try: Sys.ptask() Sys.cli_emit_progress(2) key = Io.get_data(self.o.keyfile) km = Kirmah(key) hlst = km.ck.getHashList(Sys.basename(self.a[1]), self.o.parts, True) Sys.cli_emit_progress(3) kcf = km.splitFile(self.a[1], hlst) t = int(Sys.time()) times = (t, t) p = 85 Sys.cli_emit_progress(p) Io.touch(kcf, times) frav = 0.24 for row in hlst['data']: p += frav Io.touch(row[1] + km.EXT, times) Sys.cli_emit_progress(p) if self.o.outputfile is not None: d = Sys.datetime.now() if Sys.g.DEBUG: Sys.wlog(Sys.dprint()) Sys.ptask('Preparing tark file') hlst['data'] = sorted(hlst['data'], key=lambda lst: lst[4]) with tarfile.open(self.o.outputfile, mode='w') as tar: tar.add(kcf, arcname=Sys.basename(kcf)) p = 90 for row in hlst['data']: tar.add(row[1] + km.EXT, arcname=Sys.basename(row[1] + km.EXT)) p += frav Sys.cli_emit_progress(p) Sys.pstep('Packing destination file', d, True) d = Sys.datetime.now() Sys.ptask('Finalize') for row in hlst['data']: Io.removeFile(row[1] + km.EXT) p += frav Sys.cli_emit_progress(p) Io.removeFile(kcf) Sys.pstep('Cleaning', d, True) except Exception as e: done = False if Sys.g.DEBUG: print('split exception') print(e) #~ raise e elif not Sys.g.QUIET: Sys.pwarn((str(e), )) if not Sys.g.QUIET: Sys.cli_emit_progress(100) self.onend_cmd('Kirmah Split', self.stime, done, self.o.outputfile)
def getFile(self, uid): """""" # ~ Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0, True)]) done = False key = self.idxu.index.getById(uid) row = self.idxu.index.get(key) filePath = None try: if row is not None: account = self.idxu.switchFileAccount(row[self.idxu.index.ACCOUNT]) km = Kirmah(row[self.idxu.index.KEY]) hlst = km.ck.getHashList(key, row[self.idxu.index.PARTS], True) ids = self.idxu.ih.searchBySubject(hlst["head"][2], True) Sys.cli_emit_progress(0) Sys.sleep(0.2) Sys.pwlog( [ (" Downloading : ", Const.CLZ_7), (row[self.idxu.index.LABEL] + row[self.idxu.index.EXT], Const.CLZ_2), (" (", Const.CLZ_0), (Sys.readableBytes(row[self.idxu.index.SIZE]), Const.CLZ_3), (")", Const.CLZ_0), (" please wait...", Const.CLZ_7, True), ] ) if len(ids) >= row[self.idxu.index.PARTS]: self.getFileParts(row, ids) Sys.pwlog([(" Merging parts...", Const.CLZ_7, True)]) Sys.cli_emit_progress(0) Sys.sleep(0.2) filePath = km.merge( hlst, self.deploy + row[self.idxu.index.CATG] + Sys.sep + row[self.idxu.index.LABEL], ext=row[self.idxu.index.EXT], uid=row[self.idxu.index.UID], dirs=self.inbox, ) Sys.pwlog( [ (" Deployed as ", Const.CLZ_7), (filePath, Const.CLZ_2), (" (", Const.CLZ_0), (Sys.readableBytes(Sys.getsize(filePath)), Const.CLZ_3), (") ", Const.CLZ_0, True), (" Checking integrity...", Const.CLZ_7, True), ] ) Sys.sleep(0.2) sha256 = hash_sha256_file(filePath) done = sha256 == row[ImpraIndex.HASH] done = True else: print("incomplete") except Exception as e: print(e) Sys.pwlog([(" done" if done else "ko", Const.CLZ_2 if done else Const.CLZ_1, True)]) return done, filePath
def sendFile(self, data, retry=False): """""" done = None key = None if data is not None: key, label, ext, count, catg, hlst, usr, ownerHash, sha256, size = data self.idxu.index.addUser(usr, ownerHash) account = self.idxu.switchFileAccount() sendIds = [] cancel = False d = None Sys.cli_emit_progress(0) Sys.sleep(0.2) if not retry: Sys.pwlog( [ (" Sending... ", Const.CLZ_7), (" (", Const.CLZ_0), ( " ~" + Sys.readableBytes(Sys.getsize(self.outbox + hlst["data"][0][1] + Kirmah.EXT)), Const.CLZ_3, ), (" per msg ) ", Const.CLZ_0, True), ] ) else: Sys.pwlog( [ (" Retry sending last file... ", Const.CLZ_0), (label + ext, Const.CLZ_7), (" (" + catg + ")", Const.CLZ_3, True), ] ) ignore = False for i, row in enumerate(hlst["data"]): """""" if retry: if not Io.file_exists(self.outbox + row[1] + Kirmah.EXT): continue elif not ignore: Sys.pwlog([(" Ignoring file 1 to " + str(i), Const.CLZ_1, True)]) ignore = True d = Sys.datetime.now() msg = self.idxu.mb.build(usr, "all", hlst["head"][2], self.outbox + row[1] + Kirmah.EXT) try: mid = self.idxu.ih.send(msg.as_string(), self.rootBox) except Exception as e: Sys.pwarn((("addFile : ", (str(e), Sys.CLZ_WARN_PARAM), " !"),)) Sys.echo("waiting 5 s and retry") Sys.sleep(5) # force reconnect self.impst.idxu.switchFileAccount(account, True) # retry mid = self.idxu.ih.send(msg.as_string(), self.rootBox) finally: if not mid is None: status, resp = self.idxu.ih.fetch(mid[1], "(UID BODYSTRUCTURE)", True) if status == self.idxu.ih.OK: sendIds.append((mid[1], row)) Sys.pwlog( [ (" part ", Const.CLZ_0), (str(row[0]).rjust(2, "0"), Const.CLZ_2), (" sent as msg ", Const.CLZ_0), (str(mid[1]).rjust(5, "0"), Const.CLZ_1), (" (", Const.CLZ_7), (str(int(row[4]) + 1).rjust(2, "0"), Const.CLZ_2), ("/", Const.CLZ_7), (str(count), Const.CLZ_3), (") in ", Const.CLZ_7), (Sys.getDelta(d), Const.CLZ_4, True), ] ) Sys.cli_emit_progress(int((i + 1) * 100 / len(hlst["data"]))) Sys.removeFile(self.outbox + row[1] + Kirmah.EXT) else: Sys.pwarn((("error occured when sending part ", (row[0], Sys.Clz.fgb3), " !"),)) diff = self.checkSendIds(sendIds, hlst["head"][2]) if len(sendIds) == count or retry: self.idxu.get(True) self.idxu.index.add(key, label, hlst["head"][1], ext, ownerHash, catg, sha256, size, account) done = self.idxu.update() Io.removeFile(self.addmapPath + Kirmah.EXT) # resending missing parts else: Sys.pwarn((("TODO => must resending ", ("missing", Sys.CLZ_WARN_PARAM), " parts"),)) print(diff) # clean for mid, row in sendIds: if Io.file_exists(self.outbox + row[1] + Kirmah.EXT): Sys.removeFile(self.outbox + row[1] + Kirmah.EXT) if cancel: delids = [mid for mid, row in senids] print(delids) self.idxu.ih.delete(delids, True) return done, key