def update(self):
""""""
self.switchFileAccount(self.conf.profile)
try:
if self.idx != None :
if not isinstance(self.idx,bytes):
self.idx = Io.bytes(self.idx)
self.delids.append(self.idx)
except Exception as e :
Sys.dprint('error : ')
Sys.dprint(e)
self.index.fixDuplicateIds()
#~ self.index.fixAccount('gmail5')
self.index.encrypt()
msgIndex = self.mb.buildIndex(self.pathIdx)
_, self.idx = self.ih.send(msgIndex.as_string(), self.rootBox)
date = self.ih.headerField(self.idx, 'date', True)
self.conf.sets((['uid' , self.idx , 'index'],
['date' , date , 'index'],
['time' , str(Sys.datetime.now()), 'index']))
Sys.pwlog([(' Index updated (' , Const.CLZ_0),
(str(int(self.idx)) , Const.CLZ_2),
(') ' , Const.CLZ_0),
(str(date) , Const.CLZ_7, True)])
try :
self.ih.delete(self.delids, True)
except :
Sys.dprint('error : ')
Sys.dprint(e)
self.ih.clearTrash()
return True
def build(self):
Sys.pwlog([(' Reading index, please wait...', Const.CLZ_7, True)])
self.index = ImpraIndex(self.conf.get('key','keys'), self.pathIdx, self.getIndexDefaultCatg(), self.getAccountList())
defUsers = self.conf.get('users','catg')
if not ImpraIndex.SEP_KEY_INTERN+'users' in self.index.dic:
self.index.dic[ImpraIndex.SEP_KEY_INTERN+'users'] = {}
for k in self.index.dic[ImpraIndex.SEP_KEY_INTERN+'users']:
if self.index.dic[ImpraIndex.SEP_KEY_INTERN+'users'][k] not in [ i.strip() for i in defUsers.split(',')]:
self.conf.set('users',defUsers+', '+self.index.dic[ImpraIndex.SEP_KEY_INTERN+'users'][k],'catg')
def buildFile(self, fromPath, label="", catg=""):
count = self.getCountParts(fromPath)
Sys.pwlog([(" Get Hash... ", Const.CLZ_7, True)])
sha256 = hash_sha256_file(fromPath)
Sys.pwlog([(" hash : ", Const.CLZ_0), (sha256, Const.CLZ_2, True), (" Build File...", Const.CLZ_0, True)])
kg = KeyGen(128)
size = Sys.getsize(fromPath)
row = self.idxu.index.get(sha256)
if row is None:
if label == "":
label, ext = Sys.getFileExt(Sys.basename(fromPath))
else:
label, ext = Sys.getFileExt(label)
if catg == "":
catg = self.idxu.index.getAutoCatg(ext)
size = Sys.getsize(fromPath)
Sys.pwlog(
[
(" Splitting ", Const.CLZ_1),
(label, Const.CLZ_7),
(ext, Const.CLZ_7),
(" (", Const.CLZ_0),
(Sys.readableBytes(size), Const.CLZ_3),
(")", Const.CLZ_0, True),
]
)
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
km = Kirmah(kg.key)
km.DIR_OUTBOX = self.outbox
# hlst genetate with sha256
hlst = km.ck.getHashList(sha256, int(count), True)
usr = self.idxu.conf.get("name", "infos")
ownerHash = self.idxu.mb.getHashName(usr)
km.split(fromPath, hlst)
Sys.pwlog([(" done ", Const.CLZ_2, True)])
row = [kg.key, label, ext, count, catg, hlst, usr, ownerHash, sha256, size]
self.backupAddMap(row)
else:
Sys.pwlog(
[
(" File Already exist ! ", Const.CLZ_1, True),
(" id : ".rjust(10, " "), Const.CLZ_0),
(str(row[ImpraIndex.UID]), Const.CLZ_1, True),
(" label : ".rjust(10, " "), Const.CLZ_0),
(row[ImpraIndex.LABEL], Const.CLZ_3, True),
]
)
row = None
return row
def encrypt(self, fromPath=None):
""""""
if fromPath is None :
fromPath = self.pathPlain
Sys.pwlog([(' Encrypt Index... ' , Const.CLZ_0, True)])
Io.set_data(fromPath, jdumps(self.dic))
call = ' '.join([Sys.executable, 'kirmah-cli.py', 'enc', '-qfj2' if Sys.isUnix() else '-qf', fromPath, '-z', '-r', '-m', '-o', fromPath+Kirmah.EXT, '-k', self.keyPath ])
#~ print(call)
Sys.sysCall(call)
Io.removeFile(fromPath)
Sys.pwlog([(' done', Const.CLZ_2, True)])
return Io.get_data(fromPath+Kirmah.EXT, True)
def clearTrash(self):
""""""
self.cnx.select(self.BOXS[self.TRASH])
ids = self.search('ALL',True)
if len(ids) > 0 and ids[0]!='' and ids[0]!=None:
delids = ImapHelper._getIdsList(ids)
status, resp = self.cnx.uid('store', delids, self.FLAGS, self.DELETED )
Sys.pwlog([(' Deleting msg ', Const.CLZ_0),
(delids , Const.CLZ_1),
(' '+status , Const.CLZ_7, True)])
self.cnx.expunge()
self.cnx.select(self.rootBox)
def removeFile(self, uid):
""""""
# ~ Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0, True)])
done = False
key = self.idxu.index.getById(uid)
row = self.idxu.index.get(key)
if row is not None:
account = self.idxu.switchFileAccount(row[self.idxu.index.ACCOUNT])
Sys.pwlog([(" Removing... plz wait. ", Const.CLZ_7)])
km = Kirmah(row[self.idxu.index.KEY])
hlst = km.ck.getHashList(key, row[self.idxu.index.PARTS], True)
ids = self.idxu.ih.searchBySubject(hlst["head"][2], True)
self.idxu.ih.delete(ids, True, True)
self.idxu.ih.clearTrash()
self.idxu.switchFileAccount(self.idxu.conf.profile)
self.idxu.get(True)
self.idxu.index.rem(key)
done = self.idxu.update()
return done, key
def decrypt(self, fromPath=None):
""""""
done = False
try :
if fromPath is None :
fromPath = self.path
toPath = fromPath[:-len(Kirmah.EXT)] if fromPath.endswith(Kirmah.EXT) else fromPath+'.dump'
if Io.file_exists(fromPath) :
Sys.pwlog([(' Decrypt Index... ' , Const.CLZ_0, True)])
call = ' '.join([Sys.executable, 'kirmah-cli.py', 'dec', '-qfj2' if Sys.isUnix() else '-qf', fromPath, '-z', '-r', '-m', '-o', toPath, '-k', self.keyPath ])
print(call)
Sys.sysCall(call)
data = jloads(Io.get_data(toPath))
Io.removeFile(toPath)
else :
data = {}
done = True
except ValueError as e:
raise BadKeyException(e)
Sys.pwlog([(' done'if done else ' ko' , Const.CLZ_2 if done else Const.CLZ_1, True)])
return data
def editFile(self, key, label, category):
""""""
# ~ Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0, True)])
done = False
uid = self.idxu._getId(True)
uidx = self.idxu.conf.get("uid", "index")
if int(uid) != int(uidx):
self.idxu.get(True)
row = self.idxu.index.get(key)
if row is not None:
if row[ImpraIndex.LABEL] != label or row[ImpraIndex.CATG] != category:
Sys.pwlog(
[
(" Editing file ", Const.CLZ_0),
(row[ImpraIndex.LABEL], Const.CLZ_7),
(" [", Const.CLZ_0),
(row[ImpraIndex.CATG], Const.CLZ_4),
("] to : ", Const.CLZ_0),
(label if label is not None else row[ImpraIndex.LABEL], Const.CLZ_3),
(" [", Const.CLZ_0),
(category if category is not None else row[ImpraIndex.CATG], Const.CLZ_2),
("]", Const.CLZ_0, True),
]
)
done = self.idxu.index.edit(key, label, category)
Sys.pwlog([(" done" if done else "ko", Const.CLZ_2 if done else Const.CLZ_1, True)])
if done:
Sys.pwlog([(" Updating index...", Const.CLZ_0, True)])
self.idxu.update()
return done
def switchAccount(self, conf, box='INBOX', force=False):
""""""
if force or self.cnx is None or self.cnxusr is not conf.user :
try :
Sys.pwlog([(' Attempt to login... ' , Const.CLZ_7),
('(' , Const.CLZ_0),
(conf.user , Const.CLZ_2),
('@' , Const.CLZ_0),
(conf.host , Const.CLZ_3),
(':' , Const.CLZ_0),
(conf.port , Const.CLZ_4),
(')' , Const.CLZ_0, True)])
self.cnx = ImapClient(conf.host,conf.port)
except Exception as e :
raise BadHostException()
try :
status, resp = self.cnx.login(conf.user,conf.pwd)
except Exception as e :
status = self.KO
pass
finally :
if status == self.KO :
self.cnxusr = None
raise BadLoginException(' Cannot login with '+conf.user+':'+conf.pwd)
else :
Sys.pwlog([(' Connected ', Const.CLZ_2, True),
(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0, True)])
self.cnxusr = conf.user
try :
status, resp = self.cnx.select(self.rootBox)
if status == self.KO and not self.noBoxCreat:
self.createBox(self.rootBox)
status, resp = self.cnx.select(self.rootBox)
self.initBoxNames()
except Exception as e :
print(e)
def getFileParts(self, row, ids):
""""""
done = False
if len(ids) >= row[self.idxu.index.PARTS]:
for i, uid in enumerate(ids):
d = Sys.datetime.now()
self.idxu.ih.getAttachment(uid, self.inbox, True)
Sys.pwlog(
[
(" part ", Const.CLZ_0),
(str(i + 1).rjust(2, " "), Const.CLZ_2),
(" / ", Const.CLZ_0),
(str(len(ids)), Const.CLZ_3),
(" downloaded in ", Const.CLZ_0),
(Sys.getDelta(d), Const.CLZ_4, True),
]
)
Sys.cli_emit_progress(int((i + 1) * 100 / len(ids)))
Sys.sleep(0.5)
Sys.mkdir_p(self.deploy + row[self.idxu.index.CATG])
Sys.cli_emit_progress(100)
def getInfo(self, uid):
""""""
done = False
key = self.idxu.index.getById(uid)
row = self.idxu.index.get(key)
if row is not None:
account = self.idxu.switchFileAccount(row[self.idxu.index.ACCOUNT])
km = Kirmah(row[self.idxu.index.KEY])
hlst = km.ck.getHashList(key, row[self.idxu.index.PARTS], True)
Sys.pwlog(
[
("id ".rjust(14, " ") + ": ", Const.CLZ_0),
(str(row[ImpraIndex.UID]), Const.CLZ_1, True),
("hash ".rjust(14, " ") + ": ", Const.CLZ_0),
(row[ImpraIndex.HASH], Const.CLZ_2, True),
("name ".rjust(14, " ") + ": ", Const.CLZ_0),
(row[ImpraIndex.LABEL] + row[ImpraIndex.EXT], Const.CLZ_7, True),
("size ".rjust(14, " ") + ": ", Const.CLZ_0),
(Sys.readableBytes(row[ImpraIndex.SIZE]), Const.CLZ_6, True),
("category ".rjust(14, " ") + ": ", Const.CLZ_0),
(row[ImpraIndex.CATG], Const.CLZ_5, True),
("user ".rjust(14, " ") + ": ", Const.CLZ_0),
(self.idxu.index.getUser(row[ImpraIndex.USER]), Const.CLZ_3),
(" (" + row[ImpraIndex.USER] + ")", Const.CLZ_5, True),
("account ".rjust(14, " ") + ": ", Const.CLZ_0),
(self.idxu.conf.get("user", "imap", row[ImpraIndex.ACCOUNT]), Const.CLZ_4, True),
(Const.LINE_SEP_CHAR * Const.LINE_SEP_LEN, Const.CLZ_0, True),
("subject ".rjust(14, " ") + ": ", Const.CLZ_0),
(hlst["head"][2], Const.CLZ_1, True),
]
)
ids = self.idxu.ih.searchBySubject(hlst["head"][2], True)
for i, uid in enumerate(ids):
if i < len(hlst["data"]):
Sys.pwlog(
[
("attach file ".rjust(14, " ") + ": ", Const.CLZ_0),
(hlst["data"][i][1] + Kirmah.EXT, Const.CLZ_2),
(" (", Const.CLZ_0),
(str(int(uid)), Const.CLZ_1),
(") (", Const.CLZ_0),
(str(hlst["data"][i][4]), Const.CLZ_3),
(")", Const.CLZ_0, True),
]
)
else:
Sys.pwlog(
[(" Wrong id (to del)".ljust(14, " ") + ": ", Const.CLZ_2), (str(uid), Const.CLZ_2, True)]
)
return done
def get(self, forceRefresh=False):
""""""
self.switchFileAccount(self.conf.profile)
index = None
uid = self.conf.get('uid' ,'index')
date = self.conf.get('date' ,'index')
tstamp = self.conf.get('time' ,'index')
refresh = forceRefresh
delta = None if tstamp is None else Sys.datetime.now() - Sys.datetime.strptime(tstamp[:-7], '%Y-%m-%d %H:%M:%S')
if not refresh and tstamp is not None and delta < Sys.timedelta(minutes = 3) :
# getFromFile
if uid != None and Io.file_exists(self.pathIdx): # int(self.idx) == int(uid)
self.idx = uid
Sys.pwlog([(' Get index from cache ' , Const.CLZ_7),
('(' , Const.CLZ_0),
(str(int(self.idx)) , Const.CLZ_2),
(')' , Const.CLZ_0, True)])
else: refresh = True
else: refresh = True
self.irefresh = refresh
if refresh :
Sys.pwlog([(' Checking index...', Const.CLZ_0, True)])
self._getId()
if self.idx :
if int(self.idx) != int(uid) or not Io.file_exists(self.pathIdx):
Sys.pwlog([(' Refreshing index (local:', Const.CLZ_0),
(str(int(uid)) , Const.CLZ_2),
(' / remote:' , Const.CLZ_0),
(str(int(self.idx)) , Const.CLZ_1),
(')' , Const.CLZ_0, True)])
date = self.ih.headerField(self.idx, 'date', True)
self.conf.sets((['uid' , str(int(self.idx)) , 'index'],
['date' , date , 'index'],
['time' , str(Sys.datetime.now()), 'index']))
self._saveLocalIndex()
else :
Sys.pwlog([(' Get index from cache ' , Const.CLZ_7),
('(' , Const.CLZ_0),
(str(int(self.idx)) , Const.CLZ_2),
(')' , Const.CLZ_0, True)])
self.conf.set('time',str(Sys.datetime.now()),'index')
self.build()
def run(self):
""""""
self.cancelled = False
self.evtStop.clear()
Sys.g.THREAD_CLI = self
Sys.g.GUI = True
Sys.g.GUI_PRINT_STDOUT = True
done = False
self.can_retry = True
init(conf.PRG_NAME, Sys.g.DEBUG, loglvl=Const.LOG_APP)
try :
if self.impst is None :
label = ' [[ INIT IMPRASTORAGE ]] '
label = ' '+'~'*((Const.LINE_SEP_LEN-len(label))//2-1)+label+'~'*((Const.LINE_SEP_LEN-len(label))//2-1)+' '
Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0 , True),
(label.ljust(Const.LINE_SEP_LEN, ' ') , Const.CLZ_INIT , True),
(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0 , True)])
self.impst = ImpraStorage(self.conf, wkdir='.'+Sys.sep+'wk'+Sys.sep)
done = True
except Exception as e :
self.emit('needconfig')
raise e
self.emit('indexrefreshed', done)
if done :
while not self.evtStop.is_set() or not self.cancelled:
with self.condition :
self.condition.wait_for(lambda : not self.taskQueue.empty(), 2)
if self.can_retry and self.impst.hasBackupAddMap():
self.emit('hasaddretry')
self.can_retry = False
if not self.taskQueue.empty():
task, params, idtask = self.taskQueue.get_nowait()
label = ' [[ TASK '+str(idtask)+' : '+self.TASK_LABEL[task].upper()+' ]] '
label = ' '+'>'*((Const.LINE_SEP_LEN-len(label))//2-1)+label+'<'*((Const.LINE_SEP_LEN-len(label))//2-1)+' '
Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0 , True),
(label.ljust(Const.LINE_SEP_LEN, ' ') , Const.CLZ_ACTION, True),
(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0 , True)])
try:
if task is self.TASK_WAIT :
print('wait')
Sys.sleep(params)
elif task is self.TASK_GET:
#~ mg = Manager(self.taskGet, 1, None, Sys.g.MPEVENT, uid=params)
#~ mg.run()
#~ self.emit('fileget', True, '')
#~ Thread(target=self.taskGet, name='impra-1', kwargs={'uid':params}).start()
print(params)
self.taskGet(params)
elif task is self.TASK_ADD:
self.taskAdd(params)
elif task is self.TASK_ADD_RETRY:
self.taskAddRetry(params)
elif task is self.TASK_REFRESH:
self.taskRefresh(params)
elif task is self.TASK_INFOS:
self.taskInfos(params)
elif task is self.TASK_REMOVE:
self.taskRemove(params)
elif task is self.TASK_EDIT:
self.taskEdit(params)
self.taskQueue.task_done()
except self.impst.SocketError() as e :
Sys.pwarn((('ImpraThread.run : ',(str(e),Sys.CLZ_WARN_PARAM), ' !'),))
self.impst.reconnect()
except Exception as e:
print(type(e))
Sys.pwarn((('ImpraThread.run : ',(str(e),Sys.CLZ_WARN_PARAM), ' !'),))
else :
""""""
Sys.sleep(0.5)
self.emit('completed')
def reconnect(self):
""""""
Sys.pwlog([(' Reconnecting... ', Const.CLZ_7, True)])
self.switchAccount(self.conf, self.rootBox, True)
def getFile(self, uid):
""""""
# ~ Sys.pwlog([(Const.LINE_SEP_CHAR*Const.LINE_SEP_LEN , Const.CLZ_0, True)])
done = False
key = self.idxu.index.getById(uid)
row = self.idxu.index.get(key)
filePath = None
try:
if row is not None:
account = self.idxu.switchFileAccount(row[self.idxu.index.ACCOUNT])
km = Kirmah(row[self.idxu.index.KEY])
hlst = km.ck.getHashList(key, row[self.idxu.index.PARTS], True)
ids = self.idxu.ih.searchBySubject(hlst["head"][2], True)
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
Sys.pwlog(
[
(" Downloading : ", Const.CLZ_7),
(row[self.idxu.index.LABEL] + row[self.idxu.index.EXT], Const.CLZ_2),
(" (", Const.CLZ_0),
(Sys.readableBytes(row[self.idxu.index.SIZE]), Const.CLZ_3),
(")", Const.CLZ_0),
(" please wait...", Const.CLZ_7, True),
]
)
if len(ids) >= row[self.idxu.index.PARTS]:
self.getFileParts(row, ids)
Sys.pwlog([(" Merging parts...", Const.CLZ_7, True)])
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
filePath = km.merge(
hlst,
self.deploy + row[self.idxu.index.CATG] + Sys.sep + row[self.idxu.index.LABEL],
ext=row[self.idxu.index.EXT],
uid=row[self.idxu.index.UID],
dirs=self.inbox,
)
Sys.pwlog(
[
(" Deployed as ", Const.CLZ_7),
(filePath, Const.CLZ_2),
(" (", Const.CLZ_0),
(Sys.readableBytes(Sys.getsize(filePath)), Const.CLZ_3),
(") ", Const.CLZ_0, True),
(" Checking integrity...", Const.CLZ_7, True),
]
)
Sys.sleep(0.2)
sha256 = hash_sha256_file(filePath)
done = sha256 == row[ImpraIndex.HASH]
done = True
else:
print("incomplete")
except Exception as e:
print(e)
Sys.pwlog([(" done" if done else "ko", Const.CLZ_2 if done else Const.CLZ_1, True)])
return done, filePath
def sendFile(self, data, retry=False):
""""""
done = None
key = None
if data is not None:
key, label, ext, count, catg, hlst, usr, ownerHash, sha256, size = data
self.idxu.index.addUser(usr, ownerHash)
account = self.idxu.switchFileAccount()
sendIds = []
cancel = False
d = None
Sys.cli_emit_progress(0)
Sys.sleep(0.2)
if not retry:
Sys.pwlog(
[
(" Sending... ", Const.CLZ_7),
(" (", Const.CLZ_0),
(
" ~" + Sys.readableBytes(Sys.getsize(self.outbox + hlst["data"][0][1] + Kirmah.EXT)),
Const.CLZ_3,
),
(" per msg ) ", Const.CLZ_0, True),
]
)
else:
Sys.pwlog(
[
(" Retry sending last file... ", Const.CLZ_0),
(label + ext, Const.CLZ_7),
(" (" + catg + ")", Const.CLZ_3, True),
]
)
ignore = False
for i, row in enumerate(hlst["data"]):
""""""
if retry:
if not Io.file_exists(self.outbox + row[1] + Kirmah.EXT):
continue
elif not ignore:
Sys.pwlog([(" Ignoring file 1 to " + str(i), Const.CLZ_1, True)])
ignore = True
d = Sys.datetime.now()
msg = self.idxu.mb.build(usr, "all", hlst["head"][2], self.outbox + row[1] + Kirmah.EXT)
try:
mid = self.idxu.ih.send(msg.as_string(), self.rootBox)
except Exception as e:
Sys.pwarn((("addFile : ", (str(e), Sys.CLZ_WARN_PARAM), " !"),))
Sys.echo("waiting 5 s and retry")
Sys.sleep(5)
# force reconnect
self.impst.idxu.switchFileAccount(account, True)
# retry
mid = self.idxu.ih.send(msg.as_string(), self.rootBox)
finally:
if not mid is None:
status, resp = self.idxu.ih.fetch(mid[1], "(UID BODYSTRUCTURE)", True)
if status == self.idxu.ih.OK:
sendIds.append((mid[1], row))
Sys.pwlog(
[
(" part ", Const.CLZ_0),
(str(row[0]).rjust(2, "0"), Const.CLZ_2),
(" sent as msg ", Const.CLZ_0),
(str(mid[1]).rjust(5, "0"), Const.CLZ_1),
(" (", Const.CLZ_7),
(str(int(row[4]) + 1).rjust(2, "0"), Const.CLZ_2),
("/", Const.CLZ_7),
(str(count), Const.CLZ_3),
(") in ", Const.CLZ_7),
(Sys.getDelta(d), Const.CLZ_4, True),
]
)
Sys.cli_emit_progress(int((i + 1) * 100 / len(hlst["data"])))
Sys.removeFile(self.outbox + row[1] + Kirmah.EXT)
else:
Sys.pwarn((("error occured when sending part ", (row[0], Sys.Clz.fgb3), " !"),))
diff = self.checkSendIds(sendIds, hlst["head"][2])
if len(sendIds) == count or retry:
self.idxu.get(True)
self.idxu.index.add(key, label, hlst["head"][1], ext, ownerHash, catg, sha256, size, account)
done = self.idxu.update()
Io.removeFile(self.addmapPath + Kirmah.EXT)
# resending missing parts
else:
Sys.pwarn((("TODO => must resending ", ("missing", Sys.CLZ_WARN_PARAM), " parts"),))
print(diff)
# clean
for mid, row in sendIds:
if Io.file_exists(self.outbox + row[1] + Kirmah.EXT):
Sys.removeFile(self.outbox + row[1] + Kirmah.EXT)
if cancel:
delids = [mid for mid, row in senids]
print(delids)
self.idxu.ih.delete(delids, True)
return done, key
