def Enqueue(self, key, idMessage, q, e=None, discarded=None): """ Enqueue a message into a queue file identified by a key :param key: An ASCII string for identifying a queue at server side :param idMessage: A unsigned short number to identify a message :param q: an instance of SPA.CUQueue containing a message :param e: A callback for tracking returning index :param discarded A callback for tracking socket close or request cancel event :return: true for sending the request successfully, and false for failure """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key).SaveUShort(idMessage) if q: buffer.Push(q.IntenalBuffer, q.GetSize()) ok = None if not e: ok = self.SendRequest(CAsyncQueue.idEnqueue, buffer, None, discarded) else: ok = self.SendRequest( CAsyncQueue.idEnqueue, buffer, lambda ar: e(ar.AsyncServiceHandler, ar.LoadULong()), discarded) CScopeUQueue.Unlock(buffer) return ok
def SendMeta(self, meta, index): q = CScopeUQueue.Lock() meta.SaveTo(q) q.SaveULong(index) ret = self.SendResult(q, DB_CONSTS.idRowsetHeader) CScopeUQueue.Unlock(q) return ret != CClientPeer.REQUEST_CANCELED and ret != CClientPeer.SOCKET_NOT_FOUND
def EndQueueTrans(self, rollback=False, qt=None, discarded=None): """ End enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes :param rollback: true for rollback, and false for committing :param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_NOT_STARTED_YET, and so on :param discarded A callback for tracking socket close or request cancel event :return: true for sending the request successfully, and false for failure """ buffer = CScopeUQueue.Lock().SaveBool(rollback) ok = None if not qt: ok = self.SendRequest(CAsyncQueue.idEndTrans, buffer, None, discarded) else: ok = self.SendRequest( CAsyncQueue.idEndTrans, buffer, lambda ar: qt(ar.AsyncServiceHandler, ar.LoadInt()), discarded) cq = self.AttachedClientSocket.ClientQueue if cq.Available: if rollback: cq.AbortJob() else: cq.EndJob() CScopeUQueue.Unlock(buffer) return ok
def FlushQueue(self, key, f, option=tagOptimistic.oMemoryCached, discarded=None): """ May flush memory data into either operation system memory or hard disk, and return message count and queue file size in bytes. Note the method only returns message count and queue file size in bytes if the option is oMemoryCached :param key: An ASCII string for identifying a queue at server side :param f: A callback for tracking returning message count and queue file size in bytes :param option: one of tagOptimistic options, oMemoryCached, oSystemMemoryCached and oDiskCommitted :param discarded A callback for tracking socket close or request cancel event :return: true for sending the request successfully, and false for failure """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key).SaveInt(option) ok = None if not f: ok = self.SendRequest(CAsyncQueue.idFlush, buffer, None, discarded) else: ok = self.SendRequest( CAsyncQueue.idFlush, buffer, lambda ar: f( ar.AsyncServiceHandler, ar.LoadULong(), ar.LoadULong()), discarded) CScopeUQueue.Unlock(buffer) return ok
def Enqueue(self, key, idMessage, q, e=None, discarded=None, se=None): """ Enqueue a message into a queue file identified by a key :param key: An ASCII string for identifying a queue at server side :param idMessage: A unsigned short number to identify a message :param q: None or an instance of SPA.CUQueue or SPA.CScopeUQueue containing a message :param e: A callback for tracking returning index :param discarded A callback for tracking socket close or request cancel event :param se A callback for tracking an exception from server :return: True if communication channel is sendable, and False if communication channel is not sendable """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key).SaveUShort(idMessage) if isinstance(q, CScopeUQueue): q = q.UQueue if q: buffer.Push(q.IntenalBuffer, q.GetSize()) ok = None if not e: ok = self.SendRequest(CAsyncQueue.idEnqueue, buffer, None, discarded, se) else: ok = self.SendRequest( CAsyncQueue.idEnqueue, buffer, lambda ar: e(ar.AsyncServiceHandler, ar.LoadULong()), discarded, se) CScopeUQueue.Unlock(buffer) return ok
def SendRows(self, vData): if not vData: vData = [] q = CScopeUQueue.Lock() for vt in vData: if isinstance(vt, str): if len(vt) <= DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE: q.SaveObject(vt) else: if q.Size > 0 and not self._SendRows_(q, True): CScopeUQueue.Unlock(q) return False if not self._SendBlob_(vt): CScopeUQueue.Unlock(q) return False elif isinstance(vt, bytearray): if len(vt) <= 2 * DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE: q.SaveObject(vt) else: if q.Size > 0 and not self._SendRows_(q, True): CScopeUQueue.Unlock(q) return False if not self._SendBlob_(vt): CScopeUQueue.Unlock(q) return False else: q.SaveObject(vt) ret = self.SendResult(q, DB_CONSTS.idEndRows) return ret != CClientPeer.REQUEST_CANCELED and ret != CClientPeer.SOCKET_NOT_FOUND
def CloseQueue(self, key, c=None, discarded=None, permanent=False, se=None): """ Try to close or delete a persistent queue opened at server side :param key: An ASCII string for identifying a queue at server side :param c: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_DEQUEUING, and so on :param permanent: true for deleting a queue file, and false for closing a queue file :param discarded A callback for tracking socket close or request cancel event :param se A callback for tracking an exception from server :return: True if communication channel is sendable, and False if communication channel is not sendable """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key).SaveBool(permanent) ok = None if not c: ok = self.SendRequest(CAsyncQueue.idClose, buffer, None, discarded, se) else: ok = self.SendRequest( CAsyncQueue.idClose, buffer, lambda ar: c(ar.AsyncServiceHandler, ar.LoadInt()), discarded, se) CScopeUQueue.Unlock(buffer) return ok
def StartQueueTrans(self, key, qt=None, discarded=None, se=None): """ Start enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes :param key: An ASCII string for identifying a queue at server side :param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_ALREADY_STARTED, and so on :param discarded A callback for tracking socket close or request cancel event :param se A callback for tracking an exception from server :return: True if communication channel is sendable, and False if communication channel is not sendable """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key) ok = None cq = self.Socket.ClientQueue if cq.Available: cq.StartJob() if not qt: ok = self.SendRequest(CAsyncQueue.idStartTrans, buffer, None, discarded, se) else: ok = self.SendRequest( CAsyncQueue.idStartTrans, buffer, lambda ar: qt(ar.AsyncServiceHandler, ar.LoadInt()), discarded, se) CScopeUQueue.Unlock(buffer) return ok
def Dequeue(self, key, d, timeout=0, discarded=None): """ Dequeue messages from a persistent message queue file at server side in batch :param key: An ASCII string for identifying a queue at server side :param d: A callback for tracking data like remaining message count within a server queue file, queue file size in bytes, message dequeued within this batch and bytes dequeued within this batch :param timeout: A time-out number in milliseconds :param discarded A callback for tracking socket close or request cancel event :return: true for sending the request successfully, and false for failure """ if not key: key = '' with self._csQ_: self._dDequeue_ = d self._keyQueue_ = key buffer = CScopeUQueue.Lock().SaveAString(key).SaveUInt(timeout) ok = None if not d: ok = self.SendRequest(CAsyncQueue.idDequeue, buffer, None, discarded) else: ok = self.SendRequest( CAsyncQueue.idDequeue, buffer, lambda ar: d(ar.AsyncServiceHandler, ar.LoadULong( ), ar.LoadULong(), ar.LoadUInt(), ar.LoadUInt()), discarded) CScopeUQueue.Unlock(buffer) return ok
def _spe_(self, poolId, spe, h): #h -- usocket handle #print "Pool id = " + str(poolId) + ", spe = " + str(spe) + ", usocket handle = " + str(h) handler = self._MapToHandler_(h) if spe == tagSocketPoolEvent.speTimer: if CScopeUQueue.MemoryConsumed() / 1024 > CScopeUQueue.SHARED_BUFFER_CLEAN_SIZE: CScopeUQueue.DestroyUQueuePool() elif spe == tagSocketPoolEvent.speStarted: with self._lock_: self._PoolId_ = poolId elif spe == tagSocketPoolEvent.speShutdown: with self._lock_: self._PoolId_ = 0 elif spe == tagSocketPoolEvent.speUSocketKilled: with self._lock_: del self._m_dicSocketHandler_[handler.Socket] elif spe == tagSocketPoolEvent.speUSocketCreated: cs = CClientSocket(h) ash = self._m_cls_() ccl.SetRecvTimeout(h, self._recvTimeout_) ccl.SetConnTimeout(h, self._connTimeout_) ccl.SetAutoConn(h, self._autoConn_) if ash.SvsID == 0: ash._m_nServiceId_ = self._serviceId_ if ash.SvsID <= BaseServiceID.sidStartup: raise ValueError('Service id must be larger than SocketProAdapter.BaseServiceID.sidReserved (268435456)') ash._Attach_(cs) handler = ash with self._lock_: self._m_dicSocketHandler_[cs] = ash elif spe == tagSocketPoolEvent.speConnected: if ccl.IsOpened(h): cs = handler.Socket if self.DoSslServerAuthentication is not None and cs.EncryptionMethod == tagEncryptionMethod.TLSv1 and (not self.DoSslServerAuthentication(self, cs)): return #don't set password or call SwitchTo in case failure of ssl server authentication on certificate from server ccl.SetSockOpt(h, tagSocketOption.soRcvBuf, 116800, tagSocketLevel.slSocket) ccl.SetSockOpt(h, tagSocketOption.soSndBuf, 116800, tagSocketLevel.slSocket) ccl.SetSockOpt(h, tagSocketOption.soTcpNoDelay, 1, tagSocketLevel.slTcp) ccl.SetPassword(h, cs.ConnectionContext._Password_) ok = ccl.StartBatching(h) ok = ccl.SwitchTo(h, handler.SvsID) ok = ccl.TurnOnZipAtSvr(h, cs.ConnectionContext.Zip) ok = ccl.SetSockOptAtSvr(h, tagSocketOption.soRcvBuf, 116800, tagSocketLevel.slSocket) ok = ccl.SetSockOptAtSvr(h, tagSocketOption.soSndBuf, 116800, tagSocketLevel.slSocket) ok = ccl.SetSockOptAtSvr(h, tagSocketOption.soTcpNoDelay, 1, tagSocketLevel.slTcp) ok = ccl.CommitBatching(h, False) elif spe == tagSocketPoolEvent.speQueueMergedFrom: self._hFrom = handler elif spe == tagSocketPoolEvent.speQueueMergedTo: self._hFrom._AppendTo_(handler) self._hFrom = None else: pass if self.SocketPoolEvent: self.SocketPoolEvent(self, spe, handler) self.OnSocketPoolEvent(spe, handler) if spe == tagSocketPoolEvent.speConnected and ccl.IsOpened(h): self._SetQueue_(handler.Socket)
def GetCachedTables(self, defaultDb, handler, row, rh, flags=DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES): q = CScopeUQueue.Lock() index = self.GetCallIndex(); with self._csCache: self._mapRowset[index] = Pair(rh, row) self._mapHandler[index] = handler q.SaveString(defaultDb).SaveUInt(flags).SaveULong(index) ok = self.SendRequest(DB_CONSTS.idGetCachedTables, q, None) CScopeUQueue.Unlock(q) if not ok: with self._csCache: self._mapHandler.pop(index) self._mapRowset.pop(index) return ok
def GetRentalDateTimes(self, q, reqIndex): rental_id = q.LoadLong() myDates = CRentalDateTimes(rental_id) sql = u'SELECT rental_date,return_date,last_update FROM rental where rental_id=' + str( rental_id) handler = CYourPeer.Slave.SeekByQueue() if not handler: with CScopeUQueue() as sb: myDates.SaveTo(sb.UQueue) sb.SaveInt(-2).SaveString( 'No connection to anyone of slave databases') self.SendResultIndex(reqIndex, sb, idGetRentalDateTimes) else: def ares(h, r, err, affected, fail_ok, vtId): with CScopeUQueue() as sb: myDates.SaveTo(sb.UQueue) sb.SaveInt(r).SaveString(err) self.SendResultIndex(reqIndex, sb, idGetRentalDateTimes) def rows(h, vData): myDates.Rental = vData[0] myDates.Return = vData[1] myDates.LastUpdate = vData[2] ok = handler.Execute(sql, ares, rows) # should be always true if pool has local queue for request backup assert ok
def QueryMaxMinAvgs(self, q, reqIndex): filter = q.LoadString() pmma = CMaxMinAvg() sql = "SELECT MAX(amount),MIN(amount),AVG(amount) FROM payment" if filter and len(filter) > 0: sql += (' WHERE ' + filter) handler = CYourPeer.Slave.SeekByQueue() if not handler: with CScopeUQueue() as sb: sb.SaveInt(-2).SaveString( 'No connection to anyone of slave databases') pmma.SaveTo(sb.UQueue) self.SendResultIndex(reqIndex, sb, idQueryMaxMinAvgs) else: peer_handle = self.Handle def ares(h, r, err, affected, fail_ok, vtId): # send result if front peer not closed yet if peer_handle == self.Handle: with CScopeUQueue() as sb: sb.SaveInt(r).SaveString(err) pmma.SaveTo(sb.UQueue) self.SendResultIndex(reqIndex, sb, idQueryMaxMinAvgs) def rows(h, vData): pmma.Max = float(vData[0]) pmma.Min = float(vData[1]) pmma.Avg = float(vData[2]) ok = handler.Execute(sql, ares, rows) # should be always true if pool has local queue for request backup assert ok
def EndQueueTrans(self, rollback=False, qt=None): """ End enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes :param rollback: true for rollback, and false for committing :param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_NOT_STARTED_YET, and so on :return: true for sending the request successfully, and false for failure """ buffer = CScopeUQueue.Lock().SaveBool(rollback) ok = None if not qt: ok = self.SendRequest(CAsyncQueue.idEndTrans, buffer, None) else: ok = self.SendRequest(CAsyncQueue.idEndTrans, buffer, lambda ar: qt(ar.LoadInt())) CScopeUQueue.Unlock(buffer) return ok
def ares(h, r, err, affected, fail_ok, vtId): # send result if front peer not closed yet if peer_handle == self.Handle: with CScopeUQueue() as sb: sb.SaveInt(r).SaveString(err) pmma.SaveTo(sb.UQueue) self.SendResultIndex(reqIndex, sb, idQueryMaxMinAvgs)
def SendUserMessage(self, message, userId, hint=''): if userId is None: userId = u'' sb = CScopeUQueue() q = sb.UQueue.SaveObject(message, hint) bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_) return scl.SendUserMessage(self._m_p.Handle, userId, bytes, q.GetSize())
def closed(h, canceled): # retry if front peer not closed yet if peer_handle == self.Handle: res = cs.ErrorCode errMsg = cs.ErrorMsg with CScopeUQueue() as sb0: sb0.SaveInt(res).SaveString(errMsg) vId.SaveTo(sb0.UQueue) self.SendResultIndex(reqIndex, sb0, idUploadEmployees)
def Publish(self, message, groups, hint=''): if groups is None: groups = () size = len(groups) arr = (c_uint * size)(*groups) sb = CScopeUQueue() q = sb.UQueue.SaveObject(message, hint) bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_) return scl.Speak(self._m_p.Handle, bytes, q.GetSize(), arr, size)
def MakeRequest(self, reqId, q): if isinstance(q, CScopeUQueue): q = q.UQueue elif q is None: delay = CScopeUQueue() q = delay.UQueue buffer = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) return scl.MakeRequest(self.Handle, reqId, buffer, q.GetSize())
def StartQueueTrans(self, key, qt): """ Start enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes :param key: An ASCII string for identifying a queue at server side :param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_ALREADY_STARTED, and so on :return: true for sending the request successfully, and false for failure """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key) ok = None if not qt: ok = self.SendRequest(CAsyncQueue.idStartTrans, buffer, None) else: ok = self.SendRequest(CAsyncQueue.idStartTrans, buffer, lambda ar: qt(ar.LoadInt())) CScopeUQueue.Unlock(buffer) return ok
def et(h, r, err): if r == 0: res = r errMsg = err # send result if front peer not closed yet if peer_handle == self.Handle: with CScopeUQueue() as sb0: sb0.SaveInt(res).SaveString(errMsg) vId.SaveTo(sb0.UQueue) self.SendResultIndex(reqIndex, sb0, idUploadEmployees)
def CloseQueue(self, key, c=None, permanent=False): """ Try to close or delete a persistent queue opened at server side :param key: An ASCII string for identifying a queue at server side :param c: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_DEQUEUING, and so on :param permanent: true for deleting a queue file, and false for closing a queue file :return: true for sending the request successfully, and false for failure """ if not key: key = '' buffer = CScopeUQueue.Lock().SaveAString(key).SaveBool(permanent) ok = None if not c: ok = self.SendRequest(CAsyncQueue.idClose, buffer, None) else: ok = self.SendRequest(CAsyncQueue.idClose, buffer, lambda ar: c(ar.LoadInt())) CScopeUQueue.Unlock(buffer) return ok
def _SendBlob_(self, blob): dt = tagVariantDataType.sdVT_ARRAY | tagVariantDataType.sdVT_UI1 if isinstance(blob, str): dt = tagVariantDataType.sdVT_BSTR q = CScopeUQueue.Lock() q.SaveObject(blob) dt = q.LoadUShort() bytes = q.LoadUInt() byte_len = q.GetSize() q0 = CScopeUQueue.Lock() byte_len += 10 # sizeof(ushort) + sizeof(uint) + sizeof(uint) extra 4 bytes for string null termination q0.SaveUInt(byte_len).SaveUShort(dt).SaveUInt(q.GetSize()) ret = self.SendResult(q0, DB_CONSTS.idStartBLOB) CScopeUQueue.Unlock(q0) if ret == CClientPeer.REQUEST_CANCELED or ret == CClientPeer.SOCKET_NOT_FOUND: CScopeUQueue.Unlock(q) return False byte_len = q.GetSize() while byte_len > DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE: buffer = (c_ubyte * DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE).from_buffer( q._m_bytes_, q._m_position_) ret = scl.SendReturnData(self.Handle, DB_CONSTS.idChunk, DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE, buffer) if ret == CClientPeer.REQUEST_CANCELED or ret == CClientPeer.SOCKET_NOT_FOUND: CScopeUQueue.Unlock(q) return False q.Discard(ret) byte_len = q.GetSize() ret = self.SendResult(q, DB_CONSTS.idEndBLOB) CScopeUQueue.Unlock(q) return ret != CClientPeer.REQUEST_CANCELED and ret != CClientPeer.SOCKET_NOT_FOUND
def SendResultIndex(self, reqIndex, q, reqId): delay = q if reqId == 0: reqId = self.CurrentRequestID if isinstance(q, CScopeUQueue): q = q.UQueue elif q is None: delay = CScopeUQueue() q = delay.UQueue buffer = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) return scl.SendReturnDataIndex(self.Handle, reqIndex, reqId, q.GetSize(), buffer)
def OnPostProcessing(self, hint, data): cs = self.AttachedClientSocket ctx = CContext(False, 0) with self._csFile: count = len(self._vContext) if count > 0: front = self._vContext[0] if front.Uploading: front._OpenLocalRead() else: front._OpenLocalWrite() if front.ErrCode or front.ErrMsg: ctx = front elif front.Uploading: with CScopeUQueue() as q: q.SaveString(front.FilePath).SaveUInt(front.Flags).SaveULong(front.FileSize) if not self.SendRequest(CStreamingFile.idUpload, q, None, front.Discarded, None): front.ErrCode = cs.ErrorCode front.ErrMsg = cs.ErrorMessage ctx = front else: with CScopeUQueue() as q: q.SaveString(front.LocalFile).SaveString(front.FilePath).SaveUInt(front.Flags).SaveLong(front.InitSize) if not self.SendRequest(CStreamingFile.idDownload, q, None, front.Discarded, None): front.ErrCode = cs.ErrorCode front.ErrMsg = cs.ErrorMessage ctx = front if ctx.ErrCode or ctx.ErrMsg: ctx._CloseFile() if ctx.Download: ctx.Download(self, ctx.ErrCode, ctx.ErrMsg) with self._csFile: self._vContext.popleft() if len(self._vContext) > 0: #post processing the next one ccl.PostProcessing(self.AttachedClientSocket.Handle, 0, 0) self.AttachedClientSocket.DoEcho #make sure WaitAll works correctly
def GetCachedTables(self): defaultDb = self.UQueue.LoadString() flags = self.UQueue.LoadUInt() index = self.UQueue.LoadLong() ms = tagManagementSystem.msUnknown sb = CScopeUQueue() if len(CYourPeer.FrontCachedTables) == 0 or ( flags & DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES ) != DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES: return sb.SaveInt(ms).SaveInt(0).SaveString('') if (flags & DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES ) == DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES: self.Push.Subscribe([ DB_CONSTS.CACHE_UPDATE_CHAT_GROUP_ID, DB_CONSTS.STREAMING_SQL_CHAT_GROUP_ID ]) sql = '' v = CYourPeer.FrontCachedTables for s in v: if (len(sql)) > 0: sql += ';' sql += ('SELECT * FROM ' + s) # use Lock and Unlock to avoid SQL stream overlap on a session within a multi-thread environment handler = CYourPeer.Master.Lock() if not handler: return sb.SaveInt(ms).SaveInt(-2).SaveString( 'No connection to a master database') ms = handler.DBManagementSystem try: f = handler.execute(sql, lambda h, vData: self.SendRows(vData), lambda h: self.SendMeta(h.ColumnInfo, index)) # put back locked handler and its socket back into pool for reuse as soon as possible CYourPeer.Master.Unlock(handler) try: res = f.result(30) return sb.SaveInt(ms).SaveInt(res.ec).SaveString(res.em) except TimeoutError as ex: return sb.SaveInt(ms).SaveInt(-3).SaveString(str(ex)) except (CServerError, CSocketError) as ex: return sb.SaveInt(ms).SaveInt(ex[0]).SaveString(ex[1]) except Exception as ex: return sb.SaveInt(ms).SaveInt(-1).SaveString(str(ex))
def GetMasterSlaveConnectedSessions(self): mc = CYourPeer.Master.ConnectedSockets sc = CYourPeer.Slave.ConnectedSockets sb = CScopeUQueue() sb.SaveUInt(mc).SaveUInt(sc) return sb
def _Transfer(self): index = 0 rh = None se = None cs = self.AttachedClientSocket if not cs.Sendable: return False sent_buffer_size = cs.BytesInSendingBuffer if sent_buffer_size > 3 * CStreamingFile.STREAM_CHUNK_SIZE: return True while index < len(self._vContext): context = self._vContext[index] if context.Sent: index += 1 continue if context.Uploading and context.Tried and not context.File: if index == 0: if context.Download: context.Download( self, CStreamingFile.CANNOT_OPEN_LOCAL_FILE_FOR_READING, context.ErrMsg) self._vContext.popleft() else: index += 1 continue if context.Uploading: if not context.Tried: context.Tried = True try: context.File = open(context.LocalFile, 'rb') context.File.seek(0, io.SEEK_END) context.FileSize = context.File.tell() context.File.seek(0, io.SEEK_SET) cq = self.AttachedClientSocket.ClientQueue if cq.Available: cq.StartJob() with CScopeUQueue() as q: q.SaveString(context.FilePath).SaveUInt( context.Flags).SaveULong(context.FileSize) if not self.SendRequest(CStreamingFile.idUpload, q, rh, context.Discarded, se): return False except IOError as e: context.ErrMsg = e.strerror context.File = None if not context.File: if index == 0: if context.Download: context.Download( self, CStreamingFile. CANNOT_OPEN_LOCAL_FILE_FOR_READING, context.ErrMsg) self._vContext.popleft() else: index += 1 continue else: ret = bytearray( context.File.read(CStreamingFile.STREAM_CHUNK_SIZE)) while len(ret) > 0: if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), rh, context.Discarded, se): return False sent_buffer_size = cs.BytesInSendingBuffer if len(ret) < CStreamingFile.STREAM_CHUNK_SIZE: break if sent_buffer_size >= 5 * CStreamingFile.STREAM_CHUNK_SIZE: break ret = bytearray( context.File.read( CStreamingFile.STREAM_CHUNK_SIZE)) if len(ret) < CStreamingFile.STREAM_CHUNK_SIZE: context.Sent = True if not self.SendRequest( CStreamingFile.idUploadCompleted, None, rh, context.Discarded, se): return False cq = self.AttachedClientSocket.ClientQueue if cq.Available: cq.EndJob() if sent_buffer_size >= 4 * CStreamingFile.STREAM_CHUNK_SIZE: break else: with CScopeUQueue() as q: q.SaveString(context.FilePath).SaveUInt(context.Flags) if not self.SendRequest(CStreamingFile.idDownload, q, rh, context.Discarded, se): return False context.Sent = True context.Tried = True sent_buffer_size = cs.BytesInSendingBuffer if sent_buffer_size > 3 * CStreamingFile.STREAM_CHUNK_SIZE: return True index += 1 return True
def OnPostProcessing(self, hint, data): d = 0 with self._csFile: for it in self._vContext: if d >= self._MaxDownloading: break if it.File: if it.Uploading: break else: d += 1 continue if it._HasError(): continue if it.Uploading: it._OpenLocalRead() if not it._HasError(): with CScopeUQueue() as q: q.SaveString(it.FilePath).SaveUInt( it.Flags).SaveULong(it.FileSize) if not self.SendRequest(CStreamingFile.idUpload, q, None, it.Discarded, it.Se): it.ErrCode = CStreamingFile.SESSION_CLOSED_BEFORE it.ErrMsg = CStreamingFile.SESSION_CLOSED_BEFORE_ERR_MSG if it.Fut: if not it.Fut.done(): ec = self.Socket.ErrCode try: if ec: it.Fut.set_exception( CSocketError( ec, self.Socket.ErrMsg, CStreamingFile. idUpload, True)) else: it.Fut.set_exception( CSocketError( CAsyncServiceHandler. SESSION_CLOSED_BEFORE, it.ErrMsg, CStreamingFile. idUpload, True)) except Exception as ex: pass it.Se = None it.Discarded = None continue break else: it._OpenLocalWrite() if not it._HasError(): with CScopeUQueue() as q: q.SaveString(it.LocalFile).SaveString( it.FilePath).SaveUInt(it.Flags).SaveLong( it.InitSize) if not self.SendRequest(CStreamingFile.idDownload, q, None, it.Discarded, it.Se): it.ErrCode = CStreamingFile.SESSION_CLOSED_BEFORE it.ErrMsg = CStreamingFile.SESSION_CLOSED_BEFORE_ERR_MSG if it.Fut: if not it.Fut.done(): ec = self.Socket.ErrCode try: if ec: it.Fut.set_exception( CSocketError( ec, self.Socket.ErrMsg, CStreamingFile. idDownload, True)) else: it.Fut.set_exception( CSocketError( CAsyncServiceHandler. SESSION_CLOSED_BEFORE, it.ErrMsg, CStreamingFile. idDownload, True)) except Exception as ex: pass it.Se = None it.Discarded = None continue d += 1 while len(self._vContext): it = self._vContext[0] if it._HasError(): cb = it.Download it._CloseFile() if cb: try: self._csFile.release() cb(self, it.ErrCode, it.ErrMsg) finally: self._csFile.acquire() self._vContext.popleft() else: break
def OnResultReturned(self, reqId, mc): if reqId == CStreamingFile.idDownload: res = mc.LoadInt() errMsg = mc.LoadString() dl = None with self._csFile: if len(self._vContext): context = self._vContext[0] context.ErrCode = res context.ErrMsg = errMsg dl = context.Download if dl: dl(self, res, errMsg) with self._csFile: if len(self._vContext): self._vContext.popleft()._CloseFile() self.OnPostProcessing(0, 0) elif reqId == CStreamingFile.idStartDownloading: with self._csFile: fileSize = mc.LoadULong() localFile = mc.LoadString() remoteFile = mc.LoadString() flags = mc.LoadUInt() initSize = mc.LoadLong() if len(self._vContext) == 0: ctx = CContext(False, flags) ctx.LocalFile = localFile; ctx.FilePath = remoteFile; OpenLocalWrite(ctx); ctx.InitSize = initSize; self._vContext.append(ctx) front = self._vContext[0] front.FileSize = fileSize initSize = 0 if front.InitSize > 0: initSize = front.InitSize if front.File.tell() > initSize: front.File.flush() front.File.seek(initSize) front.File.truncate(initSize) elif reqId == CStreamingFile.idDownloading: downloaded = 0 trans = None with self._csFile: if len(self._vContext): context = self._vContext[0] trans = context.Transferring context.File.write(mc.GetBuffer()) initSize = 0 if context.InitSize > 0: initSize = context.InitSize downloaded = context.File.tell() - initSize mc.SetSize(0) if trans: trans(self, downloaded) elif reqId == CStreamingFile.idUploadBackup: pass elif reqId == CStreamingFile.idUpload: cs = self.AttachedClientSocket res = mc.LoadInt() errMsg = mc.LoadString() ctx = CContext(False, 0) if res or errMsg: ctx.ErrCode = res ctx.ErrMsg = errMsg with self._csFile: if len(self._vContext) > 0: context = self._vContext[0] if mc.GetSize() > 0: context.InitSize = mc.LoadLong() context.ErrCode = res context.ErrMsg = errMsg ctx = context else: with self._csFile: if len(self._vContext) > 0: context = self._vContext[0] if mc.GetSize() > 0: context.InitSize = mc.LoadLong() context.QueueOk = cs.ClientQueue.StartJob() queue_enabled = cs.ClientQueue.Available if queue_enabled: with CScopeUQueue() as q: q.SaveString(context.FilePath).SaveUInt(context.Flags).SaveULong(context.FileSize).SaveLong(context.InitSize) self.SendRequest(CStreamingFile.idUploadBackup, q, None, context.Discarded, None) ret = bytearray(context.File.read(CStreamingFile.STREAM_CHUNK_SIZE)) while len(ret) == CStreamingFile.STREAM_CHUNK_SIZE: if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), None, context.Discarded, None): context.ErrCode = cs.ErrorCode context.ErrMsg = cs.ErrorMessage ctx = context break ret = bytearray(context.File.read(CStreamingFile.STREAM_CHUNK_SIZE)) if len(ret) < CStreamingFile.STREAM_CHUNK_SIZE: break if not queue_enabled: sent_buffer_size = cs.BytesInSendingBuffer if sent_buffer_size >= 40 * CStreamingFile.STREAM_CHUNK_SIZE: break if ctx.ErrCode or ctx.ErrMsg: pass elif len(ret) > 0: if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), None, context.Discarded, None): context.ErrCode = cs.ErrorCode context.ErrMsg = cs.ErrorMessage ctx = context if not (ctx.ErrCode or ctx.ErrMsg) and len(ret) < CStreamingFile.STREAM_CHUNK_SIZE: context.Sent = True if not self.SendRequest(CStreamingFile.idUploadCompleted, None, None, context.Discarded, None): context.ErrCode = cs.ErrorCode context.ErrMsg = cs.ErrorMessage ctx = context if context.QueueOk: cs.ClientQueue.EndJob() if ctx.ErrCode or ctx.ErrMsg: ctx._CloseFile() if ctx.Download: ctx.Download(self, ctx.ErrCode, ctx.ErrMsg) with self._csFile: self._vContext.popleft() if ctx.QueueOk: cs.ClientQueue.AbortJob() self.OnPostProcessing(0, 0) elif reqId == CStreamingFile.idUploading: cs = self.AttachedClientSocket ctx = CContext(False, 0) trans = None uploaded = mc.LoadLong() with self._csFile: if len(self._vContext) > 0: context = self._vContext[0] trans = context.Transferring if uploaded < 0: context._CloseFile() elif not context.Sent: ret = bytearray(context.File.read(CStreamingFile.STREAM_CHUNK_SIZE)) if len(ret) > 0: if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), None, context.Discarded, None): context.ErrCode = cs.ErrorCode context.ErrMsg = cs.ErrorMessage ctx = context if not (ctx.ErrCode or ctx.ErrMsg) and len(ret) < CStreamingFile.STREAM_CHUNK_SIZE: context.Sent = True if not self.SendRequest(CStreamingFile.idUploadCompleted, None, None, context.Discarded, None): context.ErrCode = cs.ErrorCode context.ErrMsg = cs.ErrorMessage ctx = context if ctx.ErrCode or ctx.ErrMsg: ctx._CloseFile() if ctx.Download: ctx.Download(self, ctx.ErrCode, ctx.ErrMsg) with self._csFile: self._vContext.popleft() self.OnPostProcessing(0, 0) elif trans: trans(self, uploaded) elif reqId == CStreamingFile.idUploadCompleted: upl = None with self._csFile: if len(self._vContext): context = self._vContext[0] if context.File: upl = context.Download else: context.Sent = False context.QueueOk = False if upl: upl(self, 0, '') with self._csFile: if len(self._vContext): context = self._vContext[0] if context.File: context._CloseFile() self._vContext.popleft() self.OnPostProcessing(0, 0) else: super(CStreamingFile, self).OnResultReturned(reqId, mc)