def WaitAll(self, timeout = 0xffffffff): h = self._m_ClientSocket_.Handle if ccl.IsBatching(h): raise ValueError("Can't call the method WaitAll while batching requests") if ccl.IsQueueStarted(h) and ccl.GetJobSize(h) > 0: raise ValueError("Can't call the method WaitAll while enqueuing transactional requests") return ccl.WaitAll(h, timeout)
def SendRequest(self, reqId, q, arh, discarded=None, efs=None): if q is None: q = CUQueue(bytearray(0)) if isinstance(q, CScopeUQueue): q = q.UQueue if not isinstance(q, CUQueue): raise ValueError('Bad input for parameter q') #http://stackoverflow.com/questions/21483482/efficient-way-to-convert-string-to-ctypes-c-ubyte-array-in-python bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) h = self._m_ClientSocket_.Handle if h == 0: return False kv = None batching = False with self._lock_Send_: if arh or discarded or efs: kv = (reqId, CResultCb(arh, discarded, efs)) with self._lock_: batching = ccl.IsBatching(h) if batching: self._m_kvBatching_.append(kv) else: self._m_kvCallback_.append(kv) if ccl.SendRequest(h, reqId, bytes, q.GetSize()): return True if kv: with self._lock_: if batching: self._m_kvBatching_.pop() else: self._m_kvCallback_.pop() return False
def _sc_(self, handler, nError): if nError == 0 and ccl.GetSSL(self._m_h_) != 0: pCertInfo = ccl.GetUCert(self._m_h_) self._m_cert_ = CClientSocket.CUCertImpl(pCertInfo, self) else: self._m_cert_ = None if not self.SocketConnected is None: self.SocketConnected(self, nError)
def WaitAll(self, timeOut=0xffffffff): if ccl.IsBatching(self._m_h_): raise Exception( "Can't call the method WaitAll during batching requests") if ccl.IsQueueStarted(self._m_h_) and ccl.GetJobSize(self._m_h_) > 0: raise Exception( "Can't call the method WaitAll during enqueuing transactional requests" ) return ccl.WaitAll(self._m_h_, timeOut)
def ShutdownPool(self): poolId = 0 ok = True with self._lock_: poolId = self._PoolId_ self._PoolId_ = 0 if (poolId != 0): ok = ccl.DisconnectAll(poolId) ok = ccl.DestroySocketPool(poolId)
def _brp_(self, handler, reqId): if reqId == tagBaseRequestID.idSwitchTo: self._m_random_ = ccl.IsRandom(self._m_h_) self._m_currSvsId = ccl.GetCurrentServiceId(self._m_h_) if self._m_ash_: self._m_ash_.OnBaseRequestProcessed(reqId) if reqId == tagBaseRequestID.idCancel: self._m_ash_.CleanCallbacks() if not self.BaseRequestProcessed is None: self.BaseRequestProcessed(self, reqId)
def Unlock(self, asyncHandler_or_clientSocket): if asyncHandler_or_clientSocket is None: return poolId = None with self._lock_: poolId = self._PoolId_ if isinstance(asyncHandler_or_clientSocket, CAsyncServiceHandler): ccl.UnlockASocket(poolId, asyncHandler_or_clientSocket.AttachedClientSocket.Handle) elif isinstance(asyncHandler_or_clientSocket, CClientSocket): ccl.UnlockASocket(poolId, asyncHandler_or_clientSocket.Handle) else: raise ValueError('Unexpected input value')
def SendRequest(self, reqId, q, arh, discarded=None, efs=None): """ Send a request onto a remote server for processing, and return immediately without blocking :param reqId: An unique request id within a service handler :param q: An instance of CScopeUQueue or CUQueue or None :param arh: A callback for tracking an instance of CAsyncResult containing an expected result :param discarded: A callback for tracking communication channel events, close and cancel :param efs: A callback for tracking an exception from server :return: True if communication channel is sendable, and False if communication channel is not sendable """ if reqId <= tagBaseRequestID.idReservedTwo: raise ValueError('Request id must be larger than 0x2001') delay = q if isinstance(q, CScopeUQueue): q = q.UQueue elif q is None: delay = CScopeUQueue() q = delay.UQueue elif not isinstance(q, CUQueue): raise ValueError('Bad input for parameter q') #http://stackoverflow.com/questions/21483482/efficient-way-to-convert-string-to-ctypes-c-ubyte-array-in-python bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) h = self._m_ClientSocket_.Handle if h == 0: return False kv = None batching = False if arh or discarded or efs: kv = (reqId, CResultCb(arh, discarded, efs)) batching = ccl.IsBatching(h) with self._lock_Send_: with self._lock_: if batching: self._m_kvBatching_.append(kv) else: self._m_kvCallback_.append(kv) if ccl.SendRequest(h, reqId, bytes, q.GetSize()): return True else: if ccl.SendRequest(h, reqId, bytes, q.GetSize()): return True if kv: with self._lock_: if batching: self._m_kvBatching_.pop() else: self._m_kvCallback_.pop() return False
def DisconnectAll(self): poolId = 0 with self._lock_: poolId = self._PoolId_ if (poolId != 0): return ccl.DisconnectAll(poolId) return True
def SendUserMessage(self, message, userId, hint=''): if userId is None: userId = u'' q = CUQueue().SaveObject(message, hint) bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_) return ccl.SendUserMessage(self._m_cs_.Handle, userId, bytes, q.GetSize())
def SendUserMessageEx(self, userId, message): if userId is None: userId = u'' if message is None: message = () msize = len(message) arrMessage = (c_ubyte * msize).from_buffer(message) return ccl.SendUserMessageEx(self._m_cs_.Handle, userId, arrMessage, msize)
def Publish(self, message, groups, hint=''): if groups is None: groups = () size = len(groups) arr = (c_uint * size)(*groups) q = CUQueue().SaveObject(message, hint) bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_) return ccl.Speak(self._m_cs_.Handle, bytes, q.GetSize(), arr, size)
def AbortBatching(self): with self._lock_: for kv in self._m_kvBatching_: if kv[1].Discarded: kv[1].Discarded(self, True) self._m_kvBatching_.clear() h = self._m_ClientSocket_.Handle return ccl.AbortBatching(h)
def SendRouteeResult(self, q, reqId=0): if q is None: q = CUQueue() if reqId == 0: reqId = self._m_ClientSocket_.CurrentRequestID h = self._m_ClientSocket_.Handle bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) return ccl.SendRouteeResult(h, reqId, bytes, q.GetSize())
def SeekByQueue(self, queueName=''): h = None if queueName is None or len(queueName) == 0: with self._lock_: merge = ccl.GetQueueAutoMergeByPool(self._PoolId_) for cs in self._m_dicSocketHandler_.keys(): if merge and cs.ConnectionState < tagConnectionState.csSwitched: continue cq = cs.ClientQueue if not cq.Available: continue if h is None: h = self._m_dicSocketHandler_[cs] elif (cq.MessageCount < h.AttachedClientSocket.ClientQueue.MessageCount) or ( (not h.AttachedClientSocket.Connected) and cs.Connected): h = self._m_dicSocketHandler_[cs] else: if CUQueue.DEFAULT_OS == tagOperationSystem.osWin: queueName = queueName.lower() rawName = '' appName = ccl.GetClientWorkDirectory().decode('latin-1') with self._lock_: for cs in self._m_dicSocketHandler_.keys(): if not cs.ClientQueue.Available: continue if cs.ClientQueue.Secure: rawName = queueName + "_" + appName + "_1.mqc" else: rawName = queueName + "_" + appName + "_0.mqc" queueFileName = cs.ClientQueue.QueueFileName length = len(queueFileName) lenRaw = len(rawName) if lenRaw > length: continue pos = queueFileName.rfind(rawName) #queue file name with full path if pos == 0: return self._m_dicSocketHandler_[cs] #queue raw name only if pos + lenRaw == length: return self._m_dicSocketHandler_[cs] return h
def PublishEx(self, message, groups): if groups is None: groups = () size = len(groups) arr = (c_uint * size)(*groups) if message is None: message = () msize = len(message) arrMessage = (c_ubyte * msize).from_buffer(message) return ccl.SpeakEx(self._m_cs_.Handle, arrMessage, msize, arr, size)
def Config(clas): """ :return: A dictionary containing SocketPro pools configuration """ with SpManager._cs_: if SpManager._sp_config: SpManager._sp_config[ 'WorkingDir'] = ccl.GetClientWorkDirectory().decode( 'latin-1') return SpManager._sp_config return None
def SendRequest(self, reqId, q, arh): if q is None: q = CUQueue(bytearray(0)) if not isinstance(q, CUQueue): raise ValueError('Bad input for parameter q') #http://stackoverflow.com/questions/21483482/efficient-way-to-convert-string-to-ctypes-c-ubyte-array-in-python bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) h = self._m_ClientSocket_.Handle if h == 0: return False with self._lock_Send_: if not (arh is None): kv = (reqId, arh) with self._lock_: if ccl.IsBatching(h): self._m_kvBatching_.append(kv) else: self._m_kvCallback_.append(kv) return ccl.SendRequest(h, reqId, bytes, q.GetSize())
def AbortJob(self): ash = self._m_cs_.CurrentHandler with ash._lock_: aborted = len(ash._m_kvCallback_) - self._nQIndex_ if ccl.AbortJob(self._m_cs_.Handle): while aborted > 0: p = ash._m_kvCallback_.pop() if p and p[1] and p[1].Discarded: p[1].Discarded(ash, True) aborted -= 1 return True return False
def __init__(self, clsAsyncHandler, autoconn=True, recvtimeout=CClientSocket.DEFAULT_RECV_TIMEOUT, conntimeout=CClientSocket.DEFAULT_CONN_TIMEOUT, serviceid=0): self._autoConn_ = autoconn self._recvTimeout_ = recvtimeout self._connTimeout_ = conntimeout self._serviceId_ = serviceid self.SocketPoolEvent = None self.DoSslServerAuthentication = None self._PoolId_ = 0 self._lock_ = threading.Lock() self._m_dicSocketHandler_ = {} self._m_cbPool_ = ccl.PSocketPoolCallback(self._spe_) self._m_mcc_ = [[]] self._m_cls_ = clsAsyncHandler
def SetConfig(midTier=False, jsonConfig=None): """ Set socket pools configuration from a JSON text file :param midTier: True if calling from a middle tier; Otherwise, false :param jsonConfig: >A file path to a JSON configuration text file, which defaults to sp_config.json at current directory :return: A dictionary containing SocketPro pools configuration """ with SpManager._cs_: if SpManager._sp_config: return SpManager._sp_config if not jsonConfig: jsonConfig = 'sp_config.json' with open(jsonConfig, 'r') as jf: s = jf.read() start = s.find('{') s = s[start:] sc = json.loads(s) if 'CertStore' in sc and sc['CertStore']: ccl.SetVerifyLocation(sc['CertStore'].encode('latin-1')) if 'WorkingDir' in sc and sc['WorkingDir']: ccl.SetClientWorkDirectory( sc['WorkingDir'].encode('latin-1')) if 'QueuePassword' in sc and sc['QueuePassword']: ccl.SetMessageQueuePassword( sc['QueuePassword'].encode('latin-1')) sc['QueuePassword'] = 1 else: sc['QueuePassword'] = 0 if 'KeysAllowed' in sc: ka = [] for s in sc['KeysAllowed']: ka.append(s.lower()) sc['KeysAllowed'] = ka SpManager._CheckErrors(sc) SpManager._sp_config = sc SpManager._sp_config['WorkingDir'] = ccl.GetClientWorkDirectory( ).decode('latin-1') return SpManager._sp_config
def _rp_(self, handler, reqId, size): ash = self._Seek_(self.CurrentServiceID) if not ash is None: mem = (c_char * size)() res = ccl.RetrieveResult(handler, mem, size) if res != size: if res == 0: return #socket closed msg = 'Wrong number of bytes retrieved (expected = ' + str(size) + ' and obtained = ' + str(res) raise ValueError(msg) q = CUQueue(mem) ash._OnRR_(reqId, q) if not self.RequestProcessed is None: self.RequestProcessed(self, reqId, size)
def AppendTo(self, queues): if queues is None: return True queues = list(queues) count = len(queues) if count == 0: return True index = 0 handles = (USocket_Client_Handle * count)() for q in queues: if isinstance(q, CClientSocket) or isinstance(q, IClientQueue): handles[index] = q.Handle else: handles[index] = q index += 1 return ccl.PushQueueTo(self._m_cs_.Handle, handles, count)
def Download(self, localFile, remoteFile, dl=None, trans=None, discarded=None, flags=FILE_OPEN_TRUNCACTED): if not localFile: return False if not remoteFile: return False context = CContext(False, flags) context.Download = dl context.Transferring = trans context.Discarded = discarded context.FilePath = remoteFile context.LocalFile = localFile with self._csFile: self._vContext.append(context) if len(self._vContext) == 1: ccl.PostProcessing(self.AttachedClientSocket.Handle, 0, 0) self.AttachedClientSocket.DoEcho #make sure WaitAll works correctly return True
def SendRouteeResult(self, q, reqId=0): """ Send a result to the other routee :param q: An instance of CScopeUQueue or CUQueue or None :param reqId: An unique request id within a service handler :return: True if socket is connected, and False if socket is closed """ delay = q if isinstance(q, CScopeUQueue): q = q.UQueue elif q is None: delay = CScopeUQueue() q = delay.UQueue elif not isinstance(q, CUQueue): raise ValueError('Bad input for parameter q') if reqId == 0: reqId = self._m_ClientSocket_.CurrentRequestID h = self._m_ClientSocket_.Handle bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_) return ccl.SendRouteeResult(h, reqId, bytes, q.GetSize())
def OnMergeTo(self, to): with to._csFile: pos = 0 count = len(to._vContext) for it in to._vContext: if not it._HasError() and not it.File: break pos += 1 left = deque(itertools.islice(to._vContext, pos)) right = deque( itertools.islice(to._vContext, pos, len(to._vContext))) with self._csFile: left.extend(self._vContext) self._vContext = deque() left.extend(right) to._vContext = left if count == 0 and len(to._vContext): ccl.PostProcessing(to.Socket.Handle, 0, 0) if not to.Socket.CountOfRequestsInQueue: to.Socket.DoEcho() #make sure WaitAll works correctly
def StartSocketPoolEx(self, cc, avg=True, ta=tagThreadApartment.taNone): ok = False temp = {} if cc is None or len(cc) == 0 or len(cc[0]) == 0: raise ValueError('Must set connection context argument properly') if self.Started: self.ShutdownPool() self._CopyCC_(cc) first = True threads = len(cc) socketsPerThread = len(cc[0]) if not self._start_(socketsPerThread, threads, avg, ta): return False with self._lock_: index = 0 keys = self._m_dicSocketHandler_.keys() for cs in keys: temp[cs] = self._m_dicSocketHandler_[cs] m = int(index % threads) n = int(index / threads) c = self._m_mcc_[m][n] if c.Host is None: raise ValueError('Host string can not be null') c.Host = c.Host.strip() if len(c.Host) == 0: raise ValueError('Host string must be a valid string') if c.Port == 0: raise ValueError("Host port can't be zero") cs.ConnectionContext = c index += 1 for cs in temp.keys(): if cs.Connected: first = False continue c = cs.ConnectionContext ccl.SetEncryptionMethod(cs.Handle, c.EncrytionMethod) ccl.SetUserID(cs.Handle, c.UserId) ccl.SetZip(cs.Handle, c.Zip) if first: ok = ccl.Connect(cs.Handle, c.Host.encode('latin-1'), c.Port, True, c.V6) if ok and ccl.WaitAll(cs.Handle, 0xffffffff): first = False else: ccl.Connect(cs.Handle, c.Host.encode('latin-1'), c.Port, False, c.V6) return self.ConnectedSockets > 0
def Download(self, localFile, remoteFile, dl=None, trans=None, discarded=None, flags=FILE_OPEN_TRUNCACTED, se=None): """ Post a context to download a remote file at server side to a local file at client side :param localFile: A path to a local file at client side for downloading :param remoteFile: A path to a remote file at server side :param up: A callback for tracking a final result of downloading, which contains an int and an error message :param trans: A callback for tracking downloading progress :param discarded: A callback for tracking communication channel events, close and cancel :param flags: An integer bit-wise option flags for one or more options such as FILE_OPEN_TRUNCACTED|FILE_OPEN_APPENDED and FILE_OPEN_SHARE_WRITE :param se: A callback for tracking an exception (CServerError) from server :return: True if successful and False if failed when localFile or remoteFile is empty """ if not localFile or str(localFile) == 0: raise ValueError('localFile cannot be empty') if not remoteFile or str(remoteFile) == 0: raise ValueError('remoteFile cannot be empty') context = CContext(False, flags) context.Download = dl context.Transferring = trans context.Discarded = discarded context.FilePath = remoteFile context.LocalFile = localFile context.Se = se with self._csFile: self._vContext.append(context) filesOpened = self._GetFilesOpened() if self._MaxDownloading > filesOpened: ccl.PostProcessing(self.Socket.Handle, 0, 0) if not filesOpened: self.Socket.DoEcho() # make sure WaitAll works correctly return True
def EnsureAppending(self, queues): if not self.Available: return False if self.QueueStatus != tagQueueStatus.qsMergePushing: return True if queues is None: return True queues = list(queues) if len(queues) == 0: return True handles = [] for q in queues: h = 0 if isinstance(q, CClientSocket) or isinstance(q, IClientQueue): h = q.Handle else: h = q if ccl.GetClientQueueStatus(h) != tagQueueStatus.qsMergeComplete: handles.append(h) if len(handles) > 0: return self.AppendTo(handles) self.Reset() return True
def OnPostProcessing(self, hint, data): cs = self.AttachedClientSocket ctx = CContext(False, 0) with self._csFile: count = len(self._vContext) if count > 0: front = self._vContext[0] if front.Uploading: front._OpenLocalRead() else: front._OpenLocalWrite() if front.ErrCode or front.ErrMsg: ctx = front elif front.Uploading: with CScopeUQueue() as q: q.SaveString(front.FilePath).SaveUInt(front.Flags).SaveULong(front.FileSize) if not self.SendRequest(CStreamingFile.idUpload, q, None, front.Discarded, None): front.ErrCode = cs.ErrorCode front.ErrMsg = cs.ErrorMessage ctx = front else: with CScopeUQueue() as q: q.SaveString(front.LocalFile).SaveString(front.FilePath).SaveUInt(front.Flags).SaveLong(front.InitSize) if not self.SendRequest(CStreamingFile.idDownload, q, None, front.Discarded, None): front.ErrCode = cs.ErrorCode front.ErrMsg = cs.ErrorMessage ctx = front if ctx.ErrCode or ctx.ErrMsg: ctx._CloseFile() if ctx.Download: ctx.Download(self, ctx.ErrCode, ctx.ErrMsg) with self._csFile: self._vContext.popleft() if len(self._vContext) > 0: #post processing the next one ccl.PostProcessing(self.AttachedClientSocket.Handle, 0, 0) self.AttachedClientSocket.DoEcho #make sure WaitAll works correctly