def WaitAll(self, timeout = 0xffffffff):
h = self._m_ClientSocket_.Handle
if ccl.IsBatching(h):
raise ValueError("Can't call the method WaitAll while batching requests")
if ccl.IsQueueStarted(h) and ccl.GetJobSize(h) > 0:
raise ValueError("Can't call the method WaitAll while enqueuing transactional requests")
return ccl.WaitAll(h, timeout)
def SendRequest(self, reqId, q, arh, discarded=None, efs=None):
if q is None:
q = CUQueue(bytearray(0))
if isinstance(q, CScopeUQueue):
q = q.UQueue
if not isinstance(q, CUQueue):
raise ValueError('Bad input for parameter q')
#http://stackoverflow.com/questions/21483482/efficient-way-to-convert-string-to-ctypes-c-ubyte-array-in-python
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_,
q._m_position_)
h = self._m_ClientSocket_.Handle
if h == 0:
return False
kv = None
batching = False
with self._lock_Send_:
if arh or discarded or efs:
kv = (reqId, CResultCb(arh, discarded, efs))
with self._lock_:
batching = ccl.IsBatching(h)
if batching:
self._m_kvBatching_.append(kv)
else:
self._m_kvCallback_.append(kv)
if ccl.SendRequest(h, reqId, bytes, q.GetSize()):
return True
if kv:
with self._lock_:
if batching:
self._m_kvBatching_.pop()
else:
self._m_kvCallback_.pop()
return False
def _sc_(self, handler, nError):
if nError == 0 and ccl.GetSSL(self._m_h_) != 0:
pCertInfo = ccl.GetUCert(self._m_h_)
self._m_cert_ = CClientSocket.CUCertImpl(pCertInfo, self)
else:
self._m_cert_ = None
if not self.SocketConnected is None:
self.SocketConnected(self, nError)
def WaitAll(self, timeOut=0xffffffff):
if ccl.IsBatching(self._m_h_):
raise Exception(
"Can't call the method WaitAll during batching requests")
if ccl.IsQueueStarted(self._m_h_) and ccl.GetJobSize(self._m_h_) > 0:
raise Exception(
"Can't call the method WaitAll during enqueuing transactional requests"
)
return ccl.WaitAll(self._m_h_, timeOut)
def ShutdownPool(self):
poolId = 0
ok = True
with self._lock_:
poolId = self._PoolId_
self._PoolId_ = 0
if (poolId != 0):
ok = ccl.DisconnectAll(poolId)
ok = ccl.DestroySocketPool(poolId)
def _brp_(self, handler, reqId):
if reqId == tagBaseRequestID.idSwitchTo:
self._m_random_ = ccl.IsRandom(self._m_h_)
self._m_currSvsId = ccl.GetCurrentServiceId(self._m_h_)
if self._m_ash_:
self._m_ash_.OnBaseRequestProcessed(reqId)
if reqId == tagBaseRequestID.idCancel:
self._m_ash_.CleanCallbacks()
if not self.BaseRequestProcessed is None:
self.BaseRequestProcessed(self, reqId)
def Unlock(self, asyncHandler_or_clientSocket):
if asyncHandler_or_clientSocket is None:
return
poolId = None
with self._lock_:
poolId = self._PoolId_
if isinstance(asyncHandler_or_clientSocket, CAsyncServiceHandler):
ccl.UnlockASocket(poolId, asyncHandler_or_clientSocket.AttachedClientSocket.Handle)
elif isinstance(asyncHandler_or_clientSocket, CClientSocket):
ccl.UnlockASocket(poolId, asyncHandler_or_clientSocket.Handle)
else:
raise ValueError('Unexpected input value')
def SendRequest(self, reqId, q, arh, discarded=None, efs=None):
"""
Send a request onto a remote server for processing, and return immediately without blocking
:param reqId: An unique request id within a service handler
:param q: An instance of CScopeUQueue or CUQueue or None
:param arh: A callback for tracking an instance of CAsyncResult containing an expected result
:param discarded: A callback for tracking communication channel events, close and cancel
:param efs: A callback for tracking an exception from server
:return: True if communication channel is sendable, and False if communication channel is not sendable
"""
if reqId <= tagBaseRequestID.idReservedTwo:
raise ValueError('Request id must be larger than 0x2001')
delay = q
if isinstance(q, CScopeUQueue):
q = q.UQueue
elif q is None:
delay = CScopeUQueue()
q = delay.UQueue
elif not isinstance(q, CUQueue):
raise ValueError('Bad input for parameter q')
#http://stackoverflow.com/questions/21483482/efficient-way-to-convert-string-to-ctypes-c-ubyte-array-in-python
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_)
h = self._m_ClientSocket_.Handle
if h == 0:
return False
kv = None
batching = False
if arh or discarded or efs:
kv = (reqId, CResultCb(arh, discarded, efs))
batching = ccl.IsBatching(h)
with self._lock_Send_:
with self._lock_:
if batching:
self._m_kvBatching_.append(kv)
else:
self._m_kvCallback_.append(kv)
if ccl.SendRequest(h, reqId, bytes, q.GetSize()):
return True
else:
if ccl.SendRequest(h, reqId, bytes, q.GetSize()):
return True
if kv:
with self._lock_:
if batching:
self._m_kvBatching_.pop()
else:
self._m_kvCallback_.pop()
return False
def DisconnectAll(self):
poolId = 0
with self._lock_:
poolId = self._PoolId_
if (poolId != 0):
return ccl.DisconnectAll(poolId)
return True
def SendUserMessage(self, message, userId, hint=''):
if userId is None:
userId = u''
q = CUQueue().SaveObject(message, hint)
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_)
return ccl.SendUserMessage(self._m_cs_.Handle, userId, bytes,
q.GetSize())
def SendUserMessageEx(self, userId, message):
if userId is None:
userId = u''
if message is None:
message = ()
msize = len(message)
arrMessage = (c_ubyte * msize).from_buffer(message)
return ccl.SendUserMessageEx(self._m_cs_.Handle, userId, arrMessage, msize)
def Publish(self, message, groups, hint=''):
if groups is None:
groups = ()
size = len(groups)
arr = (c_uint * size)(*groups)
q = CUQueue().SaveObject(message, hint)
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_)
return ccl.Speak(self._m_cs_.Handle, bytes, q.GetSize(), arr, size)
def AbortBatching(self):
with self._lock_:
for kv in self._m_kvBatching_:
if kv[1].Discarded:
kv[1].Discarded(self, True)
self._m_kvBatching_.clear()
h = self._m_ClientSocket_.Handle
return ccl.AbortBatching(h)
def SendRouteeResult(self, q, reqId=0):
if q is None:
q = CUQueue()
if reqId == 0:
reqId = self._m_ClientSocket_.CurrentRequestID
h = self._m_ClientSocket_.Handle
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_,
q._m_position_)
return ccl.SendRouteeResult(h, reqId, bytes, q.GetSize())
def SeekByQueue(self, queueName=''):
h = None
if queueName is None or len(queueName) == 0:
with self._lock_:
merge = ccl.GetQueueAutoMergeByPool(self._PoolId_)
for cs in self._m_dicSocketHandler_.keys():
if merge and cs.ConnectionState < tagConnectionState.csSwitched:
continue
cq = cs.ClientQueue
if not cq.Available:
continue
if h is None:
h = self._m_dicSocketHandler_[cs]
elif (cq.MessageCount <
h.AttachedClientSocket.ClientQueue.MessageCount) or (
(not h.AttachedClientSocket.Connected)
and cs.Connected):
h = self._m_dicSocketHandler_[cs]
else:
if CUQueue.DEFAULT_OS == tagOperationSystem.osWin:
queueName = queueName.lower()
rawName = ''
appName = ccl.GetClientWorkDirectory().decode('latin-1')
with self._lock_:
for cs in self._m_dicSocketHandler_.keys():
if not cs.ClientQueue.Available:
continue
if cs.ClientQueue.Secure:
rawName = queueName + "_" + appName + "_1.mqc"
else:
rawName = queueName + "_" + appName + "_0.mqc"
queueFileName = cs.ClientQueue.QueueFileName
length = len(queueFileName)
lenRaw = len(rawName)
if lenRaw > length:
continue
pos = queueFileName.rfind(rawName)
#queue file name with full path
if pos == 0:
return self._m_dicSocketHandler_[cs]
#queue raw name only
if pos + lenRaw == length:
return self._m_dicSocketHandler_[cs]
return h
def PublishEx(self, message, groups):
if groups is None:
groups = ()
size = len(groups)
arr = (c_uint * size)(*groups)
if message is None:
message = ()
msize = len(message)
arrMessage = (c_ubyte * msize).from_buffer(message)
return ccl.SpeakEx(self._m_cs_.Handle, arrMessage, msize, arr, size)
def Config(clas):
"""
:return: A dictionary containing SocketPro pools configuration
"""
with SpManager._cs_:
if SpManager._sp_config:
SpManager._sp_config[
'WorkingDir'] = ccl.GetClientWorkDirectory().decode(
'latin-1')
return SpManager._sp_config
return None
def SendRequest(self, reqId, q, arh):
if q is None:
q = CUQueue(bytearray(0))
if not isinstance(q, CUQueue):
raise ValueError('Bad input for parameter q')
#http://stackoverflow.com/questions/21483482/efficient-way-to-convert-string-to-ctypes-c-ubyte-array-in-python
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_,
q._m_position_)
h = self._m_ClientSocket_.Handle
if h == 0:
return False
with self._lock_Send_:
if not (arh is None):
kv = (reqId, arh)
with self._lock_:
if ccl.IsBatching(h):
self._m_kvBatching_.append(kv)
else:
self._m_kvCallback_.append(kv)
return ccl.SendRequest(h, reqId, bytes, q.GetSize())
def AbortJob(self):
ash = self._m_cs_.CurrentHandler
with ash._lock_:
aborted = len(ash._m_kvCallback_) - self._nQIndex_
if ccl.AbortJob(self._m_cs_.Handle):
while aborted > 0:
p = ash._m_kvCallback_.pop()
if p and p[1] and p[1].Discarded:
p[1].Discarded(ash, True)
aborted -= 1
return True
return False
def __init__(self, clsAsyncHandler, autoconn=True, recvtimeout=CClientSocket.DEFAULT_RECV_TIMEOUT, conntimeout=CClientSocket.DEFAULT_CONN_TIMEOUT, serviceid=0):
self._autoConn_ = autoconn
self._recvTimeout_ = recvtimeout
self._connTimeout_ = conntimeout
self._serviceId_ = serviceid
self.SocketPoolEvent = None
self.DoSslServerAuthentication = None
self._PoolId_ = 0
self._lock_ = threading.Lock()
self._m_dicSocketHandler_ = {}
self._m_cbPool_ = ccl.PSocketPoolCallback(self._spe_)
self._m_mcc_ = [[]]
self._m_cls_ = clsAsyncHandler
def SetConfig(midTier=False, jsonConfig=None):
"""
Set socket pools configuration from a JSON text file
:param midTier: True if calling from a middle tier; Otherwise, false
:param jsonConfig: >A file path to a JSON configuration text file, which defaults to sp_config.json at current directory
:return: A dictionary containing SocketPro pools configuration
"""
with SpManager._cs_:
if SpManager._sp_config:
return SpManager._sp_config
if not jsonConfig:
jsonConfig = 'sp_config.json'
with open(jsonConfig, 'r') as jf:
s = jf.read()
start = s.find('{')
s = s[start:]
sc = json.loads(s)
if 'CertStore' in sc and sc['CertStore']:
ccl.SetVerifyLocation(sc['CertStore'].encode('latin-1'))
if 'WorkingDir' in sc and sc['WorkingDir']:
ccl.SetClientWorkDirectory(
sc['WorkingDir'].encode('latin-1'))
if 'QueuePassword' in sc and sc['QueuePassword']:
ccl.SetMessageQueuePassword(
sc['QueuePassword'].encode('latin-1'))
sc['QueuePassword'] = 1
else:
sc['QueuePassword'] = 0
if 'KeysAllowed' in sc:
ka = []
for s in sc['KeysAllowed']:
ka.append(s.lower())
sc['KeysAllowed'] = ka
SpManager._CheckErrors(sc)
SpManager._sp_config = sc
SpManager._sp_config['WorkingDir'] = ccl.GetClientWorkDirectory(
).decode('latin-1')
return SpManager._sp_config
def _rp_(self, handler, reqId, size):
ash = self._Seek_(self.CurrentServiceID)
if not ash is None:
mem = (c_char * size)()
res = ccl.RetrieveResult(handler, mem, size)
if res != size:
if res == 0:
return #socket closed
msg = 'Wrong number of bytes retrieved (expected = ' + str(size) + ' and obtained = ' + str(res)
raise ValueError(msg)
q = CUQueue(mem)
ash._OnRR_(reqId, q)
if not self.RequestProcessed is None:
self.RequestProcessed(self, reqId, size)
def AppendTo(self, queues):
if queues is None:
return True
queues = list(queues)
count = len(queues)
if count == 0:
return True
index = 0
handles = (USocket_Client_Handle * count)()
for q in queues:
if isinstance(q, CClientSocket) or isinstance(q, IClientQueue):
handles[index] = q.Handle
else:
handles[index] = q
index += 1
return ccl.PushQueueTo(self._m_cs_.Handle, handles, count)
def Download(self, localFile, remoteFile, dl=None, trans=None, discarded=None, flags=FILE_OPEN_TRUNCACTED):
if not localFile:
return False
if not remoteFile:
return False
context = CContext(False, flags)
context.Download = dl
context.Transferring = trans
context.Discarded = discarded
context.FilePath = remoteFile
context.LocalFile = localFile
with self._csFile:
self._vContext.append(context)
if len(self._vContext) == 1:
ccl.PostProcessing(self.AttachedClientSocket.Handle, 0, 0)
self.AttachedClientSocket.DoEcho #make sure WaitAll works correctly
return True
def SendRouteeResult(self, q, reqId=0):
"""
Send a result to the other routee
:param q: An instance of CScopeUQueue or CUQueue or None
:param reqId: An unique request id within a service handler
:return: True if socket is connected, and False if socket is closed
"""
delay = q
if isinstance(q, CScopeUQueue):
q = q.UQueue
elif q is None:
delay = CScopeUQueue()
q = delay.UQueue
elif not isinstance(q, CUQueue):
raise ValueError('Bad input for parameter q')
if reqId == 0:
reqId = self._m_ClientSocket_.CurrentRequestID
h = self._m_ClientSocket_.Handle
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_, q._m_position_)
return ccl.SendRouteeResult(h, reqId, bytes, q.GetSize())
def OnMergeTo(self, to):
with to._csFile:
pos = 0
count = len(to._vContext)
for it in to._vContext:
if not it._HasError() and not it.File:
break
pos += 1
left = deque(itertools.islice(to._vContext, pos))
right = deque(
itertools.islice(to._vContext, pos, len(to._vContext)))
with self._csFile:
left.extend(self._vContext)
self._vContext = deque()
left.extend(right)
to._vContext = left
if count == 0 and len(to._vContext):
ccl.PostProcessing(to.Socket.Handle, 0, 0)
if not to.Socket.CountOfRequestsInQueue:
to.Socket.DoEcho() #make sure WaitAll works correctly
def StartSocketPoolEx(self, cc, avg=True, ta=tagThreadApartment.taNone):
ok = False
temp = {}
if cc is None or len(cc) == 0 or len(cc[0]) == 0:
raise ValueError('Must set connection context argument properly')
if self.Started:
self.ShutdownPool()
self._CopyCC_(cc)
first = True
threads = len(cc)
socketsPerThread = len(cc[0])
if not self._start_(socketsPerThread, threads, avg, ta):
return False
with self._lock_:
index = 0
keys = self._m_dicSocketHandler_.keys()
for cs in keys:
temp[cs] = self._m_dicSocketHandler_[cs]
m = int(index % threads)
n = int(index / threads)
c = self._m_mcc_[m][n]
if c.Host is None:
raise ValueError('Host string can not be null')
c.Host = c.Host.strip()
if len(c.Host) == 0:
raise ValueError('Host string must be a valid string')
if c.Port == 0:
raise ValueError("Host port can't be zero")
cs.ConnectionContext = c
index += 1
for cs in temp.keys():
if cs.Connected:
first = False
continue
c = cs.ConnectionContext
ccl.SetEncryptionMethod(cs.Handle, c.EncrytionMethod)
ccl.SetUserID(cs.Handle, c.UserId)
ccl.SetZip(cs.Handle, c.Zip)
if first:
ok = ccl.Connect(cs.Handle, c.Host.encode('latin-1'), c.Port,
True, c.V6)
if ok and ccl.WaitAll(cs.Handle, 0xffffffff):
first = False
else:
ccl.Connect(cs.Handle, c.Host.encode('latin-1'), c.Port, False,
c.V6)
return self.ConnectedSockets > 0
def Download(self,
localFile,
remoteFile,
dl=None,
trans=None,
discarded=None,
flags=FILE_OPEN_TRUNCACTED,
se=None):
"""
Post a context to download a remote file at server side to a local file at client side
:param localFile: A path to a local file at client side for downloading
:param remoteFile: A path to a remote file at server side
:param up: A callback for tracking a final result of downloading, which contains an int and an error message
:param trans: A callback for tracking downloading progress
:param discarded: A callback for tracking communication channel events, close and cancel
:param flags: An integer bit-wise option flags for one or more options such as
FILE_OPEN_TRUNCACTED|FILE_OPEN_APPENDED and FILE_OPEN_SHARE_WRITE
:param se: A callback for tracking an exception (CServerError) from server
:return: True if successful and False if failed when localFile or remoteFile is empty
"""
if not localFile or str(localFile) == 0:
raise ValueError('localFile cannot be empty')
if not remoteFile or str(remoteFile) == 0:
raise ValueError('remoteFile cannot be empty')
context = CContext(False, flags)
context.Download = dl
context.Transferring = trans
context.Discarded = discarded
context.FilePath = remoteFile
context.LocalFile = localFile
context.Se = se
with self._csFile:
self._vContext.append(context)
filesOpened = self._GetFilesOpened()
if self._MaxDownloading > filesOpened:
ccl.PostProcessing(self.Socket.Handle, 0, 0)
if not filesOpened:
self.Socket.DoEcho() # make sure WaitAll works correctly
return True
def EnsureAppending(self, queues):
if not self.Available:
return False
if self.QueueStatus != tagQueueStatus.qsMergePushing:
return True
if queues is None:
return True
queues = list(queues)
if len(queues) == 0:
return True
handles = []
for q in queues:
h = 0
if isinstance(q, CClientSocket) or isinstance(q, IClientQueue):
h = q.Handle
else:
h = q
if ccl.GetClientQueueStatus(h) != tagQueueStatus.qsMergeComplete:
handles.append(h)
if len(handles) > 0:
return self.AppendTo(handles)
self.Reset()
return True
def OnPostProcessing(self, hint, data):
cs = self.AttachedClientSocket
ctx = CContext(False, 0)
with self._csFile:
count = len(self._vContext)
if count > 0:
front = self._vContext[0]
if front.Uploading:
front._OpenLocalRead()
else:
front._OpenLocalWrite()
if front.ErrCode or front.ErrMsg:
ctx = front
elif front.Uploading:
with CScopeUQueue() as q:
q.SaveString(front.FilePath).SaveUInt(front.Flags).SaveULong(front.FileSize)
if not self.SendRequest(CStreamingFile.idUpload, q, None, front.Discarded, None):
front.ErrCode = cs.ErrorCode
front.ErrMsg = cs.ErrorMessage
ctx = front
else:
with CScopeUQueue() as q:
q.SaveString(front.LocalFile).SaveString(front.FilePath).SaveUInt(front.Flags).SaveLong(front.InitSize)
if not self.SendRequest(CStreamingFile.idDownload, q, None, front.Discarded, None):
front.ErrCode = cs.ErrorCode
front.ErrMsg = cs.ErrorMessage
ctx = front
if ctx.ErrCode or ctx.ErrMsg:
ctx._CloseFile()
if ctx.Download:
ctx.Download(self, ctx.ErrCode, ctx.ErrMsg)
with self._csFile:
self._vContext.popleft()
if len(self._vContext) > 0:
#post processing the next one
ccl.PostProcessing(self.AttachedClientSocket.Handle, 0, 0)
self.AttachedClientSocket.DoEcho #make sure WaitAll works correctly