def Enqueue(self, key, idMessage, q, e=None, discarded=None):
"""
Enqueue a message into a queue file identified by a key
:param key: An ASCII string for identifying a queue at server side
:param idMessage: A unsigned short number to identify a message
:param q: an instance of SPA.CUQueue containing a message
:param e: A callback for tracking returning index
:param discarded A callback for tracking socket close or request cancel event
:return: true for sending the request successfully, and false for failure
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key).SaveUShort(idMessage)
if q:
buffer.Push(q.IntenalBuffer, q.GetSize())
ok = None
if not e:
ok = self.SendRequest(CAsyncQueue.idEnqueue, buffer, None,
discarded)
else:
ok = self.SendRequest(
CAsyncQueue.idEnqueue, buffer,
lambda ar: e(ar.AsyncServiceHandler, ar.LoadULong()),
discarded)
CScopeUQueue.Unlock(buffer)
return ok
def SendMeta(self, meta, index):
q = CScopeUQueue.Lock()
meta.SaveTo(q)
q.SaveULong(index)
ret = self.SendResult(q, DB_CONSTS.idRowsetHeader)
CScopeUQueue.Unlock(q)
return ret != CClientPeer.REQUEST_CANCELED and ret != CClientPeer.SOCKET_NOT_FOUND
def EndQueueTrans(self, rollback=False, qt=None, discarded=None):
"""
End enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes
:param rollback: true for rollback, and false for committing
:param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_NOT_STARTED_YET, and so on
:param discarded A callback for tracking socket close or request cancel event
:return: true for sending the request successfully, and false for failure
"""
buffer = CScopeUQueue.Lock().SaveBool(rollback)
ok = None
if not qt:
ok = self.SendRequest(CAsyncQueue.idEndTrans, buffer, None,
discarded)
else:
ok = self.SendRequest(
CAsyncQueue.idEndTrans, buffer,
lambda ar: qt(ar.AsyncServiceHandler, ar.LoadInt()), discarded)
cq = self.AttachedClientSocket.ClientQueue
if cq.Available:
if rollback:
cq.AbortJob()
else:
cq.EndJob()
CScopeUQueue.Unlock(buffer)
return ok
def FlushQueue(self,
key,
f,
option=tagOptimistic.oMemoryCached,
discarded=None):
"""
May flush memory data into either operation system memory or hard disk, and return message count and queue file size in bytes. Note the method only returns message count and queue file size in bytes if the option is oMemoryCached
:param key: An ASCII string for identifying a queue at server side
:param f: A callback for tracking returning message count and queue file size in bytes
:param option: one of tagOptimistic options, oMemoryCached, oSystemMemoryCached and oDiskCommitted
:param discarded A callback for tracking socket close or request cancel event
:return: true for sending the request successfully, and false for failure
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key).SaveInt(option)
ok = None
if not f:
ok = self.SendRequest(CAsyncQueue.idFlush, buffer, None, discarded)
else:
ok = self.SendRequest(
CAsyncQueue.idFlush, buffer, lambda ar: f(
ar.AsyncServiceHandler, ar.LoadULong(), ar.LoadULong()),
discarded)
CScopeUQueue.Unlock(buffer)
return ok
def Enqueue(self, key, idMessage, q, e=None, discarded=None, se=None):
"""
Enqueue a message into a queue file identified by a key
:param key: An ASCII string for identifying a queue at server side
:param idMessage: A unsigned short number to identify a message
:param q: None or an instance of SPA.CUQueue or SPA.CScopeUQueue containing a message
:param e: A callback for tracking returning index
:param discarded A callback for tracking socket close or request cancel event
:param se A callback for tracking an exception from server
:return: True if communication channel is sendable, and False if communication channel is not sendable
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key).SaveUShort(idMessage)
if isinstance(q, CScopeUQueue):
q = q.UQueue
if q:
buffer.Push(q.IntenalBuffer, q.GetSize())
ok = None
if not e:
ok = self.SendRequest(CAsyncQueue.idEnqueue, buffer, None,
discarded, se)
else:
ok = self.SendRequest(
CAsyncQueue.idEnqueue, buffer,
lambda ar: e(ar.AsyncServiceHandler, ar.LoadULong()),
discarded, se)
CScopeUQueue.Unlock(buffer)
return ok
def SendRows(self, vData):
if not vData:
vData = []
q = CScopeUQueue.Lock()
for vt in vData:
if isinstance(vt, str):
if len(vt) <= DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE:
q.SaveObject(vt)
else:
if q.Size > 0 and not self._SendRows_(q, True):
CScopeUQueue.Unlock(q)
return False
if not self._SendBlob_(vt):
CScopeUQueue.Unlock(q)
return False
elif isinstance(vt, bytearray):
if len(vt) <= 2 * DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE:
q.SaveObject(vt)
else:
if q.Size > 0 and not self._SendRows_(q, True):
CScopeUQueue.Unlock(q)
return False
if not self._SendBlob_(vt):
CScopeUQueue.Unlock(q)
return False
else:
q.SaveObject(vt)
ret = self.SendResult(q, DB_CONSTS.idEndRows)
return ret != CClientPeer.REQUEST_CANCELED and ret != CClientPeer.SOCKET_NOT_FOUND
def CloseQueue(self,
key,
c=None,
discarded=None,
permanent=False,
se=None):
"""
Try to close or delete a persistent queue opened at server side
:param key: An ASCII string for identifying a queue at server side
:param c: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_DEQUEUING, and so on
:param permanent: true for deleting a queue file, and false for closing a queue file
:param discarded A callback for tracking socket close or request cancel event
:param se A callback for tracking an exception from server
:return: True if communication channel is sendable, and False if communication channel is not sendable
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key).SaveBool(permanent)
ok = None
if not c:
ok = self.SendRequest(CAsyncQueue.idClose, buffer, None, discarded,
se)
else:
ok = self.SendRequest(
CAsyncQueue.idClose, buffer,
lambda ar: c(ar.AsyncServiceHandler, ar.LoadInt()), discarded,
se)
CScopeUQueue.Unlock(buffer)
return ok
def StartQueueTrans(self, key, qt=None, discarded=None, se=None):
"""
Start enqueuing messages with transaction style.
Currently, total size of queued messages must be less than 4 G bytes
:param key: An ASCII string for identifying a queue at server side
:param qt: A callback for tracking returning error code,
which can be one of QUEUE_OK, QUEUE_TRANS_ALREADY_STARTED, and so on
:param discarded A callback for tracking socket close or request cancel event
:param se A callback for tracking an exception from server
:return: True if communication channel is sendable, and False if communication channel is not sendable
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key)
ok = None
cq = self.Socket.ClientQueue
if cq.Available:
cq.StartJob()
if not qt:
ok = self.SendRequest(CAsyncQueue.idStartTrans, buffer, None,
discarded, se)
else:
ok = self.SendRequest(
CAsyncQueue.idStartTrans, buffer,
lambda ar: qt(ar.AsyncServiceHandler, ar.LoadInt()), discarded,
se)
CScopeUQueue.Unlock(buffer)
return ok
def Dequeue(self, key, d, timeout=0, discarded=None):
"""
Dequeue messages from a persistent message queue file at server side in batch
:param key: An ASCII string for identifying a queue at server side
:param d: A callback for tracking data like remaining message count within a server queue file, queue file size in bytes, message dequeued within this batch and bytes dequeued within this batch
:param timeout: A time-out number in milliseconds
:param discarded A callback for tracking socket close or request cancel event
:return: true for sending the request successfully, and false for failure
"""
if not key:
key = ''
with self._csQ_:
self._dDequeue_ = d
self._keyQueue_ = key
buffer = CScopeUQueue.Lock().SaveAString(key).SaveUInt(timeout)
ok = None
if not d:
ok = self.SendRequest(CAsyncQueue.idDequeue, buffer, None,
discarded)
else:
ok = self.SendRequest(
CAsyncQueue.idDequeue, buffer,
lambda ar: d(ar.AsyncServiceHandler, ar.LoadULong(
), ar.LoadULong(), ar.LoadUInt(), ar.LoadUInt()), discarded)
CScopeUQueue.Unlock(buffer)
return ok
def _spe_(self, poolId, spe, h): #h -- usocket handle
#print "Pool id = " + str(poolId) + ", spe = " + str(spe) + ", usocket handle = " + str(h)
handler = self._MapToHandler_(h)
if spe == tagSocketPoolEvent.speTimer:
if CScopeUQueue.MemoryConsumed() / 1024 > CScopeUQueue.SHARED_BUFFER_CLEAN_SIZE:
CScopeUQueue.DestroyUQueuePool()
elif spe == tagSocketPoolEvent.speStarted:
with self._lock_:
self._PoolId_ = poolId
elif spe == tagSocketPoolEvent.speShutdown:
with self._lock_:
self._PoolId_ = 0
elif spe == tagSocketPoolEvent.speUSocketKilled:
with self._lock_:
del self._m_dicSocketHandler_[handler.Socket]
elif spe == tagSocketPoolEvent.speUSocketCreated:
cs = CClientSocket(h)
ash = self._m_cls_()
ccl.SetRecvTimeout(h, self._recvTimeout_)
ccl.SetConnTimeout(h, self._connTimeout_)
ccl.SetAutoConn(h, self._autoConn_)
if ash.SvsID == 0:
ash._m_nServiceId_ = self._serviceId_
if ash.SvsID <= BaseServiceID.sidStartup:
raise ValueError('Service id must be larger than SocketProAdapter.BaseServiceID.sidReserved (268435456)')
ash._Attach_(cs)
handler = ash
with self._lock_:
self._m_dicSocketHandler_[cs] = ash
elif spe == tagSocketPoolEvent.speConnected:
if ccl.IsOpened(h):
cs = handler.Socket
if self.DoSslServerAuthentication is not None and cs.EncryptionMethod == tagEncryptionMethod.TLSv1 and (not self.DoSslServerAuthentication(self, cs)):
return #don't set password or call SwitchTo in case failure of ssl server authentication on certificate from server
ccl.SetSockOpt(h, tagSocketOption.soRcvBuf, 116800, tagSocketLevel.slSocket)
ccl.SetSockOpt(h, tagSocketOption.soSndBuf, 116800, tagSocketLevel.slSocket)
ccl.SetSockOpt(h, tagSocketOption.soTcpNoDelay, 1, tagSocketLevel.slTcp)
ccl.SetPassword(h, cs.ConnectionContext._Password_)
ok = ccl.StartBatching(h)
ok = ccl.SwitchTo(h, handler.SvsID)
ok = ccl.TurnOnZipAtSvr(h, cs.ConnectionContext.Zip)
ok = ccl.SetSockOptAtSvr(h, tagSocketOption.soRcvBuf, 116800, tagSocketLevel.slSocket)
ok = ccl.SetSockOptAtSvr(h, tagSocketOption.soSndBuf, 116800, tagSocketLevel.slSocket)
ok = ccl.SetSockOptAtSvr(h, tagSocketOption.soTcpNoDelay, 1, tagSocketLevel.slTcp)
ok = ccl.CommitBatching(h, False)
elif spe == tagSocketPoolEvent.speQueueMergedFrom:
self._hFrom = handler
elif spe == tagSocketPoolEvent.speQueueMergedTo:
self._hFrom._AppendTo_(handler)
self._hFrom = None
else:
pass
if self.SocketPoolEvent:
self.SocketPoolEvent(self, spe, handler)
self.OnSocketPoolEvent(spe, handler)
if spe == tagSocketPoolEvent.speConnected and ccl.IsOpened(h):
self._SetQueue_(handler.Socket)
def GetCachedTables(self, defaultDb, handler, row, rh, flags=DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES):
q = CScopeUQueue.Lock()
index = self.GetCallIndex();
with self._csCache:
self._mapRowset[index] = Pair(rh, row)
self._mapHandler[index] = handler
q.SaveString(defaultDb).SaveUInt(flags).SaveULong(index)
ok = self.SendRequest(DB_CONSTS.idGetCachedTables, q, None)
CScopeUQueue.Unlock(q)
if not ok:
with self._csCache:
self._mapHandler.pop(index)
self._mapRowset.pop(index)
return ok
def GetRentalDateTimes(self, q, reqIndex):
rental_id = q.LoadLong()
myDates = CRentalDateTimes(rental_id)
sql = u'SELECT rental_date,return_date,last_update FROM rental where rental_id=' + str(
rental_id)
handler = CYourPeer.Slave.SeekByQueue()
if not handler:
with CScopeUQueue() as sb:
myDates.SaveTo(sb.UQueue)
sb.SaveInt(-2).SaveString(
'No connection to anyone of slave databases')
self.SendResultIndex(reqIndex, sb, idGetRentalDateTimes)
else:
def ares(h, r, err, affected, fail_ok, vtId):
with CScopeUQueue() as sb:
myDates.SaveTo(sb.UQueue)
sb.SaveInt(r).SaveString(err)
self.SendResultIndex(reqIndex, sb, idGetRentalDateTimes)
def rows(h, vData):
myDates.Rental = vData[0]
myDates.Return = vData[1]
myDates.LastUpdate = vData[2]
ok = handler.Execute(sql, ares, rows)
# should be always true if pool has local queue for request backup
assert ok
def QueryMaxMinAvgs(self, q, reqIndex):
filter = q.LoadString()
pmma = CMaxMinAvg()
sql = "SELECT MAX(amount),MIN(amount),AVG(amount) FROM payment"
if filter and len(filter) > 0:
sql += (' WHERE ' + filter)
handler = CYourPeer.Slave.SeekByQueue()
if not handler:
with CScopeUQueue() as sb:
sb.SaveInt(-2).SaveString(
'No connection to anyone of slave databases')
pmma.SaveTo(sb.UQueue)
self.SendResultIndex(reqIndex, sb, idQueryMaxMinAvgs)
else:
peer_handle = self.Handle
def ares(h, r, err, affected, fail_ok, vtId):
# send result if front peer not closed yet
if peer_handle == self.Handle:
with CScopeUQueue() as sb:
sb.SaveInt(r).SaveString(err)
pmma.SaveTo(sb.UQueue)
self.SendResultIndex(reqIndex, sb, idQueryMaxMinAvgs)
def rows(h, vData):
pmma.Max = float(vData[0])
pmma.Min = float(vData[1])
pmma.Avg = float(vData[2])
ok = handler.Execute(sql, ares, rows)
# should be always true if pool has local queue for request backup
assert ok
def EndQueueTrans(self, rollback=False, qt=None):
"""
End enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes
:param rollback: true for rollback, and false for committing
:param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_NOT_STARTED_YET, and so on
:return: true for sending the request successfully, and false for failure
"""
buffer = CScopeUQueue.Lock().SaveBool(rollback)
ok = None
if not qt:
ok = self.SendRequest(CAsyncQueue.idEndTrans, buffer, None)
else:
ok = self.SendRequest(CAsyncQueue.idEndTrans, buffer,
lambda ar: qt(ar.LoadInt()))
CScopeUQueue.Unlock(buffer)
return ok
def ares(h, r, err, affected, fail_ok, vtId):
# send result if front peer not closed yet
if peer_handle == self.Handle:
with CScopeUQueue() as sb:
sb.SaveInt(r).SaveString(err)
pmma.SaveTo(sb.UQueue)
self.SendResultIndex(reqIndex, sb, idQueryMaxMinAvgs)
def SendUserMessage(self, message, userId, hint=''):
if userId is None:
userId = u''
sb = CScopeUQueue()
q = sb.UQueue.SaveObject(message, hint)
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_)
return scl.SendUserMessage(self._m_p.Handle, userId, bytes,
q.GetSize())
def closed(h, canceled):
# retry if front peer not closed yet
if peer_handle == self.Handle:
res = cs.ErrorCode
errMsg = cs.ErrorMsg
with CScopeUQueue() as sb0:
sb0.SaveInt(res).SaveString(errMsg)
vId.SaveTo(sb0.UQueue)
self.SendResultIndex(reqIndex, sb0, idUploadEmployees)
def Publish(self, message, groups, hint=''):
if groups is None:
groups = ()
size = len(groups)
arr = (c_uint * size)(*groups)
sb = CScopeUQueue()
q = sb.UQueue.SaveObject(message, hint)
bytes = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_)
return scl.Speak(self._m_p.Handle, bytes, q.GetSize(), arr, size)
def MakeRequest(self, reqId, q):
if isinstance(q, CScopeUQueue):
q = q.UQueue
elif q is None:
delay = CScopeUQueue()
q = delay.UQueue
buffer = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_,
q._m_position_)
return scl.MakeRequest(self.Handle, reqId, buffer, q.GetSize())
def StartQueueTrans(self, key, qt):
"""
Start enqueuing messages with transaction style. Currently, total size of queued messages must be less than 4 G bytes
:param key: An ASCII string for identifying a queue at server side
:param qt: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_TRANS_ALREADY_STARTED, and so on
:return: true for sending the request successfully, and false for failure
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key)
ok = None
if not qt:
ok = self.SendRequest(CAsyncQueue.idStartTrans, buffer, None)
else:
ok = self.SendRequest(CAsyncQueue.idStartTrans, buffer,
lambda ar: qt(ar.LoadInt()))
CScopeUQueue.Unlock(buffer)
return ok
def et(h, r, err):
if r == 0:
res = r
errMsg = err
# send result if front peer not closed yet
if peer_handle == self.Handle:
with CScopeUQueue() as sb0:
sb0.SaveInt(res).SaveString(errMsg)
vId.SaveTo(sb0.UQueue)
self.SendResultIndex(reqIndex, sb0, idUploadEmployees)
def CloseQueue(self, key, c=None, permanent=False):
"""
Try to close or delete a persistent queue opened at server side
:param key: An ASCII string for identifying a queue at server side
:param c: A callback for tracking returning error code, which can be one of QUEUE_OK, QUEUE_DEQUEUING, and so on
:param permanent: true for deleting a queue file, and false for closing a queue file
:return: true for sending the request successfully, and false for failure
"""
if not key:
key = ''
buffer = CScopeUQueue.Lock().SaveAString(key).SaveBool(permanent)
ok = None
if not c:
ok = self.SendRequest(CAsyncQueue.idClose, buffer, None)
else:
ok = self.SendRequest(CAsyncQueue.idClose, buffer,
lambda ar: c(ar.LoadInt()))
CScopeUQueue.Unlock(buffer)
return ok
def _SendBlob_(self, blob):
dt = tagVariantDataType.sdVT_ARRAY | tagVariantDataType.sdVT_UI1
if isinstance(blob, str):
dt = tagVariantDataType.sdVT_BSTR
q = CScopeUQueue.Lock()
q.SaveObject(blob)
dt = q.LoadUShort()
bytes = q.LoadUInt()
byte_len = q.GetSize()
q0 = CScopeUQueue.Lock()
byte_len += 10 # sizeof(ushort) + sizeof(uint) + sizeof(uint) extra 4 bytes for string null termination
q0.SaveUInt(byte_len).SaveUShort(dt).SaveUInt(q.GetSize())
ret = self.SendResult(q0, DB_CONSTS.idStartBLOB)
CScopeUQueue.Unlock(q0)
if ret == CClientPeer.REQUEST_CANCELED or ret == CClientPeer.SOCKET_NOT_FOUND:
CScopeUQueue.Unlock(q)
return False
byte_len = q.GetSize()
while byte_len > DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE:
buffer = (c_ubyte *
DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE).from_buffer(
q._m_bytes_, q._m_position_)
ret = scl.SendReturnData(self.Handle, DB_CONSTS.idChunk,
DB_CONSTS.DEFAULT_BIG_FIELD_CHUNK_SIZE,
buffer)
if ret == CClientPeer.REQUEST_CANCELED or ret == CClientPeer.SOCKET_NOT_FOUND:
CScopeUQueue.Unlock(q)
return False
q.Discard(ret)
byte_len = q.GetSize()
ret = self.SendResult(q, DB_CONSTS.idEndBLOB)
CScopeUQueue.Unlock(q)
return ret != CClientPeer.REQUEST_CANCELED and ret != CClientPeer.SOCKET_NOT_FOUND
def SendResultIndex(self, reqIndex, q, reqId):
delay = q
if reqId == 0:
reqId = self.CurrentRequestID
if isinstance(q, CScopeUQueue):
q = q.UQueue
elif q is None:
delay = CScopeUQueue()
q = delay.UQueue
buffer = (c_ubyte * q.GetSize()).from_buffer(q._m_bytes_,
q._m_position_)
return scl.SendReturnDataIndex(self.Handle, reqIndex, reqId,
q.GetSize(), buffer)
def OnPostProcessing(self, hint, data):
cs = self.AttachedClientSocket
ctx = CContext(False, 0)
with self._csFile:
count = len(self._vContext)
if count > 0:
front = self._vContext[0]
if front.Uploading:
front._OpenLocalRead()
else:
front._OpenLocalWrite()
if front.ErrCode or front.ErrMsg:
ctx = front
elif front.Uploading:
with CScopeUQueue() as q:
q.SaveString(front.FilePath).SaveUInt(front.Flags).SaveULong(front.FileSize)
if not self.SendRequest(CStreamingFile.idUpload, q, None, front.Discarded, None):
front.ErrCode = cs.ErrorCode
front.ErrMsg = cs.ErrorMessage
ctx = front
else:
with CScopeUQueue() as q:
q.SaveString(front.LocalFile).SaveString(front.FilePath).SaveUInt(front.Flags).SaveLong(front.InitSize)
if not self.SendRequest(CStreamingFile.idDownload, q, None, front.Discarded, None):
front.ErrCode = cs.ErrorCode
front.ErrMsg = cs.ErrorMessage
ctx = front
if ctx.ErrCode or ctx.ErrMsg:
ctx._CloseFile()
if ctx.Download:
ctx.Download(self, ctx.ErrCode, ctx.ErrMsg)
with self._csFile:
self._vContext.popleft()
if len(self._vContext) > 0:
#post processing the next one
ccl.PostProcessing(self.AttachedClientSocket.Handle, 0, 0)
self.AttachedClientSocket.DoEcho #make sure WaitAll works correctly
def GetCachedTables(self):
defaultDb = self.UQueue.LoadString()
flags = self.UQueue.LoadUInt()
index = self.UQueue.LoadLong()
ms = tagManagementSystem.msUnknown
sb = CScopeUQueue()
if len(CYourPeer.FrontCachedTables) == 0 or (
flags & DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES
) != DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES:
return sb.SaveInt(ms).SaveInt(0).SaveString('')
if (flags & DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES
) == DB_CONSTS.ENABLE_TABLE_UPDATE_MESSAGES:
self.Push.Subscribe([
DB_CONSTS.CACHE_UPDATE_CHAT_GROUP_ID,
DB_CONSTS.STREAMING_SQL_CHAT_GROUP_ID
])
sql = ''
v = CYourPeer.FrontCachedTables
for s in v:
if (len(sql)) > 0:
sql += ';'
sql += ('SELECT * FROM ' + s)
# use Lock and Unlock to avoid SQL stream overlap on a session within a multi-thread environment
handler = CYourPeer.Master.Lock()
if not handler:
return sb.SaveInt(ms).SaveInt(-2).SaveString(
'No connection to a master database')
ms = handler.DBManagementSystem
try:
f = handler.execute(sql, lambda h, vData: self.SendRows(vData),
lambda h: self.SendMeta(h.ColumnInfo, index))
# put back locked handler and its socket back into pool for reuse as soon as possible
CYourPeer.Master.Unlock(handler)
try:
res = f.result(30)
return sb.SaveInt(ms).SaveInt(res.ec).SaveString(res.em)
except TimeoutError as ex:
return sb.SaveInt(ms).SaveInt(-3).SaveString(str(ex))
except (CServerError, CSocketError) as ex:
return sb.SaveInt(ms).SaveInt(ex[0]).SaveString(ex[1])
except Exception as ex:
return sb.SaveInt(ms).SaveInt(-1).SaveString(str(ex))
def GetMasterSlaveConnectedSessions(self):
mc = CYourPeer.Master.ConnectedSockets
sc = CYourPeer.Slave.ConnectedSockets
sb = CScopeUQueue()
sb.SaveUInt(mc).SaveUInt(sc)
return sb
def _Transfer(self):
index = 0
rh = None
se = None
cs = self.AttachedClientSocket
if not cs.Sendable:
return False
sent_buffer_size = cs.BytesInSendingBuffer
if sent_buffer_size > 3 * CStreamingFile.STREAM_CHUNK_SIZE:
return True
while index < len(self._vContext):
context = self._vContext[index]
if context.Sent:
index += 1
continue
if context.Uploading and context.Tried and not context.File:
if index == 0:
if context.Download:
context.Download(
self,
CStreamingFile.CANNOT_OPEN_LOCAL_FILE_FOR_READING,
context.ErrMsg)
self._vContext.popleft()
else:
index += 1
continue
if context.Uploading:
if not context.Tried:
context.Tried = True
try:
context.File = open(context.LocalFile, 'rb')
context.File.seek(0, io.SEEK_END)
context.FileSize = context.File.tell()
context.File.seek(0, io.SEEK_SET)
cq = self.AttachedClientSocket.ClientQueue
if cq.Available:
cq.StartJob()
with CScopeUQueue() as q:
q.SaveString(context.FilePath).SaveUInt(
context.Flags).SaveULong(context.FileSize)
if not self.SendRequest(CStreamingFile.idUpload, q,
rh, context.Discarded, se):
return False
except IOError as e:
context.ErrMsg = e.strerror
context.File = None
if not context.File:
if index == 0:
if context.Download:
context.Download(
self, CStreamingFile.
CANNOT_OPEN_LOCAL_FILE_FOR_READING,
context.ErrMsg)
self._vContext.popleft()
else:
index += 1
continue
else:
ret = bytearray(
context.File.read(CStreamingFile.STREAM_CHUNK_SIZE))
while len(ret) > 0:
if not self.SendRequest(CStreamingFile.idUploading,
CUQueue(ret), rh,
context.Discarded, se):
return False
sent_buffer_size = cs.BytesInSendingBuffer
if len(ret) < CStreamingFile.STREAM_CHUNK_SIZE:
break
if sent_buffer_size >= 5 * CStreamingFile.STREAM_CHUNK_SIZE:
break
ret = bytearray(
context.File.read(
CStreamingFile.STREAM_CHUNK_SIZE))
if len(ret) < CStreamingFile.STREAM_CHUNK_SIZE:
context.Sent = True
if not self.SendRequest(
CStreamingFile.idUploadCompleted, None, rh,
context.Discarded, se):
return False
cq = self.AttachedClientSocket.ClientQueue
if cq.Available:
cq.EndJob()
if sent_buffer_size >= 4 * CStreamingFile.STREAM_CHUNK_SIZE:
break
else:
with CScopeUQueue() as q:
q.SaveString(context.FilePath).SaveUInt(context.Flags)
if not self.SendRequest(CStreamingFile.idDownload, q, rh,
context.Discarded, se):
return False
context.Sent = True
context.Tried = True
sent_buffer_size = cs.BytesInSendingBuffer
if sent_buffer_size > 3 * CStreamingFile.STREAM_CHUNK_SIZE:
return True
index += 1
return True
def OnPostProcessing(self, hint, data):
d = 0
with self._csFile:
for it in self._vContext:
if d >= self._MaxDownloading:
break
if it.File:
if it.Uploading:
break
else:
d += 1
continue
if it._HasError():
continue
if it.Uploading:
it._OpenLocalRead()
if not it._HasError():
with CScopeUQueue() as q:
q.SaveString(it.FilePath).SaveUInt(
it.Flags).SaveULong(it.FileSize)
if not self.SendRequest(CStreamingFile.idUpload, q,
None, it.Discarded, it.Se):
it.ErrCode = CStreamingFile.SESSION_CLOSED_BEFORE
it.ErrMsg = CStreamingFile.SESSION_CLOSED_BEFORE_ERR_MSG
if it.Fut:
if not it.Fut.done():
ec = self.Socket.ErrCode
try:
if ec:
it.Fut.set_exception(
CSocketError(
ec, self.Socket.ErrMsg,
CStreamingFile.
idUpload, True))
else:
it.Fut.set_exception(
CSocketError(
CAsyncServiceHandler.
SESSION_CLOSED_BEFORE,
it.ErrMsg,
CStreamingFile.
idUpload, True))
except Exception as ex:
pass
it.Se = None
it.Discarded = None
continue
break
else:
it._OpenLocalWrite()
if not it._HasError():
with CScopeUQueue() as q:
q.SaveString(it.LocalFile).SaveString(
it.FilePath).SaveUInt(it.Flags).SaveLong(
it.InitSize)
if not self.SendRequest(CStreamingFile.idDownload,
q, None, it.Discarded,
it.Se):
it.ErrCode = CStreamingFile.SESSION_CLOSED_BEFORE
it.ErrMsg = CStreamingFile.SESSION_CLOSED_BEFORE_ERR_MSG
if it.Fut:
if not it.Fut.done():
ec = self.Socket.ErrCode
try:
if ec:
it.Fut.set_exception(
CSocketError(
ec, self.Socket.ErrMsg,
CStreamingFile.
idDownload, True))
else:
it.Fut.set_exception(
CSocketError(
CAsyncServiceHandler.
SESSION_CLOSED_BEFORE,
it.ErrMsg,
CStreamingFile.
idDownload, True))
except Exception as ex:
pass
it.Se = None
it.Discarded = None
continue
d += 1
while len(self._vContext):
it = self._vContext[0]
if it._HasError():
cb = it.Download
it._CloseFile()
if cb:
try:
self._csFile.release()
cb(self, it.ErrCode, it.ErrMsg)
finally:
self._csFile.acquire()
self._vContext.popleft()
else:
break
def OnResultReturned(self, reqId, mc):
if reqId == CStreamingFile.idDownload:
res = mc.LoadInt()
errMsg = mc.LoadString()
dl = None
with self._csFile:
if len(self._vContext):
context = self._vContext[0]
context.ErrCode = res
context.ErrMsg = errMsg
dl = context.Download
if dl:
dl(self, res, errMsg)
with self._csFile:
if len(self._vContext):
self._vContext.popleft()._CloseFile()
self.OnPostProcessing(0, 0)
elif reqId == CStreamingFile.idStartDownloading:
with self._csFile:
fileSize = mc.LoadULong()
localFile = mc.LoadString()
remoteFile = mc.LoadString()
flags = mc.LoadUInt()
initSize = mc.LoadLong()
if len(self._vContext) == 0:
ctx = CContext(False, flags)
ctx.LocalFile = localFile;
ctx.FilePath = remoteFile;
OpenLocalWrite(ctx);
ctx.InitSize = initSize;
self._vContext.append(ctx)
front = self._vContext[0]
front.FileSize = fileSize
initSize = 0
if front.InitSize > 0:
initSize = front.InitSize
if front.File.tell() > initSize:
front.File.flush()
front.File.seek(initSize)
front.File.truncate(initSize)
elif reqId == CStreamingFile.idDownloading:
downloaded = 0
trans = None
with self._csFile:
if len(self._vContext):
context = self._vContext[0]
trans = context.Transferring
context.File.write(mc.GetBuffer())
initSize = 0
if context.InitSize > 0:
initSize = context.InitSize
downloaded = context.File.tell() - initSize
mc.SetSize(0)
if trans:
trans(self, downloaded)
elif reqId == CStreamingFile.idUploadBackup:
pass
elif reqId == CStreamingFile.idUpload:
cs = self.AttachedClientSocket
res = mc.LoadInt()
errMsg = mc.LoadString()
ctx = CContext(False, 0)
if res or errMsg:
ctx.ErrCode = res
ctx.ErrMsg = errMsg
with self._csFile:
if len(self._vContext) > 0:
context = self._vContext[0]
if mc.GetSize() > 0:
context.InitSize = mc.LoadLong()
context.ErrCode = res
context.ErrMsg = errMsg
ctx = context
else:
with self._csFile:
if len(self._vContext) > 0:
context = self._vContext[0]
if mc.GetSize() > 0:
context.InitSize = mc.LoadLong()
context.QueueOk = cs.ClientQueue.StartJob()
queue_enabled = cs.ClientQueue.Available
if queue_enabled:
with CScopeUQueue() as q:
q.SaveString(context.FilePath).SaveUInt(context.Flags).SaveULong(context.FileSize).SaveLong(context.InitSize)
self.SendRequest(CStreamingFile.idUploadBackup, q, None, context.Discarded, None)
ret = bytearray(context.File.read(CStreamingFile.STREAM_CHUNK_SIZE))
while len(ret) == CStreamingFile.STREAM_CHUNK_SIZE:
if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), None, context.Discarded, None):
context.ErrCode = cs.ErrorCode
context.ErrMsg = cs.ErrorMessage
ctx = context
break
ret = bytearray(context.File.read(CStreamingFile.STREAM_CHUNK_SIZE))
if len(ret) < CStreamingFile.STREAM_CHUNK_SIZE:
break
if not queue_enabled:
sent_buffer_size = cs.BytesInSendingBuffer
if sent_buffer_size >= 40 * CStreamingFile.STREAM_CHUNK_SIZE:
break
if ctx.ErrCode or ctx.ErrMsg:
pass
elif len(ret) > 0:
if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), None, context.Discarded, None):
context.ErrCode = cs.ErrorCode
context.ErrMsg = cs.ErrorMessage
ctx = context
if not (ctx.ErrCode or ctx.ErrMsg) and len(ret) < CStreamingFile.STREAM_CHUNK_SIZE:
context.Sent = True
if not self.SendRequest(CStreamingFile.idUploadCompleted, None, None, context.Discarded, None):
context.ErrCode = cs.ErrorCode
context.ErrMsg = cs.ErrorMessage
ctx = context
if context.QueueOk:
cs.ClientQueue.EndJob()
if ctx.ErrCode or ctx.ErrMsg:
ctx._CloseFile()
if ctx.Download:
ctx.Download(self, ctx.ErrCode, ctx.ErrMsg)
with self._csFile:
self._vContext.popleft()
if ctx.QueueOk:
cs.ClientQueue.AbortJob()
self.OnPostProcessing(0, 0)
elif reqId == CStreamingFile.idUploading:
cs = self.AttachedClientSocket
ctx = CContext(False, 0)
trans = None
uploaded = mc.LoadLong()
with self._csFile:
if len(self._vContext) > 0:
context = self._vContext[0]
trans = context.Transferring
if uploaded < 0:
context._CloseFile()
elif not context.Sent:
ret = bytearray(context.File.read(CStreamingFile.STREAM_CHUNK_SIZE))
if len(ret) > 0:
if not self.SendRequest(CStreamingFile.idUploading, CUQueue(ret), None, context.Discarded, None):
context.ErrCode = cs.ErrorCode
context.ErrMsg = cs.ErrorMessage
ctx = context
if not (ctx.ErrCode or ctx.ErrMsg) and len(ret) < CStreamingFile.STREAM_CHUNK_SIZE:
context.Sent = True
if not self.SendRequest(CStreamingFile.idUploadCompleted, None, None, context.Discarded, None):
context.ErrCode = cs.ErrorCode
context.ErrMsg = cs.ErrorMessage
ctx = context
if ctx.ErrCode or ctx.ErrMsg:
ctx._CloseFile()
if ctx.Download:
ctx.Download(self, ctx.ErrCode, ctx.ErrMsg)
with self._csFile:
self._vContext.popleft()
self.OnPostProcessing(0, 0)
elif trans:
trans(self, uploaded)
elif reqId == CStreamingFile.idUploadCompleted:
upl = None
with self._csFile:
if len(self._vContext):
context = self._vContext[0]
if context.File:
upl = context.Download
else:
context.Sent = False
context.QueueOk = False
if upl:
upl(self, 0, '')
with self._csFile:
if len(self._vContext):
context = self._vContext[0]
if context.File:
context._CloseFile()
self._vContext.popleft()
self.OnPostProcessing(0, 0)
else:
super(CStreamingFile, self).OnResultReturned(reqId, mc)
