def update_conversations_with_new_local_key_id(old_id, new_id):
sql = 'SELECT * FROM conversations'
params = []
modifications = {}
for row in cur().execute(sql, params):
conversation_id = row[0]
new_conversation_id = conversation_id
id1, _, id2 = conversation_id.partition('&')
try:
id1 = int(id1)
id2 = int(id2)
except:
lg.exc()
continue
if id1 == old_id:
new_conversation_id = '{}&{}'.format(new_id, id2)
if id2 == old_id:
new_conversation_id = '{}&{}'.format(id1, new_id)
if conversation_id != new_conversation_id:
modifications[conversation_id] = new_conversation_id
if _Debug:
lg.args(_DebugLevel, modifications=modifications)
for old_conv_id, new_conv_id in modifications.items():
sql = 'UPDATE conversations SET conversation_id=? WHERE conversation_id=?'
params = [
new_conv_id,
old_conv_id,
]
cur().execute(sql, params)
db().commit()
if _Debug:
lg.args(_DebugLevel, sql=sql, params=params)
def _do_request_list_files(self, suppliers_list):
backup_matrix.add_list_files_query_callback(
customer_idurl=self.queue_owner_idurl,
query_path=self.queue_alias,
callback_method=self._on_list_files_response,
)
self.correctable_errors = eccmap.GetCorrectableErrors(
len(suppliers_list))
for supplier_pos, supplier_idurl in enumerate(suppliers_list):
if not supplier_idurl:
self.requested_list_files[supplier_pos] = False
continue
outpacket = p2p_service.SendListFiles(
target_supplier=supplier_idurl,
key_id=self.group_key_id,
query_items=[
self.queue_alias,
],
timeout=30,
callbacks={
commands.Fail():
lambda resp, info: self._on_list_files_failed(supplier_pos
),
None:
lambda pkt_out: self._on_list_files_failed(supplier_pos),
},
)
self.requested_list_files[
supplier_pos] = None if outpacket else False
if _Debug:
lg.args(_DebugLevel, requested=self.requested_list_files)
self.request_list_files_timer = reactor.callLater(
30, self._on_request_list_files_timeout) # @UndefinedVariable
def _on_list_files_failed(self, supplier_num):
if not self.requested_list_files:
lg.warn(
'skip ListFiles() response, requested_list_files object is empty'
)
return
if self.requested_list_files.get(supplier_num) is not None:
lg.warn(
'skip ListFiles() response, supplier record at position %d already set to %r'
% (supplier_num, self.requested_list_files.get(supplier_num)))
return
self.requested_list_files[supplier_num] = False
lst = list(self.requested_list_files.values())
if _Debug:
lg.args(_DebugLevel,
requested_list_files=lst,
supplier_num=supplier_num)
if lst.count(None) == 0:
if self.request_list_files_timer and self.request_list_files_timer.active(
):
self.request_list_files_timer.cancel()
self.request_list_files_timer = None
backup_matrix.remove_list_files_query_callback(
customer_idurl=self.queue_owner_idurl,
query_path=self.queue_alias,
callback_method=self._on_list_files_response,
)
success_list_files = lst.count(True)
if success_list_files:
self.automat('list-files-collected')
else:
self.automat('list-files-failed')
return None
def _next_source(resp, idurl, sources, pos):
global _LastTimeCached
if _Debug:
lg.args(_DebugLevel,
resp=resp,
idurl=idurl,
pos=pos,
sources=len(sources))
if pos >= len(sources):
lg.warn('[cache failed] %r and also %d other sources' %
(idurl, len(sources)))
defer_results = caching().pop(idurl, [])
for result in defer_results:
if result and not result.called:
reactor.callLater(
0, result.errback,
Exception('cache failed from multiple sources'
)) # @UndefinedVariable
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
defer_results = []
del defer_results
return None
next_idurl = sources[pos]
next_idurl = id_url.to_original(next_idurl)
if _Debug:
lg.args(_DebugLevel, next_idurl=next_idurl)
d = net_misc.getPageTwisted(url=next_idurl, timeout=timeout)
d.addCallback(_success, idurl)
d.addErrback(_next_source, idurl, sources, pos + 1)
return None
def doSendRequestService(self, *args, **kwargs):
"""
Action method.
"""
self.target_idurl.refresh()
packet_id = packetid.UniqueID()
if _Debug:
lg.args(_DebugLevel,
idurl=self.target_idurl,
service=self.target_service,
packet_id=packet_id)
service_request_payload = self.request_service_params
if callable(service_request_payload):
service_request_payload = service_request_payload(
self.target_idurl)
out_packet = p2p_service.SendRequestService(
remote_idurl=self.target_idurl,
service_name=self.target_service,
json_payload=service_request_payload,
timeout=self.request_service_timeout,
callbacks={
commands.Ack(): self._node_acked,
commands.Fail(): self._node_failed,
None: self._node_timed_out,
},
packet_id=packet_id,
)
self.requested_packet_id = out_packet.PacketID
def OnFileSendingFinished(self, pkt_out, item, status, size, error_message):
if self.shutdown:
lg.warn('skip, supplier queue is shutting down')
return
if not pkt_out.outpacket:
lg.warn('skip, outpacket is already None')
return
packetID = global_id.CanonicalID(pkt_out.outpacket.PacketID)
if status == 'finished':
if pkt_out.outpacket.Command == commands.Retrieve():
if packetID in self.fileRequestQueue:
f_down = self.fileRequestDict[packetID]
if _Debug:
lg.args(_DebugLevel, obj=f_down, status=status, packetID=packetID, event='retrieve-sent')
f_down.event('retrieve-sent', pkt_out.outpacket)
elif pkt_out.outpacket.Command == commands.Data():
if packetID in self.fileSendQueue:
f_up = self.fileSendDict[packetID]
if _Debug:
lg.args(_DebugLevel, obj=f_up, status=status, packetID=packetID, event='data-sent')
f_up.event('data-sent', pkt_out.outpacket)
else:
if pkt_out.outpacket.Command == commands.Retrieve():
if packetID in self.fileRequestQueue:
if _Debug:
lg.dbg(_DebugLevel, 'packet %r is %r during downloading from %s' % (packetID, status, self.remoteID))
f_down = self.fileRequestDict[packetID]
f_down.event('request-failed')
elif pkt_out.outpacket.Command == commands.Data():
if packetID in self.fileSendQueue:
if _Debug:
lg.dbg(_DebugLevel, 'packet %r is %r during uploading to %s' % (packetID, status, self.remoteID))
f_up = self.fileSendDict[packetID]
f_up.event('sending-failed')
def _on_rotate_broker_connected(self, response_info, broker_pos, event,
*args, **kwargs):
try:
# skip leading "accepted:" marker
cooperated_brokers = jsn.loads(
strng.to_text(response_info[0].Payload)[9:])
cooperated_brokers.pop('archive_folder_path', None)
cooperated_brokers = {
int(k): id_url.field(v)
for k, v in cooperated_brokers.items()
}
except:
lg.exc()
self.automat('broker-rotate-failed')
return
if _Debug:
lg.args(_DebugLevel,
cooperated=cooperated_brokers,
pos=broker_pos,
e=event)
if id_url.is_the_same(cooperated_brokers.get(broker_pos),
self.my_broker_idurl):
self.automat('broker-rotate-accepted',
cooperated_brokers=cooperated_brokers)
return
self.automat('broker-rotate-rejected',
cooperated_brokers=cooperated_brokers)
def OutboxStatus(self, pkt_out, status, error):
if self.shutdown:
lg.warn('supplier queue is shutting down')
return False
packetID = global_id.CanonicalID(pkt_out.outpacket.PacketID)
if status == 'finished':
if pkt_out.outpacket.Command == commands.Data():
if packetID in self.fileSendQueue:
f_up = self.fileSendDict[packetID]
if _Debug:
lg.args(_DebugLevel,
obj=f_up,
status=status,
packetID=packetID,
event='data-sent')
if error == 'unanswered':
f_up.event('timeout', pkt_out.outpacket)
else:
f_up.event('data-sent', pkt_out.outpacket)
return False
else:
if pkt_out.outpacket.Command == commands.Data():
if packetID in self.fileSendQueue:
lg.warn('packet %r is %r during uploading to %s' %
(packetID, status, self.remoteID))
f_up = self.fileSendDict[packetID]
f_up.event('sending-failed')
return False
return False
def isMoreBlocks(self, *args, **kwargs):
"""
Condition method.
"""
if _Debug:
lg.args(_DebugLevel, workBlocks=len(self.workBlocks))
return len(self.workBlocks) > 1
def DeleteBackupSendings(self, backupName):
if self.shutdown:
# if we're closing down this queue, don't do anything, but just stop all uploads
self.StopAllSindings()
return
if _Debug:
lg.args(_DebugLevel, backupName=backupName)
packetsToRemove = set()
for packetID in self.fileSendQueue:
if (backupName and packetID.count(backupName)) or not backupName:
packetsToRemove.add(packetID)
if _Debug:
lg.out(
_DebugLevel,
'io_throttle.DeleteBackupRequests %s from downloading queue'
% packetID)
for packetID in packetsToRemove:
if packetID in self.fileSendDict:
f_up = self.fileSendDict[packetID]
f_up.event('stop')
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.DeleteBackupRequests stopped %s in %s uploading queue, %d more items"
% (packetID, self.remoteID, len(self.fileSendQueue)))
if len(self.fileSendQueue) > 0:
reactor.callLater(0, self.DoSend) # @UndefinedVariable
def OnDataReceived(self, newpacket, result):
# we requested some data from a supplier, and just received it
if self.shutdown:
lg.warn('supplier queue is shutting down')
self.StopAllRequests()
return
if _Debug:
lg.args(_DebugLevel,
newpacket=newpacket,
result=result,
queue=len(self.fileRequestQueue),
remoteName=self.remoteName)
packetID = global_id.CanonicalID(newpacket.PacketID)
if (packetID
not in self.fileRequestQueue) or (packetID
not in self.fileRequestDict):
lg.err(
'unexpected %r received which is not in the downloading queue'
% newpacket)
else:
f_down = self.fileRequestDict[packetID]
if newpacket.Command == commands.Data():
wrapped_packet = signed.Unserialize(newpacket.Payload)
if not wrapped_packet or not wrapped_packet.Valid():
lg.err('incoming Data() packet is not valid')
f_down.event('fail-received', newpacket)
return
f_down.event('valid-data-received', wrapped_packet)
elif newpacket.Command == commands.Fail():
f_down.event('fail-received', newpacket)
else:
lg.err('incorrect response command: %r' % newpacket)
def finish_notification(consumer_id, queue_id, message_id, success):
if not valid_queue_id(queue_id):
raise Exception('invalid queue id')
if queue_id not in queue():
raise Exception('queue not exist')
if message_id not in queue(queue_id):
raise Exception('message not exist')
if consumer_id not in queue(queue_id)[message_id].notifications:
raise Exception('not found pending notification for given consumer')
defer_result = queue(queue_id)[message_id].notifications[consumer_id]
if defer_result is None:
raise Exception('notification already finished')
if not isinstance(defer_result, Deferred):
raise Exception('invalid notification type')
queue(queue_id)[message_id].notifications[consumer_id] = None
# queue(queue_id)[message_id].notifications.pop(consumer_id)
if success:
queue(queue_id)[message_id].success_notifications.append(consumer_id)
consumer(consumer_id).success_notifications += 1
else:
queue(queue_id)[message_id].failed_notifications.append(consumer_id)
consumer(consumer_id).failed_notifications += 1
if not defer_result.called:
lg.info('canceling non-finished notification in the queue %s' %
queue_id)
defer_result.cancel()
del defer_result
if _Debug:
lg.args(_DebugLevel,
consumer_id=consumer_id,
queue_id=queue_id,
message_id=message_id,
success=success,
notifications=len(queue(queue_id)[message_id].notifications))
return True
def start_notification(consumer_id, queue_id, message_id, callback_object):
if not valid_queue_id(queue_id):
raise Exception('invalid queue id')
if consumer_id not in consumer():
raise Exception('consumer not found')
if queue_id not in queue():
raise Exception('queue not exist')
if message_id not in queue(queue_id):
raise Exception('message not exist')
if consumer_id in queue(queue_id)[message_id].notifications:
raise Exception('notification already sent to given consumer')
queue(queue_id)[message_id].state = 'SENT'
queue(queue_id)[message_id].notifications[consumer_id] = callback_object
consumer(consumer_id).consumed_messages += 1
callback_object.addCallback(on_notification_succeed, consumer_id, queue_id,
message_id)
callback_object.addErrback(on_notification_failed, consumer_id, queue_id,
message_id)
if _Debug:
lg.args(_DebugLevel,
consumer_id=consumer_id,
queue_id=queue_id,
message_id=message_id,
notifications=len(queue(queue_id)[message_id].notifications))
return True
def close_queue(queue_id):
global _ActiveQueues
if not valid_queue_id(queue_id):
raise Exception('invalid queue id')
if queue_id not in queue():
raise Exception('queue not exist')
if _Debug:
lg.args(_DebugLevel, queue_id=queue_id)
for message_id in list(queue(queue_id).keys()):
if message_id not in queue(queue_id):
continue
for consumer_id in list(
queue(queue_id)[message_id].notifications.keys()):
msg_obj = queue(queue_id).get(message_id)
if msg_obj:
callback_object = queue(
queue_id)[message_id].notifications.get(consumer_id)
if callback_object and not callback_object.called:
lg.info(
'canceling non-finished notification in the queue %s' %
queue_id)
callback_object.cancel()
for consumer_id in consumer().keys():
if is_consumer_subscribed(consumer_id, queue_id):
unsubscribe_consumer(consumer_id, queue_id)
_ActiveQueues.pop(queue_id)
return True
def StopAllRequests(self):
for packetID in list(self.fileRequestDict.keys()):
f_down = self.fileRequestDict.get(packetID)
if f_down:
if _Debug:
lg.args(_DebugLevel, packetID=packetID, obj=f_down, event='stop')
f_down.event('stop')
def readChunk():
size = self.blockSize - self.currentBlockSize
if size < 0:
if _Debug:
lg.args(
_DebugLevel,
eccmap_nodes=self.eccmap.nodes(),
block_size=self.blockSize,
current_block_size=self.currentBlockSize,
)
raise Exception('size < 0, blockSize=%s, currentBlockSize=%s' %
(self.blockSize, self.currentBlockSize))
elif size == 0:
return succeed(b'')
if self.pipe is None:
raise Exception('backup.pipe is None')
if self.pipe.state() == 2:
if _Debug:
lg.out(
_DebugLevel,
'backup.readChunk the state is PIPE_CLOSED in %r' %
self)
return succeed(b'')
if self.pipe.state() == 0:
if _Debug:
lg.out(
_DebugLevel,
'backup.readChunk the state is PIPE_EMPTY in %r' %
self)
return succeed(b'')
if _Debug:
lg.args(_DebugLevel, size=size)
return self.pipe.read_defer(size)
def OnDataReceived(self, newpacket, result):
# we requested some data from a supplier, and just received it
if self.shutdown:
lg.warn('skip, supplier queue is shutting down')
self.StopAllRequests()
return
if _Debug:
lg.args(_DebugLevel, newpacket=newpacket, result=result, queue=len(self.fileRequestQueue), remoteName=self.remoteName)
packetID = global_id.CanonicalID(newpacket.PacketID)
if (packetID not in self.fileRequestQueue) or (packetID not in self.fileRequestDict):
latest_idurl = global_id.NormalizeGlobalID(packetID, as_field=True)['idurl'].latest
another_packetID = global_id.SubstitutePacketID(packetID, idurl=latest_idurl)
if (another_packetID in self.fileRequestQueue) and (another_packetID in self.fileRequestDict):
packetID = another_packetID
lg.warn('found incoming %r with outdated packet id, corrected: %r' % (newpacket, another_packetID, ))
if (packetID not in self.fileRequestQueue) or (packetID not in self.fileRequestDict):
lg.err('unexpected %r received which is not in the downloading queue' % newpacket)
else:
f_down = self.fileRequestDict[packetID]
if newpacket.Command == commands.Data():
wrapped_packet = signed.Unserialize(newpacket.Payload)
if not wrapped_packet or not wrapped_packet.Valid():
lg.err('incoming Data() packet is not valid')
f_down.event('fail-received', newpacket)
return
f_down.event('valid-data-received', wrapped_packet)
elif newpacket.Command == commands.Fail():
f_down.event('fail-received', newpacket)
else:
lg.err('incorrect response command: %r' % newpacket)
def process_query_item(query_path, key_alias, ownerdir):
ret = ''
ret += 'Q%s\n' % query_path
if query_path == '*':
for one_key_alias in os.listdir(ownerdir):
if not misc.ValidKeyAlias(strng.to_text(one_key_alias)):
continue
key_alias_dir = os.path.join(ownerdir, one_key_alias)
ret += TreeSummary(key_alias_dir, key_alias=one_key_alias)
if _Debug:
lg.args(_DebugLevel, query_path=query_path, result_bytes=len(ret))
return ret
# TODO: more validations to be added
clean_path = query_path.replace('.', '').replace('~', '').replace(
':', '').replace('\\', '/').lstrip('/')
path_items = clean_path.split('/')
path_items.insert(0, ownerdir)
local_path = os.path.join(*path_items)
if not os.path.exists(local_path):
lg.warn('local file or folder not exist: %r' % local_path)
return ''
if os.path.isdir(local_path):
ret += TreeSummary(local_path, key_alias=key_alias)
if _Debug:
lg.args(_DebugLevel,
query_path=query_path,
local_path=local_path,
result_bytes=len(ret))
return ret
def doCheckFinished(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.args(_DebugLevel,
backup_job=self.backup_job,
backup_max_block_num=self.backup_max_block_num,
packets_out=list(self.packets_out.values()))
if self.backup_job:
# backup is not finished yet
return
if self.backup_max_block_num not in self.packets_out:
# packets of the last block not sent yet
return
packets_in_progress = 0
for block_num in self.packets_out.keys():
packets_in_progress += list(
self.packets_out[block_num].values()).count(None)
if packets_in_progress:
# some packets are still in progress
return
for block_num in self.packets_out.keys():
block_packets_failed = list(
self.packets_out[block_num].values()).count(False)
if block_packets_failed > self.correctable_errors * 2: # because each packet also have Parity()
lg.err(
'all packets for block %d are sent, but too many errors: %d'
% (
block_num,
block_packets_failed,
))
self.automat('sending-failed')
return
self.automat('packets-delivered')
def _do_connect_with_supplier(self, supplier_idurl):
if _Debug:
lg.args(_DebugLevel,
supplier_idurl=supplier_idurl,
customer_idurl=self.customer_idurl)
sc = supplier_connector.by_idurl(supplier_idurl,
customer_idurl=self.customer_idurl)
if sc is None:
sc = supplier_connector.create(
supplier_idurl=supplier_idurl,
customer_idurl=self.customer_idurl,
needed_bytes=
0, # we only want to read the data at the moment - requesting 0 bytes from the supplier
key_id=self.key_id,
queue_subscribe=False,
)
if sc.state in [
'CONNECTED',
'QUEUE?',
]:
self.automat('supplier-connected', supplier_idurl)
else:
sc.set_callback('shared_access_coordinator',
self._on_supplier_connector_state_changed)
sc.automat('connect')
def _success(src, idurl):
global _LastTimeCached
idurl = id_url.to_original(idurl)
defer_results = caching().pop(idurl, [])
if _Debug:
lg.args(_DebugLevel,
src=type(src),
idurl=idurl,
defer_results=len(defer_results))
if UpdateAfterChecking(idurl, src):
for result in defer_results:
if result and not result.called:
reactor.callLater(0, result.callback,
src) # @UndefinedVariable
if _Debug:
lg.out(_DebugLevel, '[cached] %s' % idurl)
p2p_stats.count_identity_cache(idurl, len(src))
_LastTimeCached[idurl] = time.time()
else:
for result in defer_results:
if result and not result.called:
reactor.callLater(0, result.errback,
Exception(src)) # @UndefinedVariable
lg.warn('[cache error] %s is not valid' % idurl)
p2p_stats.count_identity_cache(idurl, 0)
_LastTimeCached.pop(idurl, None)
defer_results = []
del defer_results
return src
def ping(idurl, channel=None, ack_timeout=15, ping_retries=0, keep_alive=False):
"""
Doing handshake with remote node only if it is currently not connected.
Returns Deferred object.
"""
idurl = strng.to_bin(idurl)
if _Debug:
lg.args(_DebugLevel, idurl=idurl, keep_alive=keep_alive, channel=channel)
result = Deferred()
result.addErrback(on_ping_failed, idurl=idurl, channel=channel)
if id_url.is_empty(idurl):
result.errback(Exception('empty idurl provided'))
return result
if not id_url.is_cached(idurl):
if _Debug:
lg.dbg(_DebugLevel, 'user identity %r not cached yet, executing clean handshake' % idurl)
return handshaker.ping(
idurl=idurl,
ack_timeout=ack_timeout,
ping_retries=ping_retries,
channel=channel or 'clean_ping',
keep_alive=keep_alive,
)
idurl = id_url.field(idurl)
if not isKnown(idurl):
if not check_create(idurl, keep_alive=keep_alive):
raise Exception('can not create instance')
A(idurl, 'ping-now', result, channel=channel, ack_timeout=ack_timeout, ping_retries=ping_retries, original_idurl=idurl.to_original())
return result
def _send_new_identity(self):
"""
Send created identity to the identity servers to register it.
"""
if _Debug:
lg.out(_DebugLevel, 'id_registrator._send_new_identity')
dlist = []
payload = self.new_identity.serialize(as_text=False)
for idurl in self.new_identity.getSources(as_originals=True):
self.free_idurls.remove(strng.to_bin(idurl))
_, host, webport, filename = nameurl.UrlParse(idurl)
url = net_misc.pack_address((
host,
webport,
), proto='http')
dlist.append(
net_misc.http_post_data(
url=url,
data=payload,
connectTimeout=15,
))
if _Debug:
lg.args(_DebugLevel,
url=url,
filename=filename,
size=len(payload))
return DeferredList(dlist, fireOnOneCallback=True)
def handshake(idurl, channel=None, ack_timeout=15, ping_retries=2, keep_alive=False):
"""
Immediately doing handshake with remote node by fetching remote identity file and then
sending my own Identity() to remote peer and wait for an Ack() packet.
Returns Deferred object.
"""
idurl = strng.to_bin(idurl)
if _Debug:
lg.args(_DebugLevel, idurl=idurl, keep_alive=keep_alive, channel=channel, ack_timeout=ack_timeout, ping_retries=ping_retries)
result = Deferred()
result.addErrback(on_ping_failed, idurl=idurl, channel=channel)
if id_url.is_empty(idurl):
result.errback(Exception('empty idurl provided'))
return result
if not id_url.is_cached(idurl):
if _Debug:
lg.dbg(_DebugLevel, 'user identity %r not cached yet, executing clean handshake' % idurl)
return handshaker.ping(
idurl=idurl,
ack_timeout=ack_timeout,
ping_retries=ping_retries,
channel=channel or 'clean_handshake',
keep_alive=keep_alive,
)
idurl = id_url.field(idurl)
if not isKnown(idurl):
if not check_create(idurl, keep_alive=keep_alive):
raise Exception('can not create instance')
A(idurl, 'handshake', result, channel=channel, ack_timeout=ack_timeout, ping_retries=ping_retries, original_idurl=idurl.to_original())
return result
def verify(self, signature, message, signature_as_digits=True):
global _CryptoLog
# if _CryptoLog is None:
# _CryptoLog = os.environ.get('CRYPTO_LOG') == '1'
signature_bytes = signature
if signature_as_digits:
signature_bytes = number.long_to_bytes(signature, blocksize=4)
if not strng.is_bin(signature_bytes):
raise ValueError('signature must be byte string')
if not strng.is_bin(message):
raise ValueError('message must be byte string')
h = hashes.sha1(message, return_object=True)
result = False
try:
pkcs1_15.new(self.keyObject).verify(h, signature_bytes)
result = True
except (
ValueError,
TypeError,
):
# do not raise any exception... just return False
lg.exc('signature=%r message=%r' % (
signature,
message,
))
if _Debug:
if _CryptoLog:
lg.args(_DebugLevel, result=result, signature=signature)
return result
def query(sender_id=None, recipient_id=None, bidirectional=True, order_by_time=True, offset=None, limit=None):
"""
"""
sql = 'SELECT * FROM history WHERE payload_type=?'
params = [TYPE_MESSAGE, ]
if bidirectional and sender_id and recipient_id:
sql += ' AND sender_glob_id IN (?, ?) AND recipient_glob_id IN (?, ? )'
params += [sender_id, recipient_id, sender_id, recipient_id, ]
else:
if sender_id:
sql += ' AND sender_glob_id=?'
params += [sender_id, ]
if recipient_id:
sql += ' AND recipient_glob_id=?'
params += [recipient_id, ]
if order_by_time:
sql += ' ORDER BY payload_time DESC'
if limit is not None:
sql += ' LIMIT ?'
params += [limit, ]
if offset is not None:
sql += ' OFFSET ?'
params += [offset, ]
if _Debug:
lg.args(_DebugLevel, sql=repr(sql), params=repr(params))
for row in cur().execute(sql, params):
yield build_json_message(
data=json.loads(row[5]),
message_id=row[4],
message_time=row[3],
sender=row[0],
recipient=row[1],
)
def _do_restore_next_backup(self, backup_index):
if _Debug:
lg.args(_DebugLevel,
backup_index=backup_index,
selected_backups=len(self.selected_backups))
if backup_index >= len(self.selected_backups):
lg.info('all selected backups are processed')
self.automat('extract-all-done', self.extracted_messages)
return
backup_id = self.selected_backups[backup_index]
outfd, outfilename = tmpfile.make(
'restore',
extension='.tar.gz',
prefix=backup_id.replace('@', '_').replace('.', '_').replace(
'/', '_').replace(':', '_') + '_',
)
rw = restore_worker.RestoreWorker(backup_id,
outfd,
KeyID=self.group_key_id)
rw.MyDeferred.addCallback(self._on_restore_done, backup_id, outfd,
outfilename, backup_index)
if _Debug:
rw.MyDeferred.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='archive_reader.doStartRestoreWorker')
rw.MyDeferred.addErrback(self._on_restore_failed, backup_id, outfd,
outfilename, backup_index)
rw.automat('init')
def StopAllSindings(self):
for packetID in list(self.fileSendDict.keys()):
f_up = self.fileSendDict.get(packetID)
if f_up:
if _Debug:
lg.args(_DebugLevel, packetID=packetID, obj=f_up, event='stop')
f_up.event('stop')
def _on_read_customer_suppliers(self, dht_value):
if _Debug:
lg.args(_DebugLevel, dht_value)
if dht_value and isinstance(dht_value, dict) and len(dht_value.get('suppliers', [])) > 0:
self.automat('dht-lookup-ok', dht_value)
else:
self.automat('fail', Exception('customers suppliers not found in DHT'))
def _do_retry_one_time(self, fail_info):
to_idurl = id_url.field(fail_info['to']).to_bin()
from_idurl = id_url.field(fail_info['from']).to_bin()
_key = (fail_info['command'], fail_info['packet_id'], from_idurl,
to_idurl)
current_retries = self.packets_retries.get(_key, 0)
if _Debug:
lg.args(_DebugLevel, key=_key, retries=current_retries)
if fail_info.get('error') != 'route already closed':
if _Debug:
lg.dbg(_DebugLevel,
'failed sending routed packet : %r' % fail_info)
self._do_clean_sent_packet(fail_info)
self._do_cancel_outbox_packets(fail_info)
self.packets_retries.pop(_key, None)
return
if current_retries >= 1:
if _Debug:
lg.dbg(
_DebugLevel,
'failed sending routed packet after few attempts : %r' %
fail_info)
self.automat('retry-failed', fail_info)
self._do_clean_sent_packet(fail_info)
self._do_cancel_outbox_packets(fail_info)
self.packets_retries.pop(_key, None)
return
self.packets_retries[_key] = current_retries + 1
d = identitycache.immediatelyCaching(fail_info['to'])
d.addCallback(self._on_cache_retry_success, fail_info)
d.addErrback(self._on_cache_retry_failed, fail_info)
