def OnFoundFolderSize(pth, sz, arg):
"""
This is a callback, fired from ``lib.dirsize.ask()`` method after finish
calculating of folder size.
"""
try:
pathID, version = arg
customerGlobID, pathID = packetid.SplitPacketID(pathID)
customerIDURL = global_id.GlobalUserToIDURL(customerGlobID)
item = backup_fs.GetByID(pathID, iterID=backup_fs.fsID(customerIDURL))
if item:
item.set_size(sz)
backup_fs.Calculate()
Save()
if version:
backupID = packetid.MakeBackupID(customerGlobID, pathID, version)
job = GetRunningBackupObject(backupID)
if job:
job.totalSize = sz
if _Debug:
lg.out(_DebugLevel, 'backup_control.OnFoundFolderSize %s %d' % (backupID, sz))
except:
lg.exc()
def load_suppliers(path=None, customer_idurl=None, all_customers=False):
"""
Load suppliers list from disk.
"""
if all_customers:
for customer_id in os.listdir(settings.SuppliersDir()):
if not global_id.IsValidGlobalUser(customer_id):
lg.warn('invalid customer record %s found in %s' %
(customer_id, settings.SuppliersDir()))
continue
path = os.path.join(settings.SuppliersDir(), customer_id,
'supplierids')
lst = bpio._read_list(path)
if lst is None:
lg.warn('did not found suppliers ids at %s' % path)
continue
lst = list(map(strng.to_bin, lst))
set_suppliers(
lst, customer_idurl=global_id.GlobalUserToIDURL(customer_id))
lg.out(
4, 'contactsdb.load_suppliers %d items from %s' %
(len(lst), path))
return True
if not customer_idurl:
customer_idurl = my_id.getLocalID()
customer_idurl = strng.to_bin(customer_idurl.strip())
if path is None:
path = os.path.join(settings.SuppliersDir(),
global_id.UrlToGlobalID(customer_idurl),
'supplierids')
lst = bpio._read_list(path)
if lst is None:
lst = list()
lst = list(map(strng.to_bin, lst))
set_suppliers(lst)
lg.out(4, 'contactsdb.load_suppliers %d items from %s' % (len(lst), path))
return True
def makeFilename(customerIDURL, packetID):
"""
Must be a customer, and then we make full path filename for where this
packet is stored locally.
"""
customerGlobID, packetID = packetid.SplitPacketID(packetID)
if not packetid.Valid(packetID): # SECURITY
if packetID not in [
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
settings.BackupIndexFileName()
]:
# lg.out(1, "p2p_service.makeFilename ERROR failed packetID format: " + packetID )
return ''
if not contactsdb.is_customer(customerIDURL): # SECURITY
lg.warn("%s is not a customer" % (customerIDURL))
return ''
if customerGlobID:
customerIDURL_packet = global_id.GlobalUserToIDURL(customerGlobID)
if customerIDURL_packet != customerIDURL:
lg.warn('making filename for another customer: %s != %s' %
(customerIDURL_packet, customerIDURL))
return constructFilename(customerIDURL, packetID)
def __init__(self,
backupID,
pipe,
finishCallback=None,
blockResultCallback=None,
blockSize=None,
sourcePath=None,
keyID=None, ):
self.backupID = backupID
_parts = packetid.SplitBackupID(self.backupID)
self.customerGlobalID = _parts[0]
self.pathID = _parts[1]
self.version = _parts[2]
self.customerIDURL = global_id.GlobalUserToIDURL(self.customerGlobalID)
self.sourcePath = sourcePath
self.keyID = keyID
self.eccmap = eccmap.Current()
self.pipe = pipe
self.blockSize = blockSize
if self.blockSize is None:
self.blockSize = settings.getBackupBlockSize()
self.ask4abort = False
self.terminating = False
self.stateEOF = False
self.stateReading = False
self.closed = False
self.currentBlockData = cStringIO.StringIO()
self.currentBlockSize = 0
self.workBlocks = {}
self.blockNumber = 0
self.dataSent = 0
self.blocksSent = 0
self.totalSize = -1
self.finishCallback = finishCallback
self.blockResultCallback = blockResultCallback
automat.Automat.__init__(self, 'backup_%s' % self.version, 'AT_STARTUP', _DebugLevel)
def _on_files_received(self, newpacket, info):
import json
from logs import lg
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
try:
user_id = newpacket.PacketID.strip().split(':')[0]
if user_id == my_id.getGlobalID():
# skip my own Files() packets which comes from my suppliers
# only process list Files() from other users who granted me access
return False
key_id = user_id
if not my_keys.is_valid_key_id(key_id):
# ignore, invalid key id in packet id
return False
if not my_keys.is_key_private(key_id):
raise Exception('private key is not registered')
except Exception as exc:
lg.warn(str(exc))
p2p_service.SendFail(newpacket, str(exc))
return False
block = encrypted.Unserialize(newpacket.Payload)
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
if block.CreatorID != global_id.GlobalUserToIDURL(user_id):
lg.warn(
'invalid packet, creator ID must be present in packet ID : %s ~ %s'
% (
block.CreatorID,
user_id,
))
return False
try:
json_data = json.loads(block.Data(), encoding='utf-8')
json_data['items']
customer_idurl = block.CreatorID
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
)
except Exception as exc:
lg.exc()
p2p_service.SendFail(newpacket, str(exc))
return False
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
customer_idurl,
key_id,
))
return True
# from access import shared_access_coordinator
# this_share = shared_access_coordinator.get_active_share(key_id)
# if not this_share:
# lg.warn('share is not opened: %s' % key_id)
# p2p_service.SendFail(newpacket, 'share is not opened')
# return False
# this_share.automat('customer-list-files-received', (newpacket, info, block, ))
return True
def _itemRestored(backupID, result):
customerGlobalID, remotePath, _ = packetid.SplitBackupID(backupID)
backup_fs.ScanID(
remotePath,
customer_idurl=global_id.GlobalUserToIDURL(customerGlobalID))
backup_fs.Calculate()
def request(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from crypt import my_keys
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from userid import global_id
customer_idurl = newpacket.OwnerID
customer_id = global_id.UrlToGlobalID(customer_idurl)
bytes_for_customer = 0
try:
bytes_for_customer = int(json_payload['needed_bytes'])
except:
lg.warn("wrong payload" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong payload')
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
data_owner_idurl = None
target_customer_idurl = None
family_position = json_payload.get('position')
ecc_map = json_payload.get('ecc_map')
family_snapshot = json_payload.get('family_snapshot')
key_id = json_payload.get('key_id')
target_customer_id = json_payload.get('customer_id')
if key_id:
# this is a request from external user to access shared data stored by one of my customers
# this is "second" customer requesting data from "first" customer
if not key_id or not my_keys.is_valid_key_id(key_id):
lg.warn('missed or invalid key id')
return p2p_service.SendFail(newpacket, 'invalid key id')
target_customer_idurl = global_id.GlobalUserToIDURL(
target_customer_id)
if not contactsdb.is_customer(target_customer_idurl):
lg.warn("target user %s is not a customer" %
target_customer_id)
return p2p_service.SendFail(newpacket, 'not a customer')
if target_customer_idurl == customer_idurl:
lg.warn('customer %s requesting shared access to own files' %
customer_idurl)
return p2p_service.SendFail(newpacket, 'invalid case')
if not my_keys.is_key_registered(key_id):
lg.warn('key not registered: %s' % key_id)
p2p_service.SendFail(newpacket, 'key not registered')
return False
data_owner_idurl = my_keys.split_key_id(key_id)[1]
if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl:
# pretty complex scenario:
# external customer requesting access to data which belongs not to that customer
# this is "third" customer accessing data belongs to "second" customer
# TODO: for now just stop it
lg.warn(
'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s'
% (
key_id,
customer_idurl,
target_customer_idurl,
))
p2p_service.SendFail(newpacket, 'under construction')
return False
# do not create connection with that customer, only accept the request
lg.info(
'external customer %s requested access to shared data at %s' %
(
customer_id,
key_id,
))
return p2p_service.SendAck(newpacket, 'accepted')
# key_id is not present in the request:
# this is a request to connect new customer (or reconnect existing one) to that supplier
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong payload : %s" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.out(6, 'service_supplier.request created a new space file')
space_dict = accounting.read_customers_quotas()
try:
free_bytes = int(space_dict[b'free'])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl not in current_customers
and customer_idurl in list(space_dict.keys())):
lg.warn("broken space file")
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl in current_customers
and customer_idurl not in list(space_dict.keys())):
lg.warn("broken customers file")
return p2p_service.SendFail(newpacket, 'broken customers file')
if customer_idurl in current_customers:
free_bytes += int(space_dict.get(customer_idurl, 0))
space_dict[b'free'] = free_bytes
current_customers.remove(customer_idurl)
space_dict.pop(customer_idurl)
new_customer = False
else:
new_customer = True
lg.out(
8, ' new_customer=%s current_allocated_bytes=%s' % (
new_customer,
space_dict.get(customer_idurl),
))
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.update_customers(current_customers)
contactsdb.remove_customer_meta_info(customer_idurl)
contactsdb.save_customers()
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('new-customer-denied', dict(idurl=customer_idurl))
else:
lg.out(
8,
" OLD CUSTOMER: DENIED !!!!!!!!!!! not enough space available"
)
events.send('existing-customer-denied',
dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'deny')
space_dict[b'free'] = free_bytes - bytes_for_customer
current_customers.append(customer_idurl)
space_dict[customer_idurl] = bytes_for_customer
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
contactsdb.add_customer_meta_info(
customer_idurl, {
'ecc_map': ecc_map,
'position': family_position,
'family_snapshot': family_snapshot,
})
accounting.write_customers_quotas(space_dict)
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
try:
if not my_keys.is_key_registered(customer_public_key_id):
key_id, key_object = my_keys.read_key_info(
customer_public_key)
if not my_keys.register_key(key_id, key_object):
lg.err('failed to register customer public key')
except:
lg.exc()
else:
lg.warn('customer public key was not provided in the request')
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.out(
8,
" NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot,
)
events.send(
'new-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
family_snapshot=family_snapshot,
key_id=customer_public_key_id,
))
else:
lg.out(
8,
" OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
lg.out(
8,
" family_snapshot=%r !!!!!!!!!!!!!!" % family_snapshot)
events.send(
'existing-customer-accepted',
dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
key_id=customer_public_key_id,
family_snapshot=family_snapshot,
))
return p2p_service.SendAck(newpacket, 'accepted')
def do_send_message(json_data,
recipient_global_id,
packet_id,
timeout,
result_defer=None):
global _OutgoingMessageCallbacks
remote_idurl = global_id.GlobalUserToIDURL(recipient_global_id)
if not remote_idurl:
raise Exception('invalid recipient')
remote_identity = identitycache.FromCache(remote_idurl)
if not remote_identity:
raise Exception('remote identity object not exist in cache')
message_body = serialization.DictToBytes(
json_data,
pack_types=True,
encoding='utf-8',
)
lg.out(
4, "message.do_send_message to %s with %d bytes message" %
(recipient_global_id, len(message_body)))
try:
private_message_object = PrivateMessage(
recipient_global_id=recipient_global_id)
private_message_object.encrypt(message_body)
except Exception as exc:
lg.exc()
raise Exception('message encryption failed')
Payload = private_message_object.serialize()
lg.out(
4, " payload is %d bytes, remote idurl is %s" %
(len(Payload), remote_idurl))
outpacket = signed.Packet(
commands.Message(),
my_id.getLocalID(),
my_id.getLocalID(),
packet_id,
Payload,
remote_idurl,
)
result = gateway.outbox(
outpacket,
wide=True,
response_timeout=timeout,
callbacks={
commands.Ack():
lambda response, info: on_message_delivered(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
response,
info,
result_defer,
),
commands.Fail():
lambda response, info: on_message_failed(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
response,
info,
result_defer,
),
None:
lambda pkt_out: on_message_failed(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
result_defer,
), # timeout
})
try:
for cp in _OutgoingMessageCallbacks:
cp(json_data, private_message_object, remote_identity, outpacket,
result)
except:
lg.exc()
raise Exception('failed sending message')
return result
def send_message(json_data,
recipient_global_id,
packet_id=None,
message_ack_timeout=None,
ping_timeout=15,
ping_retries=0,
skip_handshake=False,
fire_callbacks=True,
require_handshake=False):
"""
Send command.Message() packet to remote peer. Returns Deferred object.
"""
global _LastUserPingTime
global _PingTrustIntervalSeconds
if not packet_id:
packet_id = packetid.UniqueID()
if _Debug:
lg.out(
_DebugLevel,
"message.send_message to %s with PacketID=%s timeout=%d ack_timeout=%r retries=%d"
% (
recipient_global_id,
packet_id,
ping_timeout,
message_ack_timeout,
ping_retries,
))
remote_idurl = global_id.GlobalUserToIDURL(recipient_global_id,
as_field=False)
if not remote_idurl:
lg.warn('invalid recipient')
return fail(Exception('invalid recipient'))
ret = Deferred()
if remote_idurl not in _LastUserPingTime:
is_ping_expired = True
else:
is_ping_expired = time.time(
) - _LastUserPingTime[remote_idurl] > _PingTrustIntervalSeconds
remote_identity = identitycache.FromCache(remote_idurl)
is_online = online_status.isOnline(remote_idurl)
if _Debug:
lg.out(
_DebugLevel,
" is_ping_expired=%r remote_identity=%r is_online=%r skip_handshake=%r"
% (
is_ping_expired,
bool(remote_identity),
is_online,
skip_handshake,
))
if require_handshake or remote_identity is None or (
(is_ping_expired or not is_online) and not skip_handshake):
d = online_status.handshake(
idurl=remote_idurl,
ack_timeout=ping_timeout,
ping_retries=ping_retries,
channel='send_message',
keep_alive=True,
)
d.addCallback(lambda ok: on_ping_success(ok, remote_idurl))
d.addCallback(lambda _: do_send_message(
json_data=json_data,
recipient_global_id=recipient_global_id,
packet_id=packet_id,
message_ack_timeout=message_ack_timeout,
result_defer=ret,
fire_callbacks=fire_callbacks,
))
d.addErrback(lambda err: on_message_failed(remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
result_defer=ret,
error=err))
return ret
try:
do_send_message(
json_data=json_data,
recipient_global_id=recipient_global_id,
packet_id=packet_id,
message_ack_timeout=message_ack_timeout,
result_defer=ret,
fire_callbacks=fire_callbacks,
)
except Exception as exc:
lg.warn(str(exc))
on_message_failed(remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
error=exc)
ret.errback(exc)
return ret
def do_send_message(json_data,
recipient_global_id,
packet_id,
message_ack_timeout,
result_defer=None,
fire_callbacks=True):
global _OutgoingMessageCallbacks
remote_idurl = global_id.GlobalUserToIDURL(recipient_global_id,
as_field=False)
if not remote_idurl:
raise Exception('invalid recipient')
remote_identity = identitycache.FromCache(remote_idurl)
if not remote_identity:
raise Exception('remote identity object not exist in cache')
message_body = serialization.DictToBytes(
json_data,
pack_types=True,
encoding='utf-8',
)
if _Debug:
lg.out(
_DebugLevel,
"message.do_send_message to %s with %d bytes message ack_timeout=%s"
% (recipient_global_id, len(message_body), message_ack_timeout))
try:
private_message_object = PrivateMessage(recipient=recipient_global_id)
private_message_object.encrypt(message_body)
except:
lg.exc()
raise Exception('message encryption failed')
payload = private_message_object.serialize()
if _Debug:
lg.out(
_DebugLevel, " payload is %d bytes, remote idurl is %s" %
(len(payload), remote_idurl))
callbacks = {}
if message_ack_timeout:
callbacks = {
commands.Ack():
lambda response, info: on_message_delivered(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
response,
info,
result_defer,
),
commands.Fail():
lambda response, info: on_message_failed(remote_idurl,
json_data,
recipient_global_id,
packet_id,
response,
info,
result_defer=result_defer,
error='fail received'),
None:
lambda pkt_out: on_message_failed(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
result_defer=result_defer,
error='timeout',
),
'failed':
lambda pkt_out, errmsg: on_message_failed(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
result_defer=result_defer,
error=errmsg,
),
}
result, outpacket = p2p_service.SendMessage(
remote_idurl=remote_idurl,
packet_id=packet_id,
payload=payload,
callbacks=callbacks,
response_timeout=message_ack_timeout,
)
if fire_callbacks:
try:
for cp in _OutgoingMessageCallbacks:
cp(json_data, private_message_object, remote_identity,
outpacket, result)
except:
lg.exc()
raise Exception('failed sending message')
return result
def SpaceTime():
"""
Test all packets for each customer.
Check if he use more space than we gave him and if packets is too
old.
"""
printlog('SpaceTime ' + str(time.strftime("%a, %d %b %Y %H:%M:%S +0000")))
space = accounting.read_customers_quotas()
if space is None:
printlog(
'SpaceTime ERROR customers quotas file can not be read or it is empty, skip'
)
return
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
printlog('SpaceTime ERROR customers folder not exist')
return
remove_list = {}
used_space = accounting.read_customers_usage()
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl, None)
if curspace is None:
remove_list[onecustdir] = 'not found in space file'
continue
try:
maxspaceV = int(curspace)
except:
remove_list[onecustdir] = 'wrong space value'
continue
timedict = {}
sizedict = {}
def cb(path, subpath, name):
if not os.path.isfile(path):
return True
stats = os.stat(path)
timedict[path] = stats.st_ctime
sizedict[path] = stats.st_size
for key_alias in os.listdir(onecustdir):
if not misc.ValidKeyAlias(key_alias):
remove_list[onecustdir] = 'invalid key alias'
continue
okekeydir = os.path.join(onecustdir, key_alias)
bpio.traverse_dir_recursive(cb, okekeydir)
currentV = 0
for path in sorted(list(timedict.keys()),
key=lambda x: timedict[x],
reverse=True):
filesize = sizedict.get(path, 0)
currentV += filesize
if currentV < maxspaceV:
continue
try:
os.remove(path)
printlog('SpaceTime ' + path +
' file removed (cur:%s, max: %s)' %
(str(currentV), str(maxspaceV)))
except:
printlog('SpaceTime ERROR removing ' + path)
# time.sleep(0.01)
used_space[idurl] = str(currentV)
timedict.clear()
sizedict.clear()
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
printlog('SpaceTime ' + path + ' dir removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
printlog('SpaceTime ' + path + ' file removed (%s)' %
(remove_list[path]))
except:
printlog('SpaceTime ERROR removing ' + path)
del remove_list
accounting.update_customers_usage(used_space)
def UpdateCustomers():
"""
Test packets after list of customers was changed.
"""
space, _ = accounting.read_customers_quotas()
if space is None:
if _Debug:
printlog('UpdateCustomers ERROR space file can not be read')
return False
customers_dir = settings.getCustomersFilesDir()
if not os.path.exists(customers_dir):
if _Debug:
printlog('UpdateCustomers ERROR customers folder not exist')
return False
remove_list = {}
for customer_filename in os.listdir(customers_dir):
onecustdir = os.path.join(customers_dir, customer_filename)
if not os.path.isdir(onecustdir):
remove_list[onecustdir] = 'is not a folder'
continue
# idurl = nameurl.FilenameUrl(customer_filename)
idurl = global_id.GlobalUserToIDURL(customer_filename)
if idurl is None:
remove_list[onecustdir] = 'wrong folder name'
continue
curspace = space.get(idurl.to_bin(), None)
if curspace is None:
remove_list[onecustdir] = 'is not a customer'
continue
for path in remove_list.keys():
if not os.path.exists(path):
continue
if os.path.isdir(path):
try:
bpio._dir_remove(path)
if _Debug:
printlog('UpdateCustomers %r folder removed (%s)' % (
path,
remove_list[path],
))
except:
if _Debug:
printlog('UpdateCustomers ERROR removing %r' % path)
continue
try:
if not os.access(path, os.W_OK):
os.chmod(path, 0o600)
except:
pass
try:
os.remove(path)
if _Debug:
printlog('UpdateCustomers %r file removed (%s)' % (
path,
remove_list[path],
))
except:
if _Debug:
printlog('UpdateCustomers ERROR removing %r' % path)
if _Debug:
printlog('UpdateCustomers %r' %
time.strftime("%a, %d %b %Y %H:%M:%S +0000"))
return True
def load_suppliers(path=None, customer_idurl=None, all_customers=False):
"""
Load suppliers list from disk.
"""
if all_customers:
list_local_customers = list(os.listdir(settings.SuppliersDir()))
if _Debug:
lg.out(
_DebugLevel, 'contactsdb.load_suppliers %d known customers' %
len(list_local_customers))
for customer_id in list_local_customers:
if not global_id.IsValidGlobalUser(customer_id):
lg.warn('invalid customer record %s found in %s' %
(customer_id, settings.SuppliersDir()))
continue
path = os.path.join(settings.SuppliersDir(), customer_id,
'supplierids')
lst = bpio._read_list(path)
if lst is None:
lg.warn('did not found suppliers ids at %s' % path)
continue
one_customer_idurl = global_id.GlobalUserToIDURL(customer_id)
if not id_url.is_cached(one_customer_idurl):
lg.warn('customer identity %r not cached yet' %
one_customer_idurl)
continue
if not one_customer_idurl.is_latest():
latest_customer_path = os.path.join(settings.SuppliersDir(),
one_customer_idurl.to_id())
old_customer_path = os.path.join(settings.SuppliersDir(),
customer_id)
if not os.path.exists(latest_customer_path):
os.rename(old_customer_path, latest_customer_path)
lg.info(
'detected and processed idurl rotate when loading suppliers for customer : %r -> %r'
% (customer_id, one_customer_idurl.to_id()))
else:
bpio._dir_remove(old_customer_path)
lg.warn('found old customer dir %r and removed' %
old_customer_path)
continue
lst = list(map(lambda i: i if id_url.is_cached(i) else b'', lst))
set_suppliers(lst, customer_idurl=one_customer_idurl)
if _Debug:
lg.out(
_DebugLevel,
' loaded %d known suppliers for customer %r' %
(len(lst), one_customer_idurl))
return True
if not customer_idurl:
customer_idurl = my_id.getLocalID()
customer_idurl = id_url.field(customer_idurl)
if path is None:
path = os.path.join(settings.SuppliersDir(),
global_id.UrlToGlobalID(customer_idurl),
'supplierids')
lst = bpio._read_list(path)
if lst is None:
lst = list()
lst = list(map(lambda i: i if id_url.is_cached(i) else b'', lst))
set_suppliers(lst, customer_idurl=customer_idurl)
if _Debug:
lg.out(_DebugLevel,
'contactsdb.load_suppliers %d items from %s' % (len(lst), path))
return True
