def test_file_create(self):
path = 'cat.jpeg'
parent_path = os.path.dirname(path)
customer_idurl = 'http://127.0.0.1:8084/alice.xml'
key_alias = 'master'
id_iter_iterID = backup_fs.GetIteratorsByPath(
parent_path,
iter=backup_fs.fs(customer_idurl, key_alias),
iterID=backup_fs.fsID(customer_idurl, key_alias),
)
newPathID, itemInfo, _, _ = backup_fs.PutItem(
name=os.path.basename(path),
parent_path_id=id_iter_iterID[0],
as_folder=False,
iter=id_iter_iterID[1],
iterID=id_iter_iterID[2],
key_id=None,
)
self.assertEqual(newPathID, itemInfo.path_id)
self.assertEqual(itemInfo.name(), 'cat.jpeg')
self.assertEqual(itemInfo.key_alias(), 'master')
self.assertEqual(backup_fs.fs(customer_idurl, key_alias), {
'cat.jpeg': int(newPathID),
})
self.assertEqual(
backup_fs.fsID(customer_idurl, key_alias)[int(newPathID)].name(),
'cat.jpeg')
def ReadIndex(text_data, encoding='utf-8'):
"""
Read index data base, ``input`` is a ``StringIO.StringIO`` object which
keeps the data.
This is a simple text format, see ``p2p.backup_fs.Serialize()``
method. The first line keeps revision number.
"""
global _LoadingFlag
if _LoadingFlag:
return False
_LoadingFlag = True
backup_fs.Clear()
count = 0
try:
json_data = jsn.loads(
text_data,
encoding=encoding,
)
except:
lg.exc()
json_data = text_data
if _Debug:
lg.args(_DebugLevel, json_data=json_data)
for customer_id in json_data.keys():
if customer_id == 'items':
try:
count = backup_fs.Unserialize(json_data,
from_json=True,
decoding=encoding)
except:
lg.exc()
return False
else:
customer_idurl = global_id.GlobalUserToIDURL(customer_id)
if not id_url.is_cached(customer_idurl):
lg.warn(
'identity %r is not yet cached, skip reading related catalog items'
% customer_idurl)
identitycache.immediatelyCaching(customer_idurl,
try_other_sources=False,
ignore_errors=True)
continue
try:
count = backup_fs.Unserialize(
json_data[customer_id],
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
decoding=encoding,
)
except:
lg.exc()
return False
if _Debug:
lg.out(_DebugLevel, 'backup_control.ReadIndex %d items loaded' % count)
# local_site.update_backup_fs(backup_fs.ListAllBackupIDsSQL())
# commit(new_revision)
_LoadingFlag = False
return True
def test_file_create_with_key_alias(self):
key_id = '[email protected]_8084'
key_alias = key_id.split('$')[0]
customer_idurl = 'http://127.0.0.1:8084/alice.xml'
path = 'animals/dog.png'
parent_path = os.path.dirname(path)
parentPathID, _, _, _ = backup_fs.AddDir(
parent_path,
read_stats=False,
iter=backup_fs.fs(customer_idurl, key_alias),
iterID=backup_fs.fsID(customer_idurl, key_alias),
key_id=key_id,
)
id_iter_iterID = backup_fs.GetIteratorsByPath(
parent_path,
iter=backup_fs.fs(customer_idurl, key_alias),
iterID=backup_fs.fsID(customer_idurl, key_alias),
)
newPathID, itemInfo, _, _ = backup_fs.PutItem(
name=os.path.basename(path),
parent_path_id=parentPathID,
as_folder=False,
iter=id_iter_iterID[1],
iterID=id_iter_iterID[2],
key_id=key_id,
)
self.assertEqual(newPathID, itemInfo.path_id)
self.assertEqual(itemInfo.name(), 'dog.png')
self.assertEqual(itemInfo.key_alias(), 'share_abcd')
p1, p2 = newPathID.split('/')
self.assertEqual(backup_fs.fs(customer_idurl, key_alias),
{'animals': {
0: int(p1),
'dog.png': int(p2)
}})
self.assertEqual(
backup_fs.fsID(customer_idurl, key_alias)[int(p1)]['i'].name(),
'animals')
self.assertEqual(
backup_fs.fsID(customer_idurl, key_alias)[int(p1)]['i'].key_id,
key_id)
self.assertEqual(
backup_fs.fsID(customer_idurl, key_alias)[int(p1)][int(p2)].name(),
'dog.png')
self.assertEqual(
backup_fs.fsID(customer_idurl, key_alias)[int(p1)][int(p2)].key_id,
key_id)
def ReadLocalFiles():
"""
This method scans local backups and build the whole "local" matrix.
"""
global _LocalFilesNotifyCallback
local_files().clear()
local_max_block_numbers().clear()
local_backup_size().clear()
_counter = [0]
def visit(realpath, subpath, name):
# subpath is something like 0/0/1/0/F20131120053803PM/0-1-Data
if not os.path.isfile(realpath):
return True
if realpath.startswith("newblock-"):
return False
if subpath in [
settings.BackupIndexFileName(),
settings.BackupInfoFileName(),
settings.BackupInfoFileNameOld(),
settings.BackupInfoEncryptedFileName(),
]:
return False
try:
version = subpath.split("/")[-2]
except:
return False
if not packetid.IsCanonicalVersion(version):
return True
LocalFileReport(packetID=subpath)
_counter[0] += 1
return False
bpio.traverse_dir_recursive(visit, settings.getLocalBackupsDir())
lg.out(8, "backup_matrix.ReadLocalFiles %d files indexed" % _counter[0])
if lg.is_debug(8):
try:
if sys.version_info >= (2, 6):
# localSZ = sys.getsizeof(local_files())
# remoteSZ = sys.getsizeof(remote_files())
import lib.getsizeof
localSZ = lib.getsizeof.total_size(local_files())
remoteSZ = lib.getsizeof.total_size(remote_files())
indexByName = lib.getsizeof.total_size(backup_fs.fs())
indexByID = lib.getsizeof.total_size(backup_fs.fsID())
lg.out(10, " all local info uses %d bytes in the memory" % localSZ)
lg.out(10, " all remote info uses %d bytes in the memory" % remoteSZ)
lg.out(10, " index by name takes %d bytes in the memory" % indexByName)
lg.out(10, " index by ID takes %d bytes in the memory" % indexByID)
except:
lg.exc()
if _LocalFilesNotifyCallback is not None:
_LocalFilesNotifyCallback()
def test():
"""
For tests.
"""
# backup_fs.Calculate()
# print backup_fs.counter()
# print backup_fs.numberfiles()
# print backup_fs.sizefiles()
# print backup_fs.sizebackups()
pprint.pprint(backup_fs.fsID())
pprint.pprint(backup_fs.fs())
print(backup_fs.GetByID('0'))
def ReadIndex(text_data, encoding='utf-8'):
"""
Read index data base, ``input`` is a ``StringIO.StringIO`` object which
keeps the data.
This is a simple text format, see ``p2p.backup_fs.Serialize()``
method. The first line keeps revision number.
"""
global _LoadingFlag
if _LoadingFlag:
return False
_LoadingFlag = True
backup_fs.Clear()
count = 0
try:
json_data = jsn.loads(
text_data,
encoding=encoding,
)
except:
lg.exc()
json_data = text_data
if _Debug:
import pprint
lg.out(_DebugLevel, pprint.pformat(json_data))
for customer_id in json_data.keys():
if customer_id == 'items':
try:
count = backup_fs.Unserialize(json_data,
from_json=True,
decoding=encoding)
except:
lg.exc()
return False
else:
customer_idurl = global_id.GlobalUserToIDURL(customer_id)
try:
count = backup_fs.Unserialize(
json_data[customer_id],
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
decoding=encoding,
)
except:
lg.exc()
return False
if _Debug:
lg.out(_DebugLevel, 'backup_control.ReadIndex %d items loaded' % count)
# local_site.update_backup_fs(backup_fs.ListAllBackupIDsSQL())
# commit(new_revision)
_LoadingFlag = False
return True
def test():
"""
For tests.
"""
# backup_fs.Calculate()
# print backup_fs.counter()
# print backup_fs.numberfiles()
# print backup_fs.sizefiles()
# print backup_fs.sizebackups()
import pprint
pprint.pprint(backup_fs.fsID())
pprint.pprint(backup_fs.fs())
print backup_fs.GetByID('0')
def isAnyFilesShared(self, *args, **kwargs):
"""
Condition method.
"""
if id_url.is_the_same(self.customer_idurl, my_id.getIDURL()):
if _Debug:
lg.args(_DebugLevel,
c=self.customer_idurl,
key_alias=self.key_alias,
ret='my own share')
# no need to wait for incoming list files
# TODO: check what is going to happen when new file is uploaded to the share by another user (not creator of the share)
return True
ret = backup_fs.HasChilds('',
iter=backup_fs.fs(self.customer_idurl,
self.key_alias))
if _Debug:
lg.args(_DebugLevel,
customer=self.customer_idurl,
key_alias=self.key_alias,
ret=ret)
return ret
def isAnyFilesShared(self, *args, **kwargs):
"""
Condition method.
"""
return backup_fs.HasChilds('', iter=backup_fs.fs(self.customer_idurl))
def _on_files_received(self, newpacket, info):
import json
from logs import lg
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
try:
user_id = newpacket.PacketID.strip().split(':')[0]
if user_id == my_id.getGlobalID():
# skip my own Files() packets which comes from my suppliers
# only process list Files() from other users who granted me access
return False
key_id = user_id
if not my_keys.is_valid_key_id(key_id):
# ignore, invalid key id in packet id
return False
if not my_keys.is_key_private(key_id):
raise Exception('private key is not registered')
except Exception as exc:
lg.warn(str(exc))
p2p_service.SendFail(newpacket, str(exc))
return False
block = encrypted.Unserialize(newpacket.Payload)
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
if block.CreatorID != global_id.GlobalUserToIDURL(user_id):
lg.warn(
'invalid packet, creator ID must be present in packet ID : %s ~ %s'
% (
block.CreatorID,
user_id,
))
return False
try:
json_data = json.loads(block.Data(), encoding='utf-8')
json_data['items']
customer_idurl = block.CreatorID
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(customer_idurl),
iterID=backup_fs.fsID(customer_idurl),
from_json=True,
)
except Exception as exc:
lg.exc()
p2p_service.SendFail(newpacket, str(exc))
return False
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
customer_idurl,
key_id,
))
return True
# from access import shared_access_coordinator
# this_share = shared_access_coordinator.get_active_share(key_id)
# if not this_share:
# lg.warn('share is not opened: %s' % key_id)
# p2p_service.SendFail(newpacket, 'share is not opened')
# return False
# this_share.automat('customer-list-files-received', (newpacket, info, block, ))
return True
def _on_files_received(self, newpacket, info):
from logs import lg
from lib import serialization
from main import settings
from main import events
from p2p import p2p_service
from storage import backup_fs
from storage import backup_control
from crypt import encrypted
from crypt import my_keys
from userid import my_id
from userid import global_id
from storage import backup_matrix
from supplier import list_files
from contacts import contactsdb
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getGlobalID():
lg.warn('skip %s packet which seems to came from my own supplier' %
newpacket)
# only process list Files() from other users who granted me access
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files,
keys_to_text=True)
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
events.send(
'shared-list-files-received',
dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count,
trusted_customer_idurl,
incoming_key_id,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(
raw_files, settings.ListFilesFormat())
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
real_supplier_pos = backup_matrix.DetectSupplierPosition(
supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(
external_supplier_idurl, trusted_customer_idurl)
if real_supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != real_supplier_pos:
lg.warn(
'external supplier %s position is not matching to list files, rewriting for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
contactsdb.erase_supplier(
idurl=external_supplier_idurl,
customer_idurl=trusted_customer_idurl,
)
contactsdb.add_supplier(
idurl=external_supplier_idurl,
position=real_supplier_pos,
customer_idurl=trusted_customer_idurl,
)
contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn(
'not possible to detect external supplier position for customer %s'
% trusted_customer_idurl)
# finally send ack packet back
p2p_service.SendAck(newpacket)
lg.info(
'received list of packets from external supplier %s for customer %s'
% (external_supplier_idurl, trusted_customer_idurl))
return True
def on_files_received(newpacket, info):
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
trusted_customer_idurl = list_files_global_id['idurl']
incoming_key_id = list_files_global_id['key_id']
if trusted_customer_idurl == my_id.getLocalID():
if _Debug:
lg.dbg(_DebugLevel, 'ignore %s packet which seems to came from my own supplier' % newpacket)
# only process list Files() from other customers who granted me access to their files
return False
if not my_keys.is_valid_key_id(incoming_key_id):
lg.warn('ignore, invalid key id in packet %s' % newpacket)
return False
if not my_keys.is_key_private(incoming_key_id):
lg.warn('private key is not registered : %s' % incoming_key_id)
p2p_service.SendFail(newpacket, 'private key is not registered')
return False
try:
block = encrypted.Unserialize(
newpacket.Payload,
decrypt_key=incoming_key_id,
)
except:
lg.exc(newpacket.Payload)
return False
if block is None:
lg.warn('failed reading data from %s' % newpacket.RemoteID)
return False
# if block.CreatorID != trusted_customer_idurl:
# lg.warn('invalid packet, creator ID must be present in packet ID : %s ~ %s' % (
# block.CreatorID, list_files_global_id['idurl'], ))
# return False
try:
raw_files = block.Data()
except:
lg.exc()
return False
if block.CreatorID == trusted_customer_idurl:
# this is a trusted guy sending some shared files to me
try:
json_data = serialization.BytesToDict(raw_files, keys_to_text=True, encoding='utf-8')
json_data['items']
except:
lg.exc()
return False
count = backup_fs.Unserialize(
raw_data=json_data,
iter=backup_fs.fs(trusted_customer_idurl),
iterID=backup_fs.fsID(trusted_customer_idurl),
from_json=True,
)
p2p_service.SendAck(newpacket)
if count == 0:
lg.warn('no files were imported during file sharing')
else:
backup_control.Save()
lg.info('imported %d shared files from %s, key_id=%s' % (
count, trusted_customer_idurl, incoming_key_id, ))
events.send('shared-list-files-received', dict(
customer_idurl=trusted_customer_idurl,
new_items=count,
))
return True
# otherwise this must be an external supplier sending us a files he stores for trusted customer
external_supplier_idurl = block.CreatorID
try:
supplier_raw_list_files = list_files.UnpackListFiles(raw_files, settings.ListFilesFormat())
except:
lg.exc()
return False
# need to detect supplier position from the list of packets
# and place that supplier on the correct position in contactsdb
supplier_pos = backup_matrix.DetectSupplierPosition(supplier_raw_list_files)
known_supplier_pos = contactsdb.supplier_position(external_supplier_idurl, trusted_customer_idurl)
if _Debug:
lg.args(_DebugLevel, supplier_pos=supplier_pos, known_supplier_pos=known_supplier_pos, external_supplier=external_supplier_idurl,
trusted_customer=trusted_customer_idurl, key_id=incoming_key_id)
if supplier_pos >= 0:
if known_supplier_pos >= 0 and known_supplier_pos != supplier_pos:
lg.err('known external supplier %r position %d is not matching to received list files position %d for customer %s' % (
external_supplier_idurl, known_supplier_pos, supplier_pos, trusted_customer_idurl))
# TODO: we should remove that bellow because we do not need it
# service_customer_family() should take care of suppliers list for trusted customer
# so we need to just read that list from DHT
# contactsdb.erase_supplier(
# idurl=external_supplier_idurl,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.add_supplier(
# idurl=external_supplier_idurl,
# position=supplier_pos,
# customer_idurl=trusted_customer_idurl,
# )
# contactsdb.save_suppliers(customer_idurl=trusted_customer_idurl)
else:
lg.warn('not possible to detect external supplier position for customer %s from received list files, known position is %s' % (
trusted_customer_idurl, known_supplier_pos))
supplier_pos = known_supplier_pos
remote_files_changed, _, _, _ = backup_matrix.process_raw_list_files(
supplier_num=supplier_pos,
list_files_text_body=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
is_in_sync=True,
auto_create=True,
)
if remote_files_changed:
backup_matrix.SaveLatestRawListFiles(
supplier_idurl=external_supplier_idurl,
raw_data=supplier_raw_list_files,
customer_idurl=trusted_customer_idurl,
)
# finally sending Ack() packet back
p2p_service.SendAck(newpacket)
if remote_files_changed:
lg.info('received updated list of files from external supplier %s for customer %s' % (external_supplier_idurl, trusted_customer_idurl))
return True
