def backup_outgoing_message(private_message_object, message_id):
"""
"""
if not driver.is_on('service_backups'):
lg.warn('service_backups is not started')
return False
serialized_message = private_message_object.serialize()
local_msg_folder = os.path.join(settings.ChatChannelsDir(),
private_message_object.recipient, 'out')
if not bpio._dir_exist(local_msg_folder):
bpio._dirs_make(local_msg_folder)
local_msg_filename = os.path.join(local_msg_folder, message_id)
if not bpio.WriteBinaryFile(local_msg_filename, serialized_message):
lg.warn('failed writing outgoing message locally')
return False
remote_path_for_message = os.path.join('.messages', 'out',
private_message_object.recipient,
message_id)
global_message_path = global_id.MakeGlobalID(customer=messages_key_id(),
path=remote_path_for_message)
res = api.file_create(global_message_path)
if res['status'] != 'OK':
lg.warn('failed to create path "%s" in the catalog: %s' %
(global_message_path, res['errors']))
return False
res = api.file_upload_start(local_msg_filename,
global_message_path,
wait_result=False)
if res['status'] != 'OK':
lg.warn('failed to upload message "%s": %s' %
(global_message_path, res['errors']))
return False
return True
def doSavePacket(self, *args, **kwargs):
"""
Action method.
"""
if not args or not args[0]:
raise Exception('no input found')
NewPacket, PacketID = args[0]
glob_path = global_id.ParseGlobalID(PacketID, detect_version=True)
packetID = global_id.CanonicalID(PacketID)
customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull(packetID)
if dataORparity == 'Data':
self.OnHandData[SupplierNumber] = True
elif dataORparity == 'Parity':
self.OnHandParity[SupplierNumber] = True
if not NewPacket:
lg.warn('packet %r already exists locally' % packetID)
return
filename = os.path.join(settings.getLocalBackupsDir(), customer_id, glob_path['path'])
dirpath = os.path.dirname(filename)
if not os.path.exists(dirpath):
try:
bpio._dirs_make(dirpath)
except:
lg.exc()
# either way the payload of packet is saved
if not bpio.WriteBinaryFile(filename, NewPacket.Payload):
lg.err("unable to write to %s" % filename)
return
if self.packetInCallback is not None:
self.packetInCallback(self.backup_id, NewPacket)
if _Debug:
lg.out(_DebugLevel, "restore_worker.doSavePacket %s saved to %s" % (packetID, filename))
def backup_done(bid, result):
from crypt import signed
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(),
bid)):
filepath = os.path.join(settings.getLocalBackupsDir(), bid, filename)
payld = str(bpio.ReadBinaryFile(filepath))
outpacket = signed.Packet('Data', my_id.getLocalID(),
my_id.getLocalID(), filename, payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out',
filename)
bpio.WriteBinaryFile(newfilepath, outpacket.Serialize())
# Assume we delivered all pieces from ".out" to suppliers and lost original data
# Then we requested the data back and got it into ".inp"
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), bid + '.inp'))
except:
pass
for filename in os.listdir(
os.path.join(settings.getLocalBackupsDir(), bid + '.out')):
filepath = os.path.join(settings.getLocalBackupsDir(), bid + '.out',
filename)
data = bpio.ReadBinaryFile(filepath)
inppacket = signed.Unserialize(data)
assert inppacket
assert inppacket.Valid()
newfilepath = os.path.join(settings.getLocalBackupsDir(), bid + '.inp',
filename)
bpio.WriteBinaryFile(newfilepath, inppacket.Payload)
# Now do restore from input data
backupID = bid + '.inp'
outfd, tarfilename = tmpfile.make(
'restore',
extension='.tar.gz',
prefix=backupID.replace('/', '_') + '_',
)
r = restore_worker.RestoreWorker(backupID, outfd)
r.MyDeferred.addBoth(restore_done, tarfilename)
reactor.callLater(1, r.automat, 'init')
def _file_received(self, newpacket, state):
if state in ['in queue', 'shutdown', 'exist', 'failed']:
return
if state != 'received':
lg.warn("incorrect state [%s] for packet %s" %
(str(state), str(newpacket)))
return
if not newpacket.Valid():
# TODO: if we didn't get a valid packet ... re-request it or delete
# it?
lg.warn("%s is not a valid packet: %r" %
(newpacket.PacketID, newpacket))
return
# packetID = newpacket.PacketID
packetID = global_id.CanonicalID(newpacket.PacketID)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(), customer,
remotePath)
if os.path.isfile(filename):
lg.warn("found existed file" + filename)
self.automat('inbox-data-packet', packetID)
return
# try:
# os.remove(filename)
# except:
# lg.exc()
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.out(
2,
"backup_rebuilder._file_received ERROR can not create sub dir: "
+ dirname)
return
if not bpio.WriteBinaryFile(filename, newpacket.Payload):
lg.out(2,
"backup_rebuilder._file_received ERROR writing " + filename)
return
from storage import backup_matrix
backup_matrix.LocalFileReport(packetID)
lg.out(10, "backup_rebuilder._file_received and wrote to " + filename)
self.automat('inbox-data-packet', packetID)
def _bk_done(bid, result):
from crypt import signed
customer, remotePath = packetid.SplitPacketID(bid)
try:
os.mkdir(os.path.join(settings.getLocalBackupsDir(), customer, remotePath + '.out'))
except:
pass
for filename in os.listdir(os.path.join(settings.getLocalBackupsDir(), customer, remotePath)):
filepath = os.path.join(settings.getLocalBackupsDir(), customer, remotePath, filename)
payld = bpio.ReadBinaryFile(filepath)
newpacket = signed.Packet(
'Data',
my_id.getLocalID(),
my_id.getLocalID(),
filename,
payld,
'http://megafaq.ru/cvps1010.xml')
newfilepath = os.path.join(settings.getLocalBackupsDir(), customer, remotePath + '.out', filename)
bpio.WriteBinaryFile(newfilepath, newpacket.Serialize())
reactor.stop()
def rewrite_indexes(db_instance, source_db_instance):
"""
"""
if _Debug:
lg.out(_DebugLevel, 'coins_db.rewrite_indexes')
source_location = os.path.join(source_db_instance.path, '_indexes')
source_indexes = os.listdir(source_location)
existing_location = os.path.join(db_instance.path, '_indexes')
existing_indexes = os.listdir(existing_location)
for existing_index_file in existing_indexes:
if existing_index_file != '00id.py':
index_name = existing_index_file[2:existing_index_file.index('.')]
existing_index_path = os.path.join(existing_location,
existing_index_file)
os.remove(existing_index_path)
if _Debug:
lg.out(_DebugLevel,
' removed index at %s' % existing_index_path)
buck_path = os.path.join(db_instance.path, index_name + '_buck')
if os.path.isfile(buck_path):
os.remove(buck_path)
if _Debug:
lg.out(_DebugLevel,
' also bucket at %s' % buck_path)
stor_path = os.path.join(db_instance.path, index_name + '_stor')
if os.path.isfile(stor_path):
os.remove(stor_path)
if _Debug:
lg.out(_DebugLevel,
' also storage at %s' % stor_path)
for source_index_file in source_indexes:
if source_index_file != '00id.py':
index_name = source_index_file[2:source_index_file.index('.')]
destination_index_path = os.path.join(existing_location,
source_index_file)
source_index_path = os.path.join(source_location,
source_index_file)
if not bpio.WriteTextFile(destination_index_path,
bpio.ReadTextFile(source_index_path)):
lg.warn('failed writing index to %s' % destination_index_path)
continue
destination_buck_path = os.path.join(db_instance.path,
index_name + '_buck')
source_buck_path = os.path.join(source_db_instance.path,
index_name + '_buck')
if not bpio.WriteBinaryFile(destination_buck_path,
bpio.ReadBinaryFile(source_buck_path)):
lg.warn('failed writing index bucket to %s' %
destination_buck_path)
continue
destination_stor_path = os.path.join(db_instance.path,
index_name + '_stor')
source_stor_path = os.path.join(source_db_instance.path,
index_name + '_stor')
if not bpio.WriteBinaryFile(destination_stor_path,
bpio.ReadBinaryFile(source_stor_path)):
lg.warn('failed writing index storage to %s' %
destination_stor_path)
continue
if _Debug:
lg.out(
_DebugLevel, ' wrote index %s from %s' %
(index_name, source_index_path))
def on_data(newpacket):
if id_url.to_bin(newpacket.OwnerID) == my_id.getIDURL().to_bin():
# this Data belong to us, SKIP
return False
# if not contactsdb.is_customer(newpacket.OwnerID):
# # SECURITY
# # TODO: process files from another customer : glob_path['idurl']
# lg.warn("skip, %s not a customer, packetID=%s" % (newpacket.OwnerID, newpacket.PacketID))
# # p2p_service.SendFail(newpacket, 'not a customer')
# return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
# p2p_service.SendFail(newpacket, 'incorrect path')
return False
authorized_idurl = verify_packet_ownership(newpacket)
if authorized_idurl is None:
lg.err("ownership verification failed for %r" % newpacket)
# p2p_service.SendFail(newpacket, 'ownership verification failed')
return False
filename = make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
# p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict, _ = accounting.read_customers_quotas()
if newpacket.OwnerID.to_bin() not in list(space_dict.keys()):
lg.err("customer space is broken, no info about donated space for %s" %
newpacket.OwnerID)
p2p_service.SendFail(
newpacket,
'customer space is broken, no info about donated space',
remote_idurl=authorized_idurl)
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID.to_bin() in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID.to_bin()])
bytes_donated_to_customer = int(
space_dict[newpacket.OwnerID.to_bin()])
if bytes_donated_to_customer - bytes_used_by_customer < len(data):
lg.warn("no free space left for customer data: %s" %
newpacket.OwnerID)
p2p_service.SendFail(newpacket,
'no free space left for customer data',
remote_idurl=authorized_idurl)
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket,
'write error',
remote_idurl=authorized_idurl)
return False
# Here Data() packet was stored as it is on supplier node (current machine)
del data
p2p_service.SendAck(newpacket,
response=strng.to_text(len(newpacket.Payload)),
remote_idurl=authorized_idurl)
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
# if self.publish_event_supplier_file_modified: # TODO: must remove that actually
# from main import events
# events.send('supplier-file-modified', data=dict(
# action='write',
# glob_path=glob_path['path'],
# owner_id=newpacket.OwnerID,
# ))
return True
def _on_data(self, newpacket):
import os
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from lib import jsn
from system import bpio
from main import settings
from userid import my_id
from userid import global_id
from contacts import contactsdb
from p2p import p2p_service
from storage import accounting
if newpacket.OwnerID == my_id.getLocalID():
# this Data belong to us, SKIP
return False
if not contactsdb.is_customer(newpacket.OwnerID):
# SECURITY
# TODO: process files from another customer : glob_path['idurl']
lg.warn("skip, %s not a customer, packetID=%s" %
(newpacket.OwnerID, newpacket.PacketID))
# p2p_service.SendFail(newpacket, 'not a customer')
return False
glob_path = global_id.ParseGlobalID(newpacket.PacketID)
if not glob_path['path']:
# backward compatible check
glob_path = global_id.ParseGlobalID(
my_id.getGlobalID('master') + ':' + newpacket.PacketID)
if not glob_path['path']:
lg.err("got incorrect PacketID")
p2p_service.SendFail(newpacket, 'incorrect path')
return False
filename = self._do_make_valid_filename(newpacket.OwnerID, glob_path)
if not filename:
lg.warn("got empty filename, bad customer or wrong packetID?")
p2p_service.SendFail(newpacket, 'empty filename')
return False
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
try:
bpio._dirs_make(dirname)
except:
lg.err("can not create sub dir %s" % dirname)
p2p_service.SendFail(newpacket, 'write error')
return False
data = newpacket.Serialize()
donated_bytes = settings.getDonatedBytes()
accounting.check_create_customers_quotas(donated_bytes)
space_dict = accounting.read_customers_quotas()
if newpacket.OwnerID not in list(space_dict.keys()):
lg.err("no info about donated space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no info about donated space')
return False
used_space_dict = accounting.read_customers_usage()
if newpacket.OwnerID in list(used_space_dict.keys()):
try:
bytes_used_by_customer = int(
used_space_dict[newpacket.OwnerID])
bytes_donated_to_customer = int(space_dict[newpacket.OwnerID])
if bytes_donated_to_customer - bytes_used_by_customer < len(
data):
lg.warn("no free space for %s" % newpacket.OwnerID)
p2p_service.SendFail(newpacket, 'no free space')
return False
except:
lg.exc()
if not bpio.WriteBinaryFile(filename, data):
lg.err("can not write to %s" % str(filename))
p2p_service.SendFail(newpacket, 'write error')
return False
# Here Data() packet was stored as it is on supplier node (current machine)
sz = len(data)
del data
lg.out(self.debug_level, "service_supplier._on_data %r" % newpacket)
lg.out(
self.debug_level, " from [ %s | %s ]" % (
newpacket.OwnerID,
newpacket.CreatorID,
))
lg.out(self.debug_level, " saved with %d bytes to %s" % (
sz,
filename,
))
p2p_service.SendAck(newpacket, str(len(newpacket.Payload)))
from supplier import local_tester
reactor.callLater(0, local_tester.TestSpaceTime) # @UndefinedVariable
if self.publish_event_supplier_file_modified:
from main import events
events.send('supplier-file-modified',
data=dict(
action='write',
glob_path=glob_path['path'],
owner_id=newpacket.OwnerID,
))
return True
