def set_software_server(soft, server, path, recalculate_data=False):
file = os.path.join(path, soft.filename)
if not os.path.exists(file):
raise errors.FileNotFound(file)
if soft.size != os.path.getsize(file):
return errors.GenericError(f"file is not of specified size",
file=file,
size=soft.size)
if soft.checksum != md5(file):
return errors.GenericError(f"checksum error on file", file=file)
return SoftwareServerAssociation(software=soft, server=server, path=path)
def test_send_software_FileNotFound(self, mock_post, mock_send_file,
mock_exists):
mock_post.return_value = Response(msg={'transfer_id': '1'}, code=200)
mock_exists.return_value = False
resp = self.client.post(url_for('api_1_0.send'),
json=dict(software_id=str(self.soft.id),
dest_server_id=str(self.node2.id),
dest_path=self.dest_path),
headers=self.auth.header)
self.validate_error_response(
resp,
errors.FileNotFound(os.path.join(self.source_path, self.filename)))
def test_send_file_FileNotFound(self, mock_post, mock_send_file,
mock_exists, mock_getsize, mock_md5):
mock_post.return_value = Response(msg={'id': '1'}, code=200)
mock_exists.return_value = False
mock_getsize.return_value = self.size
mock_md5.return_value = self.checksum
resp = self.client.post(url_for('api_1_0.send'),
json=dict(file=os.path.join(
self.source_path, self.filename),
dest_server_id=str(self.node2.id),
dest_path=self.dest_path),
headers=self.auth.header)
self.validate_error_response(
resp,
errors.FileNotFound(os.path.join(self.source_path, self.filename)))
def post(self):
json = request.get_json()
file = json['file']
if not os.path.exists(file):
raise errors.FileNotFound(file)
soft = Software(name=json['name'],
version=json['version'],
filename=os.path.basename(json['file']),
size=os.path.getsize(file),
checksum=md5(file),
family=json.get('family', None))
ssa = set_software_server(soft, g.server,
os.path.dirname(json['file']))
if not isinstance(ssa, SoftwareServerAssociation):
return ssa
db.session.add_all([soft, ssa])
db.session.commit()
return {'id': str(soft.id)}, 201
def send():
def search_cost(ssa, route_list):
cost = [
route['cost'] for route in route_list
if str(ssa.server.id) == route['destination_id']
]
if cost:
if cost[0] is None:
cost = 999999
else:
cost = cost[0]
else:
cost = 999999
return cost
# Validate Data
json_data = request.get_json()
dest_server = Server.query.get_or_raise(json_data['dest_server_id'])
if 'software_id' in json_data:
software = Software.query.get_or_raise(json_data['software_id'])
ssa = SoftwareServerAssociation.query.filter_by(
server=g.server, software=software).one_or_none()
# if current server does not have the software, forward request to the closest server who has it
if not ssa:
resp = ntwrk.get(dest_server, 'api_1_0.routes', timeout=5)
if resp.code == 200:
ssas = copy.copy(software.ssas)
ssas.sort(key=functools.partial(
search_cost, route_list=resp.msg['route_list']))
# unable to get route cost, we take the first option we have
else:
ssas = random.shuffle(list(software.ssas))
if not ssas or len(ssas) == 0:
raise errors.NoSoftwareServer(software_id=str(software.id))
server = ssas[
0].server # closest server from dest_server who has the software
resp = ntwrk.post(server, 'api_1_0.send', json=json_data)
resp.raise_if_not_ok()
return resp.msg, resp.code
else:
file = os.path.join(ssa.path, software.filename)
if not os.path.exists(file):
raise errors.FileNotFound(file)
size = ssa.software.size
else:
file = json_data['file']
if os.path.exists(file):
size = os.path.getsize(file)
checksum = md5(json_data.get('file'))
else:
raise errors.FileNotFound(file)
chunk_size = d.CHUNK_SIZE * 1024 * 1024
max_senders = min(json_data.get('max_senders', d.MAX_SENDERS),
d.MAX_SENDERS)
chunks = math.ceil(size / chunk_size)
if 'software_id' in json_data:
json_msg = dict(software_id=str(software.id), num_chunks=chunks)
if 'dest_path' in json_data:
json_msg['dest_path'] = json_data.get('dest_path')
else:
json_msg = dict(dest_path=json_data['dest_path'],
filename=os.path.basename(json_data.get('file')),
size=size,
checksum=checksum,
num_chunks=chunks)
# if dest_path not set, file will be sent to
if 'force' in json_data:
json_msg['force'] = json_data['force']
resp = ntwrk.post(dest_server, 'api_1_0.transferlist', json=json_msg)
resp.raise_if_not_ok()
transfer_id = resp.msg.get('id')
current_app.logger.debug(
f"Transfer {transfer_id} created. Sending {file} to {dest_server}:{json_data.get('dest_path')}."
)
if json_data.get('background', True):
executor.submit(
asyncio.run,
async_send_file(dest_server=dest_server,
transfer_id=transfer_id,
file=file,
chunk_size=chunk_size,
max_senders=max_senders,
identity=get_jwt_identity()))
else:
asyncio.run(
async_send_file(dest_server=dest_server,
transfer_id=transfer_id,
file=file,
chunk_size=chunk_size,
max_senders=max_senders,
identity=get_jwt_identity()))
if json_data.get('include_transfer_data', False):
resp = ntwrk.get(dest_server,
"api_1_0.transferresource",
view_data=dict(transfer_id=transfer_id))
if resp.code == 200:
msg = resp.msg
else:
resp.raise_if_not_ok()
else:
msg = {'transfer_id': transfer_id}
return msg, 202 if json_data.get('background', True) else 201
def join():
global fetched_catalog
if get_jwt_identity() == '00000000-0000-0000-0000-000000000004':
js = request.get_json()
current_app.logger.debug(
f"New server wanting to join: {json.dumps(js, indent=2)}")
if db.session.query(Server).filter_by(
id=js.get('id', None)).count() > 0:
raise errors.DuplicatedId(js.get('id', None))
if db.session.query(Server).filter_by(
name=js.get('name', None)).count() > 0:
raise errors.AlreadyExists('name', js.get('name', None))
s = Server.from_json(js)
s.created_on = get_now()
external_ip = ipaddress.ip_address(request.remote_addr)
if not external_ip.is_loopback and external_ip not in [
gate.ip for gate in s.gates
]:
for port in set([gate.port for gate in s.gates]):
s.add_new_gate(external_ip, port, hidden=True)
certfile = current_app.dm.config.http_conf.get('certfile', None)
keyfile = current_app.dm.config.http_conf.get('keyfile', None)
if keyfile and os.path.exists(keyfile):
with open(keyfile, 'rb') as fh:
keyfile_content = fh.read()
else:
raise errors.FileNotFound(keyfile)
if certfile and os.path.exists(certfile):
with open(certfile, 'rb') as fh:
certfile_content = fh.read()
else:
raise errors.FileNotFound(certfile)
data = {
'keyfile': base64.b64encode(keyfile_content).decode(),
'certfile': base64.b64encode(certfile_content).decode()
}
data.update(Dimension=g.dimension.to_json())
data.update(me=str(Server.get_current().id))
with _lock:
if fetched_catalog[1] is None or fetched_catalog[0] < get_now(
) - dt.timedelta(minutes=1):
c = fetch_catalog()
fetched_catalog = (get_now(), c)
else:
c = fetched_catalog[1]
data.update(catalog=c)
if _lock_delete.acquire(False):
try:
delete_old_temp_servers()
finally:
_lock_delete.release()
server_data = s.to_json(add_gates=True)
servers_to_be_created.update({s.id: server_data})
del s
return data, 200
else:
raise errors.GenericError('Invalid token', status_code=400)