def set_software_server(soft, server, path, recalculate_data=False): file = os.path.join(path, soft.filename) if not os.path.exists(file): raise errors.FileNotFound(file) if soft.size != os.path.getsize(file): return errors.GenericError(f"file is not of specified size", file=file, size=soft.size) if soft.checksum != md5(file): return errors.GenericError(f"checksum error on file", file=file) return SoftwareServerAssociation(software=soft, server=server, path=path)
def test_send_software_FileNotFound(self, mock_post, mock_send_file, mock_exists): mock_post.return_value = Response(msg={'transfer_id': '1'}, code=200) mock_exists.return_value = False resp = self.client.post(url_for('api_1_0.send'), json=dict(software_id=str(self.soft.id), dest_server_id=str(self.node2.id), dest_path=self.dest_path), headers=self.auth.header) self.validate_error_response( resp, errors.FileNotFound(os.path.join(self.source_path, self.filename)))
def test_send_file_FileNotFound(self, mock_post, mock_send_file, mock_exists, mock_getsize, mock_md5): mock_post.return_value = Response(msg={'id': '1'}, code=200) mock_exists.return_value = False mock_getsize.return_value = self.size mock_md5.return_value = self.checksum resp = self.client.post(url_for('api_1_0.send'), json=dict(file=os.path.join( self.source_path, self.filename), dest_server_id=str(self.node2.id), dest_path=self.dest_path), headers=self.auth.header) self.validate_error_response( resp, errors.FileNotFound(os.path.join(self.source_path, self.filename)))
def post(self): json = request.get_json() file = json['file'] if not os.path.exists(file): raise errors.FileNotFound(file) soft = Software(name=json['name'], version=json['version'], filename=os.path.basename(json['file']), size=os.path.getsize(file), checksum=md5(file), family=json.get('family', None)) ssa = set_software_server(soft, g.server, os.path.dirname(json['file'])) if not isinstance(ssa, SoftwareServerAssociation): return ssa db.session.add_all([soft, ssa]) db.session.commit() return {'id': str(soft.id)}, 201
def send(): def search_cost(ssa, route_list): cost = [ route['cost'] for route in route_list if str(ssa.server.id) == route['destination_id'] ] if cost: if cost[0] is None: cost = 999999 else: cost = cost[0] else: cost = 999999 return cost # Validate Data json_data = request.get_json() dest_server = Server.query.get_or_raise(json_data['dest_server_id']) if 'software_id' in json_data: software = Software.query.get_or_raise(json_data['software_id']) ssa = SoftwareServerAssociation.query.filter_by( server=g.server, software=software).one_or_none() # if current server does not have the software, forward request to the closest server who has it if not ssa: resp = ntwrk.get(dest_server, 'api_1_0.routes', timeout=5) if resp.code == 200: ssas = copy.copy(software.ssas) ssas.sort(key=functools.partial( search_cost, route_list=resp.msg['route_list'])) # unable to get route cost, we take the first option we have else: ssas = random.shuffle(list(software.ssas)) if not ssas or len(ssas) == 0: raise errors.NoSoftwareServer(software_id=str(software.id)) server = ssas[ 0].server # closest server from dest_server who has the software resp = ntwrk.post(server, 'api_1_0.send', json=json_data) resp.raise_if_not_ok() return resp.msg, resp.code else: file = os.path.join(ssa.path, software.filename) if not os.path.exists(file): raise errors.FileNotFound(file) size = ssa.software.size else: file = json_data['file'] if os.path.exists(file): size = os.path.getsize(file) checksum = md5(json_data.get('file')) else: raise errors.FileNotFound(file) chunk_size = d.CHUNK_SIZE * 1024 * 1024 max_senders = min(json_data.get('max_senders', d.MAX_SENDERS), d.MAX_SENDERS) chunks = math.ceil(size / chunk_size) if 'software_id' in json_data: json_msg = dict(software_id=str(software.id), num_chunks=chunks) if 'dest_path' in json_data: json_msg['dest_path'] = json_data.get('dest_path') else: json_msg = dict(dest_path=json_data['dest_path'], filename=os.path.basename(json_data.get('file')), size=size, checksum=checksum, num_chunks=chunks) # if dest_path not set, file will be sent to if 'force' in json_data: json_msg['force'] = json_data['force'] resp = ntwrk.post(dest_server, 'api_1_0.transferlist', json=json_msg) resp.raise_if_not_ok() transfer_id = resp.msg.get('id') current_app.logger.debug( f"Transfer {transfer_id} created. Sending {file} to {dest_server}:{json_data.get('dest_path')}." ) if json_data.get('background', True): executor.submit( asyncio.run, async_send_file(dest_server=dest_server, transfer_id=transfer_id, file=file, chunk_size=chunk_size, max_senders=max_senders, identity=get_jwt_identity())) else: asyncio.run( async_send_file(dest_server=dest_server, transfer_id=transfer_id, file=file, chunk_size=chunk_size, max_senders=max_senders, identity=get_jwt_identity())) if json_data.get('include_transfer_data', False): resp = ntwrk.get(dest_server, "api_1_0.transferresource", view_data=dict(transfer_id=transfer_id)) if resp.code == 200: msg = resp.msg else: resp.raise_if_not_ok() else: msg = {'transfer_id': transfer_id} return msg, 202 if json_data.get('background', True) else 201
def join(): global fetched_catalog if get_jwt_identity() == '00000000-0000-0000-0000-000000000004': js = request.get_json() current_app.logger.debug( f"New server wanting to join: {json.dumps(js, indent=2)}") if db.session.query(Server).filter_by( id=js.get('id', None)).count() > 0: raise errors.DuplicatedId(js.get('id', None)) if db.session.query(Server).filter_by( name=js.get('name', None)).count() > 0: raise errors.AlreadyExists('name', js.get('name', None)) s = Server.from_json(js) s.created_on = get_now() external_ip = ipaddress.ip_address(request.remote_addr) if not external_ip.is_loopback and external_ip not in [ gate.ip for gate in s.gates ]: for port in set([gate.port for gate in s.gates]): s.add_new_gate(external_ip, port, hidden=True) certfile = current_app.dm.config.http_conf.get('certfile', None) keyfile = current_app.dm.config.http_conf.get('keyfile', None) if keyfile and os.path.exists(keyfile): with open(keyfile, 'rb') as fh: keyfile_content = fh.read() else: raise errors.FileNotFound(keyfile) if certfile and os.path.exists(certfile): with open(certfile, 'rb') as fh: certfile_content = fh.read() else: raise errors.FileNotFound(certfile) data = { 'keyfile': base64.b64encode(keyfile_content).decode(), 'certfile': base64.b64encode(certfile_content).decode() } data.update(Dimension=g.dimension.to_json()) data.update(me=str(Server.get_current().id)) with _lock: if fetched_catalog[1] is None or fetched_catalog[0] < get_now( ) - dt.timedelta(minutes=1): c = fetch_catalog() fetched_catalog = (get_now(), c) else: c = fetched_catalog[1] data.update(catalog=c) if _lock_delete.acquire(False): try: delete_old_temp_servers() finally: _lock_delete.release() server_data = s.to_json(add_gates=True) servers_to_be_created.update({s.id: server_data}) del s return data, 200 else: raise errors.GenericError('Invalid token', status_code=400)