def simulate_data_upload(env: Environment, replica: FunctionReplica): node = replica.node.ether_node func = replica started = env.now if 'data.skippy.io/sends-to-storage' not in func.pod.spec.labels: return # FIXME: storage size = parse_size_string( func.pod.spec.labels['data.skippy.io/sends-to-storage']) path = func.pod.spec.labels['data.skippy.io/sends-to-storage/path'] storage_node_name = env.cluster.get_storage_nodes(path)[0] logger.debug('%.2f replica %s uploading data %s to %s', env.now, node, path, storage_node_name) if storage_node_name == node.name: # FIXME this is essentially a disk read and not a network connection yield env.timeout(size / 1.25e+8) # 1.25e+8 = 1 GBit/s return storage_node = env.cluster.get_node(storage_node_name) route = env.topology.route_by_node_name(node.name, storage_node.name) flow = SafeFlow(env, size, route) yield flow.start() for hop in route.hops: env.metrics.log_network(size, 'data_upload', hop) env.metrics.log_flow(size, env.now - started, route.source, route.destination, 'data_upload')
def from_str(memory, cpu): """ :param memory: "64Mi" :param cpu: "250m" :return: """ return Resources(int(cpu.rstrip('m')), parse_size_string(memory))
def register_images(self, env: Environment): containers: docker.ContainerRegistry = env.container_registry for image, size, arch in self.images: containers.put(ImageProperties(image, parse_size_string(size), arch=arch)) for name, tag_dict in containers.images.items(): for tag, images in tag_dict.items(): logging.info('%s, %s, %s', name, tag, images)
def setup(self, env: Environment): containers: docker.ContainerRegistry = env.container_registry containers.put( ImageProperties('resnet50-inference-cpu', parse_size_string('56M'), arch='arm32')) containers.put( ImageProperties('resnet50-inference-cpu', parse_size_string('56M'), arch='x86')) containers.put( ImageProperties('resnet50-inference-cpu', parse_size_string('56M'), arch='aarch64')) containers.put( ImageProperties('resnet50-training-cpu', parse_size_string('128M'), arch='arm32')) containers.put( ImageProperties('resnet50-training-cpu', parse_size_string('128M'), arch='x86')) containers.put( ImageProperties('resnet50-training-cpu', parse_size_string('128M'), arch='aarch64')) # log all the images in the container for name, tag_dict in containers.images.items(): for tag, images in tag_dict.items(): logger.info('%s, %s, %s', name, tag, images)
from ether.util import parse_size_string from skippy.core.storage import DataItem resnet_train_bucket = 'bucket_resnet50_train' resnet_pre_bucket = 'bucket_resnet50_pre' resnet_model_bucket = 'bucket_resnet50_model' speech_bucket = 'bucket_speech' mobilenet_bucket = 'mobilenet_bucket' resnet_train_bucket_item = DataItem(resnet_train_bucket, 'raw_data', parse_size_string('58M')) resnet_pre_bucket_item = DataItem(resnet_pre_bucket, 'raw_data', parse_size_string('14M')) resnet_model_bucket_item = DataItem(resnet_model_bucket, 'model', parse_size_string('103M')) speech_model_tflite_bucket_item = DataItem(speech_bucket, 'model_tflite', parse_size_string('48M')) speech_model_gpu_bucket_item = DataItem(speech_bucket, 'model_gpu', parse_size_string('188M')) mobilenet_model_tflite_bucket_item = DataItem(mobilenet_bucket, 'model_tflite', parse_size_string('4M')) mobilenet_model_tpu_bucket_item = DataItem(mobilenet_bucket, 'model_tpu', parse_size_string('4M')) bucket_names = [ resnet_model_bucket, resnet_train_bucket, resnet_pre_bucket, mobilenet_bucket, speech_bucket ] data_items = [ resnet_train_bucket_item, resnet_pre_bucket_item, resnet_model_bucket_item,
def create_node(name: str, cpus: int, mem: str, arch: str, labels: Dict[str, str]) -> Node: capacity = Capacity(cpu_millis=cpus * 1000, memory=parse_size_string(mem)) return Node(name, capacity=capacity, arch=arch, labels=labels)