def __init__(self, machine_configs, task_configs, algorithm, event_file):
self.env = simpy.Environment()
cluster = Cluster()
cluster.add_machines(machine_configs)
task_broker = Episode.broker_cls(self.env, task_configs)
scheduler = Scheduler(self.env, algorithm)
self.simulation = Simulation(self.env, cluster, task_broker, scheduler,
event_file)
def creat_cluster_large():
cluster = Cluster()
for i in range(5):
cluster.add_machine(Machine(mips=2000, speed=500, micost=1)) # 构建虚拟机
for i in range(5):
cluster.add_machine(Machine(mips=1000, speed=500, micost=1))
for i in range(5):
cluster.add_machine(Machine(mips=500, speed=500, micost=1))
for i in range(5):
cluster.add_machine(Machine(mips=250, speed=500, micost=1))
return cluster
def testCluster(self):
cls = Cluster(keysPath + 'leo.pem')
cls._load('127.0.0.1/storage/0')
cls._load('127.0.0.1/storage/1')
cls._load('127.0.0.1/storage/2')
cls.setHttpOpt('timeout', 2)
assert str(cls).startswith('''['ok', ['apiEcho',''')
self.assertRaises(core.cluster.RpcApiNotFound, str, cls.abc)
assert str(cls.apiEcho) == '''{'doc': 'For test.', 'params': ['msg'], 'name': 'apiEcho'}'''
assert cls.apiEcho() == [['RpcBadParams', 'apiEcho'], ['RpcBadParams', 'apiEcho'], ['RpcBadParams', 'apiEcho']]
def testRouter(self):
stg = Cluster(keysPath + 'leo.pem')
stg._load('127.0.0.1/storage/0')
stg._load('127.0.0.1/storage/1')
stg._load('127.0.0.1/storage/2')
stg._load('127.0.0.1/storage/3')
assert stg[0].format(0) == ['ok']
assert stg[1].format(1) == ['ok']
assert stg[2].format(2) == ['ok']
assert stg[3].format(3) == ['ok']
cls = StorageClient('127.0.0.1/router/0', keysPath + 'leo.pem')
cls.readers()
time.sleep(0.5)
assert len(cls.readers()[1]) == 4
assert len(cls.writers()[1]) == 4
content = 'test1234'
for i in range(0, 1):
ok, id = cls._write(content)
assert ok == 'ok'
assert cls._read(id) == content
print cls._read(id, 1).read()
def __init__(self, configuration='nevod', num=None):
if configuration == 'nevod':
self.clusters = [
Cluster([-28.4, -7.8, -7.0], 13.3, 12.4),
Cluster([-28.4, 23.8, -7.0], 13.3, 12.4),
Cluster([0.0, 0.0, 0.0], 25.1, 13.5),
Cluster([33.3, 7.8, -14.5]),
Cluster([35.0, 46.0, -14.5]),
Cluster([-2.0, -47.0, -14.5]),
Cluster([-18.0, 62.0, -14.5]),
Cluster([60.0, -2.0, -2.0]),
Cluster([60.0, 26.0, -2.0]),
Cluster([60.0, 58.0, -8.0]),
]
elif configuration == 'flat':
self.clusters = [
Cluster([-60.0, 60.0, 0.0], 20.0, 20.0),
Cluster([0.0, 60.0, 0.0], 20.0, 20.0),
Cluster([60.0, 60.0, 0.0], 20.0, 20.0),
Cluster([-60.0, 0.0, 0.0], 20.0, 20.0),
Cluster([0.0, 0.0, 0.0], 20.0, 20.0),
Cluster([60.0, 0.0, 0.0], 20.0, 20.0),
Cluster([-60.0, -60.0, 0.0], 20.0, 20.0),
Cluster([0.0, -60.0, 0.0], 20.0, 20.0),
Cluster([60.0, -60.0, 0.0], 20.0, 20.0),
]
self.num = num # Номер установки, если создали много штук
self.eas = None # ШАЛ, упавший на данную установку
self.num_of_trig_cl = None # Число сработавших кластеров
self.average_vector = None # Средний из восстановленных векторов
self.rec_vector = None # Восстановленный вектор по установке
self.psi = None # Угол отклонения среднего
self.sigma_psi = None # Среднеквадратичный угол отклонения
self.rec_power = None # Восстановленная мощность
self.rec_age = None # Восстановленный возраст
self.rec_x = None # Восстановленне координаты прихода ШАЛ
self.rec_y = None
self.rec_theta = None # Восстановленные тета и фи
self.rec_phi = None
self.exper_num_particles = [] # Экспериментальное число частиц
self.sigma_num_particles = [] # Сигма в функционале
self.average_x0 = None # Средневзвешенные x0 и y0
self.average_y0 = None
self.grid_steps = 5 # Число шагов по сетке
self.power_age_steps = 5 # Число шагов поиска мощности и возраста
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from constants import CREATE_EMR_CLUSTER
from core.cluster import Cluster
from core.elastic_cache import ElastiCacheOperator
from core.spark import SparkOperator
from jobs.ranking import guest_house_ranking as job
from schema.ranking import TABLES
from utils.configuration import batch
from utils.dependency import sequential_dependency
with DAG(job['name'], start_date=job['start_date'], schedule_interval=job['schedule_time'].get_cron, default_args=batch) as dag:
create_emr_cluster = PythonOperator(
task_id=CREATE_EMR_CLUSTER,
trigger_rule= "all_success",
python_callable= Cluster().create(task_name=job["name"], num_nodes=job["num_nodes"], emr_version="emr-6.0.0"),
op_kwargs={
"name": job["name"],
"num_nodes": job["num_nodes"],
}
)
terminate_emr_cluster = PythonOperator(
task_id=CREATE_EMR_CLUSTER,
trigger_rule="all_done",
python_callable=Cluster.terminate(),
op_kwargs={
"create_task_id" : CREATE_EMR_CLUSTER
}
)
def testCluster(self):
cls = Cluster(keysPath + 'leo.pem')
cls._load('127.0.0.1/storage/0')
cls._load('127.0.0.1/storage/1')
cls._load('127.0.0.1/storage/2')
cls.setHttpOpt('timeout', 3)
assert cls.format(1) == [[ok], [ok], [ok]]
result = cls.write(_post='eee')
assert len(result) == 3 and map(lambda x: x[0], result) == [ok, ok, ok], result
assert cls.read(result[0][1]) == ['eee', 'eee', 'eee']
assert cls.read(result[1][1]) == ['eee', 'eee', 'eee']
assert cls.read(result[2][1]) == ['eee', 'eee', 'eee']
result = cls.read(result[0][1], _lazy=1)
assert len(result) == 3 and map(lambda x: str(x.size()) + x.read(), result) == ['3eee', '3eee', '3eee']
file = open(os.path.realpath(__file__), 'rb')
result = cls.write(_post=file)
assert len(result) == 3 and map(lambda x: x[0], result) == [ok, ok, ok]
file.close()
result = cls.read(result[0][1], _lazy=1)
assert map(lambda x: x.read(12), result) == ['#coding:utf-8', '#coding:utf-8', '#coding:utf-8']
def creat_cluster():
cluster = Cluster()
for i in range(2):
cluster.add_machine(Machine(mips=600, speed=450, micost=1)) # 构建虚拟机
for i in range(2):
cluster.add_machine(Machine(mips=500, speed=450, micost=1))
for i in range(2):
cluster.add_machine(Machine(mips=400, speed=450, micost=1))
for i in range(2):
cluster.add_machine(Machine(mips=300, speed=450, micost=1))
for i in range(2):
cluster.add_machine(Machine(mips=200, speed=450, micost=1))
return cluster
def creat_cluster_large_multiple():
cluster = Cluster()
for i in range(4):
cluster.add_machine(Machine(mips=2000, speed=560,
micost=1.71)) # 构建虚拟机
for i in range(4):
cluster.add_machine(Machine(mips=1500, speed=529, micost=2.23))
for i in range(4):
cluster.add_machine(Machine(mips=1000, speed=511, micost=2.3))
for i in range(4):
cluster.add_machine(Machine(mips=500, speed=448, micost=2.19))
for i in range(4):
cluster.add_machine(Machine(mips=250, speed=467, micost=2.18))
return cluster