def example_migrate_execution():
email = "*****@*****.**"
pwd = "airflow_test_password"
server = "http://127.0.0.1:5000"
client = CornFlow(url=server)
a = client.login(email, pwd)
info_all = client.get_all_instances()
instance = info_all[-1]
info = client.get_one_instance(instance["id"])
inst_data = client.get_api_for_id("instance/", instance["id"], "data")
inst_data = inst_data.json()
execution = info["executions"][-1]
results = client.get_solution(execution["id"])
log = client.get_log(execution["id"])
# now we go to the other server:
server2 = "SOME_OTHER_SERVER"
client2 = CornFlow(url=server2)
client2.login("ADMIN_USER", "ADMIN_PASS")
# create instance in another server
instance2 = client2.create_instance(inst_data["data"],
name=info["name"],
description=info["description"])
execution_new = client.manual_execution(
instance_id=instance2["id"],
config=execution["config"],
name=execution["name"],
data=results["data"],
log_json=log["log"],
)
def run_example():
config = dict(email=email, pwd=pwd, name=name)
# client = CornFlow(url="http://127.0.0.1:5000")
client = CornFlow(url="http://34.78.205.34:5000/")
client.sign_up(**config)
client.login(email, pwd)
prob = build_pulp_problem()
# The problem data is written to an .lp file
data = prob.to_dict()
instance_id = client.create_instance(data)
config = dict(
solver="PULP_CBC_CMD",
mip=True,
msg=True,
warmStart=True,
timeLimit=10,
options=["donotexist", "thisdoesnotexist"],
keepFiles=0,
gapRel=0.1,
gapAbs=1,
threads=1,
logPath="test_export_solver_json.log",
)
execution_id = client.create_execution(instance_id, config)
status = client.get_status(execution_id)
results = client.get_solution(execution_id)
_vars, prob = pulp.LpProblem.from_dict(results["data"])
actual_vars = group_variables_by_name(_vars, ["Route", "BuildaPlant"],
replace_underscores_with_spaces=True)
actual_vars.keys()
actual_vars["BuildaPlant"]
actual_vars["Route"][("San Francisco", "Barstow")]
# The status of the solution is printed to the screen
print("Status:", pulp.LpStatus[prob.status])
# Each of the variables is printed with it's resolved optimum value
for v in prob.variables():
print(v.name, "=", v.varValue)
# The optimised objective function value is printed to the screen
print("Total Cost of Transportation = ", pulp.value(prob.objective))
# get the values for the variables:
{k: v.value() for k, v in _vars.items()}
# get the log in text format
results["log_text"]
# get the log in json format
results["log_json"]
def test_run_dag(self):
data = _load_file(PULP_EXAMPLE)
cf_client = CornFlow(url="http://127.0.0.1:5050/")
cf_login = cf_client.login("user", "UserPassword1!")
instance = cf_client.create_instance(data, "test_example",
"test_description")
execution = cf_client.create_execution(
instance_id=instance["id"],
config={
"solver": "PULP_CBC_CMD",
"timeLimit": 100
},
name="test_execution",
description="execution_description",
schema="solve_model_dag",
run=False,
)
# Check that execution is not run
status = cf_client.get_status(execution_id=execution["id"])
self.assertEqual(-4, status["state"])
# Run the execution
response = self.client.run_dag(execution_id=execution["id"])
self.assertEqual(200, response.status_code)
self.assertIn("dag_run_id", response.json().keys())
# Check that is optimal
time.sleep(10)
status = cf_client.get_status(execution_id=execution["id"])
self.assertEqual(1, status["state"])
def test_write_execution_solution(self):
client = CornFlow(url="http://127.0.0.1:5050/")
_ = client.login("user", "UserPassword1!")
data = _load_file(PULP_EXAMPLE)
instance = client.create_instance(data, "test_example",
"test_description")
execution = client.create_execution(
instance_id=instance["id"],
config={
"solver": "PULP_CBC_CMD",
"timeLimit": 60
},
name="test_execution",
description="execution_description",
schema="solve_model_dag",
)
time.sleep(15)
solution = client.get_solution(execution["id"])
payload = dict(state=1,
log_json={},
log_text="",
solution_schema="solve_model_dag")
payload["data"] = solution["data"]
response = self.client.write_solution(execution["id"], **payload)
self.assertEqual("results successfully saved", response["message"])
def test_get_execution_data(self):
client = CornFlow(url="http://127.0.0.1:5050/")
_ = client.login("user", "UserPassword1!")
data = _load_file(PULP_EXAMPLE)
instance = client.create_instance(data, "test_example",
"test_description")
execution = client.create_execution(
instance_id=instance["id"],
config={
"solver": "PULP_CBC_CMD",
"timeLimit": 60
},
name="test_execution",
description="execution_description",
schema="solve_model_dag",
)
response = self.client.get_data(execution["id"])
items = ["id", "data", "config"]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(instance["id"], response["id"])
self.assertEqual(data, response["data"])
self.assertEqual(execution["config"], response["config"])
class TestCornflowClientAdmin(TestCase):
def setUp(self):
self.client = CornFlow(url="http://127.0.0.1:5050/")
login_result = self.client.login("admin", "Adminpassword1!")
self.assertIn("id", login_result.keys())
self.assertIn("token", login_result.keys())
self.base_user_id = CornFlow(url="http://127.0.0.1:5050/").login(
"user", "UserPassword1!")["id"]
def tearDown(self):
pass
def test_get_all_users(self):
response = self.client.get_all_users()
self.assertGreaterEqual(len(response), 3)
def test_get_one_user(self):
response = self.client.get_one_user(self.base_user_id)
items = ["id", "first_name", "last_name", "username", "email"]
for item in items:
self.assertIn(item, response.json().keys())
self.assertEqual(self.base_user_id, response.json()["id"])
self.assertEqual("user", response.json()["username"])
self.assertEqual("*****@*****.**", response.json()["email"])
def run_example():
server = "https://devsm.cornflow.baobabsoluciones.app"
server = "http://127.0.0.1:5000"
server = "http://34.76.18.17:5000/"
client = CornFlow(url=server)
config = dict(email=email, pwd=pwd, name=name)
# a = client.sign_up(**config)
a = client.login(email, pwd)
insName = "j102_6"
description = "j102_6 from j10.mm"
#'j102_2.mm', 'j102_4.mm', 'j102_5.mm', 'j102_6.mm'
instance_obj = get_test_instance("j10.mm.zip", "j102_6.mm")
data = instance_obj.to_dict()
instance = client.create_instance(
data, name=insName, description=description, data_schema="hk_2020_dag"
)
config = dict(solver="default", timeLimit=10)
execution = client.create_execution(
instance["id"], config, name="default1", description="", dag_name="hk_2020_dag"
)
client.get_status(execution["id"])["state"]
data = client.get_solution(execution["id"])
def connect_to_cornflow(secrets):
"""
Create a connection to cornflow and log in with airflow admin user.
:return: A logged and connected Cornflow class instance
"""
# This secret comes from airflow configuration
print("Getting connection information from ENV VAR=CF_URI")
uri = secrets.get_conn_uri("CF_URI")
conn = urlparse(uri)
scheme = conn.scheme
if scheme == "cornflow":
scheme = "http"
url = "{}://{}".format(scheme, conn.hostname)
if conn.port:
url += ":{uri.port}".format(uri=conn)
if conn.path:
url = urljoin(url, conn.path)
airflow_user = CornFlow(url=url)
airflow_user.login(username=conn.username, pwd=conn.password)
return airflow_user
/ no_position_station[s], "c2" + str(s)
for (t1, t2) in precedences:
model_cycle_time += sum(v01TaskInStation[t1, s] * station_order[s] for s in stations) \
<= sum(v01TaskInStation[t2, s] * station_order[s] for s in stations), "c3" + str(t1) + str(t2)
model_cycle_time += vCycleTime, "Objetive"
# model_cycle_time.solve()
# Starting up cornflow client
config = dict(email=email, pwd=pwd, name=name)
client = CornFlow(url="CORNFLOW_WEB_SERVER_URL")
# client.sign_up(**config)
client.login(email, pwd)
data = model_cycle_time.to_dict()
instance_id = client.create_instance(data)
config = dict(solver="PULP_CBC_CMD",
mip=True,
msg=True,
warmStart=True,
timeLimit=10,
options=["donotexist", "thisdoesnotexist"],
keepFiles=0,
gapRel=0,
gapAbs=0,
threads=1,
logPath="test_export_solver_json.log")
def run_example():
server = "http://127.0.0.1:5000"
client = CornFlow(url=server)
config = dict(email=email, pwd=pwd, name=name)
a = client.sign_up(**config)
a = client.login(email, pwd)
import pulp
prob = pulp.LpProblem("test_export_dict_MIP", pulp.LpMinimize)
x = pulp.LpVariable("x", 0, 4)
y = pulp.LpVariable("y", -1, 1)
z = pulp.LpVariable("z", 0, None, pulp.LpInteger)
prob += x + 4 * y + 9 * z, "obj"
prob += x + y <= 5, "c1"
prob += x + z >= 10, "c2"
prob += -y + z == 7.5, "c3"
data = prob.toDict()
filename = "test_mps.mps"
insName = "test_export_dict_MIP"
description = "very small example"
instance = client.create_instance(data,
name=insName,
description=description)
# alternatively: send file
prob.writeMPS(filename=filename)
instance = client.create_instance_file(filename=filename,
name=insName,
description=description)
# edit the instance to give it a new name
client.put_api_for_id("instance/", instance["id"], dict(name="newName"))
# get info from an instance
info = client.get_one_instance(instance["id"])
# get all instances
info_all = client.get_all_instances()
# send an execution
config = dict(solver="PULP_CBC_CMD", timeLimit=10)
execution = client.create_execution(
instance["id"],
config,
name="execution1",
description="execution of a very small instance",
)
# check the status of the execution
status = client.get_status(execution["id"])
print(status["state"])
# get the execution solution
results = client.get_solution(execution["id"])
_vars, prob = pulp.LpProblem.from_dict(results["data"])
# get the values for the variables:
print({k: v.value() for k, v in _vars.items()})
# get the log in json format
log = client.get_log(execution["id"])
print(log["log"])
class TestCornflowClientService(TestCase):
def setUp(self):
self.client = CornFlow(url="http://127.0.0.1:5050/")
login_result = self.client.login("airflow", "Airflow_test_password1")
self.assertIn("id", login_result.keys())
self.assertIn("token", login_result.keys())
def tearDown(self):
pass
def test_get_execution_data(self):
client = CornFlow(url="http://127.0.0.1:5050/")
_ = client.login("user", "UserPassword1!")
data = _load_file(PULP_EXAMPLE)
instance = client.create_instance(data, "test_example",
"test_description")
execution = client.create_execution(
instance_id=instance["id"],
config={
"solver": "PULP_CBC_CMD",
"timeLimit": 60
},
name="test_execution",
description="execution_description",
schema="solve_model_dag",
)
response = self.client.get_data(execution["id"])
items = ["id", "data", "config"]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(instance["id"], response["id"])
self.assertEqual(data, response["data"])
self.assertEqual(execution["config"], response["config"])
def test_write_execution_solution(self):
client = CornFlow(url="http://127.0.0.1:5050/")
_ = client.login("user", "UserPassword1!")
data = _load_file(PULP_EXAMPLE)
instance = client.create_instance(data, "test_example",
"test_description")
execution = client.create_execution(
instance_id=instance["id"],
config={
"solver": "PULP_CBC_CMD",
"timeLimit": 60
},
name="test_execution",
description="execution_description",
schema="solve_model_dag",
)
time.sleep(15)
solution = client.get_solution(execution["id"])
payload = dict(state=1,
log_json={},
log_text="",
solution_schema="solve_model_dag")
payload["data"] = solution["data"]
response = self.client.write_solution(execution["id"], **payload)
self.assertEqual("results successfully saved", response["message"])
def test_get_deployed_dags(self):
response = self.client.get_deployed_dags()
items = ["id", "description"]
for item in items:
self.assertIn(item, response[0].keys())
deployed_dags = [v["id"] for v in response]
for dag in PUBLIC_DAGS:
self.assertIn(dag, deployed_dags)
def test_post_deployed_dag(self):
response = self.client.create_deployed_dag(
name="test_dag", description="test_dag_description")
items = ["id", "description"]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual("test_dag", response["id"])
self.assertEqual("test_dag_description", response["description"])
class TestCornflowClientUser(TestCase):
def setUp(self):
self.client = CornFlow(url="http://127.0.0.1:5050/")
login_result = self.client.login("user", "UserPassword1!")
self.assertIn("id", login_result.keys())
self.assertIn("token", login_result.keys())
self.user_id = login_result["id"]
def tearDown(self):
pass
def test_health_endpoint(self):
response = self.client.is_alive()
self.assertEqual(response["cornflow_status"], "healthy")
self.assertEqual(response["airflow_status"], "healthy")
def test_sign_up(self):
response = self.client.sign_up("test_username",
"*****@*****.**",
"TestPassword2!")
self.assertIn("id", response.json().keys())
self.assertIn("token", response.json().keys())
self.assertEqual(201, response.status_code)
def test_create_instance(self):
data = _load_file(PULP_EXAMPLE)
response = self.client.create_instance(data, "test_example",
"test_description")
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"executions",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual("test_example", response["name"])
self.assertEqual("solve_model_dag", response["schema"])
self.assertEqual("test_description", response["description"])
return response
def test_create_case(self):
data = _load_file(PULP_EXAMPLE)
response = self.client.create_case(
name="test_case",
schema="solve_model_dag",
data=data,
description="test_description",
)
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"solution_hash",
"path",
"updated_at",
"is_dir",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual("test_case", response["name"])
self.assertEqual("solve_model_dag", response["schema"])
self.assertEqual("test_description", response["description"])
return response
def test_create_instance_file(self):
response = self.client.create_instance_file(
_get_file("../data/test_mps.mps"),
name="test_filename",
description="filename_description",
)
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"executions",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual("test_filename", response["name"])
self.assertEqual("solve_model_dag", response["schema"])
self.assertEqual("filename_description", response["description"])
def test_create_execution(self):
instance = self.test_create_instance()
response = self.client.create_execution(
instance_id=instance["id"],
config={
"solver": "PULP_CBC_CMD",
"timeLimit": 60
},
name="test_execution",
description="execution_description",
schema="solve_model_dag",
)
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"config",
"instance_id",
"state",
"message",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(instance["id"], response["instance_id"])
self.assertEqual("test_execution", response["name"])
self.assertEqual("execution_description", response["description"])
self.assertEqual({
"solver": "PULP_CBC_CMD",
"timeLimit": 60
}, response["config"])
self.assertEqual(STATUS_NOT_SOLVED, response["state"])
return response
def test_execution_results(self):
execution = self.test_create_execution()
time.sleep(10)
response = self.client.get_results(execution["id"])
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"config",
"instance_id",
"state",
"message",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(execution["id"], response["id"])
self.assertEqual(STATUS_OPTIMAL, response["state"])
def test_execution_status(self):
execution = self.test_create_execution()
response = self.client.get_status(execution["id"])
items = ["id", "state", "message", "data_hash"]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(STATUS_NOT_SOLVED, response["state"])
time.sleep(10)
response = self.client.get_status(execution["id"])
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(STATUS_OPTIMAL, response["state"])
def test_stop_execution(self):
execution = self.test_create_execution()
response = self.client.stop_execution(execution["id"])
self.assertEqual(response["message"], "The execution has been stopped")
def test_get_execution_log(self):
execution = self.test_create_execution()
response = self.client.get_log(execution["id"])
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"config",
"instance_id",
"state",
"message",
"log",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(execution["id"], response["id"])
def test_get_execution_solution(self):
execution = self.test_create_execution()
time.sleep(10)
response = self.client.get_solution(execution["id"])
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"config",
"instance_id",
"state",
"message",
"data",
"checks",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(execution["id"], response["id"])
self.assertEqual(STATUS_OPTIMAL, response["state"])
return response
def test_create_case_execution(self):
execution = self.test_get_execution_solution()
response = self.client.create_case(
name="case_from_solution",
schema="solve_model_dag",
description="case_from_solution_description",
solution=execution["data"],
)
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"solution_hash",
"path",
"updated_at",
"is_dir",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual("case_from_solution", response["name"])
self.assertEqual("solve_model_dag", response["schema"])
self.assertEqual("case_from_solution_description",
response["description"])
return response
def test_get_all_instances(self):
self.test_create_instance()
self.test_create_instance()
instances = self.client.get_all_instances()
self.assertGreaterEqual(len(instances), 2)
def test_get_all_executions(self):
self.test_stop_execution()
self.test_stop_execution()
executions = self.client.get_all_executions()
self.assertGreaterEqual(len(executions), 2)
def test_get_all_cases(self):
self.test_create_case()
self.test_create_case()
cases = self.client.get_all_cases()
self.assertGreaterEqual(len(cases), 2)
def test_get_one_user(self):
response = self.client.get_one_user(self.user_id)
self.assertEqual(response.status_code, 200)
items = ["id", "first_name", "last_name", "username", "email"]
for item in items:
self.assertIn(item, response.json().keys())
self.assertEqual(self.user_id, response.json()["id"])
self.assertEqual("user", response.json()["username"])
self.assertEqual("*****@*****.**", response.json()["email"])
def test_get_one_instance(self):
instance = self.test_create_instance()
response = self.client.get_one_instance(instance["id"])
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"executions",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(instance[item], response[item])
def test_get_one_case(self):
case = self.test_create_case()
response = self.client.get_one_case(case["id"])
items = [
"id",
"name",
"description",
"created_at",
"user_id",
"data_hash",
"schema",
"solution_hash",
"path",
"updated_at",
"is_dir",
]
for item in items:
self.assertIn(item, response.keys())
self.assertEqual(case[item], response[item])