def test_run_spark_job(self):
# given ske
ske = "opends spark it-test"
Staroid().cluster().create(ske, TEST_REGION)
# init with ske name
ods.init(ske=ske)
spark = ods.spark("test", delta=True).session()
df = spark.createDataFrame([{"hello": "world"} for x in range(100)])
self.assertEqual(100, df.count())
# delta
s3_delta_table = "{}/delta-table".format(os.environ["S3_LOCATION"])
spark.range(
0, 5).write.format("delta").mode("overwrite").save(s3_delta_table)
delta = spark.read.format("delta").load(s3_delta_table)
self.assertEqual(5, delta.count())
# delete cluster instance
spark.stop()
ods.spark("test").delete()
# clean up
Staroid().cluster().delete(ske)
def test_read_config(self):
# unset env
at = None
ac = None
if "STAROID_ACCESS_TOKEN" in os.environ:
at = os.environ["STAROID_ACCESS_TOKEN"]
del os.environ["STAROID_ACCESS_TOKEN"]
if "STAROID_ACCOUNT" in os.environ:
ac = os.environ["STAROID_ACCOUNT"]
del os.environ["STAROID_ACCOUNT"]
# given
fp = tempfile.NamedTemporaryFile()
fp.write(b"access_token: abc\naccount: GITHUB/user1")
fp.flush()
# when
s = Staroid(config_path=fp.name)
# then
self.assertEqual("abc", s.get_access_token())
self.assertEqual("GITHUB/user1", s.get_account())
# restore env
if at != None:
os.environ["STAROID_ACCESS_TOKEN"] = at
if ac != None:
os.environ["STAROID_ACCOUNT"] = ac
def test_crud(self):
# given
s = Staroid(access_token=os.environ["STAROID_ACCESS_TOKEN"],
account=os.environ["STAROID_ACCOUNT"])
all_accounts = s.get_all_accounts()
clusters_before = s.cluster().get_all()
# when create
c = s.cluster().create("staroid-python it-test")
# then
self.assertEqual("staroid-python it-test", c.name())
# when already exists on create
c = s.cluster().create("staroid-python it-test")
# then
self.assertEqual("staroid-python it-test", c.name())
# when list
clusters_after = s.cluster().get_all()
# then
self.assertEqual(len(clusters_after), len(clusters_before) + 1)
# when delete
s.cluster().delete("staroid-python it-test")
# then
clusters_after_deleted = s.cluster().get_all()
self.assertEqual(len(clusters_after_deleted), len(clusters_before))
def __init__(self, provider_config, cluster_name):
NodeProvider.__init__(self, provider_config, cluster_name)
self.__cached = {}
self.__star = Staroid(access_token=provider_config["access_token"],
account=provider_config["account"])
self.__ske = self._get_config_or_env(provider_config, "ske",
"STAROID_SKE")
self.__ske_region = self._get_config_or_env(provider_config,
"ske_region",
"STAROID_SKE_REGION")
def test_download_chisel(self):
# given
tmp_dir = tempfile.mkdtemp()
s = Staroid(cache_dir=tmp_dir)
# when
chisel_path = s.get_chisel_path()
# then
self.assertIsNotNone(chisel_path)
self.assertTrue(os.path.isfile(chisel_path))
# clean up
shutil.rmtree(pathlib.Path(tmp_dir))
def test_read_default_account(self):
# given access_token is set but account is not set
ac = None
if "STAROID_ACCOUNT" in os.environ:
ac = os.environ["STAROID_ACCOUNT"]
del os.environ["STAROID_ACCOUNT"]
# when
s = Staroid()
# then
self.assertNotEqual(None, s.get_account())
# restore env
if ac != None:
os.environ["STAROID_ACCOUNT"] = ac
def test_crud_namespace(self):
# given
s = Staroid(access_token=os.environ["STAROID_ACCESS_TOKEN"], account=os.environ["STAROID_ACCOUNT"])
c = s.cluster().create("staroid-python it-test-namespace")
# when create a namespace
ns_api = s.namespace(c)
ns = ns_api.create("instance1", "GITHUB/staroids/namespace:master")
# then namespace becomes RUNNING
wait_for_phase(ns_api, ns, "RUNNING")
self.assertEqual("RUNNING", ns_api.get_by_id(ns.id()).phase())
# start shell
ns_api.shell_start("instance1")
resources = ns_api.get_all_resources("instance1")
self.assertTrue(len(resources["services"]) > 0)
# start tunnel
ns_api.start_tunnel("instance1", ["57683:localhost:57683"])
# stop tunnel
ns_api.stop_tunnel("instance1")
# stop shell
ns_api.shell_stop("instance1")
# pause
ns = ns_api.stop("instance1")
wait_for_phase(ns_api, ns, "PAUSED")
self.assertEqual("PAUSED", ns_api.get_by_id(ns.id()).phase())
# resume
ns = ns_api.start("instance1")
wait_for_phase(ns_api, ns, "RUNNING")
self.assertEqual("RUNNING", ns_api.get_by_id(ns.id()).phase())
# when delete a namespace
ns = ns_api.delete("instance1")
# then namespace becomes REMOVED
wait_for_phase(ns_api, ns, "REMOVED")
self.assertEqual("REMOVED", ns_api.get_by_id(ns.id()).phase())
# when delete
s.cluster().delete("staroid-python it-test-namespace")
def __init__(self, staroid=None, ske=None, cache_dir=None):
self.__ske = None
if staroid == None:
self._staroid = Staroid()
else:
self._staroid = staroid
if cache_dir == None:
self.__cache_dir = "{}/.ods".format(str(Path.home()))
else:
self.__cache_dir = cache_dir
# configure from env var
if "STAROID_SKE" in os.environ:
self.__ske = os.environ["STAROID_SKE"]
# configure from args
if ske != None:
self.__ske = ske
def test_initialize(self):
s = Staroid()
def test_initialize(self):
s = Staroid(access_token=os.environ["STAROID_ACCESS_TOKEN"],
account=os.environ["STAROID_ACCOUNT"])
s.get_all_accounts()
