def test_create_local_source(self):
HEADING()
StopWatch.start("create source")
self.sourcedir = path_expand("~/.cloudmesh/storage/test/")
self.create_local_file("~/.cloudmesh/storage/test/a/a.txt",
"content of a")
self.create_local_file("~/.cloudmesh/storage/test/a/b/b.txt",
"content of b")
self.create_local_file("~/.cloudmesh/storage/test/a/b/c/c.txt",
"content of c")
StopWatch.stop("create source")
# test if the files are ok
assert True
def provider_vm_create(name, ip):
HEADING()
try:
print("start", name, ip)
StopWatch.start(f"start {name}")
parameter = {'name': name, 'ip': ip}
data = provider.create(**parameter)
StopWatch.stop(f"start {name}")
except Exception as e:
Console.error(f"could not create VM {name}", traceflag=True)
print(e)
def test_list(self):
HEADING()
StopWatch.start("LIST Directory")
contents = self.p.list(self.p.service, "/")
StopWatch.stop("LIST Directory")
for c in contents:
pprint(c)
assert len(contents) > 0
found = False
for entry in contents:
if entry["cm"]["name"] == "a1.txt":
found = True
assert found
def test_get(self):
HEADING()
home = self.sourcedir
cloud_path = "/a1.txt"
local_path = f"{home}/hello.txt"
StopWatch.start("GET file")
test_file = self.p.get(cloud_path, local_path, False)
StopWatch.stop("GET file")
assert test_file is not None
content = readfile(local_path)
assert "a1.txt" in content
def setup(self):
StopWatch.start("vdir setup")
self.vdir = Vdir()
self.endpoint = 'box:/test.txt'
self.directory_and_name = '/testdir/test'
self.directory = 'testdir'
self.file = 'test'
self.create_file('~/.cloudmesh/vdir/test/test.txt', 'test file')
self.destination = path_expand("~/.cloudmesh/vdir/test")
variables = Variables()
service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=service)
self.p.put(source='~/.cloudmesh/vdir/test/test.txt', destination='/',
recursive=False)
StopWatch.stop("vdir setup")
def test_put_and_get(self):
HEADING()
home = self.sourcedir
StopWatch.start("PUT file")
test_file = self.p.put(self.p.service, f"{home}/a/a1.txt", "/")
StopWatch.stop("PUT file")
assert test_file is not None
StopWatch.start("GET file")
test_file = self.p.get(self.p.service, f"/a1.txt", f"{home}/hello.txt")
StopWatch.stop("GET file")
assert test_file is not None
content = readfile(f"{home}/hello.txt")
assert "a1.txt" in content
def run_timed(label, command, encoding=None, service=None):
"""
runs teh command and uses the StopWatch to time it
:param label: name of the StopWatch
:param command: the command to be executed
:param encoding: the encoding
:param service: a prefix to the stopwatch label
:return:
"""
_label = str(label)
print(_label, command)
StopWatch.start(f"{service} {_label}")
result = Shell.run(command, encoding)
StopWatch.stop(f"{service} {_label}")
return str(result)
def t1_create_single_node_cluster(self,
db_name='db1',
cluster_id='my-cl1',
cluster_type='single-node',
node_type='dc2.large',
user_name='awsuser',
passwd='AWSPass321'):
StopWatch.start("create single node cluster")
m = Provider()
r = m.create_single_node_cluster(db_name, cluster_id, cluster_type,
node_type, user_name, passwd)
StopWatch.stop("create single node cluster")
print(r)
assert True
def test_06_list_images(self):
HEADING()
StopWatch.start("list images2")
images = self.p.images()
# pprint(images)
sort_keys = self.p.p.output['image']['sort_keys'] # not pretty
order = self.p.p.output['image']['order'] # not pretty
header = self.p.p.output['image']['header'] # not pretty
print(
Printer.flatwrite(images,
sort_keys=sort_keys,
order=order,
header=header))
StopWatch.stop("list images2")
def doit(name):
HEADING()
try:
print("start", name)
StopWatch.start(f"start {name}")
# put your code here
StopWatch.stop(f"start {name}")
except Exception as e:
Console.error(f"issue {name}", traceflag=True)
print(e)
return name
def provider_vm_create():
HEADING()
name_generator.incr()
name = str(Name())
try:
StopWatch.start(f"start {name}")
data = provider.create()
StopWatch.stop(f"start {name}")
except Exception as e:
Console.error(f"could not create VM {name}", traceflag=True)
print(e)
Print()
data = provider.list() # update the db
def t1_run_select_query(
self,
cluster_id='my-cl1',
db_name='db1',
host='my-cl1.ced9iqbk50ks.us-west-2.redshift.amazonaws.com',
port=5439,
user_name='awsuser',
passwd='AWSPass321',
query_text="SELECT COUNT(*) FROM emp"):
StopWatch.start("run select")
m = Provider()
r = m.runselectquery_text(cluster_id, db_name, host, port, user_name,
passwd, query_text)
StopWatch.stop("run select")
print(r)
assert True
def test_stop_cluster(self, global_data):
assert global_data['cluster'] != ""
emr = Manager()
args = {'CLUSTERID': global_data['cluster']}
StopWatch.start("Stop Cluster")
cluster = emr.stop_cluster(args)
StopWatch.stop("Stop Cluster")
assert cluster is not None
assert 'cm' in cluster[0]
assert cluster[0]['data']['name'] == global_data['cluster']
global_data['cluster'] = ""
def test_internal_ping(self):
StopWatch.start("total _ping")
for host in hosts:
location = {
'ip': host,
'count': 1,
}
StopWatch.start(f"ping {host}")
result = Host._ping(location)
StopWatch.stop(f"ping {host}")
StopWatch.stop("total _ping")
assert result['success']
def enlistFlavors(self):
print("\n")
banner(f"Fetching list of flavors for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'pricing',
aws_access_key_id=str(self.ACCESS_KEY),
aws_secret_access_key=str(self.SECRET_KEY),
region_name='us-east-1'
)
StopWatch.start(f"Flavor list {self.cloudname}.")
print("CALLING FUNC.")
#flavor_list = ec2_instance.describe_services(ServiceCode='AmazonEC2')
flavor_list = ec2_instance.get_products(ServiceCode='AmazonEC2')
print("CALLED FUNC.")
opDict= []
for i in flavor_list['PriceList']:
i = json.loads(i)
opDict.append(
{
'productFamily':i['product']['productFamily'],
'memory':i['product']['attributes']['memory'],
'instanceType':i['product']['attributes']['instanceType'],
'tenancy':i['product']['attributes']['tenancy'],
'pubdate':i['publicationDate']
}
)
StopWatch.stop(f"Flavor list {self.cloudname}.")
t = format(StopWatch.get(f"Flavor list {self.cloudname}."), '.2f')
banner(f"Flavor list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} flavors fetched. Time taken {t} \nPrinting first 5 sample flavors:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Flavor_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Flavor list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def test_07_list_vm(self):
HEADING()
StopWatch.start("list vm")
vms = self.p.list()
# pprint (vms)
sort_keys = self.p.p.output['vm']['sort_keys'] # not pretty
order = self.p.p.output['vm']['order'] # not pretty
header = self.p.p.output['vm']['header'] # not pretty
print(
Printer.flatwrite(vms,
sort_keys=sort_keys,
order=order,
header=header))
StopWatch.stop("list vm")
def test_create_local_source(self):
HEADING()
sizes = [1, 10]
for size in sizes:
texttag = f"create size-{size} file"
StopWatch.start(texttag)
source = path_expand(f"~/.cloudmesh/storage/test/size-"
f"{size}-file.txt")
d = Path(os.path.dirname(path_expand(source)))
d.mkdir(parents=True, exist_ok=True)
Benchmark.file(source, size)
StopWatch.stop(texttag)
assert True
def test_googletoaws(self):
HEADING()
sourcecloud = "google"
targetcloud = "aws"
for fileSize in fileSizes:
targetFile = f"{sourcecloud}_to_{targetcloud}_fileSize_{fileSize}.txt"
sourceFile = f'local_to_google_fileSize_{fileSize}.txt'
StopWatch.start(targetFile)
provider = Provider(service=sourcecloud)
try:
testResult = provider.copyFiles(source_cloud=sourcecloud,
source_file=sourceFile,
target_cloud=targetcloud,
target_file=targetFile)
StopWatch.status(targetFile, "Success")
finally:
StopWatch.stop(targetFile)
def test_02_boot(self):
HEADING()
StopWatch.start("cms aws boot dryrun")
result = Shell.execute(
"cms aws boot --n=2 --cloud=aws --username=root --image=ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20190212 --flavor=t2.micro --public --key={} --dryrun"
.format(self.key),
shell=True)
StopWatch.stop("cms aws boot dryrun")
VERBOSE(result)
assert "create nodes" in result
assert "image - ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20190212" in result
assert "flavor - t2.micro" in result
assert "assign public ip - True" in result
assert "security groups - None" in result
assert "keypair name - " + self.key in result
def test_status(self):
HEADING(
"please patiently wait for vm to boot and proceed with other tests"
)
# wait for vms to boot for further tests
while 'pending' in Shell.execute("cms aws list test_boot_01 --refresh",
shell=True):
time.sleep(1)
StopWatch.start("cms aws status")
result = Shell.execute("cms aws status test_boot_01 --cloud=aws",
shell=True)
StopWatch.stop("cms aws status")
VERBOSE(result)
assert "{'test_boot_01': 'running'}" in result
def test_awstogoogleDir(self):
HEADING()
sourcecloud = "aws"
targetcloud = "google"
sourceFile = "uploadtest1.txt"
targetFile = "a2/testAwsToGoogle.txt"
StopWatch.start("aws_to_google_directory")
awsProvider = Provider(service=sourcecloud)
try:
testResult = awsProvider.copyFiles(source_cloud=sourcecloud,
source_file=sourceFile,
target_cloud=targetcloud,
target_file=targetFile)
StopWatch.status("aws_to_google_directory", "Success")
finally:
StopWatch.stop("aws_to_google_directory")
def test_googletoawsDir2(self):
HEADING()
sourcecloud = "google"
targetcloud = "aws"
sourceFile = "a1/testfolder/"
targetFile = "a1/testfolder2/"
StopWatch.start("google_to_aws_directory2")
awsProvider = Provider(service=sourcecloud)
try:
testResult = awsProvider.copyFiles(source_cloud=sourcecloud,
source_file=sourceFile,
target_cloud=targetcloud,
target_file=targetFile)
StopWatch.status("google_to_aws_directory2", "Success")
finally:
StopWatch.stop("google_to_aws_directory2")
def burn(self,
image="latest",
device="/dev/sda",
blocksize="4M",
progress=True,
hostname=None,
ip=None,
key=None):
"""
:param image:
:param device:
:param blocksize:
:param progress:
:param hostname:
:param ip:
:param key:
:return:
"""
# Burns the image on the specific device
mp = '/mount/pi'
# don't do the input() after burning the last card
# use a counter to check this
counter = 0
burner = Burner()
for i in range(1):
print("counter", counter)
StopWatch.start("fcreate {hostname}")
burner.burn(image, device, blocksize=blocksize)
burner.mount(device, mp)
burner.enable_ssh(mp)
burner.set_hostname(hostname, mp)
burner.set_key(key, mp)
burner.set_static_ip(ip, mp)
burner.unmount(device)
# for some reason, need to do unmount twice for it to work properly
burner.unmount(device)
StopWatch.start("fcreate {hostname}")
def test_internal_ping(self):
HEADING()
StopWatch.start("total _ping")
for host in hosts:
location = {
'ip': host,
'count': 1,
}
StopWatch.start(f"ping {host}")
result = Host._ping(location)
StopWatch.stop(f"ping {host}")
StopWatch.stop("total _ping")
if b'Access denied' in result['stdout'] and sys.platform == "win32":
print("ERROR: This test must be run in an administrative "
"terminal")
assert result['success']
def split_data_train_test(split_pct=0.3):
"""
update the negative values to requested value
:param name:
split the data into specified percentage
return:
confirms if data was split correctly and individual data sets were created
"""
split_pct=float(split_pct/100)
if hasattr(AIServObj, 'df'):
train_df, test_df= split_train_test(AIServObj.df,split_pct)
StopWatch.start('to_csv')
train_df.to_csv("aiservices-train.csv", header=False, index=False)
test_df.to_csv("aiservices-test.csv", header=False, index=False)
StopWatch.stop('to_csv')
else:
return 'Please validate the data first'
return 'Data split successfully'
def test_internal_ssh(self):
print()
StopWatch.start("total _ssh")
for host in hosts:
location = {
'host': host,
'username': os.environ['USER'],
'key': '~/.ssh/id_rsa.pub',
'command': 'uname -a'
}
StopWatch.start(f"ssh {host}")
result = Host._ssh(location)
StopWatch.stop(f"ssh {host}")
StopWatch.stop("total _ssh")
assert result.success
def test_04_boot(self):
HEADING()
StopWatch.start("cms aws boot")
result = Shell.execute(
"cms aws boot --n=2 --cloud=aws --username=root --image=ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20190212 --flavor=t2.micro --public --key={}"
.format(self.key),
shell=True)
StopWatch.stop("cms aws boot")
VERBOSE(result)
assert "cm.name" in result
assert "cm.cloud" in result
assert "state" in result
assert "image" in result
assert "public_ips" in result
assert "private_ips" in result
assert "cm.kind" in result
def test_stopwatch_loop(self):
HEADING()
cumulate = False
dt = 0.1
n = 10
for i in range(0, n):
StopWatch.start("stopwatch loop")
time.sleep(dt)
StopWatch.stop("stopwatch loop")
StopWatch.status("stopwatch loop", True)
t = StopWatch.get("stopwatch loop")
print(t)
assert t >= dt
t = StopWatch.sum("stopwatch loop", digits=4)
print(t)
assert t >= n * dt
def test_local_list(self):
HEADING()
target_CSP = "local"
target_obj = self.location
StopWatch.start("List local for awss3 provider")
provider = Provider(source=None,
source_obj=None,
target=target_CSP,
target_obj=target_obj)
result = provider.list(source=None,
source_obj=None,
target=target_CSP,
target_obj=target_obj,
recursive=True)
StopWatch.stop("List local for awss3 provider")
assert self.file_name in result[0]['name']
def run(command, verbose=True):
global benchmark
if verbose:
print(command)
try:
StopWatch.start(command)
output = subprocess.check_output(
command,
shell=True,
stderr=subprocess.STDOUT,
)
StopWatch.stop(command)
except subprocess.CalledProcessError as err:
if verbose:
print()
print(Fore.RED + f"ERROR: {err}")
sys.exit(1)
return output.decode('utf-8')