def load(self, name):
"""
Load cached model
:param name:
:return:
"""
cm = CmDatabase()
# USER env variable is required by StopWatch
if os.getenv('USER'):
# Do nothing
VERBOSE("USER env variable is already defined")
else:
os.environ['USER'] = '******'
test = cm.find(cloud="local", kind="cache", query={"name": {'$regex': f"{name}"}})
cached_file = test[0]['cached_file']
Console.info(f"Loading serialized model: {cached_file}")
StopWatch.start(f"Load pickle {name}")
deserialized_model = self._load_pickle(cached_file)
StopWatch.stop(f"Load pickle {name}")
time_taken = StopWatch.get(f"Load pickle {name}")
# TODO: figure out how useful the duration is and return to client if required
deserialized_model_dict = {
"model_name": name,
"model_object": deserialized_model,
"duration": time_taken # duration of deserialization function
}
return deserialized_model
def test_07_delete(self):
HEADING()
dst = f"{location}/destination"
src = path_expand("{dst}/created_dir")
StopWatch.start("delete")
self.p.delete(src)
StopWatch.stop("delete")
def test_recursive_delete(self):
home = self.sourcedir
src = f"a/a/b/c"
StopWatch.start("DELETE Sub-directory")
del_files = self.p.delete(src)
StopWatch.stop("DELETE Sub-directory")
assert len(del_files) > 0
def test_create_dir(self):
HEADING()
src = 'a/created_dir'
StopWatch.start("CREATE DIR")
directory = self.p.create_dir(src)
StopWatch.stop("CREATE DIR")
assert not directory or directory is None
def install(repos, dev=False):
repos = OrderedSet(repos)
for repo in repos:
StopWatch.start("install " + repo)
if dev:
banner(f"dev install -> {repo}")
Console.info(f"pip install -e .: {repo}")
print()
os.chdir(repo)
os.system("pip install -e .")
os.chdir("../")
StopWatch.status("install " + repo, True)
else:
banner(f"install -> {repo}")
Console.info(f"pip install: {repo}")
print()
os.system("pip install {repo}".format(repo=repo))
StopWatch.status("install " + repo, True)
StopWatch.stop("install " + repo)
def ping(self, processors=1):
StopWatch.start(f"total p={processors} c=1")
r = Host.ping(hosts, processors=processors, count=1)
StopWatch.stop(f"total p={processors} c=1")
StopWatch.status(f"total p={processors} c=1", r[0]['success'])
return r
def test_function(benchmark_obj):
os.system(f"cms set cloud={benchmark_obj.provider_name}")
thrown = None
benchmark_obj.setup_function()
StopWatch.start(benchmark_obj.test_name)
try:
data = benchmark_obj.function(
) #TODO allow inputs from args/kwargs too #e.g. stop, start need image name
StopWatch.stop(benchmark_obj.test_name)
valid = benchmark_obj.validation_function(data)
results = StopWatch.__str__()
except Exception as e:
print(e)
StopWatch.stop(benchmark_obj.test_name)
valid = False
thrown = e
sw = StopWatch.__str__().split(" ")
result = ComputeBenchmarkOutput(benchmark_obj.provider_name,
benchmark_obj.function_name, sw[1], sw[2],
valid, thrown if thrown else None)
return result
def test_stopwatch(self):
HEADING()
StopWatch.start("sleep 1")
time.sleep(1)
StopWatch.stop("sleep 1")
StopWatch.benchmark(sysinfo=True)
assert True
def test_status(self):
HEADING()
StopWatch.start("vdir status")
file = self.vdir.status(dir_or_name=self.file)
StopWatch.stop("vdir status")
assert file is not None
def test_list_bucket(self):
HEADING()
from cloudmesh.google.storage.Provider import Provider
provider = Provider(service=cloud)
StopWatch.start("test_list_bucket_google")
provider.list_bucket()
StopWatch.stop("test_list_bucket_google")
def test_config_subscriptable(self):
HEADING()
config = Config()
StopWatch.start("config_subscriptable")
data = config["cloudmesh"]["data"]["mongo"]
StopWatch.stop("config_subscriptable")
assert data is not None
def test_dictreplace(self):
HEADING()
config = Config()
spec = textwrap.dedent("""
cloudmesh:
profile:
name: Gregor
unordered:
name: "{cloudmesh.other.name}.postfix"
other:
name: "{cloudmesh.profile.name}"
""")
print(spec)
# spec = spec.replace("{", "{{")
# spec = spec.replace("}", "}}")
# print(spec)
StopWatch.start("dictreplace")
result = config.spec_replace(spec)
StopWatch.stop("dictreplace")
print(result)
data = yaml.load(result, Loader=yaml.SafeLoader)
pprint(data)
assert data["cloudmesh"]["unordered"]["name"] == "Gregor.postfix"
assert data["cloudmesh"]["other"]["name"] == "Gregor"
def config_n_load(self, n):
config = [None] * n
StopWatch.start("test_config_load n={n}".format(**locals()))
for i in range(1, n):
config[i] = Config()
pprint (config[i])
StopWatch.stop("test_config_load n={n}".format(**locals()))
def test_search(self):
config = Config()
StopWatch.start("search")
r = config.search("cloudmesh.cloud.*.cm.active", True)
StopWatch.stop("search")
pprint (r)
def test_set(self):
StopWatch.start("set")
config = Config()
config["cloudmesh.test.nested"] = "Gregor"
StopWatch.stop("set")
print(config["cloudmesh.test.nested"])
assert config["cloudmesh.test.nested"] == "Gregor"
def createDatasetGroup(self):
from cloudmesh.common.StopWatch import StopWatch
from StatusIndicator import StatusIndicator
import time
def create_uuid(project_name):
import random
id = random.randrange(100000)
return project_name + '_' + str(id)
self.DATASET_FREQUENCY = "D"
self.TIMESTAMP_FORMAT = "yyyy-MM-dd hh:mm:ss"
proj = 'timeseries'
self.project = create_uuid(proj)
print(self.project)
self.datasetName = self.project + '_ds'
self.datasetGroupName = self.project + '_dsg'
self.s3DataPath = "s3://" + self.bucket_name + "/" + self.key
StopWatch.start('to_dsg')
create_dataset_group_response = self.forecast_srv.create_dataset_group(
DatasetGroupName=self.datasetGroupName,
Domain="CUSTOM",
)
datasetGroupArn = create_dataset_group_response['DatasetGroupArn']
self.datasetGroupArn = datasetGroupArn
StopWatch.stop('to_dsg')
print(StopWatch.get('to_dsg'))
return self.datasetGroupArn
def test_collection(self):
HEADING()
StopWatch.start("vdir collection")
col = self.vdir.col
StopWatch.stop("vdir collection")
assert col.name == 'local-vdir'
def test_ls(self):
HEADING()
StopWatch.start("vdir ls")
results = self.vdir.ls(directory=None)
StopWatch.stop("vdir ls")
assert results is not None
def test_transfer_s3_to_azure(self):
HEADING()
source_CSP = "awss3"
target_CSP = "azure"
source_obj = str(self.file_name)
target_obj = str(Path("\\"))
stopwatch_tag = "Transfer awss3 to azure"
print("=======> ", source_obj)
print("=======> ", target_obj)
StopWatch.start(stopwatch_tag)
provider = Provider(source=source_CSP,
source_obj=source_obj,
target=target_CSP,
target_obj=target_obj)
result = provider.copy(source=source_CSP,
source_obj=source_obj,
target=target_CSP,
target_obj=target_obj,
recursive=True)
StopWatch.stop(stopwatch_tag)
pprint(result)
assert result is not None
def test_cd(self):
HEADING()
StopWatch.start("vdir cd")
self.vdir.cd(dirname=self.directory)
StopWatch.stop("vdir cd")
assert self.vdir.directory == self.directory
def exit_handler(signal_received, frame):
# Handle any cleanup here
StopWatch.start("exit")
print('SIGINT or CTRL-C detected. Exiting gracefully')
StopWatch.stop("exit")
exit(0)
def test_stopwatch_1(self):
HEADING()
StopWatch.start("stopwatch sleep 1")
time.sleep(0.1)
StopWatch.stop("stopwatch sleep 1")
StopWatch.status("stopwatch sleep 1", True)
assert True
def do_stopwatch(self, args, arguments):
"""
::
Usage:
stopwatch start TIMER
stopwatch stop TIMER
stopwatch print [TIMER]
stopwatch benchmark
Arguments:
TIMER the name of the timer
Description:
THIS IS NOT YET WORKING
starts and stops named timers and prints them
"""
t = arguments.TIMER
if arguments.start:
StopWatch.start(t)
elif arguments.stop:
StopWatch.stop(t)
elif arguments.benchmark:
StopWatch.benchmark()
elif arguments.print:
t = arguments.TIMER
if t is None:
print(StopWatch.__str__)
else:
StopWatch.print("# Timer " + t + ":", t)
def enlistImages(self):
banner(f"Fetching image list for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'ec2',
aws_access_key_id=self.ACCESS_KEY,
aws_secret_access_key=self.SECRET_KEY,
region_name=self.REGION_ID
)
StopWatch.start(f"Image list {self.cloudname}.")
image_list = ec2_instance.describe_images()
opDict= []
opDict = [{'ImageId':i.get('ImageId'), 'ImageType': i.get('ImageType'), 'Description': i.get('Description'), 'Name': i.get('Name'), 'State':i.get('State'),'Public': i.get('Public'),'CreationDate':i.get('CreationDate')} for i in image_list['Images']]
StopWatch.stop(f"Image list {self.cloudname}.")
t = format(StopWatch.get(f"Image list {self.cloudname}."), '.2f')
banner(f"Image list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} images fetched. Time taken {t} \nPrinting first 5 sample images:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Image_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Image list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def test_get(self):
config = Config()
key = "cloudmesh.version"
StopWatch.start(f"get({key})")
value = config[key]
StopWatch.stop(f"get({key})")
assert value is not None
def test_copy(self):
HEADING()
sources = ['local', 'awss3', 'azure', 'oracle', 'google']
# sources = ['local', 'oracle']
local_source = "~/.cloudmesh/storage/test"
sizes = [1]
for size in sizes:
file = f"size-{size}-file.txt"
print("0" * 100)
print(file)
print("0" * 100)
pass_flag = True
for source in sources:
targets = sources.copy()
targets.remove(source)
for target in targets:
storage = target
if source == "local":
src = str(Path(Path(local_source) / file))
else:
src = file
if target == "local":
dst = str(Path(Path(local_source) / file))
storage = source
elif target == "azure":
dst = '/'
else:
dst = file
print("0"*100)
provider = Provider(service=storage)
banner(f"copy {source}:{src} to {target}:{dst}")
texttag = f"copy {source}:{src} to {target}:{dst}"
StopWatch.start(texttag)
try:
response = provider.copy(f'{source}:{src}',
f'{target}:{dst}')
if response is None:
Console.error(f"NULL response for copy {source}:"
f"{src} to {target}:{dst}")
pass_flag = False
except Exception as e:
pass_flag = False
Console.error(f"Exception: copy {source}:{src} to "
f"{target}:{dst} - {e}")
StopWatch.stop(texttag)
print("0" * 100)
assert pass_flag
def test_search_b1_dir(self):
# search for b/b2.txt see that this one has even the dir in the search
src = '/a'
filename = '/b/b1.txt'
StopWatch.start("SEARCH file under a sub-dir --r")
search_files = self.p.search(self.p.service, src, filename, True)
StopWatch.stop("SEARCH file under a sub-dir --r")
assert search_files is not None
def test_linear_predict_benchmark(self):
StopWatch.start("LinearRegression_predict")
subprocess.call([
'cms', 'analytics', 'LinearRegression', 'predict',
'--X=[[0,0],[0,1],[0,2]]'
])
StopWatch.stop("LinearRegression_predict")
assert 1 == 1
def test_linear_score_benchmark(self):
StopWatch.start("LinearRegression_score")
subprocess.call([
'cms', 'analytics', 'LinearRegression', 'score',
'--X=user_input_data', '--y=user_output_data'
])
StopWatch.stop("LinearRegression_score")
assert 1 == 1
def test_linear_consturctor_benchmark(self):
StopWatch.start('LinearRegression_constructor')
subprocess.call([
'cms', 'analytics', 'LinearRegression', '--fit_intercept=true',
'--normalize=false'
])
StopWatch.stop('LinearRegression_constructor')
assert 1 == 1