def apkparse(self, fname):
"""
解析、获取apk信息
返回数据结构:
{
"pkg_name" : 'com.book_iuqwer',
"version_cdoe" : 1,
"version_name" : '1.0'
}
若找不到文件，将抛出OSError找不到文件或目录
若找解析apk信息错误，即apk文件不完整，将抛出ApkDumpFailError
"""
info = {}
try:
result = str(sh.head(self.aapt("dump", "badging", fname), n=1))
pattern = re.compile(
r"package: name='(.+)' versionCode='(\d*)' versionName='(.*)'")
mth = re.match(pattern, result)
info['pkg_name'] = mth.group(1)
try:
info['version_code'] = int(mth.group(2))
except Exception, e:
info['version_code'] = 0
info['version_name'] = mth.group(3)
return info
def make_ip_window():
H, W = screen.getmaxyx()
ip_win = screen.subwin(H / 2, W, H / 2 - HINT_WIDTH, 0)
ip_win.box()
try:
ip = sh.head(sh.awk(sh.getent("ahosts",
sh.hostname().strip()), "{print $1}"),
n="1").strip()
if ip == "127.0.0.1":
raise DHCPMisconfiguration()
except sh.ErrorReturnCode:
ip_win.addstr(1, 2, "===================================")
ip_win.addstr(2, 2, "Connectivity issues detected!")
ip_win.addstr(3, 2, "===================================")
ip_win.addstr(4, 2, "Check VM setup instructions")
ip_win.addstr(6, 2, "For details, see VM setup instructions")
except DHCPMisconfiguration:
ip_win.addstr(1, 2, "===================================")
ip_win.addstr(2, 2, "Connectivity issues detected!")
ip_win.addstr(3, 2, "===================================")
ip_win.addstr(4, 2, "Check connection of Host-only interface")
ip_win.addstr(5, 2, "and check DHCP is enabled for it")
ip_win.addstr(7, 2, "For details, see VM setup instructions")
else:
ip_win.addstr(1, 2, "To initiate your Hortonworks Sandbox session,")
ip_win.addstr(2, 2, "please open a browser and enter this address ")
ip_win.addstr(3, 2, "in the browser's address field: ")
ip_win.addstr(4, 2, "http://%s/" % ip)
def make_ip_window():
H, W = screen.getmaxyx()
ip_win = screen.subwin(H / 2, W, H / 2 - HINT_WIDTH, 0)
ip_win.box()
try:
ip = sh.head(
sh.awk(
sh.getent("ahosts", sh.hostname().strip()),
"{print $1}"),
n="1").strip()
if ip == "127.0.0.1":
raise DHCPMisconfiguration()
except sh.ErrorReturnCode:
ip_win.addstr(1,2,"===================================")
ip_win.addstr(2,2,"Connectivity issues detected!")
ip_win.addstr(3,2,"===================================")
ip_win.addstr(4,2,"Check VM setup instructions")
ip_win.addstr(6,2,"For details, see VM setup instructions")
except DHCPMisconfiguration:
ip_win.addstr(1,2,"===================================")
ip_win.addstr(2,2,"Connectivity issues detected!")
ip_win.addstr(3,2,"===================================")
ip_win.addstr(4,2,"Check connection of Host-only interface")
ip_win.addstr(5,2,"and check DHCP is enabled for it")
ip_win.addstr(7,2,"For details, see VM setup instructions")
else:
ip_win.addstr(1,2,"To initiate your Hortonworks Sandbox session,")
ip_win.addstr(2,2,"please open a browser and enter this address ")
ip_win.addstr(3,2,"in the browser's address field: ")
ip_win.addstr(4,2,"http://%s/" % ip)
def findImage(repository, tag):
container=""
try:
container = sh.awk("{print $3}", _in=sh.head("-n 1", _in=sh.grep(tag, _in=sh.docker("images", repository))))
except:
print "container not available"
return container.rstrip()
def get_appyters(appyter_path):
for path in map(os.path.dirname,
glob.glob(os.path.join(appyter_path, '*',
'appyter.json'))):
appyter = json.load(open(os.path.join(path, 'appyter.json'), 'r'))
yield dict(
appyter,
path=path,
long_description=open(os.path.join(path, 'README.md'), 'r').read(),
# find the oldest commit containing the appyter's appyter.json (follow for detecting renames)
creation_timestamp=str(
sh.tail(
sh.git.log(
'--follow',
r'--pretty=format:%aI',
'--',
os.path.join(path, 'appyter.json'),
_tty_out=False,
), '-n1')).strip(),
# find the most recent commit containing the appyter's directory
update_timestamp=str(
sh.head(
sh.git.log(
r'--pretty=format:%aI',
'--',
path,
_tty_out=False,
), '-n1')).strip(),
)
def findContainer(name):
container=""
try:
container = sh.awk("{print $1}", _in=sh.head("-n 1", _in=sh.grep(name, _in=sh.docker.ps())))
except:
print "container not available"
return container.rstrip()
def get_container_id():
"get container id"
id_ = sh.cut(sh.head(sh.cat("/proc/self/cgroup"), "-n", "1"), "-d", "/",
"-f4").strip()
if not id_:
return "unknown"
return id_
def _get_last_two_deploys(environment):
import sh
git = sh.git.bake(_tty_out=False)
pipe = git('tag')
pipe = sh.grep(pipe, environment)
pipe = sh.sort(pipe, '-rn')
pipe = sh.head(pipe, '-n2')
return pipe.strip().split('\n')
def findContainer(name):
container = ""
try:
container = sh.awk("{print $1}",
_in=sh.head("-n 1",
_in=sh.grep(name, _in=sh.docker.ps())))
except:
print "container not available"
return container.rstrip()
def _get_swarm_service_state(service_name):
docker = sh.Command("docker")
try:
state = sh.awk(
sh.tail(sh.head(docker(["service", "ps", service_name]), "-2"),
"-1"), "{print $6}").strip(' \t\n\r\'')
except:
state = "Not present"
return state
def get_patchtype(filepath):
mime = get_filetype(filepath)
mime_pre = mime.partition(';')[0]
if mime_pre == 'text/plain':
head = sh.head('-n1', filepath).strip()
if head == 'xxd_diff':
return 'xxd_diff'
else:
return 'patch'
elif mime.strip().endswith('=binary'):
return 'binary'
def mount_vm(self, img_src="", mount_path=""):
"""
Mount img_src at mount_path with kpartx
"""
if not os.path.exists(img_src):
sys.exit("Invalid image to mount: %s" % (img_src))
if not os.path.exists(mount_path):
sys.exit("Invalid mount path: %s" % (mount_path))
with sh.sudo:
sh.kpartx('-a', img_src)
mount_device=sh.head(sh.awk(sh.grep(sh.kpartx('-l', img_src), '-v', 'deleted'), '{print $1}'), '-1')
sh.mount("/dev/mapper/%s" % (mount_device.rstrip()), mount_path)
def _get_python_path(self):
_python_paths = [
sh.which('python'),
sh.which('python3'),
sh.which('python2')
]
python_paths = [str(path) for path in _python_paths if path]
if os.path.isfile('/usr/local/python-3.6.5/bin/python'):
python_paths.append('/usr/local/python-3.6.5/bin/python')
if os.path.isdir('/usr/local/Cellar/python'):
out = sh.find('/usr/local/Cellar/python',
'-regex',
'.*/bin/python3[0-9.]*$',
'-type',
'f',
_piped=True)
out = sh.sort(out, _piped=True)
python_paths.append(sh.head(out, '-n1').strip())
useable_pythons = []
python_paths_set = set()
for python_path in python_paths:
if python_path in python_paths_set:
continue
python_paths_set.add(python_path)
if os.path.realpath(python_path) in python_paths_set:
continue
python_paths_set.add(os.path.realpath(python_path))
useable_pythons.append(
(python_path, self._get_python_version(python_path)))
if len(useable_pythons) == 0:
print('Not found python!!')
sys.exit(1)
error = ''
while True:
message = '{}\n{}select python path [{}]: '.format(
'\n'.join([
'{}. {} (v{})'.format(i, *e)
for i, e in enumerate(useable_pythons)
]), error,
','.join([str(i) for i in range(len(useable_pythons))]))
num = int(input(message))
if num < 0 or num >= len(useable_pythons):
error = 'error: invalid input, try again!! '
continue
return useable_pythons[num]
class Test:
# assuming first - is the prefered cloud
cloudmesh_yaml = config_file("/cloudmesh.yaml")
print(cloudmesh_yaml)
cloud_label = sh.head(sh.fgrep("-", cloudmesh_yaml), "-n", "1")
cloud_label = cloud_label.replace(" - ", "").strip()
def setup(self):
"""setup azure"""
from cloudmesh.iaas.azure.cm_compute import azure
self.azure_cloud = azure("windows_azure")
def test_14_azure(self):
"""test azure"""
HEADING()
self.setup()
self.azure_cloud.refresh("images")
print(json.dumps(self.azure_cloud.dump('images'), indent=4))
def test_azure_services(self):
"""test azure services"""
HEADING()
self.setup()
self.azure_cloud.refresh("services")
print(json.dumps(self.azure_cloud.dump('services'), indent=4))
def test_create_azure_vm(self):
"""test to create azure vm"""
HEADING()
self.setup()
self.azure_cloud.vm_create()
print(json.dumps(self.azure_cloud.get_deployment(), indent=4))
def test_delete_azure_vm(self, name):
"""Test to tear down azure vm"""
HEADING()
self.setup()
self.azure_cloud.vm_delete(name)
def test_azure_deployments(self):
"""Test to list deployments"""
HEADING()
self.setup()
res = self.azure_cloud.list_deployments()
print(json.dumps(res, indent=4))
def make_status_window():
Height, Width = screen.getmaxyx()
status_win = screen.subwin(Height / 4 - Width_Factor / 2 , Width, Height / 4 , 0)
status_win.box()
try:
ip = sh.head(sh.awk(sh.getent("ahosts", sh.hostname().strip()),"{print $1}"),n="1").strip()
if ip == "127.0.0.1":
raise NetworkMisconfiguredException()
except sh.ErrorReturnCode:
status_win.addstr(1,2,"MapR-Platfora-Sandbox-For-Hadoop setup did not succseed.")
raise ServiceFailedtoStartException()
make_error_window()
except NetworkMisconfiguredException:
make_error_window()
else:
status_win.addstr(1,2,"MapR Services failed to start.", curses.A_BOLD)
make_error_window()
def _get_swarm_node_ip(container_name):
docker = sh.Command("docker")
state = "unknown"
count = 0
while state != "Running":
print "Waiting for container deployment..."
time.sleep(5)
state = _get_swarm_service_state(container_name)
count += 1
if count == 12:
print "Error: could not get swarm node IP"
exit(1)
ip = sh.awk(
sh.xargs(
sh.awk(
sh.tail(
sh.head(docker(["service", "ps", container_name]), "-2"),
"-1"), "{print $4}"), "host"),
"{print $NF}").strip(' \t\n\r\'')
return ip
def ytded(message, session):
if (session['youtube'] == True):
downloader = Youtubedl('')
downloader.fromDict(session['downloader'])
status = downloader.downloaded()
if (not status):
return "别急，还在下载"
else:
linkName = __file__[:__file__.find(__file__.split('/')[-1])]
linkName += 'static/youtubedl/'
fileName = sh.head(sh.ls('-t', linkName), '-1')
fileName = ''.join(i for i in fileName)
fileName = fileName[:fileName.find(' ')]
killProcess(downloader.pid)
bd = Baidu()
bd.upload(linkName + fileName, fileName)
return "下载完成，链接为： " + cfg.domain_youtubedl + fileName
else:
return "请先启动youtube"
def make_status_window():
Height, Width = screen.getmaxyx()
status_win = screen.subwin(Height / 4 - Width_Factor / 2, Width,
Height / 4, 0)
status_win.box()
try:
ip = sh.head(sh.awk(sh.getent("ahosts",
sh.hostname().strip()), "{print $1}"),
n="1").strip()
if ip == "127.0.0.1":
raise NetworkMisconfiguredException()
except sh.ErrorReturnCode:
status_win.addstr(1, 2, "_MAPR_BANNER_NAME_ setup did not succseed.")
raise ServiceFailedtoStartException()
make_error_window()
except NetworkMisconfiguredException:
make_error_window()
else:
status_win.addstr(1, 2, "MapR Services failed to start.",
curses.A_BOLD)
make_error_window()
def apkparse(self, fname):
"""
解析、获取apk信息
返回数据结构:
{
"pkg_name" : 'com.book_iuqwer',
"version_cdoe" : 1,
"version_name" : '1.0'
}
若找不到文件，将抛出OSError找不到文件或目录
若找解析apk信息错误，即apk文件不完整，将抛出ApkDumpFailError
"""
info = {}
try:
result = str(sh.head(self.aapt("dump" , "badging" , fname ), n=1))
pattern = re.compile(r"package: name='(.+)' versionCode='(\d*)' versionName='(.*)'")
mth = re.match( pattern , result )
info['pkg_name'] = mth.group(1)
try:
info['version_code'] = int(mth.group(2))
except Exception, e:
info['version_code'] = 0
info['version_name'] = mth.group(3)
return info
usage()
i = None
k = None
o = None
for opt, arg in opts:
if opt in ("-h", "--help"):
usage()
elif opt in ("-i", "--input"):
i = str(arg)
elif opt in ("-k", "--keyword"):
k = str(arg)
elif opt in ("-o", "--output"):
o = str(arg)
if not any(var is None for var in [i, k, o]):
return i, k, o
print("The parameters are not yet specified!")
usage()
def var_to_file(var_to_write, file_to_write):
file = open(file_to_write, 'w')
file.write(var_to_write)
file.close()
###################################################
inputFile, inputKeyWord, outputFile = main()
output = str(sh.head("-n 1", inputFile)) + str(sh.grep(inputKeyWord,
inputFile))
var_to_file(output, outputFile)
class Test:
# assuming first - is the prefered cloud
print(os.path.expandvars(cloudmesh_yaml=config_file("/cloudmesh.yaml")))
cloud_label = head(
fgrep("-", cloudmesh_yaml=config_file("/cloudmesh.yaml")), "-n", "1")
cloud_label = cloud_label.replace(" - ", "").strip()
def setup(self):
self.configuration = cm_config()
euca_clouds = []
for name in self.configuration.config['cloudmesh']['clouds'].keys():
if self.configuration.config['cloudmesh']['clouds'][name][
'cm_type'] == 'eucalyptus':
euca_clouds.append(name)
self.name = euca_clouds[0]
print("LOADING EUCA CLOUD", self.name)
self.cloud = eucalyptus(self.name)
print("LOADED CLOUD")
def tearDown(self):
pass
def start(self):
HEADING()
image = self.configuration.default(self.name)['image']
flavor = self.configuration.default(self.name)['flavor']
self.configuration.prefix = "gvonlasz-test"
self.configuration.incr()
name = self.configuration.vmname
print("STARTING IMAGE", name, image, flavor)
result = self.cloud.vm_create(name, flavor, image)
print(result)
def test_label(self):
HEADING()
print(self.cloud.label)
print(self.cloud.credentials)
assert True
# def test_01_limit(self):
# HEADING()
# print json.dumps(self.cloud.limits(), indent=4)
def test_info(self):
HEADING()
self.cloud.refresh('images')
print(json.dumps(self.cloud.dump('images'), indent=4))
# pprint(self.cloud.dump('images', with_manager=True))
pprint(self.cloud.images)
# doing a simple test as tiny is usually 512
# assert self.cloud.flavors['m1.tiny']['ram'] == 512
print("Currently running vms:", len(self.cloud.images))
# we assume cloud is always busy which may actually not true
# we shoudl start our own vm and than probe for it for now > 0 will do
assert self.cloud.images > 0
def test_list_flavors(self):
HEADING()
self.cloud.refresh('flavors')
print(json.dumps(self.cloud.dump('flavors'), indent=4))
# doing a simple test as tiny is usually 512
assert self.cloud.flavors['m1.small']['ram'] == 512
def test_list_images(self):
HEADING()
self.cloud.refresh('images')
print(json.dumps(self.cloud.dump('images'), indent=4))
# doing a simple test as tiny is usually 512
# assert self.cloud.flavors['m1.small']['ram'] == 512
assert True
def test_list_servers(self):
HEADING()
self.cloud.refresh('servers')
print(json.dumps(self.cloud.dump('servers'), indent=4))
# doing a simple test as tiny is usually 512
# assert self.cloud.flavors['m1.small']['ram'] == 512
assert True
def test_start_vm(self):
HEADING()
configuration = cm_config()
print("NAME", self.name)
print("Getting Flavours")
self.cloud.refresh('flavors')
flavor = configuration.default(self.name)['flavor']
print("Getting Images")
self.cloud.refresh('images')
image = configuration.default(self.name)['image']
print(self.cloud.flavors_cache)
print(self.cloud.images_cache)
print("STARTING IMAGE", image, flavor)
result = self.cloud.vm_create("gregor-test-001",
flavor_name=flavor,
image_id=image)
print(result)
assert len(result.keys()) > 0
def test_delete_vm(self):
HEADING()
configuration = cm_config()
print("NAME", self.name)
print("Getting Flavours")
self.cloud.refresh('flavors')
flavor = configuration.default(self.name)['flavor']
print("Getting Images")
self.cloud.refresh('images')
image = configuration.default(self.name)['image']
# print self.cloud.flavors_cache
# print self.cloud.images_cache
print("STARTING VM", image, flavor)
result = self.cloud.vm_create("gregor-test-del",
flavor_name=flavor,
image_id=image)
print(result)
print("DELETE VM", image, flavor)
self.cloud.refresh('servers')
result = self.cloud.vm_delete("gregor-test-del")
print(result)
assert len(result.keys()) > 0
def test_refresh(self):
HEADING()
self.cloud.refresh()
pprint(self.cloud.get(self.name))
assert self.cloud.images > 0
def test_05_print_vms(self):
HEADING()
self.cloud.refresh('servers')
print(json.dumps(self.cloud.dump('servers'), indent=4))
# we assume that there are always images running
assert len(self.cloud.servers) > 0
"""
def test_0??_usage(self):
result = self.cloud.usage("2000-01-01T00:00:00", "2013-12-31T00:00:00")
print json.dumps(result, indent=4)
assert ['Instances'] > 0
"""
def test_07_print_tables(self):
HEADING()
self.test_03()
table = cm_table()
columns = ["id", "name", "ram", "vcpus"]
table.create(self.cloud.flavors, columns, header=True)
print(table)
table = cm_table()
columns = ["id", "name", "ram", "vcpus"]
table.create(self.cloud.flavors, columns, format='HTML', header=True)
print(table)
table = cm_table()
columns = ["id", "name", "ram", "vcpus"]
table.create(self.cloud.flavors, columns, format='%12s', header=True)
print(table)
assert table is not None
"""
def test_07_start_delete_vm(self):
name ="%s-%04d" % (self.cloud.credential["OS_USERNAME"], 1)
out = self.cloud.vm_create(name, "m1.tiny", "6d2bca76-8fff-4d57-9f29-50378539b4fa")
pprint(out)
print json.dumps(out, indent=4)
key = out.keys()[0]
id = out[key]["id"]
print id
vm = self.cloud.vm_delete(id)
print vm
"""
def test_08_delete_vms_of_user(self):
HEADING()
self.cloud.refresh()
user_id = self.cloud.find_user_id(force=True)
vm_ids = self.cloud.find('user_id', user_id)
print("userid", user_id)
config = cm_config()
config.data['cloudmesh']['clouds'][
self.name]['credentials']['OS_USER_ID'] = user_id
config.write()
#
# delete all vms of the user
#
servers = self.cloud.servers
print(servers)
list = self.cloud.vms_delete_user()
self.cloud = openstack(self.cloud_label)
self.cloud.refresh()
self.cloud.info()
vm_ids = self.cloud.find('user_id', user_id)
self.cloud.info()
time.sleep(2)
self.cloud.refresh()
while len(vm_ids) > 0:
vm_ids = self.cloud.find('user_id', user_id)
self.cloud.refresh("servers")
self.cloud.info()
time.sleep(1)
print("vms", vm_ids)
assert vm_ids == []
def test_09_start_two_vms(self):
HEADING()
configuration = cm_config()
image = configuration.default(self.name)['image']
print("STARTING IMAGE", image)
result = self.cloud.vm_create("gregor-test-001", "m1.tiny", image)
# print result
result = self.cloud.vm_create("gregor-test-002", "m1.tiny", image)
# print result
self.cloud.refresh()
self.cloud.info()
config = cm_config()
print("CONFIG")
user_id = config.data['cloudmesh']['clouds'][
self.name]['credentials']['OS_USER_ID']
print(user_id)
vm_ids = self.cloud.find('user_id', user_id)
print(vm_ids)
assert len(vm_ids) == 2
def test_10_list_user_vms(self):
HEADING()
list = self.cloud.vms_user(refresh=True)
pprint(list)
def test_11_refresh_all(self):
HEADING()
self.cloud.refresh()
self.cloud.info()
def test_12_print_states(self):
HEADING()
self.cloud.refresh()
print(self.cloud.states)
search_states = ('ACTIVE', 'PAUSED')
state = 'ACTIVE'
userid = None
print(state in search_states)
# self.cloud.display(search_states, userid)
# print json.dumps(self.cloud.servers, indent=4)
self.cloud.display_regex("vm['status'] in ['ACTIVE']", userid)
print(json.dumps(self.cloud.dump('servers'), indent=4))
# self.cloud.display_regex("vm['status'] in ['ERROR']", userid)
# print json.dumps(self.cloud.servers, indent=4)
def test_13_meta(self):
HEADING()
self.clean()
image = self.configuration.default(self.name)['image']
flavor = self.configuration.default(self.name)['flavor']
self.configuration.prefix = "gvonlasz-test"
self.configuration.incr()
name = self.configuration.vmname
print(name)
result = self.cloud.vm_create(name, flavor, image)
id = result['id']
print(id)
result = self.cloud.wait(id, 'ACTIVE')
result = self.cloud.set_meta(id, {"owner": "gregor"})
print("RESULT", result)
meta = self.cloud.get_meta(id)
print(meta)
def info(self):
HEADING()
self.cloud.refresh()
self.cloud.info()
def clean(self):
HEADING()
self.cloud.refresh()
self.cloud.info()
user_id = self.cloud.find_user_id()
print("Cleaning", user_id)
list = self.cloud.vms_delete_user()
print("Cleaning", list)
vm_ids = self.cloud.find('user_id', user_id)
while len(vm_ids) > 0:
vm_ids = self.cloud.find('user_id', user_id)
self.cloud.refresh("servers")
self.cloud.info()
time.sleep(1)
print("vms", vm_ids)
assert vm_ids == []
dict(
path=path,
long_description=open(os.path.join(path, 'README.md'), 'r').read(),
# find the oldest commit containing the appyter's appyter.json (follow for detecting renames)
creation_timestamp=str(
sh.tail(
sh.git.log(
'--follow',
r'--pretty=format:%aI',
'--',
os.path.join(path, 'appyter.json'),
_tty_out=False,
), '-n1')).strip(),
# find the most recent commit containing the appyter's directory
update_timestamp=str(
sh.head(
sh.git.log(
r'--pretty=format:%aI',
'--',
path,
_tty_out=False,
), '-n1')).strip(),
**json.load(open(os.path.join(path, 'appyter.json'), 'r')),
)
for path in map(os.path.dirname,
glob.glob(os.path.join(appyter_path, '*', 'appyter.json')))
]
if __name__ == '__main__':
print(json.dumps(appyters))
class Test:
# assuming first - is the prefered cloud
cloudmesh_yaml = os.path.expandvars("$HOME/.futuregrid/cloudmesh.yaml")
print cloudmesh_yaml
cloud_label = head(fgrep("-", cloudmesh_yaml), "-n", "1")
cloud_label = cloud_label.replace(" - ", "").strip()
def setup(self):
"""setup the test"""
print "CONFIG"
self.configuration = cm_config()
print "OK"
self.name = self.configuration.active()[0]
print "ACTIVE CLOUD", self.name
self.cloud = openstack(self.name)
print "PPPP"
self.cloud.get_token()
pp.pprint(self.cloud.user_token)
print "LOADED CLOUD"
"""
# For multiple clouds
# e.g. india-essex and sierra-grizzly
self.names = self.configuration.active()
if len(self.names) > 1:
print "ACTIVE CLOUDS", ",".join(self.names)
self.clouds = []
for name in self.names:
self.clouds.append(openstack(name))
self.clouds[-1].get_token()
print "LOADED CLOUDS"
"""
def test_00_label(self):
"""test the label"""
HEADING()
print self.cloud_label
assert self.cloud.label == self.cloud_label
def test_01_limit(self):
"""different way to get limits"""
HEADING()
print json.dumps(self.cloud.limits(), indent=4)
def test_02_info(self):
"""get some infor about the cloud"""
HEADING()
self.cloud.refresh('images')
print json.dumps(self.cloud.dump('images'), indent=4)
# pp.pprint(self.cloud.dump('images', with_manager=True))
pp.pprint(self.cloud.images)
# doing a simple test as tiny is usually 512
# assert self.cloud.flavors['m1.tiny']['ram'] == 512
print "Currently running vms:", len(self.cloud.images)
# we assume cloud is always busy which may actually not true
# we shoudl start our own vm and than probe for it for now > 0 will do
assert self.cloud.images > 0
def test_03_flavors(self):
"""get the flavor"""
HEADING()
self.cloud.refresh('flavors')
print json.dumps(self.cloud.dump('flavors'), indent=4)
# doing a simple test as tiny is usually 512
assert self.cloud.flavor('m1.tiny')['ram'] == 512
def test_04_start_vm(self):
"""start a vm"""
HEADING()
configuration = cm_config()
image = configuration.default(self.name)['image']
keys = configuration.userkeys()
key_name = keys["default"]
key_content = keys[key_name]
# print key_name
# print key_content
print "STARTING IMAGE", image
meta = {"cmtag": "testing tag from creation via rest api"}
result = self.cloud.vm_create("fw-test-by-post-003",
"2",
image,
key_name="grizzlykey",
meta=meta)
# result = self.cloud.vm_create("fw-test-by-post-003", "100", image, meta=meta)
pp.pprint(result)
assert len(result.keys()) > 0
def test_05_get_public_ip(self):
"""get an ip"""
HEADING()
print "Obtaining public ip......"
ip = self.cloud.get_public_ip()
print "this is the ip returned:--->%s<---" % ip
def test_06_assign_public_ip(self):
"""assign an ip"""
HEADING()
print "Associate a public ip to an running instance......"
# serverid = "cc9bd86b-babf-4760-a5cd-c1f28df7506b"
# ip = "fakeip" # {u'itemNotFound': {u'message': u'floating ip not found', u'code': 404}}
# ip = "198.202.120.175"
serverid = "b088e764-681c-4448-b487-9028b23a729e"
print "In process of obtaining a new ip..."
ip = self.cloud.get_public_ip()
print "Got a new ip as: %s, now associating the ip..." % ip
print self.cloud.assign_public_ip(serverid, ip)
def test_07_list_public_ips(self):
"""list the ips"""
HEADING()
print "List all public ips allocated to the current account..."
ips = self.cloud.list_allocated_ips()
ips_id_to_instance = {}
for ip in ips:
ips_id_to_instance[ip['id']] = ip['instance_id']
print ips_id_to_instance
def test_08_release_public_ips(self):
"""release the ips"""
HEADING()
print "Release all public ips allocated to the current account but not being used..."
print "Before releasing, we have..."
self.test04_list_public_ips()
print "releasing..."
self.cloud.release_unused_public_ips()
print "after releasing..."
self.test04_list_public_ips()
def test_09_print_vms(self):
"""print the servers"""
HEADING()
self.cloud.refresh('servers')
print json.dumps(self.cloud.dump('servers'), indent=4)
# we assume that there are always images running
assert len(self.cloud.servers) > 0
def test_10_refresh(self):
"""refresh"""
HEADING()
self.cloud.refresh()
pp.pprint(self.cloud.get(self.name))
assert self.cloud.images > 0
def test_11_print_tables(self):
"""print a table"""
HEADING()
self.test_03()
table = cm_table()
columns = ["id", "name", "ram", "vcpus"]
table.create(self.cloud.flavors, columns, header=True)
print table
table = cm_table()
columns = ["id", "name", "ram", "vcpus"]
table.create(self.cloud.flavors, columns, format='HTML', header=True)
print table
table = cm_table()
columns = ["id", "name", "ram", "vcpus"]
table.create(self.cloud.flavors, columns, format='%12s', header=True)
print table
assert table is not None
def test_12_start_delete_vm(self):
"""delete a vm"""
print "no test was being done"
'''
name ="%s-%04d" % (self.cloud.credential["OS_USERNAME"], 1)
out = self.cloud.vm_create(name, "m1.tiny", "6d2bca76-8fff-4d57-9f29-50378539b4fa")
pp.pprint(out)
print json.dumps(out, indent=4)
key = out.keys()[0]
id = out[key]["id"]
print id
'''
# id = "d9e73d16-a85e-47bf-8ab4-19d7955622db"
# id = "2a00332d-6a4a-4f7e-ad46-751438cc4d5e"
# id = "af46b3cd-99d9-4609-8c27-c481b0227e15"
# id="fakeid"
# id="b1279fa1-390a-41f8-b5d2-ad9b26cfb48a"
# ret=self.cloud.vm_delete(id)
# print "--->%s<---" % ret
def test_13_delete_vms_of_user(self):
"""delete a vm of a specific user"""
HEADING()
self.cloud.refresh()
user_id = self.cloud.find_user_id(force=True)
vm_ids = self.cloud.find('user_id', user_id)
print "userid", user_id
config = cm_config()
config.data['cloudmesh']['clouds'][
self.name]['credentials']['OS_USER_ID'] = user_id
config.write()
#
# delete all vms of the user
#
servers = self.cloud.servers
print servers
list = self.cloud.vms_delete_user()
self.cloud = openstack(self.cloud_label)
self.cloud.refresh()
self.cloud.info()
vm_ids = self.cloud.find('user_id', user_id)
self.cloud.info()
time.sleep(2)
self.cloud.refresh()
while len(vm_ids) > 0:
vm_ids = self.cloud.find('user_id', user_id)
self.cloud.refresh("servers")
self.cloud.info()
time.sleep(1)
print "vms", vm_ids
assert vm_ids == []
def test_14_start_two_vms(self):
"""start two vms"""
HEADING()
configuration = cm_config()
image = configuration.default(self.name)['image']
print "STARTING IMAGE", image
result = self.cloud.vm_create("gregor-test-001", "m1.tiny", image)
# print result
result = self.cloud.vm_create("gregor-test-002", "m1.tiny", image)
# print result
self.cloud.refresh()
self.cloud.info()
config = cm_config()
print "CONFIG"
user_id = config.data['cloudmesh']['clouds'][
self.name]['credentials']['OS_USER_ID']
print user_id
vm_ids = self.cloud.find('user_id', user_id)
print vm_ids
assert len(vm_ids) == 2
def test_15_list_user_vms(self):
"""list user and vms"""
HEADING()
list = self.cloud.vms_user(refresh=True)
pp.pprint(list)
def test_16_refresh_all(self):
"""refresh all"""
HEADING()
self.cloud.refresh()
self.cloud.info()
def test_17_print_states(self):
"""print states"""
HEADING()
self.cloud.refresh()
print self.cloud.states
search_states = ('ACTIVE', 'PAUSED')
state = 'ACTIVE'
userid = None
print state in search_states
# self.cloud.display(search_states, userid)
# print json.dumps(self.cloud.servers, indent=4)
self.cloud.display_regex("vm['status'] in ['ACTIVE']", userid)
print json.dumps(self.cloud.dump('servers'), indent=4)
# self.cloud.display_regex("vm['status'] in ['ERROR']", userid)
# print json.dumps(self.cloud.servers, indent=4)
def test_18_meta(self):
"""test metadata"""
HEADING()
self.clean()
image = self.configuration.default(self.name)['image']
flavor = self.configuration.default(self.name)['flavor']
self.configuration.prefix = "gvonlasz-test"
self.configuration.incr()
name = self.configuration.vmname
print name
result = self.cloud.vm_create(name, flavor, image)
id = result['id']
print id
result = self.cloud.wait(id, 'ACTIVE')
result = self.cloud.set_meta(id, {"owner": "gregor"})
print "RESULT", result
meta = self.cloud.get_meta(id)
print meta
def test_19_list_secgroup(self):
"""list security group"""
pp.pprint(self.cloud.list_security_groups())
def test_20_create_secgroup(self):
"""test security group"""
# print "defining a group"
#mygroup = Ec2SecurityGroup("testSecGroupCM")
mygroup = Ec2SecurityGroup("default")
# print "defining a security rule"
# rule1 = Ec2SecurityGroup.Rule(8088, 8088)
# rule2 = Ec2SecurityGroup.Rule(9090, 9099, "UDP")
# rules = [rule1, rule2]
# print self.cloud.create_security_group(mygroup, rules)
# mygroup.set_rules(rules)
# print self.cloud.create_security_group(mygroup)
groupid = self.cloud.find_security_groupid_by_name(mygroup.name)
# print groupid
assert groupid is not None
#rule3 = Ec2SecurityGroup.Rule(5000, 5000)
rule3 = Ec2SecurityGroup.Rule(22, 22)
rule4 = Ec2SecurityGroup.Rule(-1, -1, 'ICMP')
print self.cloud.add_security_group_rules(groupid, [rule3, rule4])
groupid = self.cloud.find_security_groupid_by_name(
"dummy_name_not_exist")
print groupid
assert groupid is None
def test_21_usage(self):
"""test usage"""
# by default, expect getting tenant usage info
pp.pprint(self.cloud.usage())
serverid = "e24807f6-c0d1-4cd0-a9d6-14ccd06a5c79"
# getting usage data for one specific server
pp.pprint(self.cloud.usage(serverid=serverid))
pp.pprint(self.cloud.usage(serverid="fakeserverid"))
def test_22_ks_get_extensions(self):
"""test getting extensions"""
pp.pprint(self.cloud.ks_get_extensions())
def test_23_keypair_add(self):
keyname = "skwind"
keycontent = "ssh-dss AAAAB3NzaC1kc3MAAACBAPkCkTkLqyqpmjYSU0P6cLMfuY6YPrZr5NkpVVuK9nLEKxdV3oAL1EOhTvgdve6hAVENX76fJFqUeBL1POBvsV92OH3e84cwwUiXBoQ9VrFtAn0l1tsdKkiEadWakCjxp0DqkjWJz1sLDzmB37QqO16T3KI5+yFviJPkn/LUYTMrAAAAFQCbgsFrwjHQqIlOhAKwW7tCt+M+VwAAAIEArgHogg4cLFS9uNtjP4H8vz6f+kYJ2zpHwnBMA7caO9IyFwYz/0BTjF4J/qD+Gwra+PKL7llQJlOAG1Odg79RRqPgk/4LN5KuXNkNbL9GZhcqdxD+ZpUtVjs4WLXA0C1t53CQvNKhkCKEZMTvS603rfBCJ8SBc4d4x6fAD7I6+fsAAACBALuHfiLksJR9xyexDNQpmtK12aIX+xZ+ryLEGbHGd0QIwskdnB9tqFH0S9NZg//ywSqntV9PlEqacmjEsfWojlM30b1wIJl0xCa+5eZiwFm2Xfsm4OhH0NE32SCb+Zr3ho4tcizpR9PVSoTxkU97rnGj1PDrjf1ZsuaG0Dr6Fzv3"
pp.pprint(self.cloud.keypair_add(keyname, keycontent))
def test_24_keypair_list(self):
pp.pprint(self.cloud.keypair_list())
def start(self):
"""start image"""
HEADING()
image = self.configuration.default(self.name)['image']
flavor = self.configuration.default(self.name)['flavor']
self.configuration.prefix = "gvonlasz-test"
self.configuration.incr()
name = self.configuration.vmname
print "STARTING IMAGE", name, image, flavor
result = self.cloud.vm_create(name, flavor, image)
print result
def info(self):
"""info"""
HEADING()
self.cloud.refresh()
self.cloud.info()
def clean(self):
"""clean"""
HEADING()
self.cloud.refresh()
self.cloud.info()
user_id = self.cloud.find_user_id()
print "Cleaning", user_id
list = self.cloud.vms_delete_user()
print "Cleaning", list
vm_ids = self.cloud.find('user_id', user_id)
while len(vm_ids) > 0:
vm_ids = self.cloud.find('user_id', user_id)
self.cloud.refresh("servers")
self.cloud.info()
time.sleep(1)
print "vms", vm_ids
assert vm_ids == []
def test_24_get_extensions(self):
"""test geting the cloud extensions"""
HEADING()
print json.dumps(self.cloud.get_extensions(), indent=4)
assert True
def test_25_get_users(self):
"""test to get the users"""
HEADING()
self.cloud.refresh("users")
# print json.dumps(self.cloud.get_users(), indent=4)
print json.dumps(self.cloud.users, indent=4)
assert True
def test_26_get_users_all(self):
"""testto get all information for the users"""
for name in self.configuration.active():
HEADING()
cloud = openstack(name)
cloud.refresh("users")
print json.dumps(cloud.users, indent=4)
assert True
def test_27_get_limits(self):
"""test to get the limits"""
HEADING()
print json.dumps(self.cloud.get_limits(), indent=4)
assert True
def test_28_get_servers(self):
"""test to get the servers"""
HEADING()
print json.dumps(self.cloud.get_servers(), indent=4)
assert True
def test_29_get_flavors(self):
"""test to get the flavors"""
HEADING()
# self.cloud.refresh('flavors')
# print json.dumps(self.cloud.dump('flavors'), indent=4)
print json.dumps(self.cloud.get_flavors(), indent=4)
assert True
def test_30_get_images(self):
"""test to get the images"""
HEADING()
self.cloud.refresh('images')
print json.dumps(self.cloud.dump('images'), indent=4)
print json.dumps(self.cloud.get_images(), indent=4)
assert True
"""
def num_monitors():
return int(
sh.head(sh.xrandr('--listmonitors'), '-1').stdout.decode('utf-8')[-2])
import sh as bada
x = bada.curl("http://ndtv.com", "-v")
print x
print("the number of files in my currenyt directory is : \n")
#print ( bada.wc(bada.find("/home/amardeep/"), "-l"))
print(bada.grep(bada.dmidecode(), "AMD"))
print("printing my current processor and memory information: \n")
print(bada.tail(bada.head("-n", 130, "dmi.txt"), "-n", 20))
print "getting current directory \n", bada.pwd()
DATA_BASES = [el.strip() for el in DATA_BASES]
DATA_BASES = DATA_BASES[
1:] # first entry is 'Database' which is not a Database
DATA_BASES += ['All-Databases']
DATA_BASES = ['trading_oanda_d1']
DATESTAMP = sh.date("+%Y-%m-%d_%H:%M").strip()
for DB in DATA_BASES:
for DD in [DATA_DIR, LOG_DIR]:
# step a): delete all except the latest two files for each database
print(f'database: {DB}; dir: {DD}')
a = sh.find(DATA_DIR, '-maxdepth', '1', '-type', 'f', '-regextype',
'sed', '-regex', f'^/.*{DB}\-[0-9].*', '-printf',
'%Ts\t%p\n')
b = sh.sort(a, '-n')
c = sh.head(b, '-n', '-2')
d = sh.cut(c, '-f', '2-')
print(d.strip())
e = sh.xargs(d, 'rm', '-rf')
# step b): export the databases
FILENAME = Path.joinpath(DATA_DIR, f'{DB}-{DATESTAMP}.sql.gz')
print(f'FILENAME: {FILENAME}')
LOGFILENAME = Path.joinpath(LOG_DIR, f'{DB}-{DATESTAMP}.log')
print(f'LOGFILENAME: {LOGFILENAME}')
# cmd = "mysqldump -v --single-transaction --quick --lock-tables=false ${DB} 2>'${LOGFILENAME}' | pigz > '${FILENAME}' "
# sh.mysqldump('-v', '--single-transaction', '--quick', '--lock-tables=false', DB, _out=FILENAME, _err=LOGFILENAME)
sh.ls(DATA_DIR, _out=FILENAME)
print()
# b = sh.head(sh.sort(sh.find(DATA_DIR, '-type', 'd', '-regextype', 'sed', '-regex', '^/.*testDB\-[0-9].*', '-printf', '%Ts\t%p\n'), '-n'), '-n', '-2')
component_dir = join(THIS_DIR, comp)
if exists(component_dir) and isdir(component_dir): pass
else: raise Exception("component %r doesn't exist" % comp)
# do the actual building
comp_module = __import__(comp, "make")
if comp == "agent":
comp_module.build(version, clio.branch, clio.agg_url, brand=clio.brand, extension=clio.extension)
else:
comp_module.build(version, clio.branch)
build_dir = "%s/build/deb/*.deb" % comp
deb_file = head(ls(glob(build_dir), "-t1"), n=1).strip()
deb_filename = basename(deb_file)
# Update the last build version
last_build_versions[build_key] = build
if clio.repo and clio.repo == 'appliance':
print "Adding package %s to the %s repository" % (deb_file, clio.repo)
os.system("scp %s [email protected]:/tmp" % deb_file)
os.system("ssh [email protected] /usr/bin/reprepro -v -b /home/repo includedeb stable /tmp/%s" % deb_filename)
elif clio.repo and clio.repo == 'agent':
print "Adding agent packages to repository"
print "NEEDS TO BE IMPLEMENTED!"
elif clio.repo:
def __update_network_interface(self):
self.__network_interface = sh.awk(
sh.head(sh.netstat("-nr"), '-n5'), '/default/ {print $6}').strip()
# All Rights Reserved MapR
import curses
import sh
import subprocess
import os
screen = None
Width_Factor = 4
ip = sh.head(sh.awk(sh.getent("ahosts", sh.hostname().strip()), "{print $1}"), n="1").strip()
if not os.path.exists("/vmware"):
ip = "127.0.0.1"
ssh_cmd = ""
if ip == "127.0.0.1":
ssh_cmd = "ssh mapr@localhost -p 2222"
else:
ssh_cmd = "ssh mapr@%s" % (ip)
class NetworkMisconfiguredException(Exception):
pass
class ServiceFailedtoStartException(Exception):
pass
def make_welcome_window():
Height, Width = screen.getmaxyx()
welcome_win = screen.subwin(Height / 2 - Width_Factor - 2, Width, 0, 0)
#All Rights Reserved MapR
import curses
import sh
import subprocess
import os
screen = None
Width_Factor = 4
ip = sh.head(sh.awk(sh.getent("ahosts",
sh.hostname().strip()), "{print $1}"),
n="1").strip()
if not os.path.exists("/vmware"):
ip = "127.0.0.1"
ssh_cmd = ""
if ip == "127.0.0.1":
ssh_cmd = "ssh mapr@localhost -p 2222"
else:
ssh_cmd = "ssh mapr@%s" % (ip)
class NetworkMisconfiguredException(Exception):
pass
class ServiceFailedtoStartException(Exception):
pass
def make_welcome_window():
