def scp(from_path, to_path, is_local=True, is_receive=False, use_env_host=True):
if is_receive:
target = CONF._remote_tmp_dir + from_path
target_dir = target.rsplit('/', 1)[0]
run('mkdir -p {0}'.format(target_dir))
cmd = 'scp -P {0} -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" {1}@{2}:{3} {4}'.format(
api.env.port, api.env.user, api.env.host, target, to_path)
if CONF.user and CONF.password:
result = expect(
cmd,
[['* password:'******'{0}\\n'.format(CONF.password)]],
is_local=is_local)
else:
result = local(cmd)
return result
else:
cmd = 'scp -P {0} -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" {1} {2}@'.format(
api.env.port, from_path, api.env.user)
if use_env_host:
cmd += '{0}:'.format(api.env.host)
if is_local:
tmp_target = CONF._remote_tmp_dir + to_path
tmp_target_dir = tmp_target.rsplit('/', 1)[0]
run('mkdir -p {0}'.format(tmp_target_dir))
cmd += tmp_target
if CONF.user and CONF.password:
result = expect(
cmd,
[['* password:'******'{0}\\n'.format(CONF.password)]],
is_local=is_local)
else:
result = local(cmd)
sudo('cp {0} {1}'.format(tmp_target, to_path))
return result
else:
cmd += to_path
return run(cmd)
def scp(from_path,
to_path,
is_local=True,
is_receive=False,
use_env_host=True):
if is_receive:
cmd = 'scp -P {0} -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" {1}@{2}:{3} {4}'.format(
api.env.port, api.env.user, api.env.host, from_path, to_path)
if CONF.user and CONF.password:
result = expect(cmd,
[['* password:'******'{0}\\n'.format(CONF.password)]],
is_local=is_local)
else:
result = local(cmd)
return result
else:
cmd = 'scp -P {0} -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" {1} {2}@'.format(
api.env.port, from_path, api.env.user)
if use_env_host:
cmd += '{0}:'.format(api.env.host)
if is_local:
tmp_target = CONF._remote_tmp_dir + to_path
tmp_target_dir = tmp_target.rsplit('/', 1)[0]
run('mkdir -p {0}'.format(tmp_target_dir))
cmd += tmp_target
result = api.put(from_path, tmp_target)
sudo('cp {0} {1}'.format(tmp_target, to_path))
return result
else:
cmd += to_path
return run(cmd)
import base
tab=base.IdentifierTable()
import sys
argumentList = sys.argv
if len(argumentList)>1:
fext= argumentList[1].split('.')
if len(fext)>1 and fext[1]=='zs':
f = open(argumentList[1], "r")
res,err=base.run(f.read(),tab)
if err: print(err.asString())
else:
print('File format Not supported')
else:
while True:
code=input("ZETSU >>>")
res,err=base.run(code,tab)
if err: print(err.asString())
import os
import base
import importlib
if __name__ == "__main__":
my_dir_name = os.path.dirname(os.path.realpath(__file__))
base.config.load_from_yaml(my_dir_name + f'/config.singleservice.yaml')
importlib.import_module('users.api.users')
importlib.import_module('lookup.user_permissions')
importlib.import_module('contacts.api.contacts')
base.run(debug=True)
"""
import os
import string
for root, dirs, files in os.walk(
os.path.realpath(__file__ + '/../../../')):
upperfiles = [
os.path.join(root, f) for f in files
if f.lower() != f and f.endswith('.py')
]
self.assertTrue(
len(upperfiles) == 0,
"Python files with upper-case letters detected" +
str(upperfiles))
dashfiles = [
os.path.join(root, f) for f in files
if '-' in f and f.endswith('.py')
]
self.assertTrue(
len(dashfiles) == 0,
"Python files with dashes detected" + str(dashfiles))
def suite():
suite = unittest.TestSuite()
suite.addTest(TestPyFilenames())
return suite
if __name__ == "__main__":
base.run(suite())
import base
class FifoScheduler(base.Scheduler):
def interrupt(self, time, process):
if process is not None:
self.ready_processes.append(process)
if self.currently_running is not None:
if time == self.currently_running.end_time:
self._add_to_history(self.currently_running,
self.currently_running.start_time,
self.currently_running.end_time)
self.currently_running = None
if self.currently_running is None:
if len(self.ready_processes):
self.currently_running = self.ready_processes[0]
self.currently_running.start_time = time
self.currently_running.end_time = (
time + self.currently_running.burst_time)
del self.ready_processes[0]
if self.currently_running is None:
return None
else:
return self.currently_running.end_time
base.run(FifoScheduler)
ambari, iid = self.deploy_dev()
stdout = ambari.run("echo Hello world from $HOSTNAME")
self.assertTrue("ambari" in stdout.lower() or "test-" in stdout.lower(),
"Unable to contact " + ' '.join(ambari.sshcmd()) + "\n" + stdout)
self.pull(ambari)
self.wait_for_ambari(ambari)
# test other features of ambari.run
stdout = ambari.run("ls -l '/opt'")
stdout = ambari.run('ls -l "/opt"')
self.assertTrue("lost+found" in stdout, "No /opt directory :$ see " + ' '.join(ambari.sshcmd()))
ma = lD.multiremotes(["ssh:root@" + ambari.host], access_key=ambari.access_key)
stdout = ma.run("ls -l '/opt'")
self.assertTrue("lost+found" in stdout, "No /opt directory :$ see " + ' '.join(ambari.sshcmd()))
stdout = lD.run_quiet(deploy_dir + "/aws/add_ebsvol_to_instance.py --not-strict " + iid +
' \'{"Mount": "/tmp/testdir1", "Size": 1, "Attach": "/dev/sdg"}\'')
def suite(verbose=False):
suite = unittest.TestSuite()
test = DepCentos()
test.debug = verbose
suite.addTest(test)
return suite
if __name__ == "__main__":
verbose = False
if "--verbose" in sys.argv:
verbose = True
base.run(suite(verbose))
if "--verbose" in sys.argv:
verbose = True
sys.argv = [s for s in sys.argv if s != "--verbose"]
branch = "__local__"
if "--branch" in sys.argv:
branch = "__service__"
sys.argv = [s for s in sys.argv if s != "--branch"]
if "--this-branch" in sys.argv:
branch = "__local__"
sys.argv = [s for s in sys.argv if s != "--this-branch"]
if "--prod" in sys.argv:
clustername = 'prod'
sys.argv = [s for s in sys.argv if s != "--prod"]
if len(sys.argv) < 2:
raise KeyError("You must specify which blueprint/cluster to test")
service = sys.argv[1]
test = TestCluster()
if service == "FREEIPA":
test = TestFreeIPACluster()
test.service = service
test.debug = verbose
test.clustername = clustername
test.branch = branch
test.branchtype = branch
test.checklist = []
if len(sys.argv) > 2:
test.checklist = sys.argv[2:]
suite = unittest.TestSuite()
suite.addTest(test)
base.run(suite)
def run():
'''Parser Runtime Function
Here is where we will be actually running all fo the code below. This is
considered to be boilerplate and should be in every parser.
'''
base.run(Parser())
title = strip_non_ascii(product.xpath('.//*/h4/text()')[0])
price = product.xpath('.//*/span[@class="value"]/@content')
price = price[-1] if price else "N/A"
opt_text = product.xpath('.//div[@data-color]/@data-color')
opt_links = product.xpath('.//div[@data-color]/a/@href')
href_prefix = 'https://www.patagonia.com'
links = [{
'href': href_prefix + href,
'text': text
} for href, text in zip(opt_links, opt_text)]
data.append({
"title": title,
"links": links,
"price": price
})
logging.debug("fetched {} for {}".format(len(title), url))
except IndexError as e:
logging.error(e)
write_html('patagonia-error.html', page)
post_to_slack('error encountered for patagonia', cfg.slack)
sys.exit(1)
if not product_elements:
break
return data
if __name__ == "__main__":
while True:
run(CRAWLER, scrape_patagonia)
time.sleep(CHECK_INTERVAL_SECS)
from lxml import html
import time
CRAWLER = 'icebreaker-mens'
CHECK_INTERVAL_SECS = 1200
def scrape_icebreaker(cfg):
data = []
page = requests.get(cfg.url)
tree = html.fromstring(page.content)
product_elements = tree.xpath('//*/div[contains(@class, "product-tile")]')
try:
for product in product_elements:
title = strip_non_ascii(product.xpath('.//div[@class="name"]/a/text()')[0])
link = product.xpath('.//div[@class="name"]/a/@href')[0]
price = product.xpath('.//div[@class="product-price"]//span/text()')
price = price[-1].strip() if price else "N/A"
data.append({"title": title, "link": link, "price": price})
except IndexError as e:
logging.error(e)
write_html('icebreaker-error.html', page)
return data
if __name__ == "__main__":
while True:
run(CRAWLER, scrape_icebreaker)
time.sleep(CHECK_INTERVAL_SECS)
class SjfScheduler(base.Scheduler):
def interrupt(self, time, process):
if process is not None:
self.ready_processes.append(process)
self.ready_processes.sort(key=lambda x: x.burst_time)
if self.currently_running is not None:
if time == self.currently_running.end_time:
self._add_to_history(self.currently_running,
self.currently_running.start_time,
self.currently_running.end_time)
self.currently_running = None
if self.currently_running is None:
if len(self.ready_processes):
self.currently_running = self.ready_processes[0]
self.currently_running.start_time = time
self.currently_running.end_time = (
time + self.currently_running.burst_time)
del self.ready_processes[0]
if self.currently_running is None:
return None
else:
return self.currently_running.end_time
base.run(SjfScheduler)
import base
class PriorityNonPreemptiveScheduler(base.Scheduler):
def interrupt(self, time, process):
if process is not None:
self.ready_processes.append(process)
self.ready_processes.sort(key=lambda x: x.priority)
if self.currently_running is not None:
if time == self.currently_running.end_time:
self._add_to_history(self.currently_running,
self.currently_running.start_time,
self.currently_running.end_time)
self.currently_running = None
if self.currently_running is None:
if len(self.ready_processes):
self.currently_running = self.ready_processes[0]
self.currently_running.start_time = time
self.currently_running.end_time = (
time + self.currently_running.burst_time)
del self.ready_processes[0]
if self.currently_running is None:
return None
else:
return self.currently_running.end_time
base.run(PriorityNonPreemptiveScheduler)
def interrupt(self, time, process):
TIME_SLICE = 1
if process is not None:
process.remaining_time = process.burst_time
self.ready_processes.append(process)
if self.currently_running is not None:
self.currently_running.remaining_time -= (
time - self.currently_running.start_time)
self._add_to_history(self.currently_running,
self.currently_running.start_time, time)
if self.currently_running.remaining_time == 0:
self.ready_processes.remove(self.currently_running)
self.last_run -= 1
self.currently_running = None
if len(self.ready_processes):
self.last_run = (self.last_run + 1) % len(self.ready_processes)
self.currently_running = self.ready_processes[self.last_run]
self.currently_running.start_time = time
self.currently_running.end_time = (time + TIME_SLICE)
if self.currently_running is None:
return None
else:
return self.currently_running.end_time
base.run(RoundRobinScheduler)
def delete_multipart_uploaded_objects(name, client):
ans = client.delete_objects(
Bucket=name+'hehe',
Delete={
'Objects': [
{
'Key': name+'s3'
},
{
'Key': name+'custom'
}
]
}
)
print 'Delete multiple objects:', ans
# =====================================================
TESTS = [
sanity.create_bucket,
sse_s3_multipart,
sse_custom_multipart,
delete_multipart_uploaded_objects,
sanity.delete_bucket,
]
if __name__ == '__main__':
base.run(TESTS)
for start in range(len(defaultvs)):
if len(this_default) < start:
start = 0
break
if defaultvs[start] != this_default[start]:
start = max(start - 10, 0)
break
failingpyfiles[f + '/' + configname + '/' + defaultp
] = ('( ... '
+ this_default[start:start + 80]
+ '... )'
)
self.assertEqual(len(failingpyfiles), 0,
"Found " + str(len(failingpyfiles))
+ " python params file missing defaults {file/service/property: missing_str_approx }"
+ " \n" + str(failingpyfiles).replace("',", "',\n")
)
def suite():
suite = unittest.TestSuite()
suite.addTest(TestXMLCompleteness())
suite.addTest(TestXMLContent())
suite.addTest(TestMatchRequiredOrDefault())
return suite
if __name__ == "__main__":
base.run(suite())
"ambari" in stdout or "test-" in stdout.lower(),
"Unable to contact " + ' '.join(ambari.sshcmd()) + "\n" + stdout)
# Test adding more space with add_new_ebs_volume
stdout = lD.run_quiet(
deploy_dir + "/aws/add_ebsvol_to_instance.py --not-strict " + iid +
' \'{"Mount": "/tmp/testdir1", "Size": 1, "Attach": "/dev/sdf"}\'')
stdout = ambari.run("ls -l /tmp/testdir1")
self.assertFalse(
"No such file or directory" in stdout,
"Unable to crate/mount /dev/sdf " + ' '.join(ambari.sshcmd()) +
"\n" + stdout)
def suite(verbose=False, ostype="Centos7"):
suite = unittest.TestSuite()
test = DepKnown()
test.debug = verbose
test.ostype = ostype
suite.addTest(test)
return suite
if __name__ == "__main__":
verbose = False
if "--verbose" in sys.argv:
verbose = True
ostype = "Centos7"
if len(sys.argv) > 1:
ostype = sys.argv[1]
base.run(suite(verbose, ostype=ostype))
# raise Exception('For geons dataset you must choose a subset '
# 'using the --subset flag.')
if args.forcemodels: args.force = True
kwargs = {
'exp': args.exp,
'task': args.task,
'func': args.func,
'subset': args.subset,
'model_name': args.model,
'model_path': args.model_path,
'layers': layers,
'mode': args.mode,
'savedata': not args.dry,
'force': args.force,
'forcemodels': args.forcemodels,
'filter': args.filter,
'savefig': args.savefig,
'report': False,
'bootstrap': args.bootstrap,
'html': None,
'dissim': args.dissim
}
if args.exp in global_tasks:
kwargs['task'] = args.exp
base.run(**kwargs)
else:
base.run(**kwargs)
class SrtfScheduler(base.Scheduler):
def interrupt(self, time, process):
if process is not None:
process.remaining_time = process.burst_time
self.ready_processes.append(process)
if self.currently_running is not None:
self.currently_running.remaining_time -= (
time - self.currently_running.start_time)
self._add_to_history(self.currently_running,
self.currently_running.start_time, time)
if self.currently_running.remaining_time == 0:
self.ready_processes.remove(self.currently_running)
self.ready_processes.sort(key=lambda x: x.remaining_time)
self.currently_running = None
if len(self.ready_processes):
self.currently_running = self.ready_processes[0]
self.currently_running.start_time = time
self.currently_running.end_time = (
time + self.currently_running.remaining_time)
if self.currently_running is None:
return None
else:
return self.currently_running.end_time
base.run(SrtfScheduler)
class HrrnScheduler(base.Scheduler):
def interrupt(self, time, process):
sort_key = lambda x: (time - x.arrival_time) / x.burst_time
if process is not None:
process.arrival_time = time
self.ready_processes.append(process)
if self.currently_running is not None:
if time == self.currently_running.end_time:
self._add_to_history(self.currently_running,
self.currently_running.start_time,
self.currently_running.end_time)
self.currently_running = None
if self.currently_running is None:
if len(self.ready_processes):
self.ready_processes.sort(key=sort_key, reverse=True)
self.currently_running = self.ready_processes[0]
self.currently_running.start_time = time
self.currently_running.end_time = (
time + self.currently_running.burst_time)
del self.ready_processes[0]
if self.currently_running is None:
return None
else:
return self.currently_running.end_time
base.run(HrrnScheduler)
"Test the root API endpoint."
import http.client
import base
class Root(base.Base):
"Test the root API endpoint."
def test_root_data(self):
"Get API root JSON."
url = f"{base.SETTINGS['ROOT_URL']}"
response = self.GET(url)
self.check_schema(response)
if __name__ == '__main__':
base.run()
# Check multiremote functionality
allremotes, iids = self.multiremote_from_cluster_stdout(cstdout)
allremotes.cp(deploy_dir + '/remotescripts/add_incoming_port.py',
'testtest.py')
self.assertTrue("testtest.py" in ahost.run("ls -l"),
"pdcp failed to work properly")
def suite(verbose=False, branch="__local__"):
suite = unittest.TestSuite()
test = MicroCluster()
test.debug = verbose
test.branch = branch
test.branchtype = branch
suite.addTest(test)
return suite
if __name__ == "__main__":
verbose = False
if "--verbose" in sys.argv:
verbose = True
branch = "__local__"
if "--branch" in sys.argv:
branch = "__service__"
sys.argv = [s for s in sys.argv if s != "--branch"]
if "--this-branch" in sys.argv:
branch = "__local__"
sys.argv = [s for s in sys.argv if s != "--this-branch"]
base.run(suite(verbose, branch))
#!/usr/bin/env python
import os
import base
if __name__ == '__main__':
current_file_folder = os.path.dirname(os.path.realpath(__file__))
base.config.load_from_yaml(
f'{current_file_folder}/config/config.{os.getenv("ENVIRONMENT", "local")}.yaml'
)
# config.init_logging()
with open(f'{current_file_folder}/keys/users.key.pem') as pubkey:
base.store.set('users_service_public_key', pubkey.read())
base.config.load_private_key(f'{current_file_folder}/keys/users.key')
base.run(print_app_info=True, print_routes=True)
if "--verbose" in sys.argv:
verbose = True
sys.argv = [s for s in sys.argv if s != "--verbose"]
branch = "__local__"
if "--branch" in sys.argv:
branch = "__service__"
sys.argv = [s for s in sys.argv if s != "--branch"]
if "--this-branch" in sys.argv:
branch = "__local__"
sys.argv = [s for s in sys.argv if s != "--this-branch"]
if "--prod" in sys.argv:
clustername = 'prod'
sys.argv = [s for s in sys.argv if s != "--prod"]
if len(sys.argv) < 2:
raise KeyError("You must specify which blueprint/cluster to test")
service = sys.argv[1]
test = TestCluster()
if service.startswith("FREEIPA"):
test = TestFreeIPACluster()
test.service = service
test.debug = verbose
test.clustername = clustername
test.branch = branch
test.branchtype = branch
test.checklist = []
if len(sys.argv) > 2:
test.checklist = sys.argv[2:]
suite = unittest.TestSuite()
suite.addTest(test)
base.run(suite)
['eq', '$content-type', 'text/plain'],
['eq', '$success_action_redirect', 'http://github.com']
]
)
print ans
fields = ans['fields']
html = HTML[name]
html = html.replace('{URL}', ans['url']).replace('{POLICY}', fields['policy'])
if 'v4' in name:
html = html.replace('{CREDENTIAL}', fields['x-amz-credential'])\
.replace('{DATE}', fields['x-amz-date'])\
.replace('{SIGNATURE}', fields['x-amz-signature'])
else:
html = html.replace('{ACCESS_KEY}', fields['AWSAccessKeyId'])\
.replace('{SIGNATURE}', fields['signature'])
f = open('post-policy-' + name + '.html', 'w')
f.write(html)
f.close()
print '>>> Now open', 'post-policy-' + name + '.html', 'and test in browser.'
# =====================================================
TESTS = [
sanity.create_bucket,
put_bucket_cors,
generate_post_policy,
]
if __name__ == '__main__':
base.run(TESTS)
priv_ip = lA.priv_ip(iid)
stdout = ahost.run("host " + priv_ip)
self.assertTrue("domain name pointer" in stdout,
"Reverse lookup broken on testing-001.kave.io! ("
+ ' '.join(ahost.sshcmd()) + ")")
def suite(verbose=False, branch="__local__"):
suite = unittest.TestSuite()
test = MicroCluster()
test.debug = verbose
test.branch = branch
test.branchtype = branch
suite.addTest(test)
return suite
if __name__ == "__main__":
verbose = False
if "--verbose" in sys.argv:
verbose = True
branch = "__local__"
if "--branch" in sys.argv:
branch = "__service__"
sys.argv = [s for s in sys.argv if s != "--branch"]
if "--this-branch" in sys.argv:
branch = "__local__"
sys.argv = [s for s in sys.argv if s != "--this-branch"]
base.run(suite(verbose, branch))
import base
while True:
text = input('Malieca > ')
result, error = base.run('<stdin>', text)
if error: print(error.as_string())
elif result: print(repr(result))