def assert_init_invalid(username, password):
with pytest.raises(StatusCodeException) as e:
utils.get_client(username, password)
utils.assert_status_code_exception(
exception=e.value,
status_code=401,
method="POST",
url=utils.get_url("auth/login"),
data={"error": "Invalid login"},
)
def download(file_path, save_to_path):
socket.setdefaulttimeout(10)
if '/' == save_to_path[-1]:
save_to_path = save_to_path[:-1]
client = get_client()
if not client:
return
f, m = client.get_file_and_metadata(file_path)
d = f.read()
f_hash = ''
try:
with open(save_to_path) as f:
f_hash = md5_for_file(f)
except:
pass
try:
f = open(save_to_path, 'rw')
fd_hash = md5_for_file(f)
if f_hash == fd_hash:
return False
except:
f.close()
print 'downloading %s and save to %s' % (file_path, save_to_path)
f = open(save_to_path, 'w')
f.write(d)
f.close()
print 'downloaded'
return True
def alloc_pty(ctx, f, *args, **kwargs):
pid, fd = pty.fork()
if pid == pty.CHILD:
tty = os.ttyname(0)
sys.stdin = open(tty, 'r')
sys.stdout = open(tty, 'w')
sys.stderr = open(tty, 'w')
# alternative way of doing ^ is to do:
# kwargs["stdout"] = open(tty, 'w')
# kwargs["stderr"] = open(tty, 'w')
# kwargs["stdin"] = open(tty, 'r')
# Create a new client for the child process to avoid concurrency issues
client = get_client()
f(client, *args, **kwargs)
sys.exit(0)
ctx.pty = fd
util.set_pty_size(
ctx.pty,
(ctx.rows, ctx.cols)
)
ctx.pid = pid
ctx.exit_code = -1
util.wait(ctx.pty, timeout=5)
def run(self, progress_callback):
"""
Do stuff here. Can return a string or raise an exception.
The progress_callback is a function expecting 1 value: current progress
"""
client = get_client(self.config)
return str(client.model.deploy(self.config["model_id"]))
def test_logout(data):
def assert_logout(client):
utils.assert_access_token(client)
client.logout()
utils.assert_not_access_token(client)
def assert_logout_fails(client):
with pytest.raises(StatusCodeException) as e:
client.logout()
utils.assert_status_code_exception(
exception=e.value,
status_code=401,
method="POST",
url=utils.get_url("auth/logout"),
data={"msg": "Missing Authorization Header"},
)
for user_data in data["users"].values():
client = utils.get_client(user_data["username"], user_data["password"])
assert_logout(client)
client = utils.get_client_org(user_data["username"],
user_data["password"], None)
assert_logout(client)
for org_data in data["organizations"].values():
for user_data in org_data["users"].values():
client = utils.get_client_org(user_data["username"],
user_data["password"],
org_data["name"])
assert_logout(client)
client = utils.get_client_sysadmin()
client.logout()
assert_logout_fails(client)
def test_whoami(data):
def assert_whoami(client, user_data, org):
user = client.users.whoami()
utils.assert_user_data(user, user_data, org)
def assert_whoami_fails(client):
with pytest.raises(StatusCodeException) as e:
client.users.whoami()
utils.assert_status_code_exception(
exception=e.value,
status_code=401,
method="GET",
url=utils.get_url("whoami"),
data={"msg": "Missing Authorization Header"},
)
for user_data in data["users"].values():
client = utils.get_client(user_data["username"], user_data["password"])
assert_whoami(client, user_data, None)
client.logout()
for org_data in data["organizations"].values():
for user_data in org_data["users"].values():
client = utils.get_client_org(user_data["username"],
user_data["password"],
org_data["name"])
assert_whoami(client, user_data, org_data["name"])
client.logout()
client = utils.get_client_sysadmin()
client.logout()
assert_whoami_fails(client)
def test():
client = get_client()
session = randint(0, 1000)
turkomatic = Turkomatic(client,
session=str(session),
redundancy=1,
verbosity=100)
task = 'Do some research.'
overall = 'Write a 3 paragraph essay on crowdsourcing'
test_context = {
overall: {
'Do some research.': {},
'Write the stuff.': {
'do it 1': {},
'do it 2': {}
}
}
}
test_divisions = [['aowief', 'awoeif', 'aow', 'owe'],
['aoweif', 'oaw', 'oawregi'], ['24', '241']]
test_solutions = ['awoeief', 'aowasdf', '123']
async def aprint(t):
s = await t
print(s)
asyncio.run(aprint(turkomatic._subdivide(task, context=test_context)))
def confirm(ctx, code):
"""Confirms email by code"""
response = get_client().user.confirm(code=code)
msg = 'Email successfully confirmed! You are logged in as %s' % (
response['email'], )
save_session(email=response['email'], session_id=response['sessionId'])
click.echo(msg)
def main():
if "-n" in sys.argv:
dry = True
else:
dry = False
db = vmdatabase.VMDatabase()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
for base_image in provider.base_images:
if base_image.min_ready < 0:
continue
print "Working on base image %s" % base_image.name
flavor = utils.get_flavor(client, base_image.min_ram)
print "Found flavor", flavor
branches = local_prep(base_image.name)
pprint.pprint(branches)
remote_base_image = client.images.find(name=base_image.external_id)
if not dry:
timestamp = int(time.time())
remote_snap_image_name = "%sdevstack-%s-%s.template.openstack.org" % (
DEVSTACK_GATE_PREFIX,
base_image.name,
str(timestamp),
)
build_image(
provider, client, base_image, remote_base_image, flavor, remote_snap_image_name, branches, timestamp
)
def alloc_pty(ctx, f, *args, **kwargs):
pid, fd = pty.fork()
if pid == pty.CHILD:
tty = os.ttyname(0)
sys.stdin = open(tty, 'r')
sys.stdout = open(tty, 'w')
sys.stderr = open(tty, 'w')
# alternative way of doing ^ is to do:
# kwargs["stdout"] = open(tty, 'w')
# kwargs["stderr"] = open(tty, 'w')
# kwargs["stdin"] = open(tty, 'r')
# Create a new client for the child process to avoid concurrency issues
client = get_client()
f(client, *args, **kwargs)
sys.exit(0)
else:
ctx.pty = fd
util.set_pty_size(
ctx.pty,
(ctx.rows, ctx.cols)
)
ctx.pid = pid
util.wait(ctx.pty, timeout=5)
time.sleep(1) # give the terminal some time to print prompt
# util.exit_code can be called only once
ctx.exit_code = util.exit_code(ctx.pid, timeout=5)
if ctx.exit_code != 0:
raise Exception("child process did not finish correctly")
def signup(ctx, email, password):
"""Creates an account"""
response = get_client().user.create(email=email, password=password)
msg = 'Account successfully created!'
if response.get('message'):
msg += ' %s' % response.get('message')
click.echo(msg)
def main():
db = vmdatabase.VMDatabase()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
for base_image in provider.base_images:
if base_image.min_ready <= 0:
continue
print "Working on base image %s" % base_image.name
flavor = utils.get_flavor(client, base_image.min_ram)
print "Found flavor", flavor
branches = local_prep(base_image.name)
remote_base_image = client.images.find(name=base_image.external_id)
timestamp = int(time.time())
remote_snap_image_name = (
'%sdevstack-%s-%s.template.openstack.org' %
(DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp)))
remote_snap_image = build_image(provider, client, base_image,
remote_base_image, flavor,
remote_snap_image_name, branches,
timestamp)
def list(branch_parent_id):
ds = get_client()
query = ds.query(kind=kind_name, order=['sortorder'])
if branch_parent_id != 0:
#We have a valid parent # ID
#This means we are loading children of a branch
#print ("Fetching children of CategoryID")
#print(branch_parent_id)
query.add_filter('branch_parent_id', '=', branch_parent_id)
else:
#No parent branch id available
#simply list root branches
#print ("Fetching Root categories..")
query.add_filter('branch_parent_id', '=', 0)
entities = query.fetch(limit=100)
items = []
count = 0
for x in entities:
count = count + 1
items.append(x)
#print("Listing branches. branch_parent_id=")
#print(branch_parent_id)
#print("Final count of branches=")
#print(count)
return items
def login():
global client, SERVER_URL
if request.method == 'GET':
url = get_config(currentPath)
if not url:
return 'Set SERVER_URL first!<a href=/setting>setting</a>'
SERVER_URL = url
return render_template('login.html', title='login', action='/login')
if request.method == 'POST':
name = request.form['name']
psd = request.form['psd']
if not all([name, psd]):
return make_response("invalid param", 400)
try:
client = get_client(SERVER_URL, name, psd)
except Exception as e:
print(e)
return jsonify(error='error')
curr_user = User()
curr_user.id = name
login_user(curr_user)
r_next = request.args.get('next')
flag = True
for i in clients_pool:
if i['id'] == name:
flag = False
if flag:
clients_pool.append({'id': name, 'client': client})
return redirect(r_next or '/article')
def alloc_pty(ctx, f, *args, **kwargs):
pid, fd = pty.fork()
if pid == pty.CHILD:
tty = os.ttyname(0)
sys.stdin = open(tty, 'r')
sys.stdout = open(tty, 'w')
sys.stderr = open(tty, 'w')
# alternative way of doing ^ is to do:
# kwargs["stdout"] = open(tty, 'w')
# kwargs["stderr"] = open(tty, 'w')
# kwargs["stdin"] = open(tty, 'r')
# Create a new client for the child process to avoid concurrency issues
client = get_client()
f(client, *args, **kwargs)
sys.exit(0)
else:
ctx.pty = fd
util.set_pty_size(ctx.pty, (ctx.rows, ctx.cols))
ctx.pid = pid
util.wait(ctx.pty, timeout=5)
time.sleep(1) # give the terminal some time to print prompt
# util.exit_code can be called only once
ctx.exit_code = util.exit_code(ctx.pid, timeout=5)
if ctx.exit_code != 0:
raise Exception("child process did not finish correctly")
def handler(event, context, metrics):
client = get_client()
metrics.set_namespace('DynamoDB Performance Testing')
metrics.put_dimensions({"Operation": f"BatchWriteItem -- {COUNT} items"})
for i in range(10):
wrapped = wrapper(client.batch_write_item, **make_batch_items(COUNT))
latency = timeit.timeit(wrapped, number=1)
metrics.put_metric("Latency", latency * 1000, "Milliseconds")
def delete(path):
print 'delete %s' % path
socket.setdefaulttimeout(10)
client = get_client()
try:
client.file_delete(path)
except Exception, e:
print 'Error %s' % e
def move(path, to_path):
print 'move %s to %s' % (path, to_path)
socket.setdefaulttimeout(10)
client = get_client()
try:
client.file_move(path, to_path)
except Exception, e:
print 'Error %s' % e
def handler(event, context, metrics):
client = get_client()
metrics.set_namespace('DynamoDB Performance Testing')
metrics.put_dimensions({"Operation": "PutItem"})
for i in range(10):
wrapped = wrapper(client.put_item, **make_put_item())
latency = timeit.timeit(wrapped, number=1)
metrics.put_metric("Latency", latency * 1000, "Milliseconds")
def create_folder(path):
print 'create folder %s' % path
socket.setdefaulttimeout(10)
client = get_client()
try:
client.file_create_folder(path)
except Exception, e:
print 'Error %s' % e
def client(self):
if not self._client:
o = self.config
self._client = get_client(**({
'host': o.host, 'port': o.port, 'url_prefix': o.url_prefix,
'http_auth': o.http_auth, 'use_ssl': o.use_ssl,
'master_only': o.master_only, 'timeout': o.timeout
}))
return self._client
def client(self):
if not self._client:
o = self.opts
self._client = get_client(**({
'host': o.host, 'port': o.port, 'url_prefix': o.url_prefix,
'http_auth': o.http_auth, 'use_ssl': o.use_ssl,
'master_only': o.master_only, 'timeout': o.timeout
}))
return self._client
def index():
tsc = get_client()
sections = [s for s in tsc.get_endpoints() if s.type == 'service_sections']
section_url = plugin.url_for(
'list_folder',
url=json.dumps([s.links['self'] for s in sections]),
titles=json.dumps([s['display_name'] for s in sections]))
return plugin.redirect(section_url)
def test_list_orgs(data):
def assert_list_orgs(client):
orgs = client.organizations.list_organizations()
expected_len = len(data["organizations"])
actual_len = len(orgs)
assert actual_len == expected_len, f"len(orgs) {actual_len} != {expected_len}"
for org_data in data["organizations"].values():
for org in orgs:
if org.name == org_data["name"]:
utils.assert_org_data(org, org_data)
break
else:
assert False, f"Organization \"{org_data['name']}\" not found"
def assert_list_orgs_fails(client):
with pytest.raises(StatusCodeException) as e:
client.organizations.list_organizations()
utils.assert_status_code_exception(
exception=e.value,
status_code=401,
method="GET",
url=utils.get_url("organization/all"),
data={"msg": "Missing Authorization Header"},
)
def assert_list_orgs_forbidden(client):
with pytest.raises(StatusCodeException) as e:
client.organizations.list_organizations()
utils.assert_status_code_exception(
exception=e.value,
status_code=403,
method="GET",
url=utils.get_url("organization/all"),
data={
"error":
"You are not authorized to access this resource: system_admin required."
},
)
for user_data in data["users"].values():
client = utils.get_client(user_data["username"], user_data["password"])
assert_list_orgs(client)
client.logout()
for org_data in data["organizations"].values():
for user_data in org_data["users"].values():
client = utils.get_client_org(user_data["username"],
user_data["password"],
org_data["name"])
assert_list_orgs_forbidden(client)
client.logout()
client = utils.get_client_sysadmin()
client.logout()
assert_list_orgs_fails(client)
def before_all(ctx):
"""
Pulls down busybox:latest before anything is tested.
"""
ctx.client = get_client()
try:
ctx.client.inspect_image(IMAGE)
except NotFound:
ctx.client.pull(IMAGE)
def get_next_sortorder(branch_parent_id):
ds = get_client()
query = ds.query(kind=kind_name, order=['-sortorder'])
query.add_filter('branch_parent_id', '=', branch_parent_id)
entities = query.fetch(limit=1)
next_sortorder = 1
for entity in entities:
#Return the highest sortorder + 1
next_sortorder = int(entity['sortorder']) + 1
return next_sortorder
def main():
client = get_client(production=False)
session = randint(0, 1000)
turkomatic = Turkomatic(client,
session=str(session),
redundancy=3,
verbosity=100)
task = 'Write a 3 paragraph essay on "User Interfaces in Programming Languages".'
asyncio.run(turkomatic.ask_async(task, 2, quiet=False))
def handler(event, context, metrics):
client = get_client()
metrics.set_namespace('DynamoDB Performance Testing')
metrics.put_dimensions(
{"Operation": f"TransactWriteItems -- {COUNT} items, Failure {FAIL}"})
for i in range(10):
wrapped = exception_wrapper(client.transact_write_items,
**make_transact_items(COUNT, FAIL))
latency = timeit.timeit(wrapped, number=1)
metrics.put_metric("Latency", latency * 1000, "Milliseconds")
def test_get_org_by_name(data):
def assert_get_org_by_name(client, org_data):
org = client.organizations.get_organization_by_name(org_data["name"])
utils.assert_org_data(org, org_data)
def assert_get_org_by_name_invalid(client, name):
with pytest.raises(ValueError) as e:
client.organizations.get_organization_by_name(name)
assert str(e.value) == f"Invalid organization {name}"
def assert_get_org_by_name_fails(client, name):
with pytest.raises(StatusCodeException) as e:
client.organizations.get_organization_by_name(name)
utils.assert_status_code_exception(
exception=e.value,
status_code=401,
method="GET",
url=utils.get_url("organization/all"),
data={"msg": "Missing Authorization Header"},
)
def assert_get_org_by_name_forbidden(client, name):
with pytest.raises(StatusCodeException) as e:
client.organizations.get_organization_by_name(name)
utils.assert_status_code_exception(
exception=e.value,
status_code=403,
method="GET",
url=utils.get_url("organization/all"),
data={
"error":
"You are not authorized to access this resource: system_admin required."
},
)
for user_data in data["users"].values():
client = utils.get_client(user_data["username"], user_data["password"])
for org in data["organizations"].values():
assert_get_org_by_name(client, org)
assert_get_org_by_name_invalid(client, "invalid")
client.logout()
for org_data in data["organizations"].values():
for user_data in org_data["users"].values():
client = utils.get_client_org(user_data["username"],
user_data["password"],
org_data["name"])
for org in data["organizations"].values():
assert_get_org_by_name_forbidden(client, org["name"])
assert_get_org_by_name_forbidden(client, "invalid")
client.logout()
client = utils.get_client_sysadmin()
client.logout()
assert_get_org_by_name_fails(client, "invalid")
def play_file():
tsc = get_client()
url = json.loads(plugin.request.args.get('url')[0])
doc = tsc.stream_url(url)
for item in doc.data:
if item.type == 'stream_http':
return plugin.set_resolved_url({
'label': item.id,
'path': item.media_url,
})
def test_get_metadata(data, awslang):
def assert_get_metadata(client):
metalist = client.metadata.get_metadata()
# uncommment this to serialize new test input data
# with open("newlang.json", mode='w') as writer:
# from securicad.enterprise.metadata import RiskTypeJsonEncoder
# import json
# json.dump(fp=writer, obj=metalist, indent=2, cls=RiskTypeJsonEncoder)
expected_len = len(awslang)
actual_len = len(metalist)
assert (actual_len == expected_len
), f"len(metalist) {actual_len} != {expected_len}"
for i in range(actual_len):
expected_asset = awslang[i]
actual_asset = metalist[i]
assert (
actual_asset["name"] == expected_asset["name"]
), f"Unexpected asset name \"{actual_asset['name']}\" != \"{expected_asset['name']}\""
expected_attacksteps_len = len(expected_asset["attacksteps"])
actual_attacksteps_len = len(actual_asset["attacksteps"])
assert (
actual_attacksteps_len == expected_attacksteps_len
), f"len({expected_asset['name']}[\"attacksteps\"]) {actual_attacksteps_len} != {expected_attacksteps_len}"
def assert_get_metadata_fails(client):
with pytest.raises(StatusCodeException) as e:
client.metadata.get_metadata()
utils.assert_status_code_exception(
exception=e.value,
status_code=401,
method="GET",
url=utils.get_url("metadata"),
data={"msg": "Missing Authorization Header"},
)
for user_data in data["users"].values():
client = utils.get_client(user_data["username"], user_data["password"])
assert_get_metadata(client)
client.logout()
for org_data in data["organizations"].values():
for user_data in org_data["users"].values():
client = utils.get_client_org(user_data["username"],
user_data["password"],
org_data["name"])
assert_get_metadata(client)
client.logout()
client = utils.get_client_sysadmin()
client.logout()
assert_get_metadata_fails(client)
def create(id_, name, email, profile_pic):
ds = get_client()
key = ds.key('User')
entity = datastore.Entity(key=key)
entity.update({
"userid": id_,
"name": name,
"email": email,
"profile_pic": profile_pic
})
ds.put(entity)
def add(obj):
ds = get_client()
key = ds.key(kind_name)
entity = datastore.Entity(key=key)
data = {
'name': obj.name,
'branch_parent_id': obj.branch_parent_id,
'sortorder': int(obj.sortorder)
}
entity.update(data)
ds.put(entity)
return from_datastore(entity)
def from_crawler(cls, crawler, client=None, dsn=None):
dsn = crawler.settings.get("SENTRY_DSN", None)
client = get_client(dsn)
o = cls(dsn=dsn)
sentry_signals = crawler.settings.get("SENTRY_SIGNALS", [])
if len(sentry_signals):
receiver = o.signal_receiver
for signalname in sentry_signals:
signal = getattr(signals, signalname)
crawler.signals.connect(receiver, signal=signal)
return o
def from_crawler(cls, crawler, client=None, dsn=None):
dsn = crawler.settings.get("SENTRY_DSN", None)
client = get_client(dsn)
o = cls(dsn=dsn)
sentry_signals = crawler.settings.get("SENTRY_SIGNALS", [])
if len(sentry_signals):
receiver = o.signal_receiver
for signalname in sentry_signals:
signal = getattr(signals, signalname)
crawler.signals.connect(receiver, signal=signal)
return o
def get_by_path(cls, path):
client = get_client()
md = client.metadata(path)
hash = md.get('hash')
thumb_exists = md.get('thumb_exists')
bytes = md.get('bytes')
path = md.get('path')
is_dir = md.get('is_dir')
icon = md.get('icon')
root = md.get('root')
contents = md.get('contents')
return cls(hash, thumb_exists, bytes,
path, is_dir, icon, root, contents)
def upload(file_name, as_file_name):
print "uploading %s to %s" % (file_name, as_file_name)
socket.setdefaulttimeout(10)
try:
client = get_client()
if not client:
return
f = open(file_name)
client.put_file(as_file_name, f, overwrite=True)
print 'uploaded'
except Exception, e:
print 'Error %s' % e
f.close()
def _check_delete(path):
client = get_client()
if path == PATH_TO_WATCH:
return
path = path.replace(PATH_TO_WATCH, DROPBOX_ROOT_FOLDER)
try:
m = client.metadata(path)
if m.get('is_deleted'):
delete(path)
except:
try:
delete(path)
except:
pass
def aggregate_features(index, row, bar, df_q, token_q):
repo = get_client().get_repo(row['repository'])
owner = row['owner']
name = row['name']
token = token_q.get()
try:
new_data_frame = pd.DataFrame.from_dict(row).T
new_data_frame = add_graph_features(new_data_frame, index, owner, name)
new_data_frame = add_rest_features(new_data_frame, index, repo)
new_data_frame = add_custom_features(new_data_frame, index, owner, name)
if "closed_issues" in new_data_frame.columns:
new_data_frame = fix_closed_issues(new_data_frame, index)
except Exception, e:
print "Exception in aggregate_features: " + str(e)
token_q.put(token)
return
def get(user_id):
ds = get_client()
key = ds.key('User')
query = ds.query(kind='User')
query.add_filter('userid', '=', user_id)
results = list(query.fetch(limit=1))
for x in results:
if not x:
return None
#print("user rec=")
#print(x)
user = User(id_=x['userid'],
name=x['name'],
email=x['email'],
profile_pic=x['profile_pic'])
#print("user obj name=")
#print(user.name)
return user
def main(args):
client = get_client(None, production=args.production)
task = BonusTask(client,
title=args.title,
reward=args.reward,
description=args.description,
worker_ids=args.worker_ids,
duration=600,
lifetime=60 * 60 * 48)
async def proc():
async for answer, assignment in task.ask_async(assignments=len(
args.worker_ids),
verbosity=100,
message=args.message):
print('Bonus done for worker:', answer)
print(assignment)
asyncio.run(proc())
def result():
client = utils.get_client()
data = request.values.to_dict()
geocode_result = client.geocode(data['address'])
if geocode_result:
res = geocode_result[0]['geometry']
home_lat_lon = (res['location']['lat'], res['location']['lng'])
else:
raise AddressNotFoundError(data['address'], status_code=400)
target = data['desired_route_distance']
tbell_list = utils.tbell_finder(home_lat_lon, client)
path = utils.choose_tbell_sequence(
home_lat_lon,
tbell_list,
target
)
if path:
path_dict = {'status': 'ok', 'path': path}
else:
raise PathFinderError('No path found')
query_url = utils.path_dict_to_embedded_query(path_dict)
return render_template('map.html', url=query_url,
n_tbells=len(path_dict['path']) - 2)
def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
print 'Known machines (start):'
db.print_state()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
flavor = utils.get_flavor(client, 1024)
print "Found flavor", flavor
error = False
now = time.time()
for machine in provider.machines:
# Normally, reap machines that have sat in their current state
# for 24 hours, unless that state is READY.
if (REAP_ALL_SERVERS or
(machine.state != vmdatabase.READY and
now - machine.state_time > MACHINE_LIFETIME) or
machine.state == vmdatabase.DELETE):
print 'Deleting machine', machine.name
try:
delete_machine(jenkins, client, machine)
except:
error = True
traceback.print_exc()
provider_min_ready = 0
for base_image in provider.base_images:
provider_min_ready += base_image.min_ready
for snap_image in base_image.snapshot_images:
# Normally, reap images that have sat in their current state
# for 24 hours, unless the image is the current snapshot
if (REAP_ALL_IMAGES or
(snap_image != base_image.current_snapshot and
now - snap_image.state_time > MACHINE_LIFETIME)):
print 'Deleting image', snap_image.name
try:
delete_image(client, snap_image)
except:
error = True
traceback.print_exc()
# Make sure the provider has enough headroom for the min_ready
# of all base images, deleting used serverss if needed.
overcommitment = ((len(provider.machines) -
len(provider.ready_machines) + provider_min_ready) -
provider.max_servers)
while overcommitment > 0:
print 'Overcommitted by %s machines' % overcommitment
last_overcommitment = overcommitment
for machine in provider.machines:
if machine.state == vmdatabase.READY:
continue
if machine.state == vmdatabase.BUILDING:
continue
if machine.state == vmdatabase.HOLD:
continue
print 'Deleting machine', machine.name
try:
delete_machine(jenkins, client, machine)
overcommitment -= 1
except:
error = True
traceback.print_exc()
if overcommitment == last_overcommitment:
raise Exception("Unable to reduce overcommitment")
last_overcommitment = overcommitment
print
print 'Known machines (end):'
db.print_state()
if error:
sys.exit(1)
def client(self):
if not self._client:
self._client = utils.get_client(**self.config)
return self._client
def __init__(self, dsn=None, client=None):
self.client = client if client else get_client(dsn)
def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
last_name = ''
error_counts = {}
error = False
for base_image in provider.base_images:
snap_image = base_image.current_snapshot
if not snap_image:
continue
print "Working on image %s" % snap_image.name
flavor = utils.get_flavor(client, base_image.min_ram)
print "Found flavor", flavor
remote_snap_image = client.images.get(snap_image.external_id)
print "Found image", remote_snap_image
num_to_launch = calculate_deficit(provider, base_image)
for i in range(num_to_launch):
try:
server, machine = launch_node(client,
snap_image,
remote_snap_image,
flavor,
last_name)
last_name = machine.name
except:
traceback.print_exc()
error = True
while True:
utils.update_stats(provider)
building_machines = provider.building_machines
if not building_machines:
print "No more machines are building, finished."
break
print "Waiting on %s machines" % len(building_machines)
for machine in building_machines:
try:
check_machine(jenkins, client, machine, error_counts)
except:
traceback.print_exc()
print "Abandoning machine %s" % machine.id
machine.state = vmdatabase.ERROR
error = True
db.commit()
time.sleep(3)
if error:
sys.exit(1)
def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
print 'Known machines (start):'
db.print_state()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
flavor = utils.get_flavor(client, 1024)
print "Found flavor", flavor
error = False
now = time.time()
for machine in provider.machines:
# Normally, reap machines that have sat in their current state
# for 24 hours, unless that state is READY.
if (REAP_ALL_SERVERS or
(machine.state != vmdatabase.READY and
now - machine.state_time > MACHINE_LIFETIME) or
machine.state == vmdatabase.DELETE or
machine.state == vmdatabase.ERROR):
print 'Deleting machine', machine.name
try:
delete_machine(jenkins, client, machine)
except:
error = True
traceback.print_exc()
provider_min_ready = 0
for base_image in provider.base_images:
provider_min_ready += base_image.min_ready
for snap_image in base_image.snapshot_images:
# Normally, reap images that have sat in their current state
# for 24 hours, unless the image is the current snapshot
if (REAP_ALL_IMAGES or
(snap_image != base_image.current_snapshot and
now - snap_image.state_time > MACHINE_LIFETIME)):
print 'Deleting image', snap_image.name
try:
delete_image(client, snap_image)
except:
error = True
traceback.print_exc()
print
print 'Known machines (end):'
db.print_state()
utils.update_stats(provider)
if error:
sys.exit(1)