def _test_pcluster_compute_fleet(cluster, expected_num_nodes):
"""Test pcluster compute fleet commands."""
logging.info("Testing pcluster stop functionalities")
cluster.stop()
# Sample pcluster stop output:
# Compute fleet status is: RUNNING. Submitting status change request.
# Request submitted successfully. It might take a while for the transition to complete.
# Please run 'pcluster status' if you need to check compute fleet status
wait_for_num_instances_in_cluster(cluster.cfn_name, cluster.region, desired=0)
_test_describe_instances(cluster)
compute_fleet = cluster.describe_compute_fleet()
assert_that(compute_fleet["status"]).is_equal_to("STOPPED")
last_stop_time = compute_fleet["lastStatusUpdatedTime"]
logging.info("Testing pcluster start functionalities")
# Do a complicated sequence of start and stop and see if commands will still work
cluster.start()
cluster.stop()
cluster.stop()
cluster.start()
compute_fleet = cluster.describe_compute_fleet()
last_start_time = compute_fleet["lastStatusUpdatedTime"]
logging.info("Checking last status update time is updated")
assert_that(last_stop_time < last_start_time)
wait_for_num_instances_in_cluster(cluster.cfn_name, cluster.region, desired=expected_num_nodes)
_test_describe_instances(cluster)
check_status(cluster, "CREATE_COMPLETE", "running", "RUNNING")
def close_spider(self,spider):
for (id,item) in self.result_dict.items():
cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]})
utils.set_origin_value_list(item, cursor, ['display_time_begin'])
utils.set_origin_value_if_db_smaller(item, cursor, 'display_time_end')
utils.check_status(item, cursor)
if item.has_key('limit') and item['limit'] != utils.UNLIMITED_NUM:
limit = item['limit']
elif item.has_key('left_goods'):
limit = item['left_goods']
else:
limit = utils.UNLIMITED_NUM
origin_limit = utils.UNLIMITED_NUM
if cursor:
if cursor.has_key('limit') and cursor['limit'] != utils.UNLIMITED_NUM:
origin_limit = cursor['limit']
if origin_limit > limit:
limit = origin_limit
if item.has_key('left_goods'):
sale = limit - item['left_goods']
item['sale'] = sale
item['limit'] = limit
item['sale_percent'] = sale * 100 / limit
self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe)
utils.driver_quit(spider.driver)
spider.log('driver.quit()', level = log.DEBUG)
def put(self, **kwargs):
if self.check_write_access('update', **kwargs) == False:
return abort(403)
if not self.form:
return abort(405)
instance = getattr(self.model, 'find_by_%s' % self.key_name)(**kwargs)
check_status(instance)
if len(request.form) > 0:
form = self.form(request.form, obj=instance)
else:
form = self.form(ImmutableMultiDict(request.json), obj=instance)
if not form.validate():
return form.errors_as_json()
try:
self.pre_update(form, instance, **kwargs)
except FormApiError, e:
if not e.get_key() in form.errors:
form.errors[e.get_key()] = []
form.errors[e.get_key()].append(e.get_message())
return form.errors_as_json()
def test_slurm_cli_commands(
request, scheduler, region, os, pcluster_config_reader, clusters_factory, s3_bucket_factory
):
"""Test pcluster cli commands are working."""
# Use long scale down idle time so we know nodes are terminated by pcluster stop
cluster_config = pcluster_config_reader(scaledown_idletime=60)
# Using custom AMI not tagged by pcluser will generate a warning
custom_ami = retrieve_latest_ami(region, os, ami_type="official", architecture="x86_64")
config_file = "pcluster.config.with.warnings.yaml"
cluster_config_with_warning = pcluster_config_reader(config_file=config_file, custom_ami=custom_ami)
# Test below is not compatible with `--cluster` flag. Therefore, skip it if the flag is provided.
if not request.config.getoption("cluster"):
_test_create_with_warnings(cluster_config_with_warning, clusters_factory)
cluster = _test_create_cluster(clusters_factory, cluster_config, request)
_test_describe_cluster(cluster)
_test_list_cluster(cluster.name, "CREATE_COMPLETE")
_test_update_with_warnings(cluster_config_with_warning, cluster)
check_status(cluster, "CREATE_COMPLETE", "running", "RUNNING")
filters = [{}, {"node_type": "HeadNode"}, {"node_type": "Compute"}, {"queue_name": "ondemand1"}]
for filter_ in filters:
_test_describe_instances(cluster, **filter_)
_test_pcluster_export_cluster_logs(s3_bucket_factory, cluster)
check_pcluster_list_cluster_log_streams(cluster, os)
_test_pcluster_get_cluster_log_events(cluster)
_test_pcluster_get_cluster_stack_events(cluster)
_test_pcluster_compute_fleet(cluster, expected_num_nodes=2)
remote_command_executor = RemoteCommandExecutor(cluster)
assert_no_errors_in_logs(remote_command_executor, scheduler)
def close_spider(self,spider):
for (id,item) in self.result_dict.items():
cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]})
utils.set_origin_value_list(item, cursor, ['display_time_end'])
utils.check_status(item, cursor)
self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe)
utils.driver_quit(spider.driver)
spider.log('driver.quit()', level = log.DEBUG)
def close_spider(self,spider):
for (id,item) in self.result_dict.items():
cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]})
utils.check_status(item, cursor)
if not cursor or utils.is_item_changed(item, cursor):
self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe)
utils.driver_quit(spider.driver)
spider.log('driver.quit()', level = log.DEBUG)
def run(self):
last = 0
while True:
now = time.time()
looptime = now - last
if looptime < self.interval:
time.sleep(self.interval + 1 - looptime)
last = time.time()
else:
last = now
result = utils.readConfig(self.hutch, self.mtime)
if result != None:
(self.mtime, cfglist, hosts, vdict) = result
self.rmtime = {} # Force a re-read!
self.model.configuration(cfglist, hosts, vdict)
result = utils.readStatusDir(self.hutch, self.readStatusFile)
for l in result:
rdir = l['rdir']
l.update(utils.check_status(l['rhost'], l['rport'], l['rid']))
l['stattime'] = time.time()
if l['rdir'] == '/tmp':
l['rdir'] = rdir
else:
l['newstyle'] = False
self.model.running(l)
for l in self.model.cfglist:
if l['stattime'] + self.interval > time.time():
continue;
if l['hard']:
s = {'pid' : -1,
'autorestart' : False }
try:
pv = psp.Pv.Pv(l['base'] + ":HEARTBEAT")
pv.connect(1.0)
pv.disconnect()
s['status'] = utils.STATUS_RUNNING
except:
s['status'] = utils.STATUS_SHUTDOWN
s['rid'] = l['id']
s['rdir'] = l['dir']
else:
s = utils.check_status(l['host'], l['port'], l['id'])
s['stattime'] = time.time()
s['rhost'] = l['host']
s['rport'] = l['port']
if l['newstyle']:
if s['rdir'] == '/tmp':
del s['rdir']
else:
s['newstyle'] = False # We've switched from new to old?!?
self.model.running(s)
for p in self.model.children:
if p.poll() != None:
self.model.children.remove(p)
def add_customer(customer_id, **kw):
"""
Add a customer to the database
:param customer_id:
:param name:
:param lastname:
:param home_address:
:param phone_number:
:param email_address:
:param status: must be True or False, active or inactive
:param credit_limit: Must be a number, no $
:return: None
"""
try:
cust = Customer.create(customer_id=customer_id,
first_name=kw['name'],
last_name=kw['last_name'],
home_address=kw['home_address'],
phone_number=kw['phone_number'],
email_address=kw['email_address'],
status=check_status(kw['status']),
credit_limit=kw['credit_limit'])
cust.save()
except KeyError as err:
logging.error(err)
raise ValueError(config.etext['no_save'].format(customer_id))
except Exception as err:
logging.error(err)
raise
logger.info("Customer with id %s successfully added!", customer_id)
def run(self):
last = 0
while True:
now = time.time()
looptime = now - last
if looptime < self.interval:
time.sleep(self.interval + 1 - looptime)
last = time.time()
else:
last = now
result = utils.readConfig(self.hutch, self.mtime)
if result != None:
(self.mtime, cfglist, hosts, vdict) = result
self.rmtime = {} # Force a re-read!
self.model.configuration(cfglist, hosts, vdict)
result = utils.readStatusDir(self.hutch, self.readStatusFile)
for l in result:
rdir = l['rdir']
l.update(utils.check_status(l['rhost'], l['rport'], l['rid']))
l['stattime'] = time.time()
if l['rdir'] == '/tmp':
l['rdir'] = rdir
else:
l['newstyle'] = False
self.model.running(l)
for l in self.model.cfglist:
if l['stattime'] + self.interval > time.time():
continue;
s = utils.check_status(l['host'], l['port'], l['id'])
s['stattime'] = time.time()
s['rhost'] = l['host']
s['rport'] = l['port']
if l['newstyle']:
if s['rdir'] == '/tmp':
del s['rdir']
else:
s['newstyle'] = False # We've switched from new to old?!?
self.model.running(s)
for p in self.model.children:
if p.poll() != None:
self.model.children.remove(p)
def details(self, **kwargs):
if self.check_read_access('details', **kwargs) == False:
return abort(403)
instance = getattr(self.model, 'find_by_%s' % self.key_name)(**kwargs)
check_status(instance)
result = instance.to_json()
if isinstance(instance, SeenModel):
if not instance.seen:
instance.seen = 0
instance.seen = instance.seen + 1
instance.last_seen = datetime.datetime.utcnow()
instance.save()
return jsonify(result), 200
def delete(self, **kwargs):
if self.check_write_access('delete', **kwargs) == False:
return abort(403)
if not self.form:
return abort(405)
# Return 204 !
instance = self.model.find_by_id(**kwargs)
check_status(instance)
try:
self.pre_delete(instance, **kwargs)
except FormApiError, e:
if not e.get_key() in form.errors:
form.errors[e.get_key()] = []
form.errors[e.get_key()].append(e.get_message())
return form.errors_as_json()
def test_hmmpress_task_existing(self, tmpdir, datadir):
with tmpdir.as_cwd():
tf = datadir('test-profile.hmm')
for ext in self.extensions:
touch(tf + ext)
task = HMMPressTask().task(tf)
run_tasks([task], ['run'])
print(os.listdir(), file=sys.stderr)
print(task, file=sys.stderr)
status = check_status(task)
assert status.status == 'up-to-date'
def test_cmpress_task(self, tmpdir, datadir):
with tmpdir.as_cwd():
tf = datadir('test-covariance-model.cm')
task = CMPressTask().task(tf, params=self.cmpress_cfg)
run_tasks([task], ['run'])
status = check_status(task)
print(os.listdir(), file=sys.stderr)
for ext in self.extensions:
assert os.path.isfile(tf + ext)
assert status.status == 'up-to-date'
def test_hmmpress_task(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-profile.hmm', td) as tf:
task = tasks.get_hmmpress_task(tf, self.hmmpress_cfg)
run_tasks([task], ['run'])
status = check_status(task)
print(os.listdir(td), file=sys.stderr)
for ext in self.extensions:
self.assertTrue(os.path.isfile(tf + ext))
self.assertEquals(status.status, 'up-to-date')
def test_hmmpress_task(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-profile.hmm', td) as tf:
task = tasks.get_hmmpress_task(tf, self.hmmpress_cfg)
run_tasks([task], ['run'])
status = check_status(task)
print(os.listdir(td), file=sys.stderr)
for ext in self.extensions:
self.assertTrue(os.path.isfile(tf + ext))
self.assertEquals(status.status, 'up-to-date')
def test_cmpress_task_existing(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-covariance-model.cm', td) as tf:
for ext in self.extensions:
touch(tf + ext)
task = tasks.get_cmpress_task(tf, self.cmpress_cfg)
run_tasks([task], ['run'])
print(os.listdir(td), file=sys.stderr)
print(task, file=sys.stderr)
status = check_status(task)
self.assertEquals(status.status, 'up-to-date')
def test_cmpress_task_existing(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-covariance-model.cm', td) as tf:
for ext in self.extensions:
touch(tf + ext)
task = tasks.get_cmpress_task(tf, self.cmpress_cfg)
run_tasks([task], ['run'])
print(os.listdir(td), file=sys.stderr)
print(task, file=sys.stderr)
status = check_status(task)
self.assertEquals(status.status, 'up-to-date')
def test_lastdb_task_existing(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-protein.fa', td) as tf:
for ext in self.extensions:
touch(tf + ext)
task = tasks.get_lastdb_task(tf, tf, self.lastdb_cfg,
prot=True)
run_tasks([task], ['run'])
print(os.listdir(td), file=sys.stderr)
print(task, file=sys.stderr)
status = check_status(task)
self.assertEquals(status.status, 'up-to-date')
def test_lastdb_task_nucl(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-transcript.fa', td) as tf:
task = tasks.get_lastdb_task(tf, tf, self.lastdb_cfg,
prot=False)
run_tasks([task], ['run'])
status = check_status(task)
print(os.listdir(td), file=sys.stderr)
print('PATH:', os.environ['PATH'], file=sys.stderr)
for ext in self.extensions:
self.assertTrue(os.path.isfile(tf + ext))
self.assertEquals(status.status, 'up-to-date')
def update_info():
changed = []
logger.info(f"Checking at {datetime.now()}")
for server in servers:
last = server['last_status']
update = check_status(server['host'], server['port'])
print(server, last, update)
server['last_status'] = update
server['last_update'] = datetime.now()
logger.debug(json.dumps(server, indent=2, default=str))
if update != last and last is not None:
changed.append(server)
return changed
def test_lastdb_task_nucl(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-transcript.fa', td) as tf:
task = tasks.get_lastdb_task(tf,
tf,
self.lastdb_cfg,
prot=False)
run_tasks([task], ['run'])
status = check_status(task)
print(os.listdir(td), file=sys.stderr)
print('PATH:', os.environ['PATH'], file=sys.stderr)
for ext in self.extensions:
self.assertTrue(os.path.isfile(tf + ext))
self.assertEquals(status.status, 'up-to-date')
def test_lastdb_task_existing(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-protein.fa', td) as tf:
for ext in self.extensions:
touch(tf + ext)
task = tasks.get_lastdb_task(tf,
tf,
self.lastdb_cfg,
prot=True)
run_tasks([task], ['run'])
print(os.listdir(td), file=sys.stderr)
print(task, file=sys.stderr)
status = check_status(task)
self.assertEquals(status.status, 'up-to-date')
def test_lastal_task_uptodate(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-protein.fa', td) as prot, \
TemporaryFile(td) as out:
print(os.listdir(td), file=sys.stderr)
db_task = tasks.get_lastdb_task(prot, prot, self.lastdb_cfg)
aln_task = tasks.get_lastal_task(prot, prot, out,
self.lastal_cfg,
translate=False,
cutoff=None)
# Run it once
run_tasks([db_task, aln_task], ['run'])
# Now run again and check the status
run_tasks([aln_task], ['run'])
status = check_status(aln_task)
self.assertEquals(status.status, 'up-to-date')
def test_lastal_task_uptodate(self):
with TemporaryDirectory() as td:
with Move(td):
with TestData('test-protein.fa', td) as prot, \
TemporaryFile(td) as out:
print(os.listdir(td), file=sys.stderr)
db_task = tasks.get_lastdb_task(prot, prot,
self.lastdb_cfg)
aln_task = tasks.get_lastal_task(prot,
prot,
out,
self.lastal_cfg,
translate=False,
cutoff=None)
# Run it once
run_tasks([db_task, aln_task], ['run'])
# Now run again and check the status
run_tasks([aln_task], ['run'])
status = check_status(aln_task)
self.assertEquals(status.status, 'up-to-date')
def close_spider(self,spider):
for (id,item) in self.result_dict.items():
cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]})
utils.check_status(item, cursor)
if not utils.set_origin_value_if_db_smaller(item, cursor, 'display_time_end') or utils.is_item_changed(item, cursor):
self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe)
def _assert_server_status(cluster):
check_status(cluster, "CREATE_COMPLETE", "running", "RUNNING")
async def user_ques(ctx, handle, *args):
url = f'{CF_USER_STATUS}{handle}&from=1'
obj = requests.get(url)
data = json.loads(obj.text)
if data['status'] == "FAILED":
await ctx.send(f'{data["comment"]}')
return
elif len(data['result']) == 0:
await ctx.send(f'{handle} has made no submissions yet.')
return
official, virtual, practice, unoff = False, False, False, False
var = 0
embeds, val = [], ''
ques = []
for arg in args:
if arg == '+contest':
official = True
elif arg == '+virtual':
virtual = True
elif arg == '+practice':
practice = True
elif arg == '+unofficial':
unoff = True
else:
await ctx.send(
'Invalid arguments. Please enter correct arguments and then try again.'
)
return
for prob in data['result']:
idx = str(prob['contestId']) + prob['problem']['index']
part = prob["author"]["participantType"]
check = utils.check_status(official, virtual, practice, unoff, part)
if prob['verdict'] == 'OK' and check == 1 and idx not in ques:
if var % 10 == 0:
embed = discord.Embed(color=0x000000)
url = f'{CF_PROBLEM}'
url += str(prob['contestId'])
url += '/'
url += prob["problem"]["index"]
ques.append(idx)
if 'rating' not in prob["problem"]:
rat = '?'
else:
rat = prob['problem']['rating']
tmp = f'[({prob["contestId"]}{prob["problem"]["index"]}) {prob["problem"]["name"]}]({url}) [{rat}]\n'
val += tmp
var = var + 1
if var % 10 == 0:
embed.add_field(
name=f'{handle}\'s latest Accepted solutions :',
value=val,
inline=True)
val = ''
embeds.append(embed)
if var == 100:
break
if var == 0:
await ctx.send(
f'{handle} has not yet submitted a correct solution in the given range.'
)
return
if var % 10 != 0:
embed.add_field(name=f'{handle}\'s latest Accepted solutions :',
value=val,
inline=False)
embeds.append(embed)
i = 0
for embed in embeds:
embed.set_footer(text=f'\nPage : {i+1}/{len(embeds)}')
i += 1
message = await ctx.channel.send(embed=embeds[0])
emojis = ['\u23ee', '\u25c0', '\u25b6', '\u23ed']
for emoji in emojis:
await message.add_reaction(emoji)
i, emoji = 0, ''
while True:
if emoji == '\u23ee':
i = 0
await message.edit(embed=embeds[i])
if emoji == '\u25c0':
if i > 0:
i = i - 1
await message.edit(embed=embeds[i])
if emoji == '\u25b6':
if i < len(embeds) - 1:
i = i + 1
await message.edit(embed=embeds[i])
if emoji == '\u23ed':
i = len(embeds) - 1
await message.edit(embed=embeds[i])
def predicate(message):
def check(reaction, user):
if reaction.message.id != message.id or user == bot.user:
return False
for em in emojis:
if reaction.emoji == em:
return True
return False
return check
try:
react, user = await bot.wait_for('reaction_add',
timeout=15,
check=predicate(message))
except asyncio.TimeoutError:
break
emoji = str(react)
await message.remove_reaction(emoji, member=user)
for emoji in emojis:
await message.clear_reaction(emoji)
