def _test_pcluster_compute_fleet(cluster, expected_num_nodes): """Test pcluster compute fleet commands.""" logging.info("Testing pcluster stop functionalities") cluster.stop() # Sample pcluster stop output: # Compute fleet status is: RUNNING. Submitting status change request. # Request submitted successfully. It might take a while for the transition to complete. # Please run 'pcluster status' if you need to check compute fleet status wait_for_num_instances_in_cluster(cluster.cfn_name, cluster.region, desired=0) _test_describe_instances(cluster) compute_fleet = cluster.describe_compute_fleet() assert_that(compute_fleet["status"]).is_equal_to("STOPPED") last_stop_time = compute_fleet["lastStatusUpdatedTime"] logging.info("Testing pcluster start functionalities") # Do a complicated sequence of start and stop and see if commands will still work cluster.start() cluster.stop() cluster.stop() cluster.start() compute_fleet = cluster.describe_compute_fleet() last_start_time = compute_fleet["lastStatusUpdatedTime"] logging.info("Checking last status update time is updated") assert_that(last_stop_time < last_start_time) wait_for_num_instances_in_cluster(cluster.cfn_name, cluster.region, desired=expected_num_nodes) _test_describe_instances(cluster) check_status(cluster, "CREATE_COMPLETE", "running", "RUNNING")
def close_spider(self,spider): for (id,item) in self.result_dict.items(): cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]}) utils.set_origin_value_list(item, cursor, ['display_time_begin']) utils.set_origin_value_if_db_smaller(item, cursor, 'display_time_end') utils.check_status(item, cursor) if item.has_key('limit') and item['limit'] != utils.UNLIMITED_NUM: limit = item['limit'] elif item.has_key('left_goods'): limit = item['left_goods'] else: limit = utils.UNLIMITED_NUM origin_limit = utils.UNLIMITED_NUM if cursor: if cursor.has_key('limit') and cursor['limit'] != utils.UNLIMITED_NUM: origin_limit = cursor['limit'] if origin_limit > limit: limit = origin_limit if item.has_key('left_goods'): sale = limit - item['left_goods'] item['sale'] = sale item['limit'] = limit item['sale_percent'] = sale * 100 / limit self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe) utils.driver_quit(spider.driver) spider.log('driver.quit()', level = log.DEBUG)
def put(self, **kwargs): if self.check_write_access('update', **kwargs) == False: return abort(403) if not self.form: return abort(405) instance = getattr(self.model, 'find_by_%s' % self.key_name)(**kwargs) check_status(instance) if len(request.form) > 0: form = self.form(request.form, obj=instance) else: form = self.form(ImmutableMultiDict(request.json), obj=instance) if not form.validate(): return form.errors_as_json() try: self.pre_update(form, instance, **kwargs) except FormApiError, e: if not e.get_key() in form.errors: form.errors[e.get_key()] = [] form.errors[e.get_key()].append(e.get_message()) return form.errors_as_json()
def test_slurm_cli_commands( request, scheduler, region, os, pcluster_config_reader, clusters_factory, s3_bucket_factory ): """Test pcluster cli commands are working.""" # Use long scale down idle time so we know nodes are terminated by pcluster stop cluster_config = pcluster_config_reader(scaledown_idletime=60) # Using custom AMI not tagged by pcluser will generate a warning custom_ami = retrieve_latest_ami(region, os, ami_type="official", architecture="x86_64") config_file = "pcluster.config.with.warnings.yaml" cluster_config_with_warning = pcluster_config_reader(config_file=config_file, custom_ami=custom_ami) # Test below is not compatible with `--cluster` flag. Therefore, skip it if the flag is provided. if not request.config.getoption("cluster"): _test_create_with_warnings(cluster_config_with_warning, clusters_factory) cluster = _test_create_cluster(clusters_factory, cluster_config, request) _test_describe_cluster(cluster) _test_list_cluster(cluster.name, "CREATE_COMPLETE") _test_update_with_warnings(cluster_config_with_warning, cluster) check_status(cluster, "CREATE_COMPLETE", "running", "RUNNING") filters = [{}, {"node_type": "HeadNode"}, {"node_type": "Compute"}, {"queue_name": "ondemand1"}] for filter_ in filters: _test_describe_instances(cluster, **filter_) _test_pcluster_export_cluster_logs(s3_bucket_factory, cluster) check_pcluster_list_cluster_log_streams(cluster, os) _test_pcluster_get_cluster_log_events(cluster) _test_pcluster_get_cluster_stack_events(cluster) _test_pcluster_compute_fleet(cluster, expected_num_nodes=2) remote_command_executor = RemoteCommandExecutor(cluster) assert_no_errors_in_logs(remote_command_executor, scheduler)
def close_spider(self,spider): for (id,item) in self.result_dict.items(): cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]}) utils.set_origin_value_list(item, cursor, ['display_time_end']) utils.check_status(item, cursor) self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe) utils.driver_quit(spider.driver) spider.log('driver.quit()', level = log.DEBUG)
def close_spider(self,spider): for (id,item) in self.result_dict.items(): cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]}) utils.check_status(item, cursor) if not cursor or utils.is_item_changed(item, cursor): self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe) utils.driver_quit(spider.driver) spider.log('driver.quit()', level = log.DEBUG)
def run(self): last = 0 while True: now = time.time() looptime = now - last if looptime < self.interval: time.sleep(self.interval + 1 - looptime) last = time.time() else: last = now result = utils.readConfig(self.hutch, self.mtime) if result != None: (self.mtime, cfglist, hosts, vdict) = result self.rmtime = {} # Force a re-read! self.model.configuration(cfglist, hosts, vdict) result = utils.readStatusDir(self.hutch, self.readStatusFile) for l in result: rdir = l['rdir'] l.update(utils.check_status(l['rhost'], l['rport'], l['rid'])) l['stattime'] = time.time() if l['rdir'] == '/tmp': l['rdir'] = rdir else: l['newstyle'] = False self.model.running(l) for l in self.model.cfglist: if l['stattime'] + self.interval > time.time(): continue; if l['hard']: s = {'pid' : -1, 'autorestart' : False } try: pv = psp.Pv.Pv(l['base'] + ":HEARTBEAT") pv.connect(1.0) pv.disconnect() s['status'] = utils.STATUS_RUNNING except: s['status'] = utils.STATUS_SHUTDOWN s['rid'] = l['id'] s['rdir'] = l['dir'] else: s = utils.check_status(l['host'], l['port'], l['id']) s['stattime'] = time.time() s['rhost'] = l['host'] s['rport'] = l['port'] if l['newstyle']: if s['rdir'] == '/tmp': del s['rdir'] else: s['newstyle'] = False # We've switched from new to old?!? self.model.running(s) for p in self.model.children: if p.poll() != None: self.model.children.remove(p)
def add_customer(customer_id, **kw): """ Add a customer to the database :param customer_id: :param name: :param lastname: :param home_address: :param phone_number: :param email_address: :param status: must be True or False, active or inactive :param credit_limit: Must be a number, no $ :return: None """ try: cust = Customer.create(customer_id=customer_id, first_name=kw['name'], last_name=kw['last_name'], home_address=kw['home_address'], phone_number=kw['phone_number'], email_address=kw['email_address'], status=check_status(kw['status']), credit_limit=kw['credit_limit']) cust.save() except KeyError as err: logging.error(err) raise ValueError(config.etext['no_save'].format(customer_id)) except Exception as err: logging.error(err) raise logger.info("Customer with id %s successfully added!", customer_id)
def run(self): last = 0 while True: now = time.time() looptime = now - last if looptime < self.interval: time.sleep(self.interval + 1 - looptime) last = time.time() else: last = now result = utils.readConfig(self.hutch, self.mtime) if result != None: (self.mtime, cfglist, hosts, vdict) = result self.rmtime = {} # Force a re-read! self.model.configuration(cfglist, hosts, vdict) result = utils.readStatusDir(self.hutch, self.readStatusFile) for l in result: rdir = l['rdir'] l.update(utils.check_status(l['rhost'], l['rport'], l['rid'])) l['stattime'] = time.time() if l['rdir'] == '/tmp': l['rdir'] = rdir else: l['newstyle'] = False self.model.running(l) for l in self.model.cfglist: if l['stattime'] + self.interval > time.time(): continue; s = utils.check_status(l['host'], l['port'], l['id']) s['stattime'] = time.time() s['rhost'] = l['host'] s['rport'] = l['port'] if l['newstyle']: if s['rdir'] == '/tmp': del s['rdir'] else: s['newstyle'] = False # We've switched from new to old?!? self.model.running(s) for p in self.model.children: if p.poll() != None: self.model.children.remove(p)
def details(self, **kwargs): if self.check_read_access('details', **kwargs) == False: return abort(403) instance = getattr(self.model, 'find_by_%s' % self.key_name)(**kwargs) check_status(instance) result = instance.to_json() if isinstance(instance, SeenModel): if not instance.seen: instance.seen = 0 instance.seen = instance.seen + 1 instance.last_seen = datetime.datetime.utcnow() instance.save() return jsonify(result), 200
def delete(self, **kwargs): if self.check_write_access('delete', **kwargs) == False: return abort(403) if not self.form: return abort(405) # Return 204 ! instance = self.model.find_by_id(**kwargs) check_status(instance) try: self.pre_delete(instance, **kwargs) except FormApiError, e: if not e.get_key() in form.errors: form.errors[e.get_key()] = [] form.errors[e.get_key()].append(e.get_message()) return form.errors_as_json()
def test_hmmpress_task_existing(self, tmpdir, datadir): with tmpdir.as_cwd(): tf = datadir('test-profile.hmm') for ext in self.extensions: touch(tf + ext) task = HMMPressTask().task(tf) run_tasks([task], ['run']) print(os.listdir(), file=sys.stderr) print(task, file=sys.stderr) status = check_status(task) assert status.status == 'up-to-date'
def test_cmpress_task(self, tmpdir, datadir): with tmpdir.as_cwd(): tf = datadir('test-covariance-model.cm') task = CMPressTask().task(tf, params=self.cmpress_cfg) run_tasks([task], ['run']) status = check_status(task) print(os.listdir(), file=sys.stderr) for ext in self.extensions: assert os.path.isfile(tf + ext) assert status.status == 'up-to-date'
def test_hmmpress_task(self): with TemporaryDirectory() as td: with Move(td): with TestData('test-profile.hmm', td) as tf: task = tasks.get_hmmpress_task(tf, self.hmmpress_cfg) run_tasks([task], ['run']) status = check_status(task) print(os.listdir(td), file=sys.stderr) for ext in self.extensions: self.assertTrue(os.path.isfile(tf + ext)) self.assertEquals(status.status, 'up-to-date')
def test_cmpress_task_existing(self): with TemporaryDirectory() as td: with Move(td): with TestData('test-covariance-model.cm', td) as tf: for ext in self.extensions: touch(tf + ext) task = tasks.get_cmpress_task(tf, self.cmpress_cfg) run_tasks([task], ['run']) print(os.listdir(td), file=sys.stderr) print(task, file=sys.stderr) status = check_status(task) self.assertEquals(status.status, 'up-to-date')
def test_lastdb_task_existing(self): with TemporaryDirectory() as td: with Move(td): with TestData('test-protein.fa', td) as tf: for ext in self.extensions: touch(tf + ext) task = tasks.get_lastdb_task(tf, tf, self.lastdb_cfg, prot=True) run_tasks([task], ['run']) print(os.listdir(td), file=sys.stderr) print(task, file=sys.stderr) status = check_status(task) self.assertEquals(status.status, 'up-to-date')
def test_lastdb_task_nucl(self): with TemporaryDirectory() as td: with Move(td): with TestData('test-transcript.fa', td) as tf: task = tasks.get_lastdb_task(tf, tf, self.lastdb_cfg, prot=False) run_tasks([task], ['run']) status = check_status(task) print(os.listdir(td), file=sys.stderr) print('PATH:', os.environ['PATH'], file=sys.stderr) for ext in self.extensions: self.assertTrue(os.path.isfile(tf + ext)) self.assertEquals(status.status, 'up-to-date')
def update_info(): changed = [] logger.info(f"Checking at {datetime.now()}") for server in servers: last = server['last_status'] update = check_status(server['host'], server['port']) print(server, last, update) server['last_status'] = update server['last_update'] = datetime.now() logger.debug(json.dumps(server, indent=2, default=str)) if update != last and last is not None: changed.append(server) return changed
def test_lastal_task_uptodate(self): with TemporaryDirectory() as td: with Move(td): with TestData('test-protein.fa', td) as prot, \ TemporaryFile(td) as out: print(os.listdir(td), file=sys.stderr) db_task = tasks.get_lastdb_task(prot, prot, self.lastdb_cfg) aln_task = tasks.get_lastal_task(prot, prot, out, self.lastal_cfg, translate=False, cutoff=None) # Run it once run_tasks([db_task, aln_task], ['run']) # Now run again and check the status run_tasks([aln_task], ['run']) status = check_status(aln_task) self.assertEquals(status.status, 'up-to-date')
def close_spider(self,spider): for (id,item) in self.result_dict.items(): cursor = self.collection.find_one({self.uniq_key : item[self.uniq_key]}) utils.check_status(item, cursor) if not utils.set_origin_value_if_db_smaller(item, cursor, 'display_time_end') or utils.is_item_changed(item, cursor): self.collection.update({self.uniq_key: item[self.uniq_key] }, {'$set':dict(item) },upsert=True, safe=self.safe)
def _assert_server_status(cluster): check_status(cluster, "CREATE_COMPLETE", "running", "RUNNING")
async def user_ques(ctx, handle, *args): url = f'{CF_USER_STATUS}{handle}&from=1' obj = requests.get(url) data = json.loads(obj.text) if data['status'] == "FAILED": await ctx.send(f'{data["comment"]}') return elif len(data['result']) == 0: await ctx.send(f'{handle} has made no submissions yet.') return official, virtual, practice, unoff = False, False, False, False var = 0 embeds, val = [], '' ques = [] for arg in args: if arg == '+contest': official = True elif arg == '+virtual': virtual = True elif arg == '+practice': practice = True elif arg == '+unofficial': unoff = True else: await ctx.send( 'Invalid arguments. Please enter correct arguments and then try again.' ) return for prob in data['result']: idx = str(prob['contestId']) + prob['problem']['index'] part = prob["author"]["participantType"] check = utils.check_status(official, virtual, practice, unoff, part) if prob['verdict'] == 'OK' and check == 1 and idx not in ques: if var % 10 == 0: embed = discord.Embed(color=0x000000) url = f'{CF_PROBLEM}' url += str(prob['contestId']) url += '/' url += prob["problem"]["index"] ques.append(idx) if 'rating' not in prob["problem"]: rat = '?' else: rat = prob['problem']['rating'] tmp = f'[({prob["contestId"]}{prob["problem"]["index"]}) {prob["problem"]["name"]}]({url}) [{rat}]\n' val += tmp var = var + 1 if var % 10 == 0: embed.add_field( name=f'{handle}\'s latest Accepted solutions :', value=val, inline=True) val = '' embeds.append(embed) if var == 100: break if var == 0: await ctx.send( f'{handle} has not yet submitted a correct solution in the given range.' ) return if var % 10 != 0: embed.add_field(name=f'{handle}\'s latest Accepted solutions :', value=val, inline=False) embeds.append(embed) i = 0 for embed in embeds: embed.set_footer(text=f'\nPage : {i+1}/{len(embeds)}') i += 1 message = await ctx.channel.send(embed=embeds[0]) emojis = ['\u23ee', '\u25c0', '\u25b6', '\u23ed'] for emoji in emojis: await message.add_reaction(emoji) i, emoji = 0, '' while True: if emoji == '\u23ee': i = 0 await message.edit(embed=embeds[i]) if emoji == '\u25c0': if i > 0: i = i - 1 await message.edit(embed=embeds[i]) if emoji == '\u25b6': if i < len(embeds) - 1: i = i + 1 await message.edit(embed=embeds[i]) if emoji == '\u23ed': i = len(embeds) - 1 await message.edit(embed=embeds[i]) def predicate(message): def check(reaction, user): if reaction.message.id != message.id or user == bot.user: return False for em in emojis: if reaction.emoji == em: return True return False return check try: react, user = await bot.wait_for('reaction_add', timeout=15, check=predicate(message)) except asyncio.TimeoutError: break emoji = str(react) await message.remove_reaction(emoji, member=user) for emoji in emojis: await message.clear_reaction(emoji)