def run(dry_run): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() state = State( integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings ) emails = queries.get_app_interface_emails() smtp_client = SmtpClient(settings=settings) # validate no 2 emails have the same name email_names = set([e['name'] for e in emails]) if len(emails) != len(email_names): logging.error('email names must be unique.') sys.exit(1) emails_to_send = [e for e in emails if not state.exists(e['name'])] for email in emails_to_send: logging.info(['send_email', email['name'], email['subject']]) if not dry_run: names = collect_to(email['to']) subject = email['subject'] body = email['body'] smtp_client.send_mail(names, subject, body) state.add(email['name'])
def __init__(self): ns = 'voxel_map/' self._visual_threshold = rospy.get_param(ns + 'visual_threshold') voxels_per_side = rospy.get_param(ns + 'voxels_per_side') width_on_a_side = rospy.get_param(ns + 'width_on_a_side') initial_prob = rospy.get_param(ns + 'initial_prob') pD = rospy.get_param(ns + 'pD') pFA = rospy.get_param(ns + 'pFA') ps = rospy.get_param(ns + 'ps') pt = rospy.get_param(ns + 'pt') self.map = VoxelMap(voxels_per_side, width_on_a_side, initial_prob, pD, pFA, ps, pt) rospy.Subscriber("truth/NED", Odometry, callback=self.state_cb, queue_size=1) rospy.Subscriber("triang_points", PointCloud, callback=self.measurement_cb) self.map_pub = rospy.Publisher("voxel_map", PointCloud, queue_size=1) self.vis_pub = rospy.Publisher("voxel_map_visual", PointCloud, queue_size=1) self.map_center_pub = rospy.Publisher("voxel_map_center", Odometry) self.state = State() self.blank_cloud = PointCloud() zero_point = Point32(0, 0, 0) self.blank_cloud.points = [zero_point] * self.map.N**3
def run(dry_run): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() users = queries.get_users() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) mails = smtp_client.get_mails(criteria='SUBJECT "Sentry Access Request"', folder='[Gmail]/Sent Mail', settings=settings) user_names = get_sentry_users_from_mails(mails) if not dry_run: slack = init_slack_workspace(QONTRACT_INTEGRATION) for user_name in user_names: guesses = guess_user(user_name, users) if not guesses: logging.debug(f'no users guessed for {user_name}') continue slack_username = \ guesses[0].get('slack_username') or guesses[0]['org_username'] if state.exists(slack_username): continue logging.info(['help_user', slack_username]) if not dry_run: state.add(slack_username) slack.chat_post_message( f'yo <@{slack_username}>! it appears that you have ' + 'requested access to a project in Sentry. ' + 'access is managed automatically via app-interface. ' 'checkout https://url.corp.redhat.com/sentry-help')
def run(dry_run, thread_pool_size=10, internal=None, use_jump_host=True, defer=None): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() clusters = [c for c in queries.get_clusters(minimal=True) if c.get('ocm')] oc_map = OC_Map(clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings, internal=internal, use_jump_host=use_jump_host, thread_pool_size=thread_pool_size) defer(lambda: oc_map.cleanup()) state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) if not dry_run: slack = init_slack_workspace(QONTRACT_INTEGRATION) now = datetime.utcnow() for cluster in oc_map.clusters(): oc = oc_map.get(cluster) if not oc: logging.log(level=oc.log_level, msg=oc.message) continue upgrade_config = oc.get(namespace='openshift-managed-upgrade-operator', kind='UpgradeConfig', name='osd-upgrade-config', allow_not_found=True) if not upgrade_config: logging.debug(f'[{cluster}] UpgradeConfig not found.') continue upgrade_spec = upgrade_config['spec'] upgrade_at = upgrade_spec['upgradeAt'] version = upgrade_spec['desired']['version'] upgrade_at_obj = datetime.strptime(upgrade_at, '%Y-%m-%dT%H:%M:%SZ') state_key = f'{cluster}-{upgrade_at}' # if this is the first iteration in which 'now' had passed # the upgrade at date time, we send a notification if upgrade_at_obj < now: if state.exists(state_key): # already notified continue logging.info(['cluster_upgrade', cluster]) if not dry_run: state.add(state_key) usergroup = f'{cluster}-cluster' usergroup_id = slack.get_usergroup_id(usergroup) slack.chat_post_message( f'Heads up <!subteam^{usergroup_id}>! ' + f'cluster `{cluster}` is currently ' + f'being upgraded to version `{version}`')
def test_get_estimation(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10) state.get_estimation() self.assertIsInstance(state.df, DataFrame) self.assertTrue('Centroid Latitude' in state.df.columns) self.assertTrue('Centroid Longitude' in state.df.columns) self.assertTrue('Congressional District' in state.df.columns) self.assertTrue('GEOID' in state.df.columns) self.assertTrue('Predicted 2015 Population' in state.df.columns) self.assertTrue('geometry' in state.df.columns)
def test_create_run_and_return_kmeans(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10) state.load_estimation() state.K = 2 state.X = np.stack( (np.asarray(state.df['Centroid Latitude'].apply(float)), np.asarray(state.df['Centroid Longitude'].apply(float))), axis=1) state.counts = list(state.df[state.population].apply(float)) state.create_run_and_return_kmeans()
def strategy(self): # past state self.past_state = self.state # current state self.state = State( self.my_pose, # (1, 2) self.lidar_ranges, # (1, 1, 360) self.image, # (1, 3, 480, 640) self.mask, # (1, 18) ) if self.action is not None: current_score = copy.deepcopy(self.score) reward = self.get_reward(self.past_score, current_score) print("reward: {}".format(reward)) self.past_score = current_score reward = torch.LongTensor([reward]) self.agent.memorize(self.past_state, self.action, self.state, reward) # manual wall avoidance if not self.punish_if_facing_wall: avoid, linear_x, angular_z = manual_avoid_wall_2(self.lidar_ranges, dist_th=0.13, back_vel=0.2) else: avoid = False if avoid: self.action = None else: # get action from agent if self.step % 3 == 0: policy = "boltzmann" else: policy = "epsilon" self.action = self.agent.get_action(self.state, self.episode, policy, self.debug) choice = int(self.action.item()) linear_x = ACTION_LIST[choice][0] angular_z = ACTION_LIST[choice][1] print("step: {}, vel:{}, omega:{}".format(self.step, linear_x, angular_z)) # update twist twist = Twist() twist.linear.x = linear_x twist.linear.y = 0.0 twist.linear.z = 0.0 twist.angular.x = 0.0 twist.angular.y = 0.0 twist.angular.z = angular_z self.twist_pub.publish(twist) self.step += 1
def emigrate(self, fraction): """ Generates a State, describing a population slice. This population slice is then deducted from the Agent's population pool. :param fraction: float in range 0.0 < x < 1.0 denoting what fraction of the population is emigrating. """ if not 0.0 < fraction < 1.0: raise ValueError( "Emigration fraction not in valid range 0.0 < x < 1.0") # Compute emigrant slice S, E, I, R, N = self.state() n_emigrants = int(N * fraction) # int() floors/truncates the number. # Update the current Agent's population count new_state = State(S, E, I, R, N - n_emigrants) self.set_state(new_state) return State(S, E, I, R, n_emigrants)
def ls(ctx, integration): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() state = State(integration, accounts, settings=settings) keys = state.ls() # if 'integration' is defined, the 0th token is empty table_content = [ {'integration': k.split('/')[0] or integration, 'key': '/'.join(k.split('/')[1:])} for k in keys] print_output('table', table_content, ['integration', 'key'])
def test_create_unique_filename_for_state(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10) # Check new filename is a unicode string: self.assertIsInstance(state.unique_filename_for_state, unicode) # Check it really is unique: self.assertFalse( os.path.isfile(state.clustered_geojson_results_dir + state.unique_filename_for_state)) self.assertFalse( os.path.isfile(state.clustered_csv_results_dir + state.unique_filename_for_state))
def test_init(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10) self.assertTrue(state.state == 'RI') self.assertTrue(state.year == '2015') self.assertTrue(state.alpha == 0) self.assertTrue(state.alpha_increment == 0.5) self.assertTrue(state.alpha_max == 10) self.assertTrue(state.beta == 0) self.assertTrue(state.beta_increment == 0.1) self.assertTrue(state.dist == cdist) self.assertTrue(state.max_runs == 10) self.assertTrue(state.population == 'Predicted 2015 Population') self.assertIsInstance(state.cong_dist, DataFrame)
def run(dry_run=False): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() state = State( integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings ) credentials_requests = queries.get_credentials_requests() # validate no 2 requests have the same name credentials_requests_names = \ set([r['name'] for r in credentials_requests]) if len(credentials_requests) != len(credentials_requests_names): logging.error('request names must be unique.') sys.exit(1) error = False credentials_requests_to_send = \ [r for r in credentials_requests if not state.exists(r['name'])] for credentials_request_to_send in credentials_requests_to_send: user = credentials_request_to_send['user'] org_username = user['org_username'] public_gpg_key = user.get('public_gpg_key') credentials_name = credentials_request_to_send['credentials'] if not public_gpg_key: error = True logging.error( f"user {org_username} does not have a public gpg key") continue logging.info(['send_credentials', org_username, credentials_name]) if not dry_run: request_name = credentials_request_to_send['name'] names = [org_username] subject = request_name ecrypted_credentials = \ get_ecrypted_credentials(credentials_name, user, settings) if not ecrypted_credentials: error = True logging.error( f"could not get encrypted credentials {credentials_name}") continue body = MESSAGE_TEMPLATE.format( request_name, credentials_name, ecrypted_credentials) smtp_client.send_mail(names, subject, body, settings=settings) state.add(request_name) if error: sys.exit(1)
def run(dry_run): unleash_instances = queries.get_unleash_instances() accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) for unleash_instance in unleash_instances: instance_name = unleash_instance['name'] current_state = fetch_current_state(unleash_instance) previous_state = fetch_previous_state(state, instance_name) diffs = calculate_diff(current_state, previous_state) if diffs: act(dry_run, state, unleash_instance, diffs)
def test_run_stats(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10, unique_filename_for_state='RI_2017_10_07_14_17_05') state.run_stats() self.assertTrue(os.path.isfile(state.stats_filename)) self.assertTrue(os.path.isfile(state.means_filename))
def slack_notify(self, dry_run, aws_accounts, ri): result = self._get_deployment_result(dry_run, ri) state = State( integration=self.integration, accounts=aws_accounts, settings=self.settings ) for saas_file in self.saas_files: self.github = self._initiate_github(saas_file) saas_file_name = saas_file['name'] for resource_template in saas_file['resourceTemplates']: url = resource_template['url'] hash_length = resource_template['hash_length'] resource_template_name = resource_template['name'] for target in resource_template['targets']: if not target.get('notify'): continue cluster, namespace = \ self._get_cluster_and_namespace(target) target_hash = target['hash'] desired_commit_sha = \ self._get_commit_sha(url, target_hash, hash_length) state_key_format = "{}/{}/{}/{}" state_key = state_key_format.format( saas_file_name, resource_template_name, cluster, namespace ) current_commit_sha = state.get(state_key, None) if current_commit_sha != desired_commit_sha: slack_info = saas_file.get('slack') if slack_info: slack = self._init_slack(slack_info) msg_format = "[{}] {} deployment to {}/{}: {}" msg = msg_format.format( saas_file_name, resource_template_name, cluster, namespace, result ) channel = slack.chat_kwargs['channel'] logging.info(['slack_notify', channel, msg]) if not dry_run: state[state_key] = desired_commit_sha slack.chat_post_message(msg)
def plot(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10, unique_filename_for_state='RI_2017_10_07_14_17_05') state.plot() self.assertTrue( os.path.isfile(state.cong_dist_plot_dir + state.state + '_cong_dist')) self.assertTrue(os.path.isfile(state.clusters_plot_filename + '.png')) self.assertTrue(os.path.isfile(state.clusters_plot_filename + '.pdf'))
def run(dry_run): jira_boards = [j for j in queries.get_jira_boards() if j.get('slack')] accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) for index, jira_board in enumerate(jira_boards): if not is_in_shard_round_robin(jira_board['name'], index): continue jira, current_state = fetch_current_state(jira_board, settings) previous_state = fetch_previous_state(state, jira.project) if previous_state: diffs = calculate_diff(jira.server, current_state, previous_state) act(dry_run, jira_board, diffs) if not dry_run: write_state(state, jira.project, current_state)
def run(dry_run): unleash_instances = queries.get_unleash_instances() accounts = queries.get_aws_accounts() settings = queries.get_app_interface_settings() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) for unleash_instance in unleash_instances: instance_name = unleash_instance['name'] current_state = fetch_current_state(unleash_instance) if not current_state: logging.warning('not acting on empty Unleash instances. ' + 'please create a feature toggle to get started.') continue previous_state = fetch_previous_state(state, instance_name) diffs = calculate_diff(current_state, previous_state) if diffs: act(dry_run, state, unleash_instance, diffs)
def test_cluster_population_is_out_of_bounds(self): # Single values of population: state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10, min_population=1, max_population=100) state.people_per_cluster = 0 self.assertTrue(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = 101 self.assertTrue(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = 1 self.assertFalse(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = 100 self.assertFalse(state.cluster_population_is_out_of_bounds()) test = ( (state.max_population - state.min_population) * random.random() + state.min_population) state.people_per_cluster = test self.assertFalse(state.cluster_population_is_out_of_bounds()) # Multiple values of population: state.people_per_cluster = [0, 100, 50] self.assertTrue(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = [0, 101, 50] self.assertTrue(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = [1, 101, 50] self.assertTrue(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = [1, 100, 50] self.assertFalse(state.cluster_population_is_out_of_bounds()) state.people_per_cluster = [ (state.max_population - state.min_population) * random.random() + state.min_population, (state.max_population - state.min_population) * random.random() + state.min_population ] self.assertFalse(state.cluster_population_is_out_of_bounds())
def immigrate(self, immigration_slice): # Get immigrant data n_im = immigration_slice.N distribution_im = np.asarray(immigration_slice)[:4] # Get local data n_local = self.__history[-1].N distribution_local = np.asarray(self.state())[:4] # Compute new local distribution distribution_post = np.add(distribution_im * n_im, distribution_local * n_local) / (n_im + n_local) # Replace the current state with a state that includes the new immigration state_post = State(distribution_post[0], distribution_post[1], distribution_post[2], distribution_post[3], n_im + n_local) self.__history[-1] = state_post
def test_get_stats(self): state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10, unique_filename_for_state='RI_2017_10_07_14_17_05') stats_df = state.get_stats() self.assertIsInstance(stats_df, DataFrame) self.assertTrue('Population' in stats_df.columns) self.assertTrue('Number of Blocks' in stats_df.columns) self.assertTrue('Is Cluster' in stats_df.columns) self.assertTrue('Mean Pairwise Distance' in stats_df.columns) self.assertTrue( 'Cluster/Congressional District ID' in stats_df.columns)
def __init__(self): ns = 'voxel_map/' self._visual_threshold = rospy.get_param(ns + 'visual_threshold') self.map = VoxelMap() rospy.Subscriber("truth/NED", Odometry, callback=self.state_cb, queue_size=1) rospy.Subscriber("triang_points", PointCloud, callback=self.measurement_cb) self.map_pub = rospy.Publisher("voxel_map", PointCloud, queue_size=1) self.vis_pub = rospy.Publisher("voxel_map_visual", PointCloud, queue_size=1) self.state = State() self.blank_cloud = PointCloud() zero_point = Point32(0, 0, 0) self.blank_cloud.points = [zero_point] * self.map.N**3
def run(dry_run, io_dir='throughput/', defer=None): jjb, additional_repo_urls = init_jjb() defer(lambda: jjb.cleanup()) accounts = queries.get_aws_accounts() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=jjb.settings) if dry_run: validate_repos_and_admins(jjb, additional_repo_urls) jjb.generate(io_dir, 'desired') jjb.overwrite_configs(state) jjb.generate(io_dir, 'current') jjb.print_diffs(io_dir) else: jjb.update() configs = jjb.get_configs() for name, desired_config in configs.items(): state.add(name, value=desired_config, force=True)
from agent.agent import Agent from utils.state import State, Parameters from matplotlib import pyplot as plt from utils.strconv import params2str a = 0.2 # exposure rate b = 1.75 # Measure of time to infection once exposed g = 0.5 # Rate of recovery d = 0.05 # Rate at which loss of immunity occurs. r = 1 # Effectiveness of measures (range 0-1, lower is more effective) init_agent_a = State(0.99, 0, 0.01, 0, 10000) initial_params = Parameters(a, b, g, d, r) agent_a = Agent('test', init_agent_a, initial_params) print(f"Agent a initial: {init_agent_a}") # Simulate for i iterations i = initial_i = 10000 print(f"Iterations remaining {i}") iterations_list = [] agent_iterations = 0 while True: agent_a.iterate_rl() agent_iterations += 1
def _initiate_state(self, accounts): self.state = State(integration=self.integration, accounts=accounts, settings=self.settings)
from agent.agent import Agent from utils.state import State, Parameters from utils.plotting import plot_agent_history, plot_compartment_comparison # Initialize SEIRS model params a = 0.2 # exposure rate b = 1.75 # Measure of time to infection once exposed g = 0.5 # Rate of recovery d = 0 # Rate at which loss of immunity occurs. r = 1 # Cost level?? initial_params = Parameters(a, b, g, d, r) # Initialize agent states init_agent_a = State(0.99, 0, 0.01, 0, 10000) init_agent_b = State(1.0, 0, 0, 0, 10000) init_agent_c = State(1.0, 0, 0, 0, 10000) # Initialize agent objects agent_a = Agent('A', init_agent_a, initial_params) agent_b = Agent('B', init_agent_b, initial_params) agent_c = Agent('C', init_agent_c, initial_params) agents = [agent_a, agent_b, agent_c] # Print initial status print(f"Agent a initial: {init_agent_a}") print(f"Agent b initial: {init_agent_b}") print(f"Agent c initial: {init_agent_c}") # Simulate for 100 iterations for _ in range(80):
def run(dry_run=False, enable_deletion=False): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() state = State(integration=QONTRACT_INTEGRATION, accounts=accounts, settings=settings) queries_list = collect_queries() remove_candidates = [] for query in queries_list: query_name = query['name'] # Checking the sql-query state: # - No state: up for execution. # - State is a timestamp: executed and up for removal # after the JOB_TTL # - State is 'DONE': executed and removed. try: query_state = state[query_name] if query_state != 'DONE': remove_candidates.append({ 'name': query_name, 'timestamp': query_state }) continue except KeyError: pass job_yaml = process_template(query) job = yaml.safe_load(job_yaml) job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION) oc_map = OC_Map(namespaces=[query['namespace']], integration=QONTRACT_INTEGRATION, settings=queries.get_app_interface_settings(), internal=None) openshift_base.apply(dry_run=dry_run, oc_map=oc_map, cluster=query['cluster'], namespace=query['namespace']['name'], resource_type='job', resource=job_resource) if not dry_run: state[query_name] = time.time() for candidate in remove_candidates: if time.time() < candidate['timestamp'] + JOB_TTL: continue try: query = collect_queries(query_name=candidate['name'])[0] except IndexError: raise RuntimeError(f'sql-query {candidate["name"]} not present' f'in the app-interface while its Job is still ' f'not removed from the cluster. Manual clean ' f'up is needed.') oc_map = OC_Map(namespaces=[query['namespace']], integration=QONTRACT_INTEGRATION, settings=queries.get_app_interface_settings(), internal=None) openshift_base.delete(dry_run=dry_run, oc_map=oc_map, cluster=query['cluster'], namespace=query['namespace']['name'], resource_type='job', name=query['name'], enable_deletion=enable_deletion) if not dry_run: state[candidate['name']] = 'DONE'
def set(ctx, integration, key, value): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() state = State(integration, accounts, settings=settings) state.add(key, value=value, force=True)
def __init__(self): self.__name = "Pat" self.__connection = Connection() self.__prover = Thread(target=self.prove, args=()) self.__state = State()
def rm(ctx, integration, key): settings = queries.get_app_interface_settings() accounts = queries.get_aws_accounts() state = State(integration, accounts, settings=settings) state.rm(key)