def run(dry_run):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(
integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings
)
emails = queries.get_app_interface_emails()
smtp_client = SmtpClient(settings=settings)
# validate no 2 emails have the same name
email_names = set([e['name'] for e in emails])
if len(emails) != len(email_names):
logging.error('email names must be unique.')
sys.exit(1)
emails_to_send = [e for e in emails if not state.exists(e['name'])]
for email in emails_to_send:
logging.info(['send_email', email['name'], email['subject']])
if not dry_run:
names = collect_to(email['to'])
subject = email['subject']
body = email['body']
smtp_client.send_mail(names, subject, body)
state.add(email['name'])
def __init__(self):
ns = 'voxel_map/'
self._visual_threshold = rospy.get_param(ns + 'visual_threshold')
voxels_per_side = rospy.get_param(ns + 'voxels_per_side')
width_on_a_side = rospy.get_param(ns + 'width_on_a_side')
initial_prob = rospy.get_param(ns + 'initial_prob')
pD = rospy.get_param(ns + 'pD')
pFA = rospy.get_param(ns + 'pFA')
ps = rospy.get_param(ns + 'ps')
pt = rospy.get_param(ns + 'pt')
self.map = VoxelMap(voxels_per_side, width_on_a_side, initial_prob, pD,
pFA, ps, pt)
rospy.Subscriber("truth/NED",
Odometry,
callback=self.state_cb,
queue_size=1)
rospy.Subscriber("triang_points",
PointCloud,
callback=self.measurement_cb)
self.map_pub = rospy.Publisher("voxel_map", PointCloud, queue_size=1)
self.vis_pub = rospy.Publisher("voxel_map_visual",
PointCloud,
queue_size=1)
self.map_center_pub = rospy.Publisher("voxel_map_center", Odometry)
self.state = State()
self.blank_cloud = PointCloud()
zero_point = Point32(0, 0, 0)
self.blank_cloud.points = [zero_point] * self.map.N**3
def run(dry_run):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
users = queries.get_users()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
mails = smtp_client.get_mails(criteria='SUBJECT "Sentry Access Request"',
folder='[Gmail]/Sent Mail',
settings=settings)
user_names = get_sentry_users_from_mails(mails)
if not dry_run:
slack = init_slack_workspace(QONTRACT_INTEGRATION)
for user_name in user_names:
guesses = guess_user(user_name, users)
if not guesses:
logging.debug(f'no users guessed for {user_name}')
continue
slack_username = \
guesses[0].get('slack_username') or guesses[0]['org_username']
if state.exists(slack_username):
continue
logging.info(['help_user', slack_username])
if not dry_run:
state.add(slack_username)
slack.chat_post_message(
f'yo <@{slack_username}>! it appears that you have ' +
'requested access to a project in Sentry. ' +
'access is managed automatically via app-interface. '
'checkout https://url.corp.redhat.com/sentry-help')
def run(dry_run,
thread_pool_size=10,
internal=None,
use_jump_host=True,
defer=None):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
clusters = [c for c in queries.get_clusters(minimal=True) if c.get('ocm')]
oc_map = OC_Map(clusters=clusters,
integration=QONTRACT_INTEGRATION,
settings=settings,
internal=internal,
use_jump_host=use_jump_host,
thread_pool_size=thread_pool_size)
defer(lambda: oc_map.cleanup())
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
if not dry_run:
slack = init_slack_workspace(QONTRACT_INTEGRATION)
now = datetime.utcnow()
for cluster in oc_map.clusters():
oc = oc_map.get(cluster)
if not oc:
logging.log(level=oc.log_level, msg=oc.message)
continue
upgrade_config = oc.get(namespace='openshift-managed-upgrade-operator',
kind='UpgradeConfig',
name='osd-upgrade-config',
allow_not_found=True)
if not upgrade_config:
logging.debug(f'[{cluster}] UpgradeConfig not found.')
continue
upgrade_spec = upgrade_config['spec']
upgrade_at = upgrade_spec['upgradeAt']
version = upgrade_spec['desired']['version']
upgrade_at_obj = datetime.strptime(upgrade_at, '%Y-%m-%dT%H:%M:%SZ')
state_key = f'{cluster}-{upgrade_at}'
# if this is the first iteration in which 'now' had passed
# the upgrade at date time, we send a notification
if upgrade_at_obj < now:
if state.exists(state_key):
# already notified
continue
logging.info(['cluster_upgrade', cluster])
if not dry_run:
state.add(state_key)
usergroup = f'{cluster}-cluster'
usergroup_id = slack.get_usergroup_id(usergroup)
slack.chat_post_message(
f'Heads up <!subteam^{usergroup_id}>! ' +
f'cluster `{cluster}` is currently ' +
f'being upgraded to version `{version}`')
def test_get_estimation(self):
state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10)
state.get_estimation()
self.assertIsInstance(state.df, DataFrame)
self.assertTrue('Centroid Latitude' in state.df.columns)
self.assertTrue('Centroid Longitude' in state.df.columns)
self.assertTrue('Congressional District' in state.df.columns)
self.assertTrue('GEOID' in state.df.columns)
self.assertTrue('Predicted 2015 Population' in state.df.columns)
self.assertTrue('geometry' in state.df.columns)
def test_create_run_and_return_kmeans(self):
state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10)
state.load_estimation()
state.K = 2
state.X = np.stack(
(np.asarray(state.df['Centroid Latitude'].apply(float)),
np.asarray(state.df['Centroid Longitude'].apply(float))),
axis=1)
state.counts = list(state.df[state.population].apply(float))
state.create_run_and_return_kmeans()
def strategy(self):
# past state
self.past_state = self.state
# current state
self.state = State(
self.my_pose, # (1, 2)
self.lidar_ranges, # (1, 1, 360)
self.image, # (1, 3, 480, 640)
self.mask, # (1, 18)
)
if self.action is not None:
current_score = copy.deepcopy(self.score)
reward = self.get_reward(self.past_score, current_score)
print("reward: {}".format(reward))
self.past_score = current_score
reward = torch.LongTensor([reward])
self.agent.memorize(self.past_state, self.action, self.state, reward)
# manual wall avoidance
if not self.punish_if_facing_wall:
avoid, linear_x, angular_z = manual_avoid_wall_2(self.lidar_ranges, dist_th=0.13, back_vel=0.2)
else:
avoid = False
if avoid:
self.action = None
else:
# get action from agent
if self.step % 3 == 0:
policy = "boltzmann"
else:
policy = "epsilon"
self.action = self.agent.get_action(self.state, self.episode, policy, self.debug)
choice = int(self.action.item())
linear_x = ACTION_LIST[choice][0]
angular_z = ACTION_LIST[choice][1]
print("step: {}, vel:{}, omega:{}".format(self.step, linear_x, angular_z))
# update twist
twist = Twist()
twist.linear.x = linear_x
twist.linear.y = 0.0
twist.linear.z = 0.0
twist.angular.x = 0.0
twist.angular.y = 0.0
twist.angular.z = angular_z
self.twist_pub.publish(twist)
self.step += 1
def emigrate(self, fraction):
"""
Generates a State, describing a population slice. This population slice is then deducted from the Agent's
population pool.
:param fraction: float in range 0.0 < x < 1.0 denoting what fraction of the population is emigrating.
"""
if not 0.0 < fraction < 1.0:
raise ValueError(
"Emigration fraction not in valid range 0.0 < x < 1.0")
# Compute emigrant slice
S, E, I, R, N = self.state()
n_emigrants = int(N * fraction) # int() floors/truncates the number.
# Update the current Agent's population count
new_state = State(S, E, I, R, N - n_emigrants)
self.set_state(new_state)
return State(S, E, I, R, n_emigrants)
def ls(ctx, integration):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration, accounts, settings=settings)
keys = state.ls()
# if 'integration' is defined, the 0th token is empty
table_content = [
{'integration': k.split('/')[0] or integration,
'key': '/'.join(k.split('/')[1:])}
for k in keys]
print_output('table', table_content, ['integration', 'key'])
def test_create_unique_filename_for_state(self):
state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10)
# Check new filename is a unicode string:
self.assertIsInstance(state.unique_filename_for_state, unicode)
# Check it really is unique:
self.assertFalse(
os.path.isfile(state.clustered_geojson_results_dir +
state.unique_filename_for_state))
self.assertFalse(
os.path.isfile(state.clustered_csv_results_dir +
state.unique_filename_for_state))
def test_init(self):
state = State('RI', '2015', 0, 0.5, 10, 0, 0.1, cdist, 10)
self.assertTrue(state.state == 'RI')
self.assertTrue(state.year == '2015')
self.assertTrue(state.alpha == 0)
self.assertTrue(state.alpha_increment == 0.5)
self.assertTrue(state.alpha_max == 10)
self.assertTrue(state.beta == 0)
self.assertTrue(state.beta_increment == 0.1)
self.assertTrue(state.dist == cdist)
self.assertTrue(state.max_runs == 10)
self.assertTrue(state.population == 'Predicted 2015 Population')
self.assertIsInstance(state.cong_dist, DataFrame)
def run(dry_run=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(
integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings
)
credentials_requests = queries.get_credentials_requests()
# validate no 2 requests have the same name
credentials_requests_names = \
set([r['name'] for r in credentials_requests])
if len(credentials_requests) != len(credentials_requests_names):
logging.error('request names must be unique.')
sys.exit(1)
error = False
credentials_requests_to_send = \
[r for r in credentials_requests if not state.exists(r['name'])]
for credentials_request_to_send in credentials_requests_to_send:
user = credentials_request_to_send['user']
org_username = user['org_username']
public_gpg_key = user.get('public_gpg_key')
credentials_name = credentials_request_to_send['credentials']
if not public_gpg_key:
error = True
logging.error(
f"user {org_username} does not have a public gpg key")
continue
logging.info(['send_credentials', org_username, credentials_name])
if not dry_run:
request_name = credentials_request_to_send['name']
names = [org_username]
subject = request_name
ecrypted_credentials = \
get_ecrypted_credentials(credentials_name, user, settings)
if not ecrypted_credentials:
error = True
logging.error(
f"could not get encrypted credentials {credentials_name}")
continue
body = MESSAGE_TEMPLATE.format(
request_name, credentials_name, ecrypted_credentials)
smtp_client.send_mail(names, subject, body, settings=settings)
state.add(request_name)
if error:
sys.exit(1)
def run(dry_run):
unleash_instances = queries.get_unleash_instances()
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
for unleash_instance in unleash_instances:
instance_name = unleash_instance['name']
current_state = fetch_current_state(unleash_instance)
previous_state = fetch_previous_state(state, instance_name)
diffs = calculate_diff(current_state, previous_state)
if diffs:
act(dry_run, state, unleash_instance, diffs)
def test_run_stats(self):
state = State('RI',
'2015',
0,
0.5,
10,
0,
0.1,
cdist,
10,
unique_filename_for_state='RI_2017_10_07_14_17_05')
state.run_stats()
self.assertTrue(os.path.isfile(state.stats_filename))
self.assertTrue(os.path.isfile(state.means_filename))
def slack_notify(self, dry_run, aws_accounts, ri):
result = self._get_deployment_result(dry_run, ri)
state = State(
integration=self.integration,
accounts=aws_accounts,
settings=self.settings
)
for saas_file in self.saas_files:
self.github = self._initiate_github(saas_file)
saas_file_name = saas_file['name']
for resource_template in saas_file['resourceTemplates']:
url = resource_template['url']
hash_length = resource_template['hash_length']
resource_template_name = resource_template['name']
for target in resource_template['targets']:
if not target.get('notify'):
continue
cluster, namespace = \
self._get_cluster_and_namespace(target)
target_hash = target['hash']
desired_commit_sha = \
self._get_commit_sha(url, target_hash, hash_length)
state_key_format = "{}/{}/{}/{}"
state_key = state_key_format.format(
saas_file_name,
resource_template_name,
cluster,
namespace
)
current_commit_sha = state.get(state_key, None)
if current_commit_sha != desired_commit_sha:
slack_info = saas_file.get('slack')
if slack_info:
slack = self._init_slack(slack_info)
msg_format = "[{}] {} deployment to {}/{}: {}"
msg = msg_format.format(
saas_file_name,
resource_template_name,
cluster,
namespace,
result
)
channel = slack.chat_kwargs['channel']
logging.info(['slack_notify', channel, msg])
if not dry_run:
state[state_key] = desired_commit_sha
slack.chat_post_message(msg)
def plot(self):
state = State('RI',
'2015',
0,
0.5,
10,
0,
0.1,
cdist,
10,
unique_filename_for_state='RI_2017_10_07_14_17_05')
state.plot()
self.assertTrue(
os.path.isfile(state.cong_dist_plot_dir + state.state +
'_cong_dist'))
self.assertTrue(os.path.isfile(state.clusters_plot_filename + '.png'))
self.assertTrue(os.path.isfile(state.clusters_plot_filename + '.pdf'))
def run(dry_run):
jira_boards = [j for j in queries.get_jira_boards() if j.get('slack')]
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
for index, jira_board in enumerate(jira_boards):
if not is_in_shard_round_robin(jira_board['name'], index):
continue
jira, current_state = fetch_current_state(jira_board, settings)
previous_state = fetch_previous_state(state, jira.project)
if previous_state:
diffs = calculate_diff(jira.server, current_state, previous_state)
act(dry_run, jira_board, diffs)
if not dry_run:
write_state(state, jira.project, current_state)
def run(dry_run):
unleash_instances = queries.get_unleash_instances()
accounts = queries.get_aws_accounts()
settings = queries.get_app_interface_settings()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
for unleash_instance in unleash_instances:
instance_name = unleash_instance['name']
current_state = fetch_current_state(unleash_instance)
if not current_state:
logging.warning('not acting on empty Unleash instances. ' +
'please create a feature toggle to get started.')
continue
previous_state = fetch_previous_state(state, instance_name)
diffs = calculate_diff(current_state, previous_state)
if diffs:
act(dry_run, state, unleash_instance, diffs)
def test_cluster_population_is_out_of_bounds(self):
# Single values of population:
state = State('RI',
'2015',
0,
0.5,
10,
0,
0.1,
cdist,
10,
min_population=1,
max_population=100)
state.people_per_cluster = 0
self.assertTrue(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = 101
self.assertTrue(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = 1
self.assertFalse(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = 100
self.assertFalse(state.cluster_population_is_out_of_bounds())
test = (
(state.max_population - state.min_population) * random.random() +
state.min_population)
state.people_per_cluster = test
self.assertFalse(state.cluster_population_is_out_of_bounds())
# Multiple values of population:
state.people_per_cluster = [0, 100, 50]
self.assertTrue(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = [0, 101, 50]
self.assertTrue(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = [1, 101, 50]
self.assertTrue(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = [1, 100, 50]
self.assertFalse(state.cluster_population_is_out_of_bounds())
state.people_per_cluster = [
(state.max_population - state.min_population) * random.random() +
state.min_population,
(state.max_population - state.min_population) * random.random() +
state.min_population
]
self.assertFalse(state.cluster_population_is_out_of_bounds())
def immigrate(self, immigration_slice):
# Get immigrant data
n_im = immigration_slice.N
distribution_im = np.asarray(immigration_slice)[:4]
# Get local data
n_local = self.__history[-1].N
distribution_local = np.asarray(self.state())[:4]
# Compute new local distribution
distribution_post = np.add(distribution_im * n_im, distribution_local *
n_local) / (n_im + n_local)
# Replace the current state with a state that includes the new immigration
state_post = State(distribution_post[0], distribution_post[1],
distribution_post[2], distribution_post[3],
n_im + n_local)
self.__history[-1] = state_post
def test_get_stats(self):
state = State('RI',
'2015',
0,
0.5,
10,
0,
0.1,
cdist,
10,
unique_filename_for_state='RI_2017_10_07_14_17_05')
stats_df = state.get_stats()
self.assertIsInstance(stats_df, DataFrame)
self.assertTrue('Population' in stats_df.columns)
self.assertTrue('Number of Blocks' in stats_df.columns)
self.assertTrue('Is Cluster' in stats_df.columns)
self.assertTrue('Mean Pairwise Distance' in stats_df.columns)
self.assertTrue(
'Cluster/Congressional District ID' in stats_df.columns)
def __init__(self):
ns = 'voxel_map/'
self._visual_threshold = rospy.get_param(ns + 'visual_threshold')
self.map = VoxelMap()
rospy.Subscriber("truth/NED",
Odometry,
callback=self.state_cb,
queue_size=1)
rospy.Subscriber("triang_points",
PointCloud,
callback=self.measurement_cb)
self.map_pub = rospy.Publisher("voxel_map", PointCloud, queue_size=1)
self.vis_pub = rospy.Publisher("voxel_map_visual",
PointCloud,
queue_size=1)
self.state = State()
self.blank_cloud = PointCloud()
zero_point = Point32(0, 0, 0)
self.blank_cloud.points = [zero_point] * self.map.N**3
def run(dry_run, io_dir='throughput/', defer=None):
jjb, additional_repo_urls = init_jjb()
defer(lambda: jjb.cleanup())
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=jjb.settings)
if dry_run:
validate_repos_and_admins(jjb, additional_repo_urls)
jjb.generate(io_dir, 'desired')
jjb.overwrite_configs(state)
jjb.generate(io_dir, 'current')
jjb.print_diffs(io_dir)
else:
jjb.update()
configs = jjb.get_configs()
for name, desired_config in configs.items():
state.add(name, value=desired_config, force=True)
from agent.agent import Agent
from utils.state import State, Parameters
from matplotlib import pyplot as plt
from utils.strconv import params2str
a = 0.2 # exposure rate
b = 1.75 # Measure of time to infection once exposed
g = 0.5 # Rate of recovery
d = 0.05 # Rate at which loss of immunity occurs.
r = 1 # Effectiveness of measures (range 0-1, lower is more effective)
init_agent_a = State(0.99, 0, 0.01, 0, 10000)
initial_params = Parameters(a, b, g, d, r)
agent_a = Agent('test', init_agent_a, initial_params)
print(f"Agent a initial: {init_agent_a}")
# Simulate for i iterations
i = initial_i = 10000
print(f"Iterations remaining {i}")
iterations_list = []
agent_iterations = 0
while True:
agent_a.iterate_rl()
agent_iterations += 1
def _initiate_state(self, accounts):
self.state = State(integration=self.integration,
accounts=accounts,
settings=self.settings)
from agent.agent import Agent
from utils.state import State, Parameters
from utils.plotting import plot_agent_history, plot_compartment_comparison
# Initialize SEIRS model params
a = 0.2 # exposure rate
b = 1.75 # Measure of time to infection once exposed
g = 0.5 # Rate of recovery
d = 0 # Rate at which loss of immunity occurs.
r = 1 # Cost level??
initial_params = Parameters(a, b, g, d, r)
# Initialize agent states
init_agent_a = State(0.99, 0, 0.01, 0, 10000)
init_agent_b = State(1.0, 0, 0, 0, 10000)
init_agent_c = State(1.0, 0, 0, 0, 10000)
# Initialize agent objects
agent_a = Agent('A', init_agent_a, initial_params)
agent_b = Agent('B', init_agent_b, initial_params)
agent_c = Agent('C', init_agent_c, initial_params)
agents = [agent_a, agent_b, agent_c]
# Print initial status
print(f"Agent a initial: {init_agent_a}")
print(f"Agent b initial: {init_agent_b}")
print(f"Agent c initial: {init_agent_c}")
# Simulate for 100 iterations
for _ in range(80):
def run(dry_run=False, enable_deletion=False):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration=QONTRACT_INTEGRATION,
accounts=accounts,
settings=settings)
queries_list = collect_queries()
remove_candidates = []
for query in queries_list:
query_name = query['name']
# Checking the sql-query state:
# - No state: up for execution.
# - State is a timestamp: executed and up for removal
# after the JOB_TTL
# - State is 'DONE': executed and removed.
try:
query_state = state[query_name]
if query_state != 'DONE':
remove_candidates.append({
'name': query_name,
'timestamp': query_state
})
continue
except KeyError:
pass
job_yaml = process_template(query)
job = yaml.safe_load(job_yaml)
job_resource = OpenshiftResource(job, QONTRACT_INTEGRATION,
QONTRACT_INTEGRATION_VERSION)
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
openshift_base.apply(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='job',
resource=job_resource)
if not dry_run:
state[query_name] = time.time()
for candidate in remove_candidates:
if time.time() < candidate['timestamp'] + JOB_TTL:
continue
try:
query = collect_queries(query_name=candidate['name'])[0]
except IndexError:
raise RuntimeError(f'sql-query {candidate["name"]} not present'
f'in the app-interface while its Job is still '
f'not removed from the cluster. Manual clean '
f'up is needed.')
oc_map = OC_Map(namespaces=[query['namespace']],
integration=QONTRACT_INTEGRATION,
settings=queries.get_app_interface_settings(),
internal=None)
openshift_base.delete(dry_run=dry_run,
oc_map=oc_map,
cluster=query['cluster'],
namespace=query['namespace']['name'],
resource_type='job',
name=query['name'],
enable_deletion=enable_deletion)
if not dry_run:
state[candidate['name']] = 'DONE'
def set(ctx, integration, key, value):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration, accounts, settings=settings)
state.add(key, value=value, force=True)
def __init__(self):
self.__name = "Pat"
self.__connection = Connection()
self.__prover = Thread(target=self.prove, args=())
self.__state = State()
def rm(ctx, integration, key):
settings = queries.get_app_interface_settings()
accounts = queries.get_aws_accounts()
state = State(integration, accounts, settings=settings)
state.rm(key)
