def ask_firm_init(player_id):
# ------- Get needed objects ----------------------------------------------------------------- #
p = Players.objects.get(player_id=player_id)
rd = Round.objects.get(round_id=p.round_id)
# ------- Log stuff -------------------------------------------------------------------------- #
utils.log("Player {} for room {} and round {} ask init: t is {}.".format(
player_id, p.room_id, rd.round_id, rd.t),
f=utils.fname(), path=__path__)
# ------- Maybe client has to wait the other player ------------------------------------------ #
if player.dialog.client_has_to_wait_over_player(player_id=player_id, called_from=__path__ + ':' + utils.fname()):
if not room.dialog.is_trial(player_id, utils.fname()):
opp_progression = player.dialog.get_opponent_progression(
player_id=player_id,
called_from=__path__ + ':' + utils.fname()
)
return parameters.error["wait"], opp_progression # Tuple is necessary!! '-1' hold for wait
# Get information necessary for firm initialization
d = data.get_init_info(player_id=player_id)
return rd.t, d["firm_state"], d["player"]["position"], d["player"]["price"], d["player"]["profits"], \
d["opp"]["position"], d["opp"]["price"], d["opp"]["profits"], rd.ending_t
def topic_page(p, topic_name=None):
if not p.isdigit():
p = '1'
page = int(p)
pre_page = 2
if topic_name is None:
query = Post.query
node_list = Node.query.all()
log(query)
else:
topic = Topic.query.filter_by(en_name=topic_name).first()
log(topic)
if topic is None:
abort(404)
query = topic.posts
node_list = topic.nodes.all()
paginate = query.order_by(Post.created_time.desc()).paginate(page, pre_page, False)
post_list = paginate.items
topic_list = Topic.query.all()
data = {
'post_list': post_list,
'paginate': paginate,
'topic_list': topic_list,
'node_list': node_list,
'selected_topic': topic_name,
}
return data
def create_network_interface(self, network_client, interface_name, ip_name,
subnet, parameters):
""" Creates the Public IP Address resource and uses that to create the
Network Interface.
Args:
network_client: A NetworkManagementClient instance.
interface_name: The name to use for the Network Interface resource.
ip_name: The name to use for the Public IP Address resource.
subnet: The Subnet resource from the Virtual Network created.
parameters: A dict, containing all the parameters necessary to
authenticate this user with Azure.
"""
group_name = parameters[self.PARAM_RESOURCE_GROUP]
region = parameters[self.PARAM_ZONE]
utils.log("Creating/Updating the Public IP Address '{}'".format(ip_name))
ip_address = PublicIPAddress(
location=region, public_ip_allocation_method=IPAllocationMethod.dynamic,
idle_timeout_in_minutes=4)
result = network_client.public_ip_addresses.create_or_update(
group_name, ip_name, ip_address)
self.sleep_until_update_operation_done(result, ip_name)
public_ip_address = network_client.public_ip_addresses.get(group_name, ip_name)
utils.log("Creating/Updating the Network Interface '{}'".format(interface_name))
network_interface_ip_conf = NetworkInterfaceIPConfiguration(
name=interface_name, private_ip_allocation_method=IPAllocationMethod.dynamic,
subnet=subnet, public_ip_address=PublicIPAddress(id=(public_ip_address.id)))
result = network_client.network_interfaces.create_or_update(
group_name, interface_name, NetworkInterface(location=region,
ip_configurations=[network_interface_ip_conf]))
self.sleep_until_update_operation_done(result, interface_name)
def __spawn_vms(self, agent, num_vms, parameters, reservation_id):
"""
Private method for starting a set of VMs
Args:
agent Infrastructure agent in charge of current operation
num_vms No. of VMs to be spawned
parameters A dictionary of parameters
reservation_id Reservation ID of the current run request
"""
status_info = self.reservations.get(reservation_id)
try:
security_configured = agent.configure_instance_security(parameters)
instance_info = agent.run_instances(num_vms, parameters,
security_configured)
ids = instance_info[0]
public_ips = instance_info[1]
private_ips = instance_info[2]
status_info['state'] = self.STATE_RUNNING
status_info['vm_info'] = {
'public_ips': public_ips,
'private_ips': private_ips,
'instance_ids': ids
}
utils.log('Successfully finished request {0}.'.format(reservation_id))
except AgentRuntimeException as exception:
status_info['state'] = self.STATE_FAILED
status_info['reason'] = str(exception)
self.reservations.put(reservation_id, status_info)
async def place(self, ctx: commands.Context, role, *names):
event = EventModel.load(ctx.channel.id)
if event is None:
await ctx.author.send(errors.NONEXISTENT_EVENT)
return False
if date.is_date_time_expired(event.date, event.time):
await ctx.author.send(errors.EXPIRED_EVENT)
return False
role = role.title()
if role and role not in settings.ROLES.ALL:
await ctx.author.send(errors.NONEXISTENT_ROLE)
return
for name in names:
members = ctx.channel.guild.fetch_members()
member = await members.find(
lambda m: utils.fuzzy_string_match(m.display_name, name))
if member is None:
await ctx.author.send(errors.NONEXISTENT_MEMBER)
return
await self.add_member(ctx.channel, event, member, role)
utils.log(ctx.author.display_name, "placed player", name, "...")
embed = view.create(ctx.channel.id, ctx.guild.emojis, self.bot.user.id)
await utils.show_event(channel=ctx.channel,
client=self.bot,
embed=embed)
def group_message(self, message):
"""
Send json to a group, called by self._group_send
method and WSDialog.group_send from outside
:param message:
:return:
"""
data = message['data']
group = message['group']
is_json = message['json']
if is_json:
try:
if len(str(data)) < 100:
utils.log(f'Sending to group {group}: {data}',
f=self._group_send)
except UnicodeEncodeError:
print('Error printing request.')
self.send_json(data)
else:
self.send(data)
def email_valid(email):
result = dict(valid=True, msg=dict())
# str_valid = True
# # email = email.strip()
# # for c in email:
# # if c not in email_valid_str:
# # str_valid =False
#
# split_email = email.split('@', 1)
# split_valid = len(split_email) == 2
# not_exist = False
# if split_valid and split_email[0] != '' and split_email[1] != '':
# host = split_email[1]
# split_host = host.split('.', 1)
# split_valid = len(split_host) == 2
# if split_valid and split_host[0] != '' and split_host[1] != '':
u = User.query.filter_by(email=email).first()
log(u)
if u is not None:
result['valid'] = False
result['msg']['.email-message'] = '邮箱已被注册'
# not_exist = True
# else:
# str_valid = False
# result['valid'] = not_exist
# msg = result['msg']
# if not str_valid:
# msg['.email-message'] = '请输入正确的邮箱'
# elif not not_exist:
# msg['.email-message'] = '邮箱已被注册'
return result
def login(form, r=dict):
if r['valid']:
log("login")
db_u = User.query.filter_by(username=form['login_username']).first()
log('login success')
login_user(db_u)
return r
def register(form, r=dict):
log('register', r['valid'], r['msg'])
if r['valid']:
u = User(form)
log('register')
u.save()
return r
def read_dataset():
global dataset
file_path = os.path.join('../build/data', args.scope,
'labeled/Learning_set-Bearing3_1-acc.csv')
table = np.genfromtxt(
file_path,
delimiter=',',
skip_header=1,
usecols=(1, 2) # x_acc, y_acc
)
for sample_from_idx in range(0, table.shape[0] - args.step_size + 1):
table_sample = table[sample_from_idx:sample_from_idx + args.step_size]
dataset['ordered'].append(table_sample[:, 0:2])
if sample_from_idx % 100000 == 0:
log('%2.0f%% loaded' %
((float(sample_from_idx) /
(table.shape[0] - args.step_size + 1)) * 100))
# train, validate split
dataset_health, dataset['anomalous'] = np.split(
dataset['ordered'], [int(.9 * len(dataset['ordered']))])
# shuffle
np.random.shuffle(dataset_health)
dataset['train'], dataset['validate'] = np.split(
dataset_health, [int(.9 * len(dataset_health))])
np.random.shuffle(dataset['train'])
dataset['ordered'] = np.array(dataset['ordered'])
def wrapper(*args, **kwargs):
form = args[0]
old = form.get('old_password')
new = form.get('change_password')
log(old, new)
# if g.user.password == old:
# nn_valid = True
# else:
# nn_valid_msg = nickname_valid(nickname)
# nn_valid = nn_valid_msg['valid']
# msg.update(nn_valid_msg['msg'])
# if g.user.email == email:
# e_valid = True
# else:
# e_valid_msg = email_valid(email)
# e_valid = e_valid_msg['valid']
# msg.update(e_valid_msg['msg'])
# r = dict(
# valid=e_valid and nn_valid,
# msg=msg
# )
# u = User.query.filter_by(username=username).first()
# log('valid',u)
# valid = False
# log(valid)
r = dict(valid=False, msg=dict())
msg = r['msg']
if g.user.password != old:
msg['.password-message'] = '当前密码错误'
elif old == new:
msg['.password-message'] = '不能与原密码相同'
r['valid'] = (g.user.password == old) and (old != new)
args = form, r
return function(*args, **kwargs)
def main():
dest_dir = prepare_directory(
os.path.join('../../build/data', SCOPE, 'initialized'))
for dataset in DATASETS_TO_PARSE:
filenames = glob.glob(os.path.join(SRC_DIR, dataset, '*.csv'))
for filename in filenames:
log('parsing ' + filename + '...')
dfChunks = pd.read_csv(filename,
header=None,
names=INPUT_CSV_COLUMNS,
chunksize=CHUNK_SIZE)
i = 0
for dfChunk in dfChunks:
i = i + 1
header = i is 1
mode = 'a' if i > 1 else 'w'
log('parsing chunk %d' % i)
dfChunk['datetime'] = pd.to_datetime(dfChunk['timestamp'],
unit='ms')
dfChunk['u'] = dfChunk['u'] * 0.1
dfChunk['v'] = dfChunk['v'] * 0.1
dfChunk['w'] = dfChunk['w'] * 0.1
dfChunk['x'] = dfChunk['x'] * 0.004
dfChunk['y'] = dfChunk['y'] * 0.004
dfChunk['z'] = dfChunk['z'] * 0.004
dfChunk.to_csv(os.path.join(
dest_dir, '%s-%s' % (dataset, os.path.basename(filename))),
mode=mode,
header=header,
index=False,
columns=OUTPUT_CSV_COLUMNS)
def verify_rule_resfinally_reject_result(self, id_account, id_borrow, result='reject', status='new'):
query_rule_resfin = "select * from vnrisk_rule_resfinally where id_account={} and id_borrow={} order by id desc limit 1;".format(id_account, id_borrow)
time.sleep(2)
resfinally_result = self.risk_conn.fetchone(query_rule_resfin)
utils.log("rule_resfinally result:\n{}".format(resfinally_result))
actual_result, actual_status, blocking_day = resfinally_result['result'], resfinally_result['status'], resfinally_result['blocking_day']
self.assertTrue(actual_result == result and actual_status == status and blocking_day >= 3, 'rule result Not correct! please check.')
def attach_disk(self, parameters, disk_name, instance_id):
""" Attaches the Elastic Block Store volume specified in 'disk_name' to this
virtual machine.
Args:
parameters: A dict with keys for each parameter needed to connect to AWS.
disk_name: A str naming the EBS mount to attach to this machine.
instance_id: A str naming the id of the instance that the disk should be
attached to. In practice, callers add disks to their own instances.
Returns:
The location on the local filesystem where the disk has been attached.
"""
# In Amazon Web Services, if we're running on a Xen Paravirtualized machine,
# then devices get added starting at /dev/xvda. If not, they get added at
# /dev/sda. Find out which one we're on so that we know where the disk will
# get attached to.
if glob.glob("/dev/xvd*"):
mount_point = '/dev/xvdc'
else:
mount_point = '/dev/sdc'
try:
conn = self.open_connection(parameters)
utils.log('Attaching volume {0} to instance {1}, at {2}'.format(
disk_name, instance_id, mount_point))
conn.attach_volume(disk_name, instance_id, mount_point)
return mount_point
except EC2ResponseError as exception:
utils.log('An error occurred when trying to attach volume {0} to ' \
'instance {1} at /dev/sdc'.format(disk_name, instance_id))
self.handle_failure('EC2 response error while attaching volume:' +
exception.error_message)
def wrapper(*args, **kwargs):
form = args[0]
username = form.get('register_username')
nickname = form.get('register_nickname')
password = form.get('register_password')
# confirm_password = form.get('confirm')
email = form.get('register_email')
un_valid_msg = username_valid(username)
nn_valid_msg = nickname_valid(nickname)
pw_valid_msg = password_valid(password)
e_valid_msg = email_valid(email)
un_valid = un_valid_msg['valid']
nn_valid = nn_valid_msg['valid']
pw_valid = pw_valid_msg['valid']
e_valid = e_valid_msg['valid']
msg = dict()
msg.update(un_valid_msg['msg'])
msg.update(nn_valid_msg['msg'])
msg.update(pw_valid_msg['msg'])
msg.update(e_valid_msg['msg'])
log(un_valid, pw_valid, e_valid, nn_valid)
r = dict(valid=un_valid and pw_valid and e_valid and nn_valid, msg=msg)
args = form, r
return function(*args, **kwargs)
def describe_instances_old(self, parameters):
"""
Retrieves the list of running instances that have been instantiated using a
particular EC2 keyname. The target keyname is read from the input parameter
map. (Also see documentation for the BaseAgent class)
Args:
parameters A dictionary containing the 'keyname' parameter
Returns:
A tuple of the form (public_ips, private_ips, instances) where each
member is a list.
"""
instance_ids = []
public_ips = []
private_ips = []
conn = self.open_connection(parameters)
reservations = conn.get_all_instances()
instances = [i for r in reservations for i in r.instances]
utils.log("Looking for instances with key = {0}".format(parameters[self.PARAM_KEYNAME]))
for i in instances:
if i.state == 'running' and i.key_name == parameters[self.PARAM_KEYNAME]:
instance_ids.append(i.id)
public_ips.append(i.public_dns_name)
private_ips.append(i.private_dns_name)
return public_ips, private_ips, instance_ids
def post(self, request, *args, **kwargs):
"""
Room creation process
:param request: using POST request
:return: html room form template (success or fail)
"""
if request.POST["form_function"] == "room_organisation":
form = RoomForm(request.POST)
if form.is_valid():
with transaction.atomic():
try:
# Create room
room.dashboard.create(form.get_data())
except IntegrityError:
utils.log("Room already exists!", f=utils.fname(), path=__path__, level=2)
return redirect("/room_management")
else:
context = {"subtitle": "Set parameters and create a room", "form": form}
return render(request, self.template_name, context)
else:
raise Exception("Error validating the form.")
def run_instances(self, count, parameters, security_configured):
"""
Spawns the specified number of OpenStack instances using the parameters
provided. This method is blocking in that it waits until the
requested VMs are properly booted up. However if the requested
VMs cannot be procured within 1800 seconds, this method will treat
it as an error and return. (Also see documentation for the BaseAgent
class).
This method differs from its OpenStack counterpart because OpenStack
does not support spot instances.
Args:
count: Number of VMs to spawn.
parameters: A dictionary of parameters. This must contain 'keyname',
'group', 'image_id' and 'instance_type' parameters.
security_configured: Uses this boolean value as an heuristic to
detect brand new AppScale deployments.
Returns:
A tuple of the form (instances, public_ips, private_ips).
"""
if parameters[self.PARAM_SPOT] == "True":
parameters[self.PARAM_SPOT] = 'False'
utils.log("OpenStack does not support spot instances")
super.run_instances(self, count, parameters, security_configured)
def search_kyc_reason_api(self, url, header, params):
utils.log(">>search kyc reason records...")
kyc_list_resp = self.req.get(url, headers=header, params=params)
self.assertEqual(
json.loads(kyc_list_resp.text)['msg'], 'success',
'call search kyc reason api failed!')
return kyc_list_resp.text
def main():
args = get_args()
centroids = {}
df_breakpoints = pd.read_csv(args.src_breakpoint)
for column in args.columns:
log('parsing column %s ...' % (column, ))
data_points = np.array([])
for src in args.srcs:
log('parsing %s ...' % (src, ))
df = pd.read_csv(src, usecols=[column])
data_points = np.concatenate((data_points, df[column]), axis=0)
init_centroids = breakpoints_to_centroids(df_breakpoints[column].values)
kmeans = KMeans(
init=np.array(init_centroids).reshape((-1, 1)),
n_clusters=args.symbol_size,
random_state=0,
n_jobs=1,
verbose=0,
max_iter=500
).fit(np.array(data_points).reshape((-1, 1)))
centroids[column] = np.array(kmeans.cluster_centers_).reshape(-1)
df_centroids = pd.DataFrame(centroids)
dest_dir = prepare_directory(args.dest_dir)
df_centroids.to_csv(
os.path.join(
dest_dir,
'centroid-{0}.csv'.format(args.symbol_size)
),
header=True,
index=False
)
def attach_disk(self, parameters, disk_name, instance_id):
""" Attaches the Elastic Block Store volume specified in 'disk_name' to this
virtual machine.
Args:
parameters: A dict with keys for each parameter needed to connect to AWS.
disk_name: A str naming the EBS mount to attach to this machine.
instance_id: A str naming the id of the instance that the disk should be
attached to. In practice, callers add disks to their own instances.
Returns:
The location on the local filesystem where the disk has been attached.
"""
# In Amazon Web Services, if we're running on a Xen Paravirtualized machine,
# then devices get added starting at /dev/xvda. If not, they get added at
# /dev/sda. Find out which one we're on so that we know where the disk will
# get attached to.
if glob.glob("/dev/xvd*"):
mount_point = '/dev/xvdc'
else:
mount_point = '/dev/sdc'
try:
conn = self.open_connection(parameters)
utils.log('Attaching volume {0} to instance {1}, at {2}'.format(
disk_name, instance_id, mount_point))
conn.attach_volume(disk_name, instance_id, mount_point)
return mount_point
except EC2ResponseError as exception:
utils.log('An error occurred when trying to attach volume {0} to ' \
'instance {1} at /dev/sdc'.format(disk_name, instance_id))
self.handle_failure('EC2 response error while attaching volume:' +
exception.error_message)
def include_players_into_round_compositions(room_id, player_id, called_from):
utils.log("Called by '{}'".format(called_from),
f=utils.fname(),
path=__path__)
return composition.include_players_into_round_compositions(
room_id, player_id)
def test_vn_es_data_consistence_with_work(self):
"""验证ES数据内容与源表work一致"""
utils.log("verify es details data is consistence with source work.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['es_work'],
self.crm_data['ESSrcDetail']['Check'][constants.CN]['work'],
self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['workQuery']),
self.crm_data['ESSrcDetail']['Src'][constants.CN]['work'])
def test_select_save_people(self):
"""
select people and save people library
"""
utils.log("1. Call search people api.")
utils.log(
"Select people: gender= Male, birthday<=2006-10-03, marita status in ('Single','Married'), "
"last application status not in (Failure, Rejected),last login date >current_date-20"
)
# search_save_payload = self.crm_data['CRM']['search']['payload1']
search_save_payload = self.crm_data['CRM']['search']['payload2']
search_resp, select_count = self.search_people_api(search_save_payload)
self.assertGreater(int(select_count), 0,
"Select People Failed!!! The count less than 0")
utils.log("2. Save searched people library.")
save_resp, people_lib_id = self.save_people_api(
{"labelListJson": search_save_payload},
self.crm_data['CRM']['save']['controlGroupJson'],
int(select_count))
self.assertGreater(int(people_lib_id), 1,
"Save People Library Failed!!!")
utils.log("3. Verify new save people library exist in db.")
self.verify_people_lib_id_in_db(people_lib_id, self.desc_prefix)
utils.log("4. Add people library task.")
self.add_people_lib_task(people_lib_id)
def make_sleepy(self,parameters, instance_id):
utils.log( "Making instance {0} sleepy...".format(instance_id))
credentials = parameters[self.PARAM_CREDENTIALS]
ec2 = boto.connect_cloudwatch(
str(credentials[self.PARAM_CREDS_PUBLIC]),
str(credentials[self.PARAM_CREDS_PRIVATE])
)
region = "us-east-1"
terminate_arn = 'arn:aws:automate:{0}:ec2:terminate'.format(region)
alarm_name = 'ec2_shutdown_sleepy_{0}'.format(instance_id)
# define our alarm to terminate the instance if it gets sleepy
# i.e. if CPU utilisation is less than 10% for 1 x 4 hr intervals
sleepy_alarm = MetricAlarm(
name=alarm_name,
namespace='AWS/EC2',
metric='CPUUtilization',
statistic='Average',
comparison='<',
threshold='10',
period='3600',
evaluation_periods=4,
alarm_actions=[terminate_arn],
dimensions={'InstanceId':instance_id}
)
# create the alarm.. Zzzz!
ec2.create_alarm(sleepy_alarm)
def terminate_instances(self, parameters):
"""
Stop one or more EC2 instances. The input instance IDs are
fetched from the 'instance_ids' parameter in the input parameters
map. (Also see documentation for the BaseAgent class)
If parameters contains the 'prefix' key, instances will be terminated
based on whether or not the keypair used to start them begins with
the given prefix.
Args:
parameters A dictionary of parameters
"""
conn = self.open_connection(parameters)
instance_ids = []
if self.PARAM_KEY_PREFIX in parameters:
prefix = parameters[self.PARAM_KEY_PREFIX]
reservations = conn.get_all_instances()
instances = [i for r in reservations for i in r.instances]
for i in instances:
if i.key_name is not None and i.key_name.startswith(prefix):
instance_ids.append(i.id)
else:
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
terminated_instances = conn.terminate_instances(instance_ids)
for instance in terminated_instances:
instance.update()
while instance.state != 'terminated':
time.sleep(5)
instance.update()
utils.log('Instance {0} was terminated'.format(instance.id))
return True
def get_target_row(self, row_unique_key_name, row_unique_key):
"""
get target row in data grid matched unique key value(eg: type, ruleId)
"""
target_row, target_row_id, action_column_index = [], None, -1
trs = self.get_data_grid_rows()
headers = self.get_data_grid_headers()
for index, header in enumerate(headers):
if header == constants.ActionsColumnName or header == constants.ActionColumnName:
action_column_index = index # get the [Action] column index
break
for tr in trs:
tds_in_tr = self.slb.get_elements_via_element(tr, datagrid.td)
target_row_id = self.slb.get_element_text(
tds_in_tr[0]) # first column value, help find row on ui
for td in tds_in_tr:
if self.slb.get_element_text(td) == row_unique_key:
utils.log("find the data record with {}={}".format(
row_unique_key_name, row_unique_key))
target_row.append(tr)
break
if target_row:
break
if not target_row:
utils.warn("Not find any data record with {}={} !".format(
row_unique_key_name, row_unique_key))
return
return target_row, target_row_id, action_column_index
def attach_disk(self, parameters, disk_name, instance_id):
""" Attaches the Elastic Block Store volume specified in 'disk_name' to this
virtual machine.
Args:
parameters: A dict with keys for each parameter needed to connect to AWS.
disk_name: A str naming the EBS mount to attach to this machine.
instance_id: A str naming the id of the instance that the disk should be
attached to. In practice, callers add disks to their own instances.
"""
try:
conn = self.open_connection(parameters)
utils.log('Attaching volume {0} to instance {1}, at /dev/sdc'.format(
disk_name, instance_id))
conn.attach_volume(disk_name, instance_id, '/dev/sdc')
while True:
utils.log('Waiting for disk to finish attaching.')
status = conn.get_all_volumes(disk_name)[0].status
utils.log('Volume {0} reports its status as {1}'.format(disk_name,
status))
if status == 'in-use':
break
utils.sleep(1)
utils.log('Volume {0} is attached and ready for use.')
return '/dev/sdc'
except EC2ResponseError as exception:
utils.log('An error occurred when trying to attach volume {0} to ' \
'instance {1} at /dev/sdc'.format(disk_name, instance_id))
self.handle_failure('EC2 response error while attaching volume:' +
exception.error_message)
def run_instances(self, count, parameters, security_configured):
"""
Spawns the specified number of OpenStack instances using the parameters
provided. This method is blocking in that it waits until the
requested VMs are properly booted up. However if the requested
VMs cannot be procured within 1800 seconds, this method will treat
it as an error and return. (Also see documentation for the BaseAgent
class).
This method differs from its OpenStack counterpart because OpenStack
does not support spot instances.
Args:
count: Number of VMs to spawn.
parameters: A dictionary of parameters. This must contain 'keyname',
'group', 'image_id' and 'instance_type' parameters.
security_configured: Uses this boolean value as an heuristic to
detect brand new AppScale deployments.
Returns:
A tuple of the form (instances, public_ips, private_ips).
"""
if parameters[self.PARAM_SPOT] == "True":
parameters[self.PARAM_SPOT] = 'False'
utils.log("OpenStack does not support spot instances")
super.run_instances(self, count, parameters, security_configured)
async def create_event(self,
ctx: commands.Context,
name,
date,
time,
description=""):
date = try_parse_day(date)
if not is_valid_date_time(date, time):
await ctx.author.send(errors.INVALID_DATE.format(date, time))
return
if is_date_time_expired(date, time):
await ctx.author.send(errors.EXPIRED_DATE.format(date, time))
return
event = EventModel(self.bot.user.id, name, date, time, description,
ctx.channel.id, 0)
event.save()
embed = view.create(ctx.channel.id, ctx.guild.emojis, self.bot.user.id)
await utils.show_event(channel=ctx.channel,
client=self.bot,
embed=embed,
new_event=True)
await utils.send_announcement(
ctx,
settings.MESSAGE.NEW_EVENT.format(event.name, event.date,
event.time, ctx.channel.mention))
utils.log(ctx.author.display_name, "created event", event.name, "...")
async def edit(self, ctx: commands.Context, *raw_aspects):
event = EventModel.load(ctx.channel.id)
if event is None:
await ctx.author.send(errors.NONEXISTENT_EVENT)
return
parsed_aspect_dict, invalids = EventAspectParser.parse(raw_aspects)
for name, value in parsed_aspect_dict.items():
event.set_attribute(name, value)
if len(invalids) > 0:
await ctx.author.send(errors.INVALID_ASPECTS + '\n' +
'\n'.join(invalids))
if is_date_time_expired(event.date, event.time):
await ctx.author.send(
errors.EXPIRED_DATE.format(event.date, event.time))
return
event.save(append=True)
embed = view.create(ctx.channel.id, ctx.guild.emojis, self.bot.user.id)
await utils.show_event(channel=ctx.channel,
client=self.bot,
embed=embed,
new_event=False)
utils.log(ctx.author.display_name, "edited event", event.name, "...")
def __spawn_vms(self, agent, num_vms, parameters, reservation_id):
"""
Private method for starting a set of VMs
Args:
agent Infrastructure agent in charge of current operation
num_vms No. of VMs to be spawned
parameters A dictionary of parameters
reservation_id Reservation ID of the current run request
"""
status_info = self.reservations.get(reservation_id)
try:
security_configured = agent.configure_instance_security(parameters)
instance_info = agent.run_instances(num_vms, parameters,
security_configured)
ids = instance_info[0]
public_ips = instance_info[1]
private_ips = instance_info[2]
status_info['state'] = self.STATE_RUNNING
status_info['vm_info'] = {
'public_ips': public_ips,
'private_ips': private_ips,
'instance_ids': ids
}
utils.log(
'Successfully finished request {0}.'.format(reservation_id))
except AgentRuntimeException as exception:
status_info['state'] = self.STATE_FAILED
status_info['reason'] = str(exception)
self.reservations.put(reservation_id, status_info)
def click_report_biz_list(self):
utils.log("click [Report Biz List] tab.")
if not self.is_report_menu_expanded():
utils.debug("expand menu [Report Schedule]")
self.slb.click(report_schedule.report_toggle)
self.slb.move_to_click(report_schedule.report_schedule_link)
self.slb.sleep(1)
def test_vn_es_data_consistence_with_person(self):
"""验证ES数据内容与源表person一致"""
utils.log("verify es details data is consistence with source person.")
self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['person'],
self.crm_data['ESSrcDetail']['Check'][constants.CN]['person'],
self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['query']),
self.crm_data['ESSrcDetail']['Src'][constants.CN]['person'])
def create_scratch_disk(self, parameters):
""" Creates a disk from a given machine image.
GCE does not support scratch disks on API version v1 and higher. We create
a persistent disk upon creation to act like one to keep the abstraction used
in other infrastructures.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
Returns:
A str, the url to the disk to use.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
disk_name = self.generate_disk_name(parameters)
project_name = parameters[self.PARAM_PROJECT]
project_url = '{0}{1}'.format(self.GCE_URL,
parameters[self.PARAM_PROJECT])
source_image_url = '{0}{1}/global/images/{2}'.format(
self.GCE_URL, project_name, parameters[self.PARAM_IMAGE_ID])
request = gce_service.disks().insert(project=project_name,
zone=parameters[self.PARAM_ZONE],
body={'name': disk_name},
sourceImage=source_image_url)
response = request.execute(http=auth_http)
utils.log(str(response))
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
disk_url = "{0}/zones/{1}/disks/{2}".format(
project_url, parameters[self.PARAM_ZONE], disk_name)
return disk_url
def test_id_storage_wide_total_consistent(self):
"""验证窄表与宽表数据总量一致"""
utils.log("verify JP user_tag_storage and user_tag_wide total count equals")
self.verify_storage_wide_consistent(
'',
"select count(1) from idcrmtest_ext.user_tag_storage where tag_id='accountname'",
"select count(1) from idcrmtest_ext.user_tag_wide")
def set_view_table_email_header(self, email_header):
utils.log(
"input Table Report Email Header in [View] report dialog. as below\n{}"
.format(email_header))
self.slb.type(
report_schedule.view_html_text_area.format('Email Header'),
email_header)
def validate_application_for_deployment(self, secret, app):
if self.secret != secret:
return self.__generate_response(False, self.REASON_BAD_SECRET)
name = app['name']
version = app['version']
dependencies = app['dependencies']
api_list = app['api_list']
api_list_without_specs = self.__remove_specs(api_list)
owner = app['owner']
if dependencies:
dep_invalid = self.adaptor.validate_application_dependencies(name, version,
api_list_without_specs, dependencies)
if dep_invalid:
detail = { 'detail' : dep_invalid }
return self.__generate_response(False, self.REASON_BAD_DEPENDENCIES, detail)
pre_validation_errors = []
for api in api_list:
api_name = api['name']
api_spec = api['specification']
if not self.__is_api_name_valid(api_name):
pre_validation_errors.append('Invalid characters in API name: {0}'.format(api_name))
spec_valid, spec_errors = swagger.validate_swagger_description(api_spec)
if not spec_valid:
pre_validation_errors.append(spec_errors)
if pre_validation_errors:
detail = { 'detail' : '|'.join(pre_validation_errors) }
return self.__generate_response(False, self.REASON_BAD_API_METADATA, detail)
passed, message = self.policy_engine.run_policy_enforcement(name, version, dependencies,
api_list_without_specs, owner)
if not passed:
detail = { 'detail' : message }
return self.__generate_response(False, self.REASON_API_POLICY_VIOLATION, detail)
post_validation_errors = []
for api in api_list:
api_name = api['name']
api_version = api['version']
api_spec = api['specification']
if self.adaptor.is_api_available(api_name, api_version):
passed, message = self.__invoke_api_validations(api_name, api_version, api_spec)
else:
passed, message = self.__handle_new_api(api_name, api_version, api_spec)
if not passed:
post_validation_errors.append(message)
if post_validation_errors:
detail = { 'detail' : '|'.join(post_validation_errors)}
return self.__generate_response(False, self.REASON_API_VALIDATION_FAILED, detail)
if not self.adaptor.record_application_dependencies(name, version, api_list_without_specs, dependencies):
utils.log("Failed to record dependencies for {0}-v{1}".format(name, version))
return self.__generate_response(False, self.REASON_DEPENDENCY_RECORDING_FAILED)
return self.__generate_response(True, self.REASON_API_VALIDATION_SUCCESS)
def stop(self):
"""
Stop the EAGER service.
"""
if self.started:
utils.log('Stopping EAGER service')
self.started = False
self.server.shutdown()
else:
utils.log('Warning - Stop called on already stopped server')
def stop(self):
"""
Stop the infrastructure manager service.
"""
if self.started:
utils.log('Stopping AppScale Infrastructure Manager')
self.started = False
self.server.shutdown()
else:
utils.log('Warning - Stop called on already stopped server')
def setProperty(self, prop, value):
if prop == None or value == None:
if prop in self.properties:
del self.properties[prop]
utils.logTrace('Removing property ' + repr(prop) + ' value: ' + repr(value))
if prop in self.properties:
old = ''
if self.properties[prop] != None:
old = self.properties[prop]
utils.log('Token "' + self.name + '" Replacing ' + prop + ' old value: ' + repr(old) + ' new value ' + repr(value))
self.properties[prop] = value
def addToken(self, token, lock=False):
try:
if lock:
self.tlLock.acquire()
tokenName = token.getName()
if not tokenName in self.list:
self.list[tokenName] = []
self.list[tokenName].append(token)
utils.log('Adding token ' + tokenName)
finally:
if lock:
self.tlLock.release()
def handle_failure(self, msg):
"""
Log the specified error message and raise an AgentRuntimeException
Args:
msg An error message to be logged and included in the raised exception
Raises:
AgentRuntimeException Contains the input error message
"""
utils.log(msg)
raise AgentRuntimeException(msg)
def start(self):
"""
Start the EAGER service. This method blocks
as long as the service is alive. The caller should handle the
threading requirements
"""
if self.started:
utils.log('Warning - Start called on already running server')
else:
utils.log('Starting EAGER service on port: ' + str(self.port))
self.started = True
while self.started:
self.server.serve_forever()
def __init__(self, initialTokenAssignments):
utils.log('New TokenList')
self.initialTokenAssignments = initialTokenAssignments
self.list = {}
self.requiredTokensByState = {}
if initialTokenAssignments != None:
for tokenName, attributes in self.initialTokenAssignments.iteritems():
numTokens = attributes[0]
if not tokenName in self.list:
self.list[tokenName] = []
for t in xrange(int(numTokens)):
token = Token(tokenName, copy.deepcopy(attributes[1]))
token.unlock()
self.list[tokenName].append(token)
def run_instances_response(self, command, output):
"""
Local utility method to parse the validate the output of ec2-run-instances
command.
Args:
command Exact command executed
output Output of the command
Returns:
A tuple of the form (status,command) where status is a boolean value
indicating the success/failure status of the output and command is
the modified command to be retried in case the previous attempt had
failed.
"""
if output.find('Please try again later') != -1:
utils.log('Error with run instances: {0}. Will try again in a moment.'.format(output))
return False, command
elif output.find('try --addressing private') != -1:
utils.log('Need to retry with addressing private. Will try again in a moment.')
return False, command + ' --addressing private'
elif output.find('PROBLEM') != -1:
utils.log('Error: {0}'.format(output))
sys.exit('Saw the following error from EC2 tools: {0}'.format(output))
else:
utils.log('Run instances message sent successfully. Waiting for the image to start up.')
return True, command
def attach_disk(self, parameters, disk_name, instance_id):
""" Attaches the persistent disk specified in 'disk_name' to this virtual
machine.
Args:
parameters: A dict with keys for each parameter needed to connect to
Google Compute Engine.
disk_name: A str naming the persistent disk to attach to this machine.
instance_id: A str naming the id of the instance that the disk should be
attached to. In practice, callers add disks to their own instance.
Returns:
A str indicating where the persistent disk has been attached to.
"""
gce_service, credentials = self.open_connection(parameters)
http = httplib2.Http()
auth_http = credentials.authorize(http)
project = parameters[self.PARAM_PROJECT]
zone = parameters[self.PARAM_ZONE]
# If the disk is already attached, return the mount point.
request = gce_service.instances().get(project=project, zone=zone,
instance=instance_id)
disks = request.execute(auth_http)['disks']
for disk in disks:
path = disk['source'].split('/')
if project == path[-5] and zone == path[-3] and disk_name == path[-1]:
device_name = '/dev/{}'.format(disk['deviceName'])
utils.log('Disk is already attached at {}'.format(device_name))
return device_name
request = gce_service.instances().attachDisk(
project=project,
zone=zone,
instance=instance_id,
body={
'kind' : 'compute#attachedDisk',
'type' : 'PERSISTENT',
'mode' : 'READ_WRITE',
'source' : "https://www.googleapis.com/compute/{0}/projects/{1}" \
"/zones/{2}/disks/{3}".format(self.API_VERSION, project,
zone, disk_name),
'deviceName' : 'sdb'
}
)
response = request.execute(auth_http)
utils.log(str(response))
self.ensure_operation_succeeds(gce_service, auth_http, response,
parameters[self.PARAM_PROJECT])
return '/dev/sdb'
def set_environment_variables(self, parameters):
"""
Set the EC2 specific environment variables. Required values for the
environment variables are read from the 'credentials' parameter of
the parameters map. (Also see documentation for the BaseAgent class)
Args:
parameters A dictionary containing the 'credentials' key
"""
if os.environ.has_key('EC2_JVM_ARGS'):
del(os.environ['EC2_JVM_ARGS'])
variables = parameters[self.PARAM_CREDENTIALS]
for key, value in variables.items():
if value is None:
utils.log('None value detected for the credential: {0}.'.format(key))
continue
if key.find('KEY') != -1:
utils.log('Setting {0} to {1} in our environment.'.format(
key, utils.obscure_string(value)))
else:
utils.log('Setting {0} to {1} in our environment.'.format(key, value))
os.environ[key] = value
ec2_keys_dir = os.path.abspath('/etc/appscale/keys/cloud1')
os.environ['EC2_PRIVATE_KEY'] = ec2_keys_dir + '/mykey.pem'
os.environ['EC2_CERT'] = ec2_keys_dir + '/mycert.pem'
utils.log('Setting private key to: {0} and certificate to: {1}'.format(
os.environ['EC2_PRIVATE_KEY'], os.environ['EC2_CERT']))
def remove(self, token):
try:
self.tlLock.acquire()
n = 0
if not token.getName() in self.list:
utils.logTrace('token ' + token.getName() + ' not found')
return
l = self.list[token.getName()]
if not token in l:
utils.log('token ' + token.getName() + ' not found in remove')
ndx = l.index(token)
del l[ndx]
finally:
self.tlLock.release()
def terminate_instances(self, parameters):
"""
Stop one of more EC2 instances using. The input instance IDs are
fetched from the 'instance_ids' parameters in the input map. (Also
see documentation for the BaseAgent class)
Args:
parameters A dictionary of parameters
"""
instance_ids = parameters[self.PARAM_INSTANCE_IDS]
conn = self.open_connection(parameters)
terminated_instances = conn.terminate_instances(instance_ids)
for instance in terminated_instances:
utils.log('Instance {0} was terminated'.format(instance.id))
def __init__(self):
parent_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
self.policy_store_dir = os.path.join(parent_dir, self.POLICY_STORE_DIR)
self.active_policies = []
self.inactive_policies = []
self.lock = Lock()
if os.path.exists(self.policy_store_dir):
for policy_file in os.listdir(self.policy_store_dir):
if policy_file.endswith('.a.py'):
full_path = os.path.join(self.policy_store_dir, policy_file)
try:
policy = Policy(full_path)
self.active_policies.append(policy)
except Exception as ex:
utils.log("Error while loading policy '{0}': {1}".format(policy_file, str(ex)))
elif policy_file.endswith('.i.py'):
full_path = os.path.join(self.policy_store_dir, policy_file)
try:
policy = Policy(full_path)
self.inactive_policies.append(policy)
except Exception as ex:
utils.log("Error while loading policy '{0}': {1}".format(policy_file, str(ex)))
if self.active_policies:
msg_suffix = '1 policy'
if len(self.active_policies) > 1:
msg_suffix = '{0} policies'.format(len(self.active_policies))
utils.log("Initialized policy engine with {0}.".format(msg_suffix))
else:
utils.log("No active policies found.")
def evaluate(self, app, errors):
globals_map = globals().copy()
globals_map['app'] = app
globals_map['assert_app_dependency'] = assert_app_dependency
globals_map['assert_not_app_dependency'] = assert_not_app_dependency
globals_map['assert_app_dependency_in_range'] = assert_app_dependency_in_range
globals_map['assert_true'] = assert_true
globals_map['assert_false'] = assert_false
globals_map['compare_versions'] = compare_versions
try:
exec(self.source_code, globals_map, {})
except EagerPolicyAssertionException as ex:
errors.append('[{0}] {1}'.format(self.name, ex))
except Exception as ex:
utils.log('[{0}] Unexpected policy exception: {1}'.format(self.name, ex))
def configure_instance_security(self, parameters):
"""
Setup EC2 security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
EC2 key-pair and a security group. Security group will be granted permissions
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters A dictionary of parameters
"""
keyname = parameters[self.PARAM_KEYNAME]
#keyname = "cloudrunner-keypair"
group = parameters[self.PARAM_GROUP]
#group = "cloudrunner-security-group"
key_path = '{0}.key'.format(keyname)
ssh_key = os.path.abspath(key_path)
utils.log(
'About to spawn EC2 instances - Expecting to find a key at {0}'.format(ssh_key)
)
try:
conn = self.open_connection(parameters)
if os.path.exists(ssh_key):
utils.log('SSH keys found in the local system - Not initializing EC2 security')
else:
utils.log('Creating key pair: ' + keyname)
key_pair = conn.create_key_pair(keyname)
utils.write_key_file(ssh_key, key_pair.material)
security_groups = conn.get_all_security_groups()
group_exists = False
for security_group in security_groups:
if security_group.name == group:
group_exists = True
break
if not group_exists:
utils.log('Creating security group: ' + group)
newgroup = conn.create_security_group(group, 'CloudRunner security group')
newgroup.authorize('tcp', 22, 22, '0.0.0.0/0')
newgroup.authorize('tcp', 5672, 5672, '0.0.0.0/0')
newgroup.authorize('tcp', 6379, 6379, '0.0.0.0/0')
newgroup.authorize('tcp', 11211, 11211, '0.0.0.0/0')
newgroup.authorize('tcp', 55672, 55672, '0.0.0.0/0')
result = {
'success': True,
'absolute_key_path': ssh_key
}
return result
except EC2ResponseError as exception:
self.handle_failure(
'EC2 response error while initializing security: {0}'.format(exception.error_message)
)
except Exception as exception:
self.handle_failure(
'Error while initializing EC2 security: {0}'.format(exception.message)
)
def dump(self, title = None, indent = 5):
if not debug: return
pad = ''
for i in xrange(indent): pad += ' '
msg = '\n'
try:
if title != None and len(title) > 0:
msg += title + '\n'
msg += 'Token List Dump'.rjust(indent+15, ' ') + '\n'
for tokenName,tokens in self.list.items():
for token in tokens:
msg += token.dump('', indent)
utils.log(msg)
return msg
finally:
pass
def add(self, tokenName, amount, properties = {}):
try:
self.tlLock.acquire()
n = 0
if not tokenName in self.list:
self.list[tokenName] = []
for i in xrange(amount):
token = Token(tokenName)
token.unlock()
for propName, propValue in properties.items():
token.setProperty(propName, propValue)
self.list[tokenName].append(token)
utils.log('Adding token ' + tokenName)
return n
finally:
self.tlLock.release()
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, ssl=True):
"""
Initialize a new instance of the EAGER service.
Args:
host Hostname to which the service should bind (Optional). Defaults
to 0.0.0.0.
port Port of the service (Optional). Default to 18444.
ssl True if SSL should be engaged or False otherwise (Optional).
Defaults to True. When engaged, this implementation expects
to find the necessary SSL certificates in the /etc/appscale/certs
directory.
"""
self.host = host
self.port = port
secret = None
while True:
try:
secret = utils.get_secret(self.APPSCALE_DIR + 'secret.key')
break
except Exception:
utils.log('Waiting for the secret key to become available')
utils.sleep(5)
utils.log('Found the secret set to: {0}'.format(secret))
SOAPpy.Config.simplify_objects = True
if ssl:
utils.log('Checking for the certificate and private key')
cert = self.APPSCALE_DIR + 'certs/mycert.pem'
key = self.APPSCALE_DIR + 'certs/mykey.pem'
while True:
if os.path.exists(cert) and os.path.exists(key):
break
else:
utils.log('Waiting for certificates')
utils.sleep(5)
ssl_context = SSL.Context()
ssl_context.load_cert(cert, key)
self.server = SOAPpy.SOAPServer((host, port), ssl_context=ssl_context)
else:
self.server = SOAPpy.SOAPServer((host, port))
e = Eager()
self.server.registerFunction(e.ping)
self.server.registerFunction(e.validate_application_for_deployment)
self.server.registerFunction(e.publish_api_list)
self.server.registerFunction(e.add_policy)
self.server.registerFunction(e.remove_policy)
self.server.registerFunction(e.enable_policy)
self.server.registerFunction(e.disable_policy)
self.server.registerFunction(e.list_policy)
self.server.registerFunction(e.info_policy)
self.started = False
def configure_instance_security(self, parameters):
"""
Setup EC2 security keys and groups. Required input values are read from
the parameters dictionary. More specifically, this method expects to
find a 'keyname' parameter and a 'group' parameter in the parameters
dictionary. Using these provided values, this method will create a new
EC2 key-pair and a security group. Security group will be granted permissions
to access any port on the instantiated VMs. (Also see documentation for the
BaseAgent class)
Args:
parameters A dictionary of parameters
"""
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
key_path = '/etc/appscale/keys/cloud1/{0}.key'.format(keyname)
ssh_key = os.path.abspath(key_path)
utils.log('About to spawn EC2 instances - ' \
'Expecting to find a key at {0}'.format(ssh_key))
if os.path.exists(ssh_key):
utils.log('SSH keys found in the local system - '
'Not initializing EC2 security')
return False
try:
conn = self.open_connection(parameters)
key_pair = conn.get_key_pair(keyname)
if key_pair is None:
utils.log('Creating key pair: ' + keyname)
key_pair = conn.create_key_pair(keyname)
utils.write_key_file(ssh_key, key_pair.material)
security_groups = conn.get_all_security_groups()
group_exists = False
for security_group in security_groups:
if security_group.name == group:
group_exists = True
break
if not group_exists:
utils.log('Creating security group: ' + group)
conn.create_security_group(group, 'AppScale security group')
conn.authorize_security_group(group, from_port=1,
to_port=65535, ip_protocol='udp')
conn.authorize_security_group(group, from_port=1,
to_port=65535, ip_protocol='tcp')
conn.authorize_security_group(group, ip_protocol='icmp',
cidr_ip='0.0.0.0/0')
return True
except EC2ResponseError as exception:
self.handle_failure('EC2 response error while initializing '
'security: ' + exception.error_message)
except Exception as exception:
self.handle_failure('Error while initializing EC2 '
'security: ' + exception.message)
def __generate_response(self, status, msg, extra=None):
"""
Generate an infrastructure manager service response
Args:
status A boolean value indicating the status
msg A reason message (useful if this a failed operation)
extra Any extra fields to be included in the response (Optional)
Returns:
A dictionary containing the operation response
"""
utils.log("Sending success = {0}, reason = {1}".format(status, msg))
response = {'success': status, 'reason': msg}
if extra is not None:
for key, value in extra.items():
response[key] = value
return response
def sleep_until_update_operation_done(self, result, resource_name):
""" Sleeps until the create/update operation for the resource is completed
successfully.
Args:
result: An instance, of the AzureOperationPoller to poll for the status
of the operation being performed.
resource_name: The name of the resource being updated.
"""
time_start = time.time()
while not result.done():
utils.log("Waiting {0} second(s) for {1} to be created/updated.".
format(self.SLEEP_TIME, resource_name))
time.sleep(self.SLEEP_TIME)
total_sleep_time = time.time() - time_start
if total_sleep_time > self.MAX_SLEEP_TIME:
utils.log("Waited {0} second(s) for {1} to be created/updated. "
"Operation has timed out.".format(total_sleep_time, resource_name))
break
