def ask_firm_init(player_id): # ------- Get needed objects ----------------------------------------------------------------- # p = Players.objects.get(player_id=player_id) rd = Round.objects.get(round_id=p.round_id) # ------- Log stuff -------------------------------------------------------------------------- # utils.log("Player {} for room {} and round {} ask init: t is {}.".format( player_id, p.room_id, rd.round_id, rd.t), f=utils.fname(), path=__path__) # ------- Maybe client has to wait the other player ------------------------------------------ # if player.dialog.client_has_to_wait_over_player(player_id=player_id, called_from=__path__ + ':' + utils.fname()): if not room.dialog.is_trial(player_id, utils.fname()): opp_progression = player.dialog.get_opponent_progression( player_id=player_id, called_from=__path__ + ':' + utils.fname() ) return parameters.error["wait"], opp_progression # Tuple is necessary!! '-1' hold for wait # Get information necessary for firm initialization d = data.get_init_info(player_id=player_id) return rd.t, d["firm_state"], d["player"]["position"], d["player"]["price"], d["player"]["profits"], \ d["opp"]["position"], d["opp"]["price"], d["opp"]["profits"], rd.ending_t
def topic_page(p, topic_name=None): if not p.isdigit(): p = '1' page = int(p) pre_page = 2 if topic_name is None: query = Post.query node_list = Node.query.all() log(query) else: topic = Topic.query.filter_by(en_name=topic_name).first() log(topic) if topic is None: abort(404) query = topic.posts node_list = topic.nodes.all() paginate = query.order_by(Post.created_time.desc()).paginate(page, pre_page, False) post_list = paginate.items topic_list = Topic.query.all() data = { 'post_list': post_list, 'paginate': paginate, 'topic_list': topic_list, 'node_list': node_list, 'selected_topic': topic_name, } return data
def create_network_interface(self, network_client, interface_name, ip_name, subnet, parameters): """ Creates the Public IP Address resource and uses that to create the Network Interface. Args: network_client: A NetworkManagementClient instance. interface_name: The name to use for the Network Interface resource. ip_name: The name to use for the Public IP Address resource. subnet: The Subnet resource from the Virtual Network created. parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. """ group_name = parameters[self.PARAM_RESOURCE_GROUP] region = parameters[self.PARAM_ZONE] utils.log("Creating/Updating the Public IP Address '{}'".format(ip_name)) ip_address = PublicIPAddress( location=region, public_ip_allocation_method=IPAllocationMethod.dynamic, idle_timeout_in_minutes=4) result = network_client.public_ip_addresses.create_or_update( group_name, ip_name, ip_address) self.sleep_until_update_operation_done(result, ip_name) public_ip_address = network_client.public_ip_addresses.get(group_name, ip_name) utils.log("Creating/Updating the Network Interface '{}'".format(interface_name)) network_interface_ip_conf = NetworkInterfaceIPConfiguration( name=interface_name, private_ip_allocation_method=IPAllocationMethod.dynamic, subnet=subnet, public_ip_address=PublicIPAddress(id=(public_ip_address.id))) result = network_client.network_interfaces.create_or_update( group_name, interface_name, NetworkInterface(location=region, ip_configurations=[network_interface_ip_conf])) self.sleep_until_update_operation_done(result, interface_name)
def __spawn_vms(self, agent, num_vms, parameters, reservation_id): """ Private method for starting a set of VMs Args: agent Infrastructure agent in charge of current operation num_vms No. of VMs to be spawned parameters A dictionary of parameters reservation_id Reservation ID of the current run request """ status_info = self.reservations.get(reservation_id) try: security_configured = agent.configure_instance_security(parameters) instance_info = agent.run_instances(num_vms, parameters, security_configured) ids = instance_info[0] public_ips = instance_info[1] private_ips = instance_info[2] status_info['state'] = self.STATE_RUNNING status_info['vm_info'] = { 'public_ips': public_ips, 'private_ips': private_ips, 'instance_ids': ids } utils.log('Successfully finished request {0}.'.format(reservation_id)) except AgentRuntimeException as exception: status_info['state'] = self.STATE_FAILED status_info['reason'] = str(exception) self.reservations.put(reservation_id, status_info)
async def place(self, ctx: commands.Context, role, *names): event = EventModel.load(ctx.channel.id) if event is None: await ctx.author.send(errors.NONEXISTENT_EVENT) return False if date.is_date_time_expired(event.date, event.time): await ctx.author.send(errors.EXPIRED_EVENT) return False role = role.title() if role and role not in settings.ROLES.ALL: await ctx.author.send(errors.NONEXISTENT_ROLE) return for name in names: members = ctx.channel.guild.fetch_members() member = await members.find( lambda m: utils.fuzzy_string_match(m.display_name, name)) if member is None: await ctx.author.send(errors.NONEXISTENT_MEMBER) return await self.add_member(ctx.channel, event, member, role) utils.log(ctx.author.display_name, "placed player", name, "...") embed = view.create(ctx.channel.id, ctx.guild.emojis, self.bot.user.id) await utils.show_event(channel=ctx.channel, client=self.bot, embed=embed)
def group_message(self, message): """ Send json to a group, called by self._group_send method and WSDialog.group_send from outside :param message: :return: """ data = message['data'] group = message['group'] is_json = message['json'] if is_json: try: if len(str(data)) < 100: utils.log(f'Sending to group {group}: {data}', f=self._group_send) except UnicodeEncodeError: print('Error printing request.') self.send_json(data) else: self.send(data)
def email_valid(email): result = dict(valid=True, msg=dict()) # str_valid = True # # email = email.strip() # # for c in email: # # if c not in email_valid_str: # # str_valid =False # # split_email = email.split('@', 1) # split_valid = len(split_email) == 2 # not_exist = False # if split_valid and split_email[0] != '' and split_email[1] != '': # host = split_email[1] # split_host = host.split('.', 1) # split_valid = len(split_host) == 2 # if split_valid and split_host[0] != '' and split_host[1] != '': u = User.query.filter_by(email=email).first() log(u) if u is not None: result['valid'] = False result['msg']['.email-message'] = '邮箱已被注册' # not_exist = True # else: # str_valid = False # result['valid'] = not_exist # msg = result['msg'] # if not str_valid: # msg['.email-message'] = '请输入正确的邮箱' # elif not not_exist: # msg['.email-message'] = '邮箱已被注册' return result
def login(form, r=dict): if r['valid']: log("login") db_u = User.query.filter_by(username=form['login_username']).first() log('login success') login_user(db_u) return r
def register(form, r=dict): log('register', r['valid'], r['msg']) if r['valid']: u = User(form) log('register') u.save() return r
def read_dataset(): global dataset file_path = os.path.join('../build/data', args.scope, 'labeled/Learning_set-Bearing3_1-acc.csv') table = np.genfromtxt( file_path, delimiter=',', skip_header=1, usecols=(1, 2) # x_acc, y_acc ) for sample_from_idx in range(0, table.shape[0] - args.step_size + 1): table_sample = table[sample_from_idx:sample_from_idx + args.step_size] dataset['ordered'].append(table_sample[:, 0:2]) if sample_from_idx % 100000 == 0: log('%2.0f%% loaded' % ((float(sample_from_idx) / (table.shape[0] - args.step_size + 1)) * 100)) # train, validate split dataset_health, dataset['anomalous'] = np.split( dataset['ordered'], [int(.9 * len(dataset['ordered']))]) # shuffle np.random.shuffle(dataset_health) dataset['train'], dataset['validate'] = np.split( dataset_health, [int(.9 * len(dataset_health))]) np.random.shuffle(dataset['train']) dataset['ordered'] = np.array(dataset['ordered'])
def wrapper(*args, **kwargs): form = args[0] old = form.get('old_password') new = form.get('change_password') log(old, new) # if g.user.password == old: # nn_valid = True # else: # nn_valid_msg = nickname_valid(nickname) # nn_valid = nn_valid_msg['valid'] # msg.update(nn_valid_msg['msg']) # if g.user.email == email: # e_valid = True # else: # e_valid_msg = email_valid(email) # e_valid = e_valid_msg['valid'] # msg.update(e_valid_msg['msg']) # r = dict( # valid=e_valid and nn_valid, # msg=msg # ) # u = User.query.filter_by(username=username).first() # log('valid',u) # valid = False # log(valid) r = dict(valid=False, msg=dict()) msg = r['msg'] if g.user.password != old: msg['.password-message'] = '当前密码错误' elif old == new: msg['.password-message'] = '不能与原密码相同' r['valid'] = (g.user.password == old) and (old != new) args = form, r return function(*args, **kwargs)
def main(): dest_dir = prepare_directory( os.path.join('../../build/data', SCOPE, 'initialized')) for dataset in DATASETS_TO_PARSE: filenames = glob.glob(os.path.join(SRC_DIR, dataset, '*.csv')) for filename in filenames: log('parsing ' + filename + '...') dfChunks = pd.read_csv(filename, header=None, names=INPUT_CSV_COLUMNS, chunksize=CHUNK_SIZE) i = 0 for dfChunk in dfChunks: i = i + 1 header = i is 1 mode = 'a' if i > 1 else 'w' log('parsing chunk %d' % i) dfChunk['datetime'] = pd.to_datetime(dfChunk['timestamp'], unit='ms') dfChunk['u'] = dfChunk['u'] * 0.1 dfChunk['v'] = dfChunk['v'] * 0.1 dfChunk['w'] = dfChunk['w'] * 0.1 dfChunk['x'] = dfChunk['x'] * 0.004 dfChunk['y'] = dfChunk['y'] * 0.004 dfChunk['z'] = dfChunk['z'] * 0.004 dfChunk.to_csv(os.path.join( dest_dir, '%s-%s' % (dataset, os.path.basename(filename))), mode=mode, header=header, index=False, columns=OUTPUT_CSV_COLUMNS)
def verify_rule_resfinally_reject_result(self, id_account, id_borrow, result='reject', status='new'): query_rule_resfin = "select * from vnrisk_rule_resfinally where id_account={} and id_borrow={} order by id desc limit 1;".format(id_account, id_borrow) time.sleep(2) resfinally_result = self.risk_conn.fetchone(query_rule_resfin) utils.log("rule_resfinally result:\n{}".format(resfinally_result)) actual_result, actual_status, blocking_day = resfinally_result['result'], resfinally_result['status'], resfinally_result['blocking_day'] self.assertTrue(actual_result == result and actual_status == status and blocking_day >= 3, 'rule result Not correct! please check.')
def attach_disk(self, parameters, disk_name, instance_id): """ Attaches the Elastic Block Store volume specified in 'disk_name' to this virtual machine. Args: parameters: A dict with keys for each parameter needed to connect to AWS. disk_name: A str naming the EBS mount to attach to this machine. instance_id: A str naming the id of the instance that the disk should be attached to. In practice, callers add disks to their own instances. Returns: The location on the local filesystem where the disk has been attached. """ # In Amazon Web Services, if we're running on a Xen Paravirtualized machine, # then devices get added starting at /dev/xvda. If not, they get added at # /dev/sda. Find out which one we're on so that we know where the disk will # get attached to. if glob.glob("/dev/xvd*"): mount_point = '/dev/xvdc' else: mount_point = '/dev/sdc' try: conn = self.open_connection(parameters) utils.log('Attaching volume {0} to instance {1}, at {2}'.format( disk_name, instance_id, mount_point)) conn.attach_volume(disk_name, instance_id, mount_point) return mount_point except EC2ResponseError as exception: utils.log('An error occurred when trying to attach volume {0} to ' \ 'instance {1} at /dev/sdc'.format(disk_name, instance_id)) self.handle_failure('EC2 response error while attaching volume:' + exception.error_message)
def wrapper(*args, **kwargs): form = args[0] username = form.get('register_username') nickname = form.get('register_nickname') password = form.get('register_password') # confirm_password = form.get('confirm') email = form.get('register_email') un_valid_msg = username_valid(username) nn_valid_msg = nickname_valid(nickname) pw_valid_msg = password_valid(password) e_valid_msg = email_valid(email) un_valid = un_valid_msg['valid'] nn_valid = nn_valid_msg['valid'] pw_valid = pw_valid_msg['valid'] e_valid = e_valid_msg['valid'] msg = dict() msg.update(un_valid_msg['msg']) msg.update(nn_valid_msg['msg']) msg.update(pw_valid_msg['msg']) msg.update(e_valid_msg['msg']) log(un_valid, pw_valid, e_valid, nn_valid) r = dict(valid=un_valid and pw_valid and e_valid and nn_valid, msg=msg) args = form, r return function(*args, **kwargs)
def describe_instances_old(self, parameters): """ Retrieves the list of running instances that have been instantiated using a particular EC2 keyname. The target keyname is read from the input parameter map. (Also see documentation for the BaseAgent class) Args: parameters A dictionary containing the 'keyname' parameter Returns: A tuple of the form (public_ips, private_ips, instances) where each member is a list. """ instance_ids = [] public_ips = [] private_ips = [] conn = self.open_connection(parameters) reservations = conn.get_all_instances() instances = [i for r in reservations for i in r.instances] utils.log("Looking for instances with key = {0}".format(parameters[self.PARAM_KEYNAME])) for i in instances: if i.state == 'running' and i.key_name == parameters[self.PARAM_KEYNAME]: instance_ids.append(i.id) public_ips.append(i.public_dns_name) private_ips.append(i.private_dns_name) return public_ips, private_ips, instance_ids
def post(self, request, *args, **kwargs): """ Room creation process :param request: using POST request :return: html room form template (success or fail) """ if request.POST["form_function"] == "room_organisation": form = RoomForm(request.POST) if form.is_valid(): with transaction.atomic(): try: # Create room room.dashboard.create(form.get_data()) except IntegrityError: utils.log("Room already exists!", f=utils.fname(), path=__path__, level=2) return redirect("/room_management") else: context = {"subtitle": "Set parameters and create a room", "form": form} return render(request, self.template_name, context) else: raise Exception("Error validating the form.")
def run_instances(self, count, parameters, security_configured): """ Spawns the specified number of OpenStack instances using the parameters provided. This method is blocking in that it waits until the requested VMs are properly booted up. However if the requested VMs cannot be procured within 1800 seconds, this method will treat it as an error and return. (Also see documentation for the BaseAgent class). This method differs from its OpenStack counterpart because OpenStack does not support spot instances. Args: count: Number of VMs to spawn. parameters: A dictionary of parameters. This must contain 'keyname', 'group', 'image_id' and 'instance_type' parameters. security_configured: Uses this boolean value as an heuristic to detect brand new AppScale deployments. Returns: A tuple of the form (instances, public_ips, private_ips). """ if parameters[self.PARAM_SPOT] == "True": parameters[self.PARAM_SPOT] = 'False' utils.log("OpenStack does not support spot instances") super.run_instances(self, count, parameters, security_configured)
def search_kyc_reason_api(self, url, header, params): utils.log(">>search kyc reason records...") kyc_list_resp = self.req.get(url, headers=header, params=params) self.assertEqual( json.loads(kyc_list_resp.text)['msg'], 'success', 'call search kyc reason api failed!') return kyc_list_resp.text
def main(): args = get_args() centroids = {} df_breakpoints = pd.read_csv(args.src_breakpoint) for column in args.columns: log('parsing column %s ...' % (column, )) data_points = np.array([]) for src in args.srcs: log('parsing %s ...' % (src, )) df = pd.read_csv(src, usecols=[column]) data_points = np.concatenate((data_points, df[column]), axis=0) init_centroids = breakpoints_to_centroids(df_breakpoints[column].values) kmeans = KMeans( init=np.array(init_centroids).reshape((-1, 1)), n_clusters=args.symbol_size, random_state=0, n_jobs=1, verbose=0, max_iter=500 ).fit(np.array(data_points).reshape((-1, 1))) centroids[column] = np.array(kmeans.cluster_centers_).reshape(-1) df_centroids = pd.DataFrame(centroids) dest_dir = prepare_directory(args.dest_dir) df_centroids.to_csv( os.path.join( dest_dir, 'centroid-{0}.csv'.format(args.symbol_size) ), header=True, index=False )
def include_players_into_round_compositions(room_id, player_id, called_from): utils.log("Called by '{}'".format(called_from), f=utils.fname(), path=__path__) return composition.include_players_into_round_compositions( room_id, player_id)
def test_vn_es_data_consistence_with_work(self): """验证ES数据内容与源表work一致""" utils.log("verify es details data is consistence with source work.") self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['es_work'], self.crm_data['ESSrcDetail']['Check'][constants.CN]['work'], self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['workQuery']), self.crm_data['ESSrcDetail']['Src'][constants.CN]['work'])
def test_select_save_people(self): """ select people and save people library """ utils.log("1. Call search people api.") utils.log( "Select people: gender= Male, birthday<=2006-10-03, marita status in ('Single','Married'), " "last application status not in (Failure, Rejected),last login date >current_date-20" ) # search_save_payload = self.crm_data['CRM']['search']['payload1'] search_save_payload = self.crm_data['CRM']['search']['payload2'] search_resp, select_count = self.search_people_api(search_save_payload) self.assertGreater(int(select_count), 0, "Select People Failed!!! The count less than 0") utils.log("2. Save searched people library.") save_resp, people_lib_id = self.save_people_api( {"labelListJson": search_save_payload}, self.crm_data['CRM']['save']['controlGroupJson'], int(select_count)) self.assertGreater(int(people_lib_id), 1, "Save People Library Failed!!!") utils.log("3. Verify new save people library exist in db.") self.verify_people_lib_id_in_db(people_lib_id, self.desc_prefix) utils.log("4. Add people library task.") self.add_people_lib_task(people_lib_id)
def make_sleepy(self,parameters, instance_id): utils.log( "Making instance {0} sleepy...".format(instance_id)) credentials = parameters[self.PARAM_CREDENTIALS] ec2 = boto.connect_cloudwatch( str(credentials[self.PARAM_CREDS_PUBLIC]), str(credentials[self.PARAM_CREDS_PRIVATE]) ) region = "us-east-1" terminate_arn = 'arn:aws:automate:{0}:ec2:terminate'.format(region) alarm_name = 'ec2_shutdown_sleepy_{0}'.format(instance_id) # define our alarm to terminate the instance if it gets sleepy # i.e. if CPU utilisation is less than 10% for 1 x 4 hr intervals sleepy_alarm = MetricAlarm( name=alarm_name, namespace='AWS/EC2', metric='CPUUtilization', statistic='Average', comparison='<', threshold='10', period='3600', evaluation_periods=4, alarm_actions=[terminate_arn], dimensions={'InstanceId':instance_id} ) # create the alarm.. Zzzz! ec2.create_alarm(sleepy_alarm)
def terminate_instances(self, parameters): """ Stop one or more EC2 instances. The input instance IDs are fetched from the 'instance_ids' parameter in the input parameters map. (Also see documentation for the BaseAgent class) If parameters contains the 'prefix' key, instances will be terminated based on whether or not the keypair used to start them begins with the given prefix. Args: parameters A dictionary of parameters """ conn = self.open_connection(parameters) instance_ids = [] if self.PARAM_KEY_PREFIX in parameters: prefix = parameters[self.PARAM_KEY_PREFIX] reservations = conn.get_all_instances() instances = [i for r in reservations for i in r.instances] for i in instances: if i.key_name is not None and i.key_name.startswith(prefix): instance_ids.append(i.id) else: instance_ids = parameters[self.PARAM_INSTANCE_IDS] terminated_instances = conn.terminate_instances(instance_ids) for instance in terminated_instances: instance.update() while instance.state != 'terminated': time.sleep(5) instance.update() utils.log('Instance {0} was terminated'.format(instance.id)) return True
def get_target_row(self, row_unique_key_name, row_unique_key): """ get target row in data grid matched unique key value(eg: type, ruleId) """ target_row, target_row_id, action_column_index = [], None, -1 trs = self.get_data_grid_rows() headers = self.get_data_grid_headers() for index, header in enumerate(headers): if header == constants.ActionsColumnName or header == constants.ActionColumnName: action_column_index = index # get the [Action] column index break for tr in trs: tds_in_tr = self.slb.get_elements_via_element(tr, datagrid.td) target_row_id = self.slb.get_element_text( tds_in_tr[0]) # first column value, help find row on ui for td in tds_in_tr: if self.slb.get_element_text(td) == row_unique_key: utils.log("find the data record with {}={}".format( row_unique_key_name, row_unique_key)) target_row.append(tr) break if target_row: break if not target_row: utils.warn("Not find any data record with {}={} !".format( row_unique_key_name, row_unique_key)) return return target_row, target_row_id, action_column_index
def attach_disk(self, parameters, disk_name, instance_id): """ Attaches the Elastic Block Store volume specified in 'disk_name' to this virtual machine. Args: parameters: A dict with keys for each parameter needed to connect to AWS. disk_name: A str naming the EBS mount to attach to this machine. instance_id: A str naming the id of the instance that the disk should be attached to. In practice, callers add disks to their own instances. """ try: conn = self.open_connection(parameters) utils.log('Attaching volume {0} to instance {1}, at /dev/sdc'.format( disk_name, instance_id)) conn.attach_volume(disk_name, instance_id, '/dev/sdc') while True: utils.log('Waiting for disk to finish attaching.') status = conn.get_all_volumes(disk_name)[0].status utils.log('Volume {0} reports its status as {1}'.format(disk_name, status)) if status == 'in-use': break utils.sleep(1) utils.log('Volume {0} is attached and ready for use.') return '/dev/sdc' except EC2ResponseError as exception: utils.log('An error occurred when trying to attach volume {0} to ' \ 'instance {1} at /dev/sdc'.format(disk_name, instance_id)) self.handle_failure('EC2 response error while attaching volume:' + exception.error_message)
async def create_event(self, ctx: commands.Context, name, date, time, description=""): date = try_parse_day(date) if not is_valid_date_time(date, time): await ctx.author.send(errors.INVALID_DATE.format(date, time)) return if is_date_time_expired(date, time): await ctx.author.send(errors.EXPIRED_DATE.format(date, time)) return event = EventModel(self.bot.user.id, name, date, time, description, ctx.channel.id, 0) event.save() embed = view.create(ctx.channel.id, ctx.guild.emojis, self.bot.user.id) await utils.show_event(channel=ctx.channel, client=self.bot, embed=embed, new_event=True) await utils.send_announcement( ctx, settings.MESSAGE.NEW_EVENT.format(event.name, event.date, event.time, ctx.channel.mention)) utils.log(ctx.author.display_name, "created event", event.name, "...")
async def edit(self, ctx: commands.Context, *raw_aspects): event = EventModel.load(ctx.channel.id) if event is None: await ctx.author.send(errors.NONEXISTENT_EVENT) return parsed_aspect_dict, invalids = EventAspectParser.parse(raw_aspects) for name, value in parsed_aspect_dict.items(): event.set_attribute(name, value) if len(invalids) > 0: await ctx.author.send(errors.INVALID_ASPECTS + '\n' + '\n'.join(invalids)) if is_date_time_expired(event.date, event.time): await ctx.author.send( errors.EXPIRED_DATE.format(event.date, event.time)) return event.save(append=True) embed = view.create(ctx.channel.id, ctx.guild.emojis, self.bot.user.id) await utils.show_event(channel=ctx.channel, client=self.bot, embed=embed, new_event=False) utils.log(ctx.author.display_name, "edited event", event.name, "...")
def __spawn_vms(self, agent, num_vms, parameters, reservation_id): """ Private method for starting a set of VMs Args: agent Infrastructure agent in charge of current operation num_vms No. of VMs to be spawned parameters A dictionary of parameters reservation_id Reservation ID of the current run request """ status_info = self.reservations.get(reservation_id) try: security_configured = agent.configure_instance_security(parameters) instance_info = agent.run_instances(num_vms, parameters, security_configured) ids = instance_info[0] public_ips = instance_info[1] private_ips = instance_info[2] status_info['state'] = self.STATE_RUNNING status_info['vm_info'] = { 'public_ips': public_ips, 'private_ips': private_ips, 'instance_ids': ids } utils.log( 'Successfully finished request {0}.'.format(reservation_id)) except AgentRuntimeException as exception: status_info['state'] = self.STATE_FAILED status_info['reason'] = str(exception) self.reservations.put(reservation_id, status_info)
def click_report_biz_list(self): utils.log("click [Report Biz List] tab.") if not self.is_report_menu_expanded(): utils.debug("expand menu [Report Schedule]") self.slb.click(report_schedule.report_toggle) self.slb.move_to_click(report_schedule.report_schedule_link) self.slb.sleep(1)
def test_vn_es_data_consistence_with_person(self): """验证ES数据内容与源表person一致""" utils.log("verify es details data is consistence with source person.") self.verify_es_details_with_src(self.crm_data['ESSrcDetail']['Check'][constants.CN]['person'], self.crm_data['ESSrcDetail']['Check'][constants.CN]['person'], self.get_es_data_details_list(self.crm_data['ESSrcDetail']['ES'][constants.CN]['query']), self.crm_data['ESSrcDetail']['Src'][constants.CN]['person'])
def create_scratch_disk(self, parameters): """ Creates a disk from a given machine image. GCE does not support scratch disks on API version v1 and higher. We create a persistent disk upon creation to act like one to keep the abstraction used in other infrastructures. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. Returns: A str, the url to the disk to use. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) disk_name = self.generate_disk_name(parameters) project_name = parameters[self.PARAM_PROJECT] project_url = '{0}{1}'.format(self.GCE_URL, parameters[self.PARAM_PROJECT]) source_image_url = '{0}{1}/global/images/{2}'.format( self.GCE_URL, project_name, parameters[self.PARAM_IMAGE_ID]) request = gce_service.disks().insert(project=project_name, zone=parameters[self.PARAM_ZONE], body={'name': disk_name}, sourceImage=source_image_url) response = request.execute(http=auth_http) utils.log(str(response)) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT]) disk_url = "{0}/zones/{1}/disks/{2}".format( project_url, parameters[self.PARAM_ZONE], disk_name) return disk_url
def test_id_storage_wide_total_consistent(self): """验证窄表与宽表数据总量一致""" utils.log("verify JP user_tag_storage and user_tag_wide total count equals") self.verify_storage_wide_consistent( '', "select count(1) from idcrmtest_ext.user_tag_storage where tag_id='accountname'", "select count(1) from idcrmtest_ext.user_tag_wide")
def set_view_table_email_header(self, email_header): utils.log( "input Table Report Email Header in [View] report dialog. as below\n{}" .format(email_header)) self.slb.type( report_schedule.view_html_text_area.format('Email Header'), email_header)
def validate_application_for_deployment(self, secret, app): if self.secret != secret: return self.__generate_response(False, self.REASON_BAD_SECRET) name = app['name'] version = app['version'] dependencies = app['dependencies'] api_list = app['api_list'] api_list_without_specs = self.__remove_specs(api_list) owner = app['owner'] if dependencies: dep_invalid = self.adaptor.validate_application_dependencies(name, version, api_list_without_specs, dependencies) if dep_invalid: detail = { 'detail' : dep_invalid } return self.__generate_response(False, self.REASON_BAD_DEPENDENCIES, detail) pre_validation_errors = [] for api in api_list: api_name = api['name'] api_spec = api['specification'] if not self.__is_api_name_valid(api_name): pre_validation_errors.append('Invalid characters in API name: {0}'.format(api_name)) spec_valid, spec_errors = swagger.validate_swagger_description(api_spec) if not spec_valid: pre_validation_errors.append(spec_errors) if pre_validation_errors: detail = { 'detail' : '|'.join(pre_validation_errors) } return self.__generate_response(False, self.REASON_BAD_API_METADATA, detail) passed, message = self.policy_engine.run_policy_enforcement(name, version, dependencies, api_list_without_specs, owner) if not passed: detail = { 'detail' : message } return self.__generate_response(False, self.REASON_API_POLICY_VIOLATION, detail) post_validation_errors = [] for api in api_list: api_name = api['name'] api_version = api['version'] api_spec = api['specification'] if self.adaptor.is_api_available(api_name, api_version): passed, message = self.__invoke_api_validations(api_name, api_version, api_spec) else: passed, message = self.__handle_new_api(api_name, api_version, api_spec) if not passed: post_validation_errors.append(message) if post_validation_errors: detail = { 'detail' : '|'.join(post_validation_errors)} return self.__generate_response(False, self.REASON_API_VALIDATION_FAILED, detail) if not self.adaptor.record_application_dependencies(name, version, api_list_without_specs, dependencies): utils.log("Failed to record dependencies for {0}-v{1}".format(name, version)) return self.__generate_response(False, self.REASON_DEPENDENCY_RECORDING_FAILED) return self.__generate_response(True, self.REASON_API_VALIDATION_SUCCESS)
def stop(self): """ Stop the EAGER service. """ if self.started: utils.log('Stopping EAGER service') self.started = False self.server.shutdown() else: utils.log('Warning - Stop called on already stopped server')
def stop(self): """ Stop the infrastructure manager service. """ if self.started: utils.log('Stopping AppScale Infrastructure Manager') self.started = False self.server.shutdown() else: utils.log('Warning - Stop called on already stopped server')
def setProperty(self, prop, value): if prop == None or value == None: if prop in self.properties: del self.properties[prop] utils.logTrace('Removing property ' + repr(prop) + ' value: ' + repr(value)) if prop in self.properties: old = '' if self.properties[prop] != None: old = self.properties[prop] utils.log('Token "' + self.name + '" Replacing ' + prop + ' old value: ' + repr(old) + ' new value ' + repr(value)) self.properties[prop] = value
def addToken(self, token, lock=False): try: if lock: self.tlLock.acquire() tokenName = token.getName() if not tokenName in self.list: self.list[tokenName] = [] self.list[tokenName].append(token) utils.log('Adding token ' + tokenName) finally: if lock: self.tlLock.release()
def handle_failure(self, msg): """ Log the specified error message and raise an AgentRuntimeException Args: msg An error message to be logged and included in the raised exception Raises: AgentRuntimeException Contains the input error message """ utils.log(msg) raise AgentRuntimeException(msg)
def start(self): """ Start the EAGER service. This method blocks as long as the service is alive. The caller should handle the threading requirements """ if self.started: utils.log('Warning - Start called on already running server') else: utils.log('Starting EAGER service on port: ' + str(self.port)) self.started = True while self.started: self.server.serve_forever()
def __init__(self, initialTokenAssignments): utils.log('New TokenList') self.initialTokenAssignments = initialTokenAssignments self.list = {} self.requiredTokensByState = {} if initialTokenAssignments != None: for tokenName, attributes in self.initialTokenAssignments.iteritems(): numTokens = attributes[0] if not tokenName in self.list: self.list[tokenName] = [] for t in xrange(int(numTokens)): token = Token(tokenName, copy.deepcopy(attributes[1])) token.unlock() self.list[tokenName].append(token)
def run_instances_response(self, command, output): """ Local utility method to parse the validate the output of ec2-run-instances command. Args: command Exact command executed output Output of the command Returns: A tuple of the form (status,command) where status is a boolean value indicating the success/failure status of the output and command is the modified command to be retried in case the previous attempt had failed. """ if output.find('Please try again later') != -1: utils.log('Error with run instances: {0}. Will try again in a moment.'.format(output)) return False, command elif output.find('try --addressing private') != -1: utils.log('Need to retry with addressing private. Will try again in a moment.') return False, command + ' --addressing private' elif output.find('PROBLEM') != -1: utils.log('Error: {0}'.format(output)) sys.exit('Saw the following error from EC2 tools: {0}'.format(output)) else: utils.log('Run instances message sent successfully. Waiting for the image to start up.') return True, command
def attach_disk(self, parameters, disk_name, instance_id): """ Attaches the persistent disk specified in 'disk_name' to this virtual machine. Args: parameters: A dict with keys for each parameter needed to connect to Google Compute Engine. disk_name: A str naming the persistent disk to attach to this machine. instance_id: A str naming the id of the instance that the disk should be attached to. In practice, callers add disks to their own instance. Returns: A str indicating where the persistent disk has been attached to. """ gce_service, credentials = self.open_connection(parameters) http = httplib2.Http() auth_http = credentials.authorize(http) project = parameters[self.PARAM_PROJECT] zone = parameters[self.PARAM_ZONE] # If the disk is already attached, return the mount point. request = gce_service.instances().get(project=project, zone=zone, instance=instance_id) disks = request.execute(auth_http)['disks'] for disk in disks: path = disk['source'].split('/') if project == path[-5] and zone == path[-3] and disk_name == path[-1]: device_name = '/dev/{}'.format(disk['deviceName']) utils.log('Disk is already attached at {}'.format(device_name)) return device_name request = gce_service.instances().attachDisk( project=project, zone=zone, instance=instance_id, body={ 'kind' : 'compute#attachedDisk', 'type' : 'PERSISTENT', 'mode' : 'READ_WRITE', 'source' : "https://www.googleapis.com/compute/{0}/projects/{1}" \ "/zones/{2}/disks/{3}".format(self.API_VERSION, project, zone, disk_name), 'deviceName' : 'sdb' } ) response = request.execute(auth_http) utils.log(str(response)) self.ensure_operation_succeeds(gce_service, auth_http, response, parameters[self.PARAM_PROJECT]) return '/dev/sdb'
def set_environment_variables(self, parameters): """ Set the EC2 specific environment variables. Required values for the environment variables are read from the 'credentials' parameter of the parameters map. (Also see documentation for the BaseAgent class) Args: parameters A dictionary containing the 'credentials' key """ if os.environ.has_key('EC2_JVM_ARGS'): del(os.environ['EC2_JVM_ARGS']) variables = parameters[self.PARAM_CREDENTIALS] for key, value in variables.items(): if value is None: utils.log('None value detected for the credential: {0}.'.format(key)) continue if key.find('KEY') != -1: utils.log('Setting {0} to {1} in our environment.'.format( key, utils.obscure_string(value))) else: utils.log('Setting {0} to {1} in our environment.'.format(key, value)) os.environ[key] = value ec2_keys_dir = os.path.abspath('/etc/appscale/keys/cloud1') os.environ['EC2_PRIVATE_KEY'] = ec2_keys_dir + '/mykey.pem' os.environ['EC2_CERT'] = ec2_keys_dir + '/mycert.pem' utils.log('Setting private key to: {0} and certificate to: {1}'.format( os.environ['EC2_PRIVATE_KEY'], os.environ['EC2_CERT']))
def remove(self, token): try: self.tlLock.acquire() n = 0 if not token.getName() in self.list: utils.logTrace('token ' + token.getName() + ' not found') return l = self.list[token.getName()] if not token in l: utils.log('token ' + token.getName() + ' not found in remove') ndx = l.index(token) del l[ndx] finally: self.tlLock.release()
def terminate_instances(self, parameters): """ Stop one of more EC2 instances using. The input instance IDs are fetched from the 'instance_ids' parameters in the input map. (Also see documentation for the BaseAgent class) Args: parameters A dictionary of parameters """ instance_ids = parameters[self.PARAM_INSTANCE_IDS] conn = self.open_connection(parameters) terminated_instances = conn.terminate_instances(instance_ids) for instance in terminated_instances: utils.log('Instance {0} was terminated'.format(instance.id))
def __init__(self): parent_dir = os.path.dirname(os.path.realpath(sys.argv[0])) self.policy_store_dir = os.path.join(parent_dir, self.POLICY_STORE_DIR) self.active_policies = [] self.inactive_policies = [] self.lock = Lock() if os.path.exists(self.policy_store_dir): for policy_file in os.listdir(self.policy_store_dir): if policy_file.endswith('.a.py'): full_path = os.path.join(self.policy_store_dir, policy_file) try: policy = Policy(full_path) self.active_policies.append(policy) except Exception as ex: utils.log("Error while loading policy '{0}': {1}".format(policy_file, str(ex))) elif policy_file.endswith('.i.py'): full_path = os.path.join(self.policy_store_dir, policy_file) try: policy = Policy(full_path) self.inactive_policies.append(policy) except Exception as ex: utils.log("Error while loading policy '{0}': {1}".format(policy_file, str(ex))) if self.active_policies: msg_suffix = '1 policy' if len(self.active_policies) > 1: msg_suffix = '{0} policies'.format(len(self.active_policies)) utils.log("Initialized policy engine with {0}.".format(msg_suffix)) else: utils.log("No active policies found.")
def evaluate(self, app, errors): globals_map = globals().copy() globals_map['app'] = app globals_map['assert_app_dependency'] = assert_app_dependency globals_map['assert_not_app_dependency'] = assert_not_app_dependency globals_map['assert_app_dependency_in_range'] = assert_app_dependency_in_range globals_map['assert_true'] = assert_true globals_map['assert_false'] = assert_false globals_map['compare_versions'] = compare_versions try: exec(self.source_code, globals_map, {}) except EagerPolicyAssertionException as ex: errors.append('[{0}] {1}'.format(self.name, ex)) except Exception as ex: utils.log('[{0}] Unexpected policy exception: {1}'.format(self.name, ex))
def configure_instance_security(self, parameters): """ Setup EC2 security keys and groups. Required input values are read from the parameters dictionary. More specifically, this method expects to find a 'keyname' parameter and a 'group' parameter in the parameters dictionary. Using these provided values, this method will create a new EC2 key-pair and a security group. Security group will be granted permissions to access any port on the instantiated VMs. (Also see documentation for the BaseAgent class) Args: parameters A dictionary of parameters """ keyname = parameters[self.PARAM_KEYNAME] #keyname = "cloudrunner-keypair" group = parameters[self.PARAM_GROUP] #group = "cloudrunner-security-group" key_path = '{0}.key'.format(keyname) ssh_key = os.path.abspath(key_path) utils.log( 'About to spawn EC2 instances - Expecting to find a key at {0}'.format(ssh_key) ) try: conn = self.open_connection(parameters) if os.path.exists(ssh_key): utils.log('SSH keys found in the local system - Not initializing EC2 security') else: utils.log('Creating key pair: ' + keyname) key_pair = conn.create_key_pair(keyname) utils.write_key_file(ssh_key, key_pair.material) security_groups = conn.get_all_security_groups() group_exists = False for security_group in security_groups: if security_group.name == group: group_exists = True break if not group_exists: utils.log('Creating security group: ' + group) newgroup = conn.create_security_group(group, 'CloudRunner security group') newgroup.authorize('tcp', 22, 22, '0.0.0.0/0') newgroup.authorize('tcp', 5672, 5672, '0.0.0.0/0') newgroup.authorize('tcp', 6379, 6379, '0.0.0.0/0') newgroup.authorize('tcp', 11211, 11211, '0.0.0.0/0') newgroup.authorize('tcp', 55672, 55672, '0.0.0.0/0') result = { 'success': True, 'absolute_key_path': ssh_key } return result except EC2ResponseError as exception: self.handle_failure( 'EC2 response error while initializing security: {0}'.format(exception.error_message) ) except Exception as exception: self.handle_failure( 'Error while initializing EC2 security: {0}'.format(exception.message) )
def dump(self, title = None, indent = 5): if not debug: return pad = '' for i in xrange(indent): pad += ' ' msg = '\n' try: if title != None and len(title) > 0: msg += title + '\n' msg += 'Token List Dump'.rjust(indent+15, ' ') + '\n' for tokenName,tokens in self.list.items(): for token in tokens: msg += token.dump('', indent) utils.log(msg) return msg finally: pass
def add(self, tokenName, amount, properties = {}): try: self.tlLock.acquire() n = 0 if not tokenName in self.list: self.list[tokenName] = [] for i in xrange(amount): token = Token(tokenName) token.unlock() for propName, propValue in properties.items(): token.setProperty(propName, propValue) self.list[tokenName].append(token) utils.log('Adding token ' + tokenName) return n finally: self.tlLock.release()
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, ssl=True): """ Initialize a new instance of the EAGER service. Args: host Hostname to which the service should bind (Optional). Defaults to 0.0.0.0. port Port of the service (Optional). Default to 18444. ssl True if SSL should be engaged or False otherwise (Optional). Defaults to True. When engaged, this implementation expects to find the necessary SSL certificates in the /etc/appscale/certs directory. """ self.host = host self.port = port secret = None while True: try: secret = utils.get_secret(self.APPSCALE_DIR + 'secret.key') break except Exception: utils.log('Waiting for the secret key to become available') utils.sleep(5) utils.log('Found the secret set to: {0}'.format(secret)) SOAPpy.Config.simplify_objects = True if ssl: utils.log('Checking for the certificate and private key') cert = self.APPSCALE_DIR + 'certs/mycert.pem' key = self.APPSCALE_DIR + 'certs/mykey.pem' while True: if os.path.exists(cert) and os.path.exists(key): break else: utils.log('Waiting for certificates') utils.sleep(5) ssl_context = SSL.Context() ssl_context.load_cert(cert, key) self.server = SOAPpy.SOAPServer((host, port), ssl_context=ssl_context) else: self.server = SOAPpy.SOAPServer((host, port)) e = Eager() self.server.registerFunction(e.ping) self.server.registerFunction(e.validate_application_for_deployment) self.server.registerFunction(e.publish_api_list) self.server.registerFunction(e.add_policy) self.server.registerFunction(e.remove_policy) self.server.registerFunction(e.enable_policy) self.server.registerFunction(e.disable_policy) self.server.registerFunction(e.list_policy) self.server.registerFunction(e.info_policy) self.started = False
def configure_instance_security(self, parameters): """ Setup EC2 security keys and groups. Required input values are read from the parameters dictionary. More specifically, this method expects to find a 'keyname' parameter and a 'group' parameter in the parameters dictionary. Using these provided values, this method will create a new EC2 key-pair and a security group. Security group will be granted permissions to access any port on the instantiated VMs. (Also see documentation for the BaseAgent class) Args: parameters A dictionary of parameters """ keyname = parameters[self.PARAM_KEYNAME] group = parameters[self.PARAM_GROUP] key_path = '/etc/appscale/keys/cloud1/{0}.key'.format(keyname) ssh_key = os.path.abspath(key_path) utils.log('About to spawn EC2 instances - ' \ 'Expecting to find a key at {0}'.format(ssh_key)) if os.path.exists(ssh_key): utils.log('SSH keys found in the local system - ' 'Not initializing EC2 security') return False try: conn = self.open_connection(parameters) key_pair = conn.get_key_pair(keyname) if key_pair is None: utils.log('Creating key pair: ' + keyname) key_pair = conn.create_key_pair(keyname) utils.write_key_file(ssh_key, key_pair.material) security_groups = conn.get_all_security_groups() group_exists = False for security_group in security_groups: if security_group.name == group: group_exists = True break if not group_exists: utils.log('Creating security group: ' + group) conn.create_security_group(group, 'AppScale security group') conn.authorize_security_group(group, from_port=1, to_port=65535, ip_protocol='udp') conn.authorize_security_group(group, from_port=1, to_port=65535, ip_protocol='tcp') conn.authorize_security_group(group, ip_protocol='icmp', cidr_ip='0.0.0.0/0') return True except EC2ResponseError as exception: self.handle_failure('EC2 response error while initializing ' 'security: ' + exception.error_message) except Exception as exception: self.handle_failure('Error while initializing EC2 ' 'security: ' + exception.message)
def __generate_response(self, status, msg, extra=None): """ Generate an infrastructure manager service response Args: status A boolean value indicating the status msg A reason message (useful if this a failed operation) extra Any extra fields to be included in the response (Optional) Returns: A dictionary containing the operation response """ utils.log("Sending success = {0}, reason = {1}".format(status, msg)) response = {'success': status, 'reason': msg} if extra is not None: for key, value in extra.items(): response[key] = value return response
def sleep_until_update_operation_done(self, result, resource_name): """ Sleeps until the create/update operation for the resource is completed successfully. Args: result: An instance, of the AzureOperationPoller to poll for the status of the operation being performed. resource_name: The name of the resource being updated. """ time_start = time.time() while not result.done(): utils.log("Waiting {0} second(s) for {1} to be created/updated.". format(self.SLEEP_TIME, resource_name)) time.sleep(self.SLEEP_TIME) total_sleep_time = time.time() - time_start if total_sleep_time > self.MAX_SLEEP_TIME: utils.log("Waited {0} second(s) for {1} to be created/updated. " "Operation has timed out.".format(total_sleep_time, resource_name)) break