def discover_activities(extension_mod_name: str, activity_type: str) -> DiscoveredActivities: """ Discover exported activities from the given extension module name. """ try: mod = importlib.import_module(extension_mod_name) except ImportError: raise DiscoveryFailed("could not import extension module '{m}'".format( m=extension_mod_name)) activities = [] try: exported = getattr(mod, "__all__") except AttributeError: logger.warn("'{m}' does not expose the __all__ attribute. " "It is required to determine what functions are actually " "exported as activities.".format(m=extension_mod_name)) return activities funcs = inspect.getmembers(mod, inspect.isfunction) for (name, func) in funcs: if exported and name not in exported: # do not return "private" functions continue sig = inspect.signature(func) activity = { "type": activity_type, "name": name, "mod": mod.__name__, "doc": inspect.getdoc(func), "arguments": [] } if sig.return_annotation is not inspect.Signature.empty: activity["return_type"] = portable_type_name(sig.return_annotation) for param in sig.parameters.values(): if param.kind in (param.KEYWORD_ONLY, param.VAR_KEYWORD): continue arg = { "name": param.name, } if param.default is not inspect.Parameter.empty: arg["default"] = param.default if param.annotation is not inspect.Parameter.empty: arg["type"] = portable_type_name(param.annotation) activity["arguments"].append(arg) activities.append(activity) return activities
def index_remove_coin(self, coin): if self.coin_supported_check(coin.upper()): if DatabaseManager.delete_index_coin_model(coin.upper()): DatabaseManager.delete_realized_gain_model(coin.upper()) logger.info("Coin " + coin.upper() + " removed from index") else: # Already Exist logger.warn("Coin not in index") else: logger.warn("Coin not supported")
def is_cookie_ok(self): url = 'http://photo.weibo.com/albums/get_all?uid=6069778559&page=1&count=20' print(url) rs = self.sess.get(url, headers=self.get_headers) if rs.history and rs.history[0].status_code == 302: # if not rs.status_code == 200: log.warn('session Expired, relogin.') self.login() os._exit(-1) else: log.debug('session is ok!!!')
def parse(self, response): # Number of main page scrapped logger.warn('Parse page ({})') # Get the url of each reference on the current page links = get_info.get_article_urls(response) # Get pagination information next_page = get_info.get_next_page_of_articles(response) next_page_number = get_info.get_next_page_number(next_page)
def run(args): cli = Cli("example") num1 = int(args["NUM1"]) num2 = int(args["NUM2"]) total = num1 + num2 total_minus_one = total - 1 logger.info(f"{num1} plus {num2} is {total}") logger.info(f"minus 1 that's {total_minus_one}") logger.warn("QUICK MATHS")
def is_configured(self, config): if not config['func_name']: logger.error('No function assigned for %s/parser_config.json' % self.raw_dir) return False if not config['func_hash']: logger.warn('No hash for function %s. Assigning current hash' % config['func_name']) return True return True
async def get_water_level(self): try: response = await self._bus.req('tank.water', {'command': 'get'}) if response['status'] != 'ok': logger.warn("Cannot get 'tank.water' status: %s", response['message']) return None return response['water'] except futures.TimeoutError: logger.warn("Cannot get 'tank.water' status: request timeout") return None
def is_cookie_ok(self): url = 'http://photo.weibo.com/albums/get_all?uid={}&page=1&count=20'.format( self.my_info.get('uid')) # log.info('TEST of access to {}'.format(url)) rs = self.sess.get(url, headers=self.get_headers) if rs.history and rs.history[0].status_code == 302: log.warn('✘ [session Expired], re-login.') base.force_quit() else: log.info('✔ [Web session] is ok!!!')
async def start_kernel(msg, send, context): logger.info('start_kernel') logger.warn(msg) context.realtime_evaluation_mode = msg['realtimeEvaluation'] set_state(context, RESTARTING) await stop_kernel(msg, send, context) context.a_queued = context.b_queued = context.realtime_evaluation_mode context.kernel_manager.start_kernel() context.jupyter_client = context.kernel_manager.client() context.iopub_listener = asyncio.create_task(iopub_listener(send, context)) asyncio.create_task(wait_until_kernel_ready(send, context))
def clip_gradient(self): if self.gradient_clip_value is not None: max_norm = max(self.gradient_norm_queue) total_norm = torch.nn.utils.clip_grad_norm_( self.model.parameters(), max_norm * self.gradient_clip_value) self.gradient_norm_queue.append( min(total_norm, max_norm * 2.0, 1.0)) if total_norm > max_norm * self.gradient_clip_value: logger.warn( F'Clipping gradients with total norm {round(total_norm, 5)} ' F'and max norm {round(max_norm, 5)}')
def push_last(self, item): with self.cond: ret = None if len(self) + 1 >= self.max_size: ret = self.arr.pop() logger.warn('current list size out of max size, remove %s', ret) self.arr.append(item) if len(self) == 1: self.cond.notify() return ret
def load_sources(dataset_name: str) -> Optional[References]: """Load a file with sources for a standard GEM dataset (attempt download), return None if not present. Note that it can be the same files as for references -- the difference is in the source/target fields inside the JSON structure.""" if dataset_name in _DATASET_SOURCES_URLS: try: dataset_file = ensure_download('references', dataset_name + '.json', _DATASET_SOURCES_URLS[dataset_name]) return Sources(dataset_file) except Exception as e: logger.warn(f'Could not download references for {dataset_name}: {str(e)}') return None return None
def __init__(self, directory_path): self.path = directory_path self.output_file_dict = AnalysisDirectory._validator.collect_output_file_dict(self.path) try: AnalysisDirectory._validator.complete(self.path, self.output_file_dict) self.is_complete = True except RuntimeError as e: logger.warn(str(e)) self.is_complete = False if self.is_complete: self._completion_time = AnalysisDirectory._validator.completion_time(self.path, self.is_complete, self.output_file_dict)
async def get_temperature(self): try: response = await self._bus.req('output.temperature', {'command': 'get'}) if response['status'] != 'ok': logger.warn("Cannot get output temperature: %s", response['message']) return None return response['temperature'] except futures.TimeoutError: logger.warn("Cannot get output temperature: request timeout") return None
def index_rebalance_tick_update(self, tickcount): if isinstance(int(tickcount), (float, int, complex, long)): DatabaseManager.update_index_info_model( indexInfo.Active, indexInfo.TotalBTCVal, indexInfo.TotalUSDVal, indexInfo.TotalRealizedGain, indexInfo.TotalUnrealizedGain, round(float(percentage), 2), indexInfo.OrderTimeout, indexInfo.OrderRetryAmount, int(tickcount)) logger.info("Index rebalance time set to " + str(tickcount) + " minutes.") else: logger.warn("Tick count isn't a number")
def is_r_installed(): installed = True FNULL = open(os.devnull, 'w') try: sub.call(['Rscript'], stdout=FNULL, stderr=sub.STDOUT) except sub.CalledProcessError: installed = False except OSError: installed = False logging.warn( 'R or Rscript not installed. Will not be able to use linear model') return installed
def stress_vmss_instance_cpu(filter: str = None, duration: int = 120, timeout: int = 60, instance_criteria: Iterable[Mapping[str, any]] = None, configuration: Configuration = None, secrets: Secrets = None): logger.warn("Deprecated usage of activity 'stress_vmss_instance_cpu'." " Please use activity 'stress_cpu' in favor since this" " activity will be removed in a future release.") return stress_cpu(filter, duration, timeout, instance_criteria, configuration, secrets)
def node_failure_in_rabbit_cluster(configuration: Configuration = None, secrets: Secrets = None): rmq_client = rmq_client_connect(configuration, secrets) client = ssh.connect(secrets) stdin, stdout, stderr = client.exec_command( 'sudo /usr/sbin/rabbitmqctl stop_app') o, r = stdout.read().decode('utf-8'), stderr.read().decode('utf-8') client.close() logger.info(o) if r != "": logger.warn(r)
def _assign_param_grid(self) -> None: assert self.context.node_index == 0 param_grids = [] for i in range(len(self.context.nodes)): param_grids.append([]) node_assignments = cycle(range(len(self.context.nodes))) Cs = [0.01, 0.1, 1, 10] gammas = [0.001, 0.01, 0.1, 1, 10] for C in Cs: param_grids[next(node_assignments)].append({ 'C': [C], 'kernel': ['linear'] }) if not self._linear_only: for C in Cs: for gamma in gammas: param_grids[next(node_assignments)].append({ 'C': [C], 'gamma': [gamma], 'kernel': ['rbf'] }) self._set_param_grid_inner(param_grids[0]) for i in range(1, len(self.context.nodes)): while True: try: message = Any() message.Pack( SVMTrainerMessage(setParamGrid=SetParamGrid( grid=json.dumps(param_grids[i])))) self.context.nodes[i].internal.MessageInternal( InternalMessage(searchId=self._search_id, trainerIndex=self._trainer_index, message=message)) break except Exception as e: logger.warn( 'Could not set param grid - node {} might still not be up' .format(self.context.nodes[i].url)) logger.exception(e) time.sleep(5) logger.info('Set param grid on node {}'.format( self.context.nodes[i].url)) self._param_grid_event.set()
def accimage_loader(path): import accimage try: start = time.time() img = accimage.Image(path) end = time.time() logger.debug('accimage decode {:.3f} ms'.format(1000 * (end - start))) return img except IOError: logger.warn('accimage failed to decode {}'.format(path)) # Potentially a decoding problem, fall back to PIL.Image return pil_loader(path)
def facebook_postback_handler(event, page=None): """Handle facebook postbacks.""" # For now only deal with GET_STARTED postback if event.postback["payload"] != "GET_STARTED": # pylint: disable=deprecated-method logger.warn(f"Postback {event.postback['payload']} not supported") return user = User.get_or_create_facebook_user(page, event.sender_id) Log.log_message(user, f"POSTBACK: {event.postback['payload']}") logger.info(f"Postback {event.postback['payload']} received " f"from user {user.id}") if not facebook_handle_maintenance(page, event): handle_event(page, user, event)
def should_parse(self, config): if self.force: logger.warn('Force flag detected. Ignoring has and parsing %s' % self.raw_dir) return True if config['func_hash'] != self.get_function_hash(config['func_name']): logger.warn('Function hash in %s is different, should parse' % self.raw_dir) self.set_config(func_name=config['func_name']) return True logger.info('Parser did not change since last run') return False
def load_references(dataset_name: str) -> Optional[References]: """Load a file with references for a standard GEM dataset (attempt download), return None if not present.""" if dataset_name in _DATASET_REFERENCES_URLS: try: dataset_file = ensure_download( 'references', dataset_name + '.json', _DATASET_REFERENCES_URLS[dataset_name]) return References(dataset_file) except Exception as e: logger.warn( f'Could not download references for {dataset_name}: {str(e)}') return None return None
def handle_set(prefix, topic, payload): """ <peripheral type="button" alternative_press=""> <output type="integer" value="1" destination="/set/BAR/markers/capture1"/> </peripheral> <peripheral type="button" alternative_press=""> <output type="integer" value="+=1" destination="/set/BAR/markers/capture1"/> </peripheral> """ clobber = True payload = payload.decode() address = topic.replace(prefix, "") if address.startswith("/"): address = address[1:] logger.info("set {}{}{}".format(prefix, topic, payload)) address, sub_address = address.split("/") value = None symbol = None if "+=" in payload: symbol = "+=" elif "-=" in payload: symbol = "-=" elif "*=" in payload: symbol = "*=" if symbol is None: r.hmset("state:{}".format(address), dict({sub_address: payload})) else: try: try: stored = int(r.hget("state:{}".format(address), sub_address)) except Exception as ex: logger.warn(ex) if clobber: stored = 0 else: return value = int(payload.partition(symbol)[-1]) if symbol == "+=": stored += value elif symbol == "-=": stored -= value elif symbol == "*=": stored *= value #elif symbol == "++": #string concat # stored += value r.hmset("state:{}".format(address), dict({sub_address: stored})) except Exception as ex: logger.warn(ex)
def probe_app_can_connect_and_send_message_to_rabbit( configuration: Configuration = None, secrets: Secrets = None): if not secrets: raise ActivityFailed( "Please set the secrets entry to specify the SSH client settings") # Create ssh client client = ssh.connect(secrets) # Get RabbitMQ connection params from secrets rmq_api_rest_endpoint = os.getenv(secrets.get("rabbitmq_restendpoint")) rmq_username = os.getenv(secrets.get("rabbitmq_username")) rmq_password = os.getenv(secrets.get("rabbitmq_password")) rmq_vhost_url = os.getenv( secrets.get("rabbitmq_host")) + "/api/healthchecks/node" rabbit_creds = rmq_username + ":" + rmq_password wait_time = int(os.getenv('WAIT_TIME')) if wait_time > 0: logger.info("Waiting for " + str(wait_time) + " secs before connecting to rabbit") time.sleep(wait_time) rabbitmq_connect_curl = 'curl -s -u {rabbitmq_creds} {rabbitmq_host}'.format( rabbitmq_creds=rabbit_creds, rabbitmq_host=rmq_vhost_url) stdin, stdout, stderr = client.exec_command(rabbitmq_connect_curl) o, r = stdout.read().decode('utf-8'), stderr.read().decode('utf-8') client.close() logger.info(o) error = False if r != "": logger.warn(r) error = True rmq_client = rmq_client_connect(configuration, secrets) rmq_client.create_vhost("ce_vhost") rmq_client.create_exchange("ce_vhost", "ce_exc", "direct") rmq_client.create_queue("ce_vhost", "ce_que") rmq_client.create_binding("ce_vhost", "ce_exc", "ce_que", "ce.rtkey") if not rmq_client.publish('ce_vhost', 'ce_exc', 'ce.rtkey', 'chaos experiment message'): error = True rmq_client.delete_vhost("ce_vhost") return not error
def __call__(self, item): rv, jpg_arr = cv2.imencode('.jpg', item.array) assert rv, "Fail to re-encode into jpg" file_like = io.BytesIO(jpg_arr.tostring()) try: r = self.request_session.post(self.detect_url, files={'image': file_like}) assert r.ok result = r.json() if result['success']: detections = result count_hit = 0 vis_arr = item.array for box, score, class_name in zip( detections['detection_boxes'], detections['detection_scores'], detections['detection_names']): if score < self.confidence: break for t in self.targets: if t in class_name: # hit count_hit += 1 if s3dexp.config.VISUALIZE_RESULT: h, w = vis_arr.shape[:2] top, left, bottom, right = box # TF return between 0~1 top, bottom = top * h, bottom * h left, right = left * w, right * w top, left, bottom, right = map( int, (top, left, bottom, right)) vis_arr = cv2.rectangle( vis_arr, (left, top), (right, bottom), (0, 255, 0), 3) if count_hit > 0: if s3dexp.config.VISUALIZE_RESULT: vis_path = 'vis-detect-{}-{}.jpg'.format( self.targets[0], os.path.basename(item.src)) logger.warn("Saving visualized detection to {}".format( vis_path)) cv2.imwrite(vis_path, vis_arr) return True except: raise return False
def load_module(self, module_name: str, module_path: str) -> bool: if os.path.exists(module_path): spec = importlib.util.spec_from_file_location("Module", module_path) if spec is None: logger.warn(f"The module {module_name} needs to have a class called Module") return False module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) self.loaded_modules[module_name] = module.Module() return True else: logger.warn(f'Module: {module_name} not found') return False
def run_steady_state_hypothesis(experiment: Experiment, configuration: Configuration, secrets: Secrets, dry: bool = False): """ Run all probes in the hypothesis and fail the experiment as soon as any of the probe fails or is outside the tolerance zone. """ state = { "steady_state_met": None, "probes": [] } hypo = experiment.get("steady-state-hypothesis") if not hypo: logger.info( "No steady state hypothesis defined. That's ok, just exploring.") return logger.info("Steady state hypothesis: {h}".format(h=hypo.get("title"))) probes = hypo.get("probes", []) for activity in probes: run = execute_activity( activity, configuration=configuration, secrets=secrets, dry=dry) state["probes"].append(run) if run["status"] == "failed": run["tolerance_met"] = False state["steady_state_met"] = False logger.warn("Probe terminated unexpectedly, " "so its tolerance could not be validated") return state run["tolerance_met"] = True if dry: # do not check for tolerance when dry mode is on continue tolerance = activity.get("tolerance") logger.debug("allowed tolerance is {t}".format(t=str(tolerance))) if not within_tolerance(tolerance, run["output"]): run["tolerance_met"] = False state["steady_state_met"] = False return state state["steady_state_met"] = True logger.info("Steady state hypothesis is met!") return state
def disk_read(base_dir, disk, ext='jpg', sort_inode=False, store_result=True): logger.warn("Make sure you cleaned the OS page buffer!") base_dir = os.path.realpath(base_dir) paths = list(recursive_glob(base_dir, '*.{}'.format(ext))) if sort_inode: paths = sorted(paths, key=lambda p: os.stat(p).st_ino) logger.info("Sort by inode num.") else: # deterministic pseudo-random random.seed(42) random.shuffle(paths) results = [] for p in paths: tic = time.time() fd = os.open(p, os.O_RDONLY) size = os.path.getsize(p) buf = os.read(fd, size) os.close(fd) elapsed = time.time() - tic logger.debug("{}: {} bytes {} ms".format(p, len(buf), elapsed * 1000)) vals_dict = {'size': size} if sort_inode: vals_dict['seq_read_ms'] = elapsed * 1000 else: vals_dict['rand_read_ms'] = elapsed * 1000 results.append({ 'keys_dict': { 'path': p, 'disk': disk }, 'vals_dict': vals_dict }) if store_result: logger.info("Going to write {} results to DB".format(len(results))) sess = dbutils.get_session() for r in results: dbutils.insert_or_update_one(sess, models.DiskReadProfile, keys_dict=r['keys_dict'], vals_dict=r['vals_dict']) sess.commit() sess.close()
def fix_missed(blizzard_characters: List): db = blizzard_db() q = DatabaseUtils.core_query(db.query(CharacterModel)).all() for character in q: found = False for b_ch in blizzard_characters: if b_ch['character']['name'] == character.name: found = True if not found: logger.warn(character.name + " escape guild") db.query(CharacterModel).filter( CharacterModel.wow_id == character.wow_id).update( {'state': 0}) db.commit()