def work (unit): data = [] if isinstance(unit, Post): if not unit.image: logger.warning('unit %s is not an image.', unit) return filename = get_filename ( directory, unit, keep_names ) if not os.path.exists(filename): logger.info('downloading %s', unit.image) image_data = unit.image.download(bypass_cache=True) return filename, image_data logger.debug('%s already downloaded', filename) return logger.info('working %r', unit) for e in unit.process(): value = work(e) if not value is None: data.append(value) #pool.push(work, e) return data
def parse_subreddit(subreddit, timeframe): """ Parses top 1,000 posts from subreddit within time frame. """ total_post_count = 0 current_post_index = 0 while True: query_text = '/r/%s/top?t=%s' % (subreddit, timeframe) if total_post_count == 0: logger.info('Loading first page of %s' % query_text) posts = reddit.get(query_text) elif reddit.has_next(): logger.info('[+] Loading next page of %s' % query_text) posts = reddit.get_next() else: # No more pages to load break if posts is None or not posts: logger.warning('No posts found') return total_post_count += len(posts) for post in posts: current_post_index += 1 logger.info( '[%3d/%3d] Scraping http://redd.it/%s %s' % (current_post_index, total_post_count, post.id, post.url[:50])) parse_post(post)
def lcdTimeout(): logger.warning('Connection Timeout') lcd.backlight(lcdColors[4]) lcd.clear() lcd.setCursor(0, 0) lcd.message('No connection\nCheck back soon') sleep(timeoutInterval)
def lcdTimeout(): logger.warning('Connection Timeout') lcd.backlight(lcdColors[4]) lcd.clear() lcd.setCursor(0,0) lcd.message('No connection\nCheck back soon') sleep(timeoutInterval)
def run(cansocket): logger.warning("Inducing JSM error:") # send in less than 1ms theses frames to induce # JSM error for _ in range(5): can2RNET.cansend(cansocket, FRAME_JSM_INDUCE_ERROR) # now let's take over by sending our own # joystick frame @100Hz mintime = .01 nexttime = time() + mintime while True: # get new XY joystick increment joystick_x, joystick_y = get_new_joystick_position() # building joy frame joyframe = createJoyFrame(joystick_x, joystick_y) # sending frame can2RNET.cansend(cansocket, joyframe) # .. at 100 Hz .. nexttime += mintime t = time() if t < nexttime: sleep(nexttime - t) else: nexttime += mintime
def validate_and_prepare_args(args): # check that only one of skip_tools_file and required_tools_file has been provided if args.skip_tools_file is not None and args.required_tools_file is not None: raise ApplicationException( "You have provided both a file with tools to ignore and a file with required tools.\n" "Only one of -s/--skip-tools, -r/--required-tools can be provided." ) # flatten macros_files to make sure that we have a list containing file names and not a list of lists utils.flatten_list_of_lists(args, "macros_files") # check that the arguments point to a valid, existing path input_variables_to_check = [ "skip_tools_file", "required_tools_file", "macros_files", "formats_file" ] for variable_name in input_variables_to_check: utils.validate_argument_is_valid_path(args, variable_name) # check that the provided output files, if provided, contain a valid file path (i.e., not a folder) output_variables_to_check = [ "data_types_destination", "tool_conf_destination" ] for variable_name in output_variables_to_check: file_name = getattr(args, variable_name) if file_name is not None and os.path.isdir(file_name): raise ApplicationException( "The provided output file name (%s) points to a directory." % file_name) if not args.macros_files: # list is empty, provide the default value logger.warning("Using default macros from galaxy/macros.xml", 0) args.macros_files = ["galaxy/macros.xml"]
def downloads(): # Get content of downloads directory dl_dir = "static/downloads/" dir_content = os.listdir(dl_dir) # Make paths relative to working directory # Only allow csv files files = [ (name, os.path.join(dl_dir, name)) for name in dir_content if name.find(".csv") != -1 ] # Stat files # Remove any dirs placed accidentally files = [ (f, full, os.stat(full)) for f, full in files if os.path.isfile(full) ] if len(files) == 0: logger.warning("No export file to display in /dl") return render_template("downloads.html", export_file_stats=files)
def update_file(self, db: Session, response: Response, *, id: int, obj_in: schema_file.FileUpdateIn): """ 通过 文件id 更新文件信息 :param id: 文件id :param title: 标题/名字 :param cover 文件封面 :param description: 描述 :return: 文件信息, 提示信息 """ db_file = crud_file.get_file_by_id(db, id=id) if not db_file: message = "文件不存在" response.status_code = status.HTTP_404_NOT_FOUND logger.warning(message) return None, message db_file_obj = crud_file.update_file_by_id(db, id=id, obj_in=obj_in) if not db_file_obj: message = "文件信息更新失败" response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR logger.error(message) else: message = "文件信息更新成功" response.status_code = status.HTTP_200_OK return db_file_obj, message
def post(self): username = self.get_argument('username', '') password = self.get_argument('password', '') res = { 'code': 0, } msg = make_check(username, password) if msg: res['code'] = 1 res['msg'] = msg return self.finish(res) sql = 'select password from users where username = "******"' % (username) data = db.get_one(sql) if data and encryption(password) == data.get('password'): logger.info('[SUCCESS] %s 登录成功' % username) res['msg'] = '登录成功' else: logger.warning('[ERROR] 账号或密码错误') res['code'] = 1 res['msg'] = '账号或密码错误' return self.finish(res)
def resolve_param_mapping(param, ctd_model, fix_underscore=False): # go through all mappings and find if the given param appears as a reference name in a mapping element param_mapping = None ctd_model_cli = [] if hasattr(ctd_model, "cli"): ctd_model_cli = ctd_model.cli for cli_element in ctd_model_cli: for mapping_element in cli_element.mappings: if mapping_element.reference_name == param.name: if param_mapping is not None: logger.warning( "The parameter %s has more than one mapping in the <cli> section. " "The first found mapping, %s, will be used." % (param.name, param_mapping), 1) else: param_mapping = cli_element.option_identifier if param_mapping is not None: ret = param_mapping else: ret = param.name if fix_underscore and ret.startswith("_"): return ret[1:] else: return ret
def validate_and_prepare_args(args): # check that only one of skip_tools_file and required_tools_file has been provided if args.skip_tools_file is not None and args.required_tools_file is not None: raise ApplicationException( "You have provided both a file with tools to ignore and a file with required tools.\n" "Only one of -s/--skip-tools, -r/--required-tools can be provided.") # flatten macros_files to make sure that we have a list containing file names and not a list of lists utils.flatten_list_of_lists(args, "macros_files") # check that the arguments point to a valid, existing path input_variables_to_check = ["skip_tools_file", "required_tools_file", "macros_files", "formats_file"] for variable_name in input_variables_to_check: utils.validate_argument_is_valid_path(args, variable_name) # check that the provided output files, if provided, contain a valid file path (i.e., not a folder) output_variables_to_check = ["data_types_destination", "tool_conf_destination"] for variable_name in output_variables_to_check: file_name = getattr(args, variable_name) if file_name is not None and os.path.isdir(file_name): raise ApplicationException("The provided output file name (%s) points to a directory." % file_name) if not args.macros_files: # list is empty, provide the default value logger.warning("Using default macros from galaxy/macros.xml", 0) args.macros_files = ["galaxy/macros.xml"]
def parse_file_formats(formats_file): supported_formats = {} if formats_file is not None: line_number = 0 with open(formats_file) as f: for line in f: line_number += 1 if line is None or not line.strip() or line.strip().startswith("#"): # ignore (it'd be weird to have something like: # if line is not None and not (not line.strip()) ... pass else: # not an empty line, no comment # strip the line and split by whitespace parsed_formats = line.strip().split() # valid lines contain either one or four columns if not (len(parsed_formats) == 1 or len(parsed_formats) == 3 or len(parsed_formats) == 4): logger.warning( "Invalid line at line number %d of the given formats file. Line will be ignored:\n%s" % (line_number, line), 0) # ignore the line continue elif len(parsed_formats) == 1: supported_formats[parsed_formats[0]] = DataType(parsed_formats[0], parsed_formats[0]) else: mimetype = None # check if mimetype was provided if len(parsed_formats) == 4: mimetype = parsed_formats[3] supported_formats[parsed_formats[0]] = DataType(parsed_formats[0], parsed_formats[1], parsed_formats[2], mimetype) return supported_formats
def reset_password(params): user_name = params.get("user_name", None) password = params.get("password", None) new_password = params.get("new_password", None) logger.info("reset password, user[%s], passwd[%s], new_passwd[%s]", user_name, password, new_password) if user_name and password and new_password: if password == new_password: logger.warning("The same password, skip") sys.exit(47) ldap_client = None try: ldap_client = new_ldap_client() if not ldap_client.user_exist(user_name): sys.exit(44) ret = ldap_client.reset_password(user_name, password, new_password) if ret == 1: sys.exit(41) logger.info("Reset password, done.") except Exception: logger.error("Failed reset password of user[%s]: [%s]", user_name, traceback.format_exc()) sys.exit(1) finally: if ldap_client: ldap_client.close() else: logger.error("Reset password, lack param.") sys.exit(40)
def patch_task(self, payload): success, status_code, response = self.query_api( "PATCH", f"/tasks/{self.task_id}", payload=payload) if not success or status_code != requests.codes.NO_CONTENT: logger.warning( f"couldn't patch task status={payload['event']} HTTP {status_code}: {response}" )
def parse_macros_files(macros_file_names): macros_to_expand = list() for macros_file_name in macros_file_names: try: macros_file = open(macros_file_name) logger.info("Loading macros from %s" % macros_file_name, 0) root = parse(macros_file).getroot() for xml_element in root.findall("xml"): name = xml_element.attrib["name"] if name in macros_to_expand: logger.warning( "Macro %s has already been found. Duplicate found in file %s." % (name, macros_file_name), 0) else: logger.info("Macro %s found" % name, 1) macros_to_expand.append(name) except ParseError, e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) except IOError, e: raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e))
def parse_log( in_dir: str, out_dir: str, measure_type: common.MeasureType) -> Tuple[pd.DataFrame, pd.DataFrame]: df_runs = None df_stats = None dfs = __parse_log(in_dir, measure_type) if dfs is not None: df_runs, df_stats = dfs else: logger.warning("No logging data") if df_runs is None: df_runs = pd.DataFrame(columns=['name'], index=pd.TimedeltaIndex([], name='time')) if df_stats is None: df_stats = pd.DataFrame(columns=['cpu_load', 'ram_usage'], index=pd.TimedeltaIndex([], name='time')) logger.debug("Fixing log data types") df_runs = fix_dtypes(df_runs) df_stats = fix_dtypes(df_stats) logger.info("Saving ping data") df_runs.to_pickle(os.path.join(out_dir, 'runs.pkl')) df_stats.to_pickle(os.path.join(out_dir, 'stats.pkl')) with open(os.path.join(out_dir, 'runs.csv'), 'w+') as out_file: df_runs.to_csv(out_file) with open(os.path.join(out_dir, 'stats.csv'), 'w+') as out_file: df_stats.to_csv(out_file) return df_runs, df_stats
def __parse_slice(parse_func: Callable[..., pd.DataFrame], in_dir: str, scenarios: List[Tuple[str, Dict]], df_cols: List[str], protocol: str, entity: str) -> pd.DataFrame: """ Parse a slice of the protocol entity results using the given function. :param parse_func: The function to parse a single scenario. :param in_dir: The directory containing the measurement results. :param scenarios: The scenarios to parse within the in_dir. :param df_cols: The column names for columns in the resulting dataframe. :param protocol: The name of the protocol that is being parsed. :param entity: Then name of the entity that is being parsed. :return: A dataframe containing the combined results of the specified scenarios. """ df_slice = pd.DataFrame(columns=df_cols) for folder, config in scenarios: for pep in (False, True): df = parse_func(in_dir, folder, pep=pep) if df is not None: df_slice = extend_df(df_slice, df, protocol=protocol, pep=pep, **config) else: logger.warning("No data %s%s %s data in %s", protocol, " (pep)" if pep else "", entity, folder) return df_slice
def __parse_log( in_dir: str, measure_type: common.MeasureType ) -> Optional[Tuple[pd.DataFrame, pd.DataFrame]]: logger.info("Parsing log file") path = None if measure_type == common.MeasureType.OPENSAND: path = os.path.join(in_dir, "opensand.log") elif measure_type == common.MeasureType.NETEM: path = os.path.join(in_dir, "measure.log") if not os.path.isfile(path): logger.warning("No log file found") return None runs_data = [] stats_data = [] start_time = None with open(path) as file: for line in file: if start_time is None: start_time = datetime.strptime( ' '.join(line.split(' ', 2)[:2]), "%Y-%m-%d %H:%M:%S%z") match = re.match(r"^([0-9-+ :]+) \[INFO]: (.* run \d+/\d+)$", line) if match: runs_data.append({ 'time': datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S%z") - start_time, 'name': match.group(2), }) else: match = re.search( r"^([0-9-+ :]+) \[STAT]: CPU load \(1m avg\): (\d+(?:\.\d+)?), RAM usage: (\d+)MB$", line) if match: stats_data.append({ 'time': datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S%z") - start_time, 'cpu_load': match.group(2), 'ram_usage': match.group(3), }) runs_df = None if len(runs_data) > 0: runs_df = pd.DataFrame(runs_data) runs_df.set_index('time', inplace=True) stats_df = None if len(stats_data) > 0: stats_df = pd.DataFrame(stats_data) stats_df.set_index('time', inplace=True) return runs_df, stats_df
def __parse_quic_server_from_scenario(in_dir: str, scenario_name: str, pep: bool = False) -> pd.DataFrame: """ Parse the quic server results in the given scenario. :param in_dir: The directory containing all measurement results :param scenario_name: The name of the scenario to parse :param pep: Whether to parse QUIC or QUIC (PEP) files :return: A dataframe containing the parsed results of the specified scenario. """ logger.debug("Parsing quic%s server files in %s", " (pep)" if pep else "", scenario_name) df = pd.DataFrame( columns=['run', 'second', 'cwnd', 'packets_sent', 'packets_lost']) for file_name in os.listdir(os.path.join(in_dir, scenario_name)): path = os.path.join(in_dir, scenario_name, file_name) if not os.path.isfile(path): continue match = re.search( r"^quic%s_(\d+)_server\.txt$" % ("_pep" if pep else "", ), file_name) if not match: continue logger.debug("%s: Parsing '%s'", scenario_name, file_name) run = int(match.group(1)) with open(path) as file: for line in file: line_match = re.search( r"^connection \d+ second (\d+(?:\.\d+)?):.*send window: (\d+).*packets sent: (\d+).*packets lost: (\d+)$", line.strip()) if not line_match: continue df = df.append( { 'run': run, 'second': float(line_match.group(1)), 'cwnd': int(line_match.group(2)), 'packets_sent': int(line_match.group(3)), 'packets_lost': int(line_match.group(4)) }, ignore_index=True) with_na = len(df.index) df.dropna(subset=['cwnd', 'packets_sent', 'packets_lost'], inplace=True) without_na = len(df.index) if with_na != without_na: logger.warning("%s: Dropped %d lines with NaN values", scenario_name, with_na - without_na) if df.empty: logger.warning("%s: No quic%s server data found", scenario_name, " (pep)" if pep else "") return df
async def async_scrape(url): try: return await scrape(url) except ScraperError: logger.warning("Unhandled ScraperError for %s" % url) return None except InvalidUrl: logger.warning("InvalidUrl for %s" % url) return None
def cleanup_workdir(self): logger.info(f"Removing task workdir {self.workdir}") zim_files = [(f.name, format_size(f.stat().st_size)) for f in self.task_workdir.glob("*.zim")] if zim_files: logger.warning(f"ZIM files exists. removing anyway: {zim_files}") try: shutil.rmtree(self.task_workdir) except Exception as exc: logger.error(f"Failed to remove workdir: {exc}")
async def get_channel_info(self, channel): async with self._client as client: try: data = await client( functions.channels.GetFullChannelRequest(channel=channel)) except ValueError as e: logger.warning(str(e)) return None return json.loads(data.to_json())
def stop(self, timeout=5): """ stopping everything before exit (on term or end of task) """ logger.info("Stopping all containers and actions") self.should_stop = True for step in ("dnscache", "scraper", "uploader"): try: getattr(self, f"stop_{step}")(timeout) except Exception as exc: logger.warning(f"Failed to stop {step}: {exc}") logger.exception(exc)
def fetch_and_save_nightlies(start_date=date.today(), total_days=-14): for date_str, nightly_id in iterate_nightlies(start_date, total_days): url = pipelines_api + str(nightly_id) + "/jobs" logger.debug("Fetching URL: " + url) r = requests.get(url, headers={"PRIVATE-TOKEN": token}) if r.status_code == 404: logger.error("Error fetching %s. Skipping" % url) continue j = r.json() logger.debug("Got JSON: " + str(j)) for project in MENDER_QA_TEST_SUITES: test_job = [jj["id"] for jj in j if jj["name"] == project["job"]] if len(test_job) == 0: # RPi is not build by default if project["job"] == "test_accep_raspberrypi3": logger_func = logger.warning else: logger_func = logger.error logger_func("Cannot find %s in job list" % project["job"]) continue test_job = test_job[0] logger.info("Fetching XML results for %s" % project["job"]) url = artifacts_api_fmt.format( job_id=test_job, artifact_filename=project["results_file"] + ".xml") logger.debug("Fetching URL: " + url) r = requests.get(url, headers={"PRIVATE-TOKEN": token}) if r.status_code == 404: # BBB and RPi are not tested if project["job"] in [ "test_accep_beagleboneblack", "test_accep_raspberrypi3", ]: logger_func = logger.warning else: logger_func = logger.error logger_func("Cannot get results file %s from %s " % (project["results_file"], project["job"])) continue filename = os.path.join( TEST_RESULTS_DIR, str(project["id"]) + "-" + project["results_file"] + "@" + date_str + ".xml") if not os.path.exists(filename): logger.info("Saving report in " + filename) with open(filename, "wb") as fd: fd.write(r.content) else: logger.warning("Report " + os.path.basename(filename) + " already exists, skipping")
def get_task(self): logger.info(f"Fetching task details for {self.task_id}") success, status_code, response = self.query_api("GET", f"/tasks/{self.task_id}") if success and status_code == requests.codes.OK: self.task = response return if status_code == requests.codes.NOT_FOUND: logger.warning(f"task {self.task_id} doesn't exist") else: logger.warning(f"couldn't retrieve task detail for {self.task_id}")
def make_check(username, password): try: login_check(username, msg='用户名') login_check(password, msg='密码') except FiledsError as e: logger.warning('[ERROR] %s' % e) return str(e) except Exception as e: logger.error('[ERROR] %s' % str(e)) raise return None
def _extract_param_cli_name(param, ctd_model, fix_underscore=False): # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy) if type(param.parent) == ParameterGroup: if hasattr(ctd_model, "cli") and ctd_model.cli: logger.warning( "Using nested parameter sections (NODE elements) is not compatible with <cli>", 1) return ":".join(extract_param_path( param, fix_underscore)[:-1]) + ":" + resolve_param_mapping( param, ctd_model, fix_underscore) else: return resolve_param_mapping(param, ctd_model, fix_underscore)
def create_output_node(parent, param, model, supported_file_formats): data_node = add_child_node(parent, "data") data_node.attrib["name"] = get_galaxy_parameter_name(param) if data_node.attrib["name"].startswith('param_out_'): data_node.attrib[ "label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib[ "name"][10:] data_format = "data" if param.restrictions is not None: if type(param.restrictions) is _FileFormat: # set the first data output node to the first file format # check if there are formats that have not been registered yet... output = list() for format_name in param.restrictions.formats: if not format_name in supported_file_formats.keys(): output.append(str(format_name)) # warn only if there's about to complain if output: logger.warning( "Parameter " + param.name + " has the following unsupported format(s):" + ','.join(output), 1) data_format = ','.join(output) formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) try: data_format = formats.pop() except KeyError: # there is not much we can do, other than catching the exception pass # if there are more than one output file formats try to take the format from the input parameter if formats: corresponding_input = get_input_with_same_restrictions( param, model, supported_file_formats) if corresponding_input is not None: data_format = "input" data_node.attrib[ "metadata_source"] = get_galaxy_parameter_name( corresponding_input) else: raise InvalidModelException( "Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % { "type": type(param.restrictions), "name": param.name }) data_node.attrib["format"] = data_format return data_node
def parse_ping( in_dir: str, out_dir: str, scenarios: Dict[str, Dict], config_cols: List[str], multi_process: bool = False) -> Tuple[pd.DataFrame, pd.DataFrame]: """ Parse all ping results. :param in_dir: The directory containing the measurement results. :param out_dir: The directory to save the parsed results to. :param scenarios: The scenarios to parse within the in_dir. :param config_cols: The column names for columns taken from the scenario configuration. :param multi_process: Whether to allow multiprocessing. :return: Two dataframes containing the combined results from all scenarios, one with the raw data and one with the summary data. """ logger.info("Parsing ping results") df_ping_raw = pd.DataFrame(columns=[*config_cols, 'seq', 'ttl', 'rtt']) df_ping_summary = pd.DataFrame(columns=[ *config_cols, 'packets_sent', 'packets_received', 'rtt_min', 'rtt_avg', 'rtt_max', 'rtt_mdev' ]) for folder, config in scenarios.items(): dfs = __parse_ping_from_scenario(in_dir, folder) if dfs is not None: df_ping_raw = extend_df(df_ping_raw, dfs[0], protocol='icmp', pep=False, **config) df_ping_summary = extend_df(df_ping_summary, dfs[1], protocol='icmp', pep=False, **config) else: logger.warning("No data ping data in %s", folder) logger.debug("Fixing ping data types") df_ping_raw = fix_dtypes(df_ping_raw) df_ping_summary = fix_dtypes(df_ping_summary) logger.info("Saving ping data") df_ping_raw.to_pickle(os.path.join(out_dir, 'ping_raw.pkl')) df_ping_summary.to_pickle(os.path.join(out_dir, 'ping_summary.pkl')) with open(os.path.join(out_dir, 'ping_raw.csv'), 'w+') as out_file: df_ping_raw.to_csv(out_file) with open(os.path.join(out_dir, 'ping_summary.csv'), 'w+') as out_file: df_ping_summary.to_csv(out_file) return df_ping_raw, df_ping_summary
def get_max_or_break_tie(transition_probs: Dict[int, float]) -> int: """ Get the key with max value. In case of a tie breaks randomly """ probs = list(transition_probs.values()) # solve probability ties max_p = np.max(probs) tie_states = [s for s, p in transition_probs.items() if p == max_p] # select accordingly if (len(tie_states) > 1): # select randomly between the ties logger.warning("Selectin state randomly") return np.random.choice(tie_states) else: return tie_states[0]
def error_connection(self): """ Display timeout message and sleep """ logger.warning('Connection Timeout') self.win.fill(self.c.WHITE) self.win.blit(FONT32.render("Could not fetch", 1, self.c.BLACK), (25,70)) self.win.blit(FONT32.render("data from source", 1, self.c.BLACK), (25,120)) self.buttons = [ IconButton((40, 213), 24, self.draw_main, SpChar.CANCEL, 48, 'WHITE', 'GRAY'), IconButton((100, 213), 24, self.refresh_data, SpChar.RELOAD, 48, 'WHITE', 'GRAY') ] self.reset_update_time(cfg.timeout_interval) self.on_main = True
def post(self): title = self.get_argument('title', '') class_id = self.get_argument('class_id', '') photo = self.get_argument('photo', '') summary = self.get_argument('summary', '') res = { 'code': 0 } try: title_check = FiledsCheck(title, msg='文章标题', max_length=30) title_check.check_null() title_check.check_length() class_id_check = FiledsCheck(class_id, msg='所属分类') class_id_check.check_null() except FiledsError as msg: res['code'] = 1 res['msg'] = str(msg) logger.warning('[ERROR] %s' % str(msg)) return self.finish(res) data = { 'title': title, 'class_id': class_id, 'photo': photo, 'summary': summary, 'author': 'admin', 'create_time': datetime.datetime.now() } try: sql = 'insert into article (title, class_id, photo, summary, author, create_time) ' \ 'values ("{title}", {class_id}, "{photo}", "{summary}", "{author}", "{create_time}")' count = db.insert(sql.format(**data)) if count: logger.info('[SUCCESS] %s 添加成功' % title) res['msg'] = '添加成功!' except MysqlError as e: logger.error('[ERROR] %s 添加失败' % title) res['code'] = 1 res['msg'] = '添加失败,请重新添加!' print(e) except Exception as e: logger.error('[ERROR] %s 添加失败' % title) res['code'] = 1 res['msg'] = '添加失败,请重新添加!' print(e) return self.finish(res)
def upload_translations(crowdin_api, source_dir): locales = os.listdir(source_dir) ensure_locales(crowdin_api, locales) for locale in locales: if locale == "en_US": continue path = os.path.join(source_dir, locale, "messages.json") if not os.path.exists(path): logger.warning("Skipping file that doesn't exist: %s" % path) continue logger.info("Uploading %s" % path) with open(path, "rb") as f: crowdin_api.request( "POST", "upload-translation", files=[("messages.json", f.read(), "application/json")], data={"language": crowdinLocale(locale)} )
def _get_playlist(self, params={}): params_tmp = self.douban_fm_default_params params_tmp.update(params) params_data = urllib.urlencode(params_tmp) url = '?'.join(('http://%s%s' % (self.douban_fm_host, self.douban_fm_playlist_path), params_data)) l.debug('load playlist: %s' % url) res = self.http_session.get(url) if 'start="deleted"' in (res.headers.get('set-cookie') or ''): self.http_session.cookies.pop('start') cookies_text = '; '.join(['='.join(kv) for kv in self.http_session.cookies.items()]) with open(self.COOKIE_PATH,'w') as fh: fh.write(cookies_text) res_json = json.loads(res.text) if int(res_json.get('r')) == 0: return res_json['song'] elif int(res_json.get('r')) == 1: l.warning('cannot parse response json:\n {err}'.format(**res_json)) return []
def create_output_node(parent, param, model, supported_file_formats): data_node = add_child_node(parent, "data") data_node.attrib["name"] = get_galaxy_parameter_name(param) data_format = "data" if param.restrictions is not None: if type(param.restrictions) is _FileFormat: # set the first data output node to the first file format # check if there are formats that have not been registered yet... output = list() for format_name in param.restrictions.formats: if not format_name in supported_file_formats.keys(): output.append(str(format_name)) # warn only if there's about to complain if output: logger.warning("Parameter " + param.name + " has the following unsupported format(s):" + ','.join(output), 1) data_format = ','.join(output) formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) try: data_format = formats.pop() except KeyError: # there is not much we can do, other than catching the exception pass # if there are more than one output file formats try to take the format from the input parameter if formats: corresponding_input = get_input_with_same_restrictions(param, model, supported_file_formats) if corresponding_input is not None: data_format = "input" data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) data_node.attrib["format"] = data_format # TODO: find a smarter label ? return data_node
def parse_macros_files(macros_file_names): macros_to_expand = set() for macros_file_name in macros_file_names: try: macros_file = open(macros_file_name) logger.info("Loading macros from %s" % macros_file_name, 0) root = parse(macros_file).getroot() for xml_element in root.findall("xml"): name = xml_element.attrib["name"] if name in macros_to_expand: logger.warning("Macro %s has already been found. Duplicate found in file %s." % (name, macros_file_name), 0) else: logger.info("Macro %s found" % name, 1) macros_to_expand.add(name) except ParseError, e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) except IOError, e: raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e))
def init(self): self.get_source( 'releases', 'flatfile', { 'location': self.config['url'], 'parser': 'json', }, ) self.message_defaults.update({ 'kind': 'Kernel', 'name': self.config['moniker'], 'weight': 1, 'location': 'http://www.kernel.org/', }) if self.config['check_current']: if platform.system() != 'Linux': logger.warning('This is not a Linux system; disabling kernel comparison') self.config['check_current'] = False else: self.current = platform.release().split('_', 1)[0] if self.current.count('.') and 'rc' not in self.current: self.current = self.current + '.0'
def prepareMonitor(self): logger.info('Prepare Monitor with parameters:\nname:\t%s\ntag:\t%s\ntype:\t%s\nduration:\t%s min\nmonitorID:\t%s' , constants.MONITOR_NAME, constants.MONITOR_TAG, constants.MONITOR_TYPE, constants.DURATION, self.__monitorId) if self.__monitorId > 0: logger.info('Check correctness of monitor ID (%s)', self.__monitorId) try: info = self.__api.requestMonitorInfo(self.__monitorId) except Exception as e: logger.warning(e) info = None logger.info('Monitor info: %s', str(info)) if info == None or not isinstance(info, dict) or not (info['name'] == constants.MONITOR_NAME and info['tag'] == constants.MONITOR_TAG): logger.warning('Incorrect monitor ID') self.__monitorId = 0 if self.__monitorId <= 0: logger.info('Trying to get monitor info by name...') try: info = self.__api.requestMonitors(monitorName=constants.MONITOR_NAME, monitorTag=constants.MONITOR_TAG, monitorType=constants.MONITOR_TYPE) except Exception as e: logger.warning(e) info = None logger.info('monitorInfo: %s', str(info)) if info and len(info) > 0 and isinstance(info, list) and isinstance(info[0], dict) and info[0]['id'] > 0: self.__monitorId = info[0]['id'] else: logger.info('Monitor is not exist. Create a new one.') try: info = self.__api.addMonitor(monitorName=constants.MONITOR_NAME, monitorTag=constants.MONITOR_TAG, monitorType=constants.MONITOR_TYPE, isMultiValue = constants.MULTIVALUE, resultParams = constants.RESULT_PARAMS, additionalResultParams = constants.ADDITIONAL_PARAMS) except Exception as e: logger.warning(e) info = None logger.info('addMonitor: %s',str(info)) if not info : return False self.__monitorId = info['data'] logger.info('monitor ID: %s', self.__monitorId) if info and self.__monitorId > 0 : self.replaceInFile('./monitor_constants.py', 'MONITOR_ID=\d{1,}', 'MONITOR_ID='+str(self.__monitorId)) return (self.__monitorId > 0)
# public address address = args['address'][0] logger.debug('ADDRESS {}'.format(address)) # private key privkey = args['privkey'][0] # block explorer qr_link = None if explorer is not None: qr_link = explorer.format(address) logger.debug('EXPLORER {}'.format(qr_link)) else: logger.warning('no block explorer found for currency {}'.format(symbol)) qr_link = address # validate icon file file_icon = None if args['icon'] is not None: (file_icon,) = args['icon'] else: file_icon = './images/icon-{}.png'.format(symbol.lower()) if (os.path.isfile(file_icon)): logger.debug('ICON FILE {}'.format(file_icon)) else: logger.critical('icon file does not exist: {}'.format(file_icon)) exit(1)
def create_param_attribute_list(param_node, param, supported_file_formats): param_node.attrib["name"] = get_galaxy_parameter_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" % {"type": param.type, "name": param.name}) if param.is_list: param_type = "text" if is_selection_parameter(param): param_type = "select" if len(param.restrictions.choices) < 5: param_node.attrib["display"] = "radio" if is_boolean_parameter(param): param_type = "boolean" if param.type is _InFile: # assume it's just text unless restrictions are provided param_format = "txt" if param.restrictions is not None: # join all formats of the file, take mapping from supported_file if available for an entry if type(param.restrictions) is _FileFormat: param_format = ",".join([get_supported_file_type(i, supported_file_formats) if get_supported_file_type(i, supported_file_formats) else i for i in param.restrictions.formats]) else: raise InvalidModelException("Expected 'file type' restrictions for input file [%(name)s], " "but instead got [%(type)s]" % {"name": param.name, "type": type(param.restrictions)}) param_node.attrib["type"] = "data" param_node.attrib["format"] = param_format # in the case of multiple input set multiple flag if param.is_list: param_node.attrib["multiple"] = "true" else: param_node.attrib["type"] = param_type # check for parameters with restricted values (which will correspond to a "select" in galaxy) if param.restrictions is not None: # it could be either _Choices or _NumericRange, with special case for boolean types if param_type == "boolean": create_boolean_parameter(param_node, param) elif type(param.restrictions) is _Choices: # create as many <option> elements as restriction values for choice in param.restrictions.choices: option_node = add_child_node(param_node, "option", OrderedDict([("value", str(choice))])) option_node.text = str(choice) # preselect the default value if param.default == choice: option_node.attrib["selected"] = "true" elif type(param.restrictions) is _NumericRange: if param.type is not int and param.type is not float: raise InvalidModelException("Expected either 'int' or 'float' in the numeric range restriction for " "parameter [%(name)s], but instead got [%(type)s]" % {"name": param.name, "type": type(param.restrictions)}) # extract the min and max values and add them as attributes # validate the provided min and max values if param.restrictions.n_min is not None: param_node.attrib["min"] = str(param.restrictions.n_min) if param.restrictions.n_max is not None: param_node.attrib["max"] = str(param.restrictions.n_max) elif type(param.restrictions) is _FileFormat: param_node.attrib["format"] = ','.join([get_supported_file_type(i, supported_file_formats) if get_supported_file_type(i, supported_file_formats) else i for i in param.restrictions.formats]) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] for parameter [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) if param_type == "select" and param.default in param.restrictions.choices: param_node.attrib["optional"] = "False" else: param_node.attrib["optional"] = str(not param.required) if param_type == "text": # add size attribute... this is the length of a textbox field in Galaxy (it could also be 15x2, for instance) param_node.attrib["size"] = "30" # add sanitizer nodes, this is needed for special character like "[" # which are used for example by FeatureFinderMultiplex sanitizer_node = SubElement(param_node, "sanitizer") valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) add_child_node(valid_node, "remove", OrderedDict([("value", '\'')])) add_child_node(valid_node, "remove", OrderedDict([("value", '"')])) # check for default value if param.default is not None and param.default is not _Null: if type(param.default) is list: # we ASSUME that a list of parameters looks like: # $ tool -ignore He Ar Xe # meaning, that, for example, Helium, Argon and Xenon will be ignored param_node.attrib["value"] = ' '.join(map(str, param.default)) elif param_type != "boolean": param_node.attrib["value"] = str(param.default) else: # simple boolean with a default if param.default is True: param_node.attrib["checked"] = "true" else: if param.type is int or param.type is float: # galaxy requires "value" to be included for int/float # since no default was included, we need to figure out one in a clever way... but let the user know # that we are "thinking" for him/her logger.warning("Generating default value for parameter [%s]. " "Galaxy requires the attribute 'value' to be set for integer/floats. " "Edit the CTD file and provide a suitable default value." % param.name, 1) # check if there's a min/max and try to use them default_value = None if param.restrictions is not None: if type(param.restrictions) is _NumericRange: default_value = param.restrictions.n_min if default_value is None: default_value = param.restrictions.n_max if default_value is None: # no min/max provided... just use 0 and see what happens default_value = 0 else: # should never be here, since we have validated this anyway... # this code is here just for documentation purposes # however, better safe than sorry! # (it could be that the code changes and then we have an ugly scenario) raise InvalidModelException("Expected either a numeric range for parameter [%(name)s], " "but instead got [%(type)s]" % {"name": param.name, "type": type(param.restrictions)}) else: # no restrictions and no default value provided... # make up something default_value = 0 param_node.attrib["value"] = str(default_value) label = "%s parameter" % param.name help_text = "" if param.description is not None: label, help_text = generate_label_and_help(param.description) param_node.attrib["label"] = label param_node.attrib["help"] = "(-%s)" % param.name + " " + help_text