def run(self): while True: # Check for dhcping. output = subprocess.check_output("which dhcping", shell = True) if not output: log.fatal("dhcping not found.") try: output = subprocess.check_output("dhcping %s" % (self.wnce_addr), shell = True, stderr = subprocess.STDOUT) except subprocess.CalledProcessError as e: output = str(e.output) lines = output.split("\n") responses = [] for line in lines: if "Got answer from:" in line: response_addr = line.split(": ")[1] responses.append(response_addr) log.info("Got dhcp response from %s." % (response_addr)) # If it somehow gets reset, the second address will be the one it has. bad_addrs = [self.wnce_addr, "192.168.1.251"] for addr in bad_addrs: if addr in responses: log.error("WNCE IS RUNNING ROGUE DHCP SERVER!") # Hibernating will automatically cut power to the wnce. It will, # however, also interrupt power to us. neato_system.hibernate() time.sleep(60)
def run(self, *args, **kwargs) -> None: self.first_execution = time.perf_counter() # monotonic on both Windows and Linux which is :thumbsup: self.first_execution_dt = datetime.datetime.utcnow() if not kwargs.get("bot", True): log.fatal("tried to login with a non-bot token (this framework is designed to run with a bot account)") raise UserBotError("Non-bot accounts are not supported") # checks to make sure everything is a coroutine if any([not asyncio.iscoroutinefunction(x) for x in self._ready_handlers]): log.critical("not all ready functions are coroutines") raise HandlerError("not all ready functions are coroutines") if any([not asyncio.iscoroutinefunction(x) for x in self._shutdown_handlers]): log.critical("not all shutdown functions are coroutines") raise HandlerError("not all shutdown functions are coroutines") if any([not asyncio.iscoroutinefunction(x) for x in self._message_handlers]): log.critical("not all message handlers are coroutines") raise HandlerError("not all message handlers are coroutines") if any([not asyncio.iscoroutinefunction(x) for x in self._member_join_handlers]): log.critical("not all member join handlers are coroutines") raise HandlerError("not all member join handlers are coroutines") if any([not asyncio.iscoroutinefunction(x) for x in self._member_remove_handlers]): log.critical("not all member leave handlers are coroutines") raise HandlerError(f"not all member leave handlers are coroutines") if any([not asyncio.iscoroutinefunction(x) for x in self._reaction_add_handlers]): log.critical("not all reaction add handlers are coroutines") raise HandlerError(f"not all reaction add handlers are coroutines") if any([not asyncio.iscoroutinefunction(x) for x in self._reaction_remove_handlers]): log.critical("not all reaction remove handlers are coroutines") raise HandlerError(f"not all reaction remove handlers are coroutines") log.debug("all functions good to run (are coroutines)") log.info(f"Bot started at {str(self.first_execution_dt)} ({self.first_execution})") super().run(*args, **kwargs)
def insert_into(table_name, data): try: cursor.execute(f"INSERT INTO {table_name} {data}") conn.commit() except mysql.connector.errors.IntegrityError as e: # if e.errno == 1062: log.fatal(e.msg)
def update_user_cart(user_id, operation, isbn, count=None): """ :param count: :param isbn: :param user_id: :param operation: 0 delete 1 update """ now = '{0:%Y%m%d%H%M%S}'.format(datetime.datetime.now()) # user: eu.Customer = user_cache[user_id] user: eu.Customer = _get_user(user_id) if user.type == "0000": return False if operation == 0: if user.add_shopping_cart(isbn, now): log.info(f"user {user_id} add book {isbn} to shopping cart") return True log.fatal(f"user {user_id} add book {isbn} to shopping cart failed") return False elif operation == 1: if user.update_shopping_cart(isbn, count): log.info(f"user {user_id} update book {isbn} to {count}") return True log.fatal(f"user {user_id} update book {isbn} to {count} failed") return False
def __sanityCheckVectorLength(self, meta): for testcase in meta.testcases.list: step = 1 for test_entry in testcase.content: if type(test_entry) is CLS.TestStep: set_list = test_entry.set_list expect_list = test_entry.expect_list if self.__sanityCheckVectorLengthInStepList( meta, set_list, step) == False: return False if self.__sanityCheckVectorLengthInStepList( meta, expect_list, step) == False: return False elif type(test_entry) is CLS.TestSequence: vector_list = test_entry.list for vector in vector_list: vector = vector[5:] signals = vector.split(" ") pos = 0 for signal_val in signals: meta_signal = meta.signals.getSignalAtPosition(pos) if len(signal_val) != meta_signal.size: log.fatal( "sanityCheckVectorLength: signal={0:s} with value={1:s} in step={2:d} has incorrect length" .format(meta_signal.name, signal_val, step)) return False pos += meta_signal.size step += 1 return True
def run(self): while True: # Check for dhcping. output = subprocess.check_output("which dhcping", shell=True) if not output: log.fatal("dhcping not found.") try: output = subprocess.check_output("dhcping %s" % (self.wnce_addr), shell=True, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: output = str(e.output) lines = output.split("\n") responses = [] for line in lines: if "Got answer from:" in line: response_addr = line.split(": ")[1] responses.append(response_addr) log.info("Got dhcp response from %s." % (response_addr)) # If it somehow gets reset, the second address will be the one it has. bad_addrs = [self.wnce_addr, "192.168.1.251"] for addr in bad_addrs: if addr in responses: log.error("WNCE IS RUNNING ROGUE DHCP SERVER!") # Hibernating will automatically cut power to the wnce. It will, # however, also interrupt power to us. neato_system.hibernate() time.sleep(60)
def apply_constraint_region(self, property_name: str, region: list = None): """ :param property_name: str of property_name(column name) :param region: list of two number | None. Default None :return: condition for property_name """ if region is None or (region[0] is None and region[1] is None): return self try: self._where_check() if self._where: log.debug("where add apply_constraint_region") if None not in region: high = float(max(region)) low = float(min(region)) self.constraints += f" AND {low}<={property_name} AND {property_name}<={high}" elif region[0] is None: high = float(region[1]) self.constraints += f" AND {property_name}<={high}" elif region[1] is None: low = float(region[0]) self.constraints += f" AND {low}<={property_name}" except ValueError as e: self._revert_where_check() log.fatal(f"Invalid region constraint for {property_name}, get {region}.\nError: {e}") finally: return self
def check_filter_record(record, tLogTypes=LOG_TYPES, dtStartFilter=None, dtStopFilter=None, tFilters=(), sFilterLogic=AND_FILTER_LOGIC): """ Проверка соответствия записи фильтрам. @param record: Текущая проверяемая запись. @param tLogTypes: Кортеж/список типов сообщений. @param dtStartFilter: Начальная дата/время фильтра по времени. Если не определено, то выбор происходит с начала файла. @param dtStopFilter: Конечная дата/время фильтра по времени. Если не определено, то выбор происходит до конца файла. @param tFilters: Кортеж/список дополнительных методов фильтрации. Методы фильтрации задаются как lambda или функции, которые принимают Словарь записи, а возвращают True-запись попадает в выбор/False - не попадает. @param sFilterLogic: Комманда способа обработки дополнительных фильтров AND - Чтобы запись попала в выбор необходимо положительное выполнение всех фильтров, OR - Чтобы запись попала в выбор достаточно положительное выполнение одного фильтра. @return: True - Запись соответствует всем фильтрам и должна быть добавлена в выбор/ False - Запись не соответствует какому либо фильтру. """ msg_type = record.get('type', '') if msg_type not in tLogTypes: return False result = True dt = record.get('dt', None) if dtStartFilter: if dt: result = result and (dt >= dtStartFilter) else: log.warning(u'Ошибка определения даты записи %s' % str(record)) return False if dtStopFilter: if dt: result = result and (dt <= dtStopFilter) else: log.warning(u'Ошибка определения даты записи %s' % str(record)) return False # Дополнительные фильтры if tFilters: try: filter_result = [fltr(record) for fltr in tFilters] except: log.fatal(u'Ошибка проверки фильтров') return False if sFilterLogic == AND_FILTER_LOGIC: result = result and all(filter_result) elif sFilterLogic == OR_FILTER_LOGIC: result = result and any(filter_result) return result
def __sanityCheckVectorLengthInStepList(self, meta, sig_list, step): for entry in sig_list: signal = meta.signals.getSignal(entry) if signal.role.endswith("_vec"): if len(sig_list[entry]) != signal.size: log.fatal( "sanityCheckVectorLength: signal={0:s} with value={1:s} in step={2:d} has incorrect length" .format(entry, sig_list[entry], step)) return False return True
def add_history_record(self, isbn, visit_time): try: self.cursor.execute( f"INSERT INTO browser_history(ISBN, time) VALUES ({isbn}, {visit_time})" ) self.db.commit() return True except IntegrityError: log.fatal(f"Book [{isbn}] doesn't exist.") return False
def create_table(self, table_name, definition) -> bool: """ :param table_name: :param definition: :return: True(success), False(failed) """ try: self.cursor.execute(f"CREATE TABLE {table_name} ({definition})") except mysql.connector.errors.ProgrammingError as e: log.fatal(f"Create table failed: {e.msg}") return False return True
def __diff_to_current(self, dependencies): """Calculated the diff of the parsed dependencies to the current dependencies""" arduino_dir = os.environ["ARDUINO_DIR"] if arduino_dir is None: log.fatal("No ARDUINO_DIR environment variable set") arduino_lib_path = os.path.join(arduino_dir, "libraries") current_dir = os.getcwd() unsynced_dependencies = [] for dependency in dependencies: # Move to the arduino library directory os.chdir(arduino_lib_path) dependency_path = Path(os.path.join(os.getcwd(), dependency.name)) # Short-circuit if the path doesn't exist if not dependency_path.exists(): unsynced_dependencies.append(dependency) continue # If it's a directory, we need to check the version if dependency_path.is_dir(): # Look for the .warm_dependency - file describing the current revision warm_path = os.path.join(str(dependency_path), ".warm_dependency") if not Path(warm_path).exists(): unsynced_dependencies.append(dependency) continue else: # Read in the file to determine revision. This is a data class serialised. # Note the uncautious use of eval here. This file only ever exists on the # users machine, so if they decide to inject malicious code then it's only # affecting themselves. with open(warm_path) as warm_properties: current_dep = eval(warm_properties.read().strip()) if not current_dep == dependency: unsynced_dependencies.append(dependency) continue # Back to where we started os.chdir(current_dir) return unsynced_dependencies
def run(): arg_parser = argparse.ArgumentParser(description=app_description, epilog=app_epilog,\ prog=app_prog, formatter_class=argparse.RawDescriptionHelpFormatter) arg_parser.add_argument("comp", help="component name <required>") arg_parser.add_argument("--com", help="UART communication port <required>", required=True) arg_parser.add_argument("--tc", help="run only given testcases; comma delimeted, e.g. 1,2-5,7") arg_parser.add_argument("--verbose", help="logging verbosity (note, debug)", nargs='?', choices=['note', 'debug']) arg_parser.add_argument("--version", action="version", version="%(prog)s-" + app_version) args = arg_parser.parse_args() comm = args.com comp = args.comp tc_list = None if args.tc: tc_list = [] for tc in args.tc.split(','): if '-' in tc: r = tc.split('-') b = int(r[0]) e = int(r[1]) for i in range(b, e + 1): tc_list.append(i) else: tc_list.append(int(tc)) tc_set = set(tc_list) tc_list = list (tc_set) tc_list.sort() if args.verbose == 'note': verbose = log.Verbosity.NOTE elif args.verbose == 'debug': verbose = log.Verbosity.DEBUG else: verbose = log.Verbosity.INFO app_dir = os.path.dirname(exec_name) target_sim_path = os.path.realpath(app_dir + "/../../target_sim") if not os.path.isdir(target_sim_path): log.fatal("Directory {0:s} doesn't exist".format(target_sim_path)) app = Impl(target_sim_path, comm, comp, tc_list, verbose) log.info("running hwsim") app.run()
def update_shopping_cart(self, isbn, count): if int(count) == 0: self.cursor.execute( f"DELETE FROM shopping_cart WHERE ISBN='{isbn}'") self.db.commit() return True elif int(count) > 0: self.cursor.execute( f"UPDATE shopping_cart SET count={count} WHERE ISBN='{isbn}'") self.db.commit() return True else: log.fatal( f"Update shopping_cart for user [{self.user_id}: {self.user_name}] failed, get count={count}" ) return False
def __init__(self, run_dir: str, run_description: str, problem_type: str, load_checkpoint: bool, model: nn.Module, optimizer: optim.Optimizer, loss: nn.Module, lr_scheduler: optim.lr_scheduler._LRScheduler, metrics: List[Tuple[str, Callable[[torch.Tensor, torch.Tensor], Any]]], cudnn_autotune: bool): # Enable autotuning in CUDNN. Note that it is not a good idea to enable # this when input sizes or network structure may be different iteration # to iteration. torch.backends.cudnn.benchmark = cudnn_autotune run_dir = os.path.expanduser(run_dir) assert os.path.isdir(run_dir) self.run_dir = run_dir self.devices = [ torch.device("cuda:{}".format(d)) for d in range(torch.cuda.device_count()) ] self.model = model if len(self.devices) > 1 and not isinstance(model, nn.DataParallel): log.fatal( "{} GPUs found, but model is not DataParallel. " "Wrap it into DataParallel for greater throughput.".format( torch.cuda.device_count())) self.optimizer = optimizer self.loss_function = loss self.lr_scheduler = lr_scheduler # May be None self.metrics = metrics self.epoch = 0 self.global_step = 0 self.best_loss = sys.float_info.max if problem_type not in [ "multiclass_classification", "multilabel_classification" ]: raise ValueError( "Unsupported problem type: {}".format(problem_type)) self.problem_type = problem_type # E.g. "multilabel_classification" self.run_description = run_description self.checkpoint_path = os.path.join(self.run_dir, CHECKPOINT_FILE_NAME) self.best_checkpoint_path = os.path.join(self.run_dir, BEST_CHECKPOINT_FILE_NAME) self.average_data_duration = performance.MovingAverage(32) self.average_compute_duration = performance.MovingAverage(32) # Checkpoint reload, if requested. if load_checkpoint: self._load_checkpoint()
def query(self, query_dn, rrtype='A'): ns = self.ns_cycle.next() self._query_timing(query_dn, rrtype, ns) while True: # XXX # need to block signals because dnspython doesn't handle EINTR # correctly log.logger.block_signals() res = self._sendquery(query_dn, ns, rrtype) log.logger.unblock_signals() if not isinstance(res, N3MapError): ns.retries = 0 return res if isinstance(res, TimeOutError): ns.add_timeouterror(self.max_retries) ns = self.ns_cycle.next() continue if isinstance(res, QueryError): log.fatal("received bad response from server: ", str(q.ns))
def __loadSignalMap(self, fpath): map_file = open(fpath) line = map_file.readline() map_file.close() signals = line.split(' ') sig_map = [] hiz_map = [] for s in range(len(signals)): e = signals[s].split(':') n = e[0].split("-") sig_map.append((int(n[0]), n[1], int(e[1]))) if n[1][0] == 'Z': if len(e) == 3: hiz_map.append((int(e[0].split("-")[0]), int(e[2]))) else: log.fatal("HI-Z signal \"{0:s}\" do not have validation pin defined".format(n[1])) return sig_map, hiz_map
def apply_constraint_value(self, property_names: [str], values=None): """ :param property_name:str of property_name(column name) :param values: :return: condition for property_name """ if values is None: return self try: self._where_check() # if type(values) == list: # if len(values) == 0: # return self if not isinstance(values, Iterable) or type(values) == str: values = [values] if not isinstance(property_names, Iterable) or type(property_names) == str: property_names = [property_names] for property_name in property_names: cons = "" for value in values: if type(value) == int or type(value) == float: cons += f" {property_name}={value} OR" else: cons += f" {property_name}='{value}' OR" # cons = f" {property_name}={value} OR" cons = re.sub(r"^ *OR *", "", cons) cons = re.sub(r" *OR *$", "", cons) self.constraints += f" AND {cons} " # if self._where: # log.debug("where add apply_constraint_value") # else: # if type(values) == int or type(values) == float: # self.constraints += f" AND {property_name}={values}" # else: # self.constraints += f" AND {property_name}='{values}'" except ValueError as e: self._revert_where_check() log.fatal(f"Invalid value constraint for {property_name}, get {values}.\nError: {e}") finally: return self
def purchase(self, isbn, purchase_time, count): """ Add this record to book_e_shop.shopping_history Add count to book_info.sold_count :param isbn: :param purchase_time: :param count: :return: True | False """ try: # check storage first e_shop_cursor.execute( f"SELECT storage_count FROM book_info WHERE ISBN='{isbn}'") storage_count = e_shop_cursor.fetchone()[0] if int(storage_count) < int(count): log.fatal( f"Not enough storage for {isbn}, found {storage_count} require {count}" ) return False # check end self.cursor.execute( f"INSERT INTO purchase_history(ISBN, time, count) VALUES ({isbn}, {purchase_time}, {count})" ) self.db.commit() self.cursor.execute(f"DELETE FROM shopping_cart WHERE ISBN={isbn}") self.db.commit() e_shop_cursor.execute( f"INSERT INTO shopping_history(isbn, count, time) VALUES({isbn}, {count}, {purchase_time})" ) book_e_shop.commit() e_shop_cursor.execute( f"UPDATE book_info SET sold_count=sold_count+{count} WHERE ISBN={isbn}" ) book_e_shop.commit() e_shop_cursor.execute( f"UPDATE book_info SET storage_count=storage_count-{count} WHERE ISBN={isbn}" ) book_e_shop.commit() return True except mysql.connector.errors.IntegrityError: log.fatal(f"Book [{isbn}] not exist") return False
def add_shopping_cart(self, isbn, add_time): try: self.cursor.execute( f"INSERT INTO shopping_cart(isbn, count, time) VALUES ({isbn}, 1, {add_time})" ) except mysql.connector.errors.IntegrityError as e: if e.errno == 1452: log.fatal(f"Book [{isbn}] doesn't exist.") return False else: # TODO: too complex self.cursor.execute( f"SELECT count FROM shopping_cart WHERE ISBN='{isbn}'") count = self.cursor.fetchone()[0] self.cursor.execute( f"UPDATE shopping_cart SET count={count + 1} WHERE ISBN='{isbn}'" ) finally: self.db.commit() return True
def compile(protocol, args): """Compile C-language output""" if args.out == '-': outfile = sys.stdout else: outfile = open(args.out, 'w') if not outfile: fatal('compile', f'Could not open {args.out}') outfile.write( f'/** Generated by XMLProtoc {joinstr(".", *__version__)} on {isonow()} */\n' ) # noqa if args.prepend: outfile.write(open(args.prepend, 'r').read()) protocol.accept(MessageDefinePrinterVisitor(outfile)) if args.append: outfile.write(open(args.append, 'r').read())
def parse_msg_record(line, encoding=DEFAULT_ENCODING): """ Распарсить строку файла журнала сообщений программы. @param line: Текущая обрабатываемая линия файла журнала сообщений программы. @return: {'dt': Время регистрации в datetime.datetime, 'type': Тип сообщения INFO, WARNING и т.п., 'text': Текст сообщения, 'short': Краткий текст сообщения ограниченный MSG_LEN_LIMIT} """ dt_txt = line[:20].strip() # log.debug('Data: %s' % dt_txt) try: dt = datetime.datetime.strptime(dt_txt, DATETIME_LOG_FMT) except: log.fatal(u'Ошибка парсинга времени лога <%s>' % dt_txt) raise msg_type = get_msg_log_type(line) msg = line[21 + len(msg_type):].strip() short_msg = u'' # Переведем все в unicode try: if isinstance(msg, str): msg = unicode(msg, encoding) except: log.fatal(u'Ошибка перевода строки в unicode') msg = u'' try: if isinstance(msg, unicode): short_msg = msg[:MSG_LEN_LIMIT] + ( u'...' if len(msg) > MSG_LEN_LIMIT else u'') except: log.fatal(u'Ошибка перевода строки в unicode') return dict(dt=dt, type=msg_type, text=msg, short=short_msg)
def __init__(self, target_sim_path, comm, comp, tc, verbose): log.setVerbose(verbose) self.tc = tc self.failed_testcases = 0 self.device_info = None self.communication = Communication(comm, self) self.comp = comp self.target_sim_path = target_sim_path self.metadata_file_path = target_sim_path + "/" + comp + ".mi" self.map_file_path = target_sim_path + "/" + comp + ".map" self.def_vector_path = target_sim_path + "/" + comp + "_df.vec" if not os.path.exists(self.metadata_file_path): log.fatal("Couldn't find metadata file: " + self.metadata_file_path) if not os.path.exists(self.def_vector_path): log.fatal("Couldn't find default vector file: " + self.def_vector_path) if not os.path.exists(self.map_file_path): log.fatal("Couldn't find map file: " + self.map_file_path)
def get_records_log_file(sLogFileName, tLogTypes=LOG_TYPES, dtStartFilter=None, dtStopFilter=None, tFilters=(), sFilterLogic=AND_FILTER_LOGIC, encoding=DEFAULT_ENCODING): """ Прочитать из файла журнала сообщений программы сообщения указанных типов. @param sLogFileName: Полное имя log файла. @param tLogTypes: Кортеж/список типов сообщений. @param dtStartFilter: Начальная дата/время фильтра по времени. Если не определено, то выбор происходит с начала файла. @param dtStopFilter: Конечная дата/время фильтра по времени. Если не определено, то выбор происходит до конца файла. @param tFilters: Кортеж/список дополнительных методов фильтрации. Методы фильтрации задаются как lambda или функции, которые принимают Словарь записи, а возвращают True-запись попадает в выбор/False - не попадает. @param sFilterLogic: Комманда способа обработки дополнительных фильтров AND - Чтобы запись попала в выбор необходимо положительное выполнение всех фильтров, OR - Чтобы запись попала в выбор достаточно положительное выполнение одного фильтра. @param encoding: Кодировка файла журнала. @return: Список словарей формата: [{'dt': Время регистрации в datetime.datetime, 'type': Тип сообщения INFO, WARNING и т.п., 'text': Текст сообщения, 'short': Краткий текст сообщения ограниченный MSG_LEN_LIMIT}, ...] """ if not sLogFileName: log.warning( u'Не определен файл журнала сообщений программы для чтения') return list() if not os.path.exists(sLogFileName): log.warning(u'Файл журнала сообщений программы <%s> не найден' % sLogFileName) return list() log_file = None try: records = list() record = dict() log_file = open(sLogFileName, 'r') for line in log_file: # Определяем является ли текущая линия началом нового сообщения или продолжением предыдущего if is_new_msg(line): record = parse_msg_record(line, encoding=encoding) if check_filter_record(record, tLogTypes, dtStartFilter, dtStopFilter, tFilters, sFilterLogic): records.append(record) else: if isinstance(line, str): line = unicode(line, encoding) # record['text'] = record.get('text', u'') + LINE_SEPARATOR + line record['text'] = record.get('text', u'') + line record['short'] += (u'...' if not record.get( 'short', u'').endswith(u'...') else u'') log_file.close() return records except: if log_file: log_file.close() log.fatal(u'Ошибка чтения записей журнала сообщений программы <%s>' % sLogFileName) return list()
if args["doubleVerbose"]: log.default.level_console = log.VERB_DEBUG log.default.level_full = True if args["verbose"]: log.default.level_console = log.VERB_INFO log.debug(args) valid = True if args["command"] == "check-file": valid = main(args["infile"], args["use_schema"]) if valid: log.notice("Check %s: PASS" % (args["infile"])) else: log.fatal("Check %s: FAIL" % (args["infile"])) elif args["command"] == "check-schemas": valid = check_schemas(args["path"]) elif args["command"] == "index": index_files(args["path"]) elif args["command"] == "scan-index": scan_index(args["infile"], args["use_mntner"]) elif args["command"] == "scan": import time log.notice("## Scan Started at %s" %(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))) ck = scan_files(args["path"], args["use_mntner"], args["use_file"]) log.notice("## Scan Completed at %s" %(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())))
def __parse_dependencies(self): """Parse the dependency file and return a list of all found dependencies""" # Look for dependencies.warm in current directory log.info("Looking in the current directory for dependencies.warm") file_list = [ f for f in os.listdir('.') if os.path.isfile(f) and f == 'dependencies.warm' ] dep_file = file_list[0] if file_list else None if dep_file is None: log.fatal( "Can't find dependencies.warm in the current directory - is it named properly?" ) log.info("Found dependency file") all_deps = [] current_working_dir = os.getcwd() # Use a temporary directory as the outer completion such that we use the same directory for all # git repo resolutions try: with TemporaryDirectory() as tempdir: with open(dep_file) as f: for line_number, line in enumerate(f): # If the line is a comment, skip it if line.strip().startswith('//'): continue # The format of a dependency is as follows: # # windykeyboards/butt-in: 1.0 # [org or owner name] / [repo name] : [tag | commit hash | branch | plus] # # The following reads each dependency, detects version type, and constructs a # named tuple describing the dependency. try: owner = line.strip().split('/')[0] repo_name = line.strip().split('/')[1].split( ':')[0].strip() raw_version = line.strip().split(':')[1].strip() except: log.warn( "Malformed dependency on line {line}".format( line=line_number + 1)) continue # In future we can support more than just github. For now, hardcode a git https url git_url = "https://github.com/{owner}/{repo}.git".format( owner=owner, repo=repo_name) parsed_version = self.__parse_version( git_url, raw_version, tempdir) if parsed_version is None: log.warn("Malformed version on line {line}".format( line=line_number + 1)) continue version_type = parsed_version["type"] version = parsed_version["version"] commit_hash = parsed_version["commit_hash"] all_deps.append( Dependency(name=repo_name, git_url=git_url, version_type=version_type, version=version, current_hash=commit_hash)) except PermissionError as e: log.warn( "Deletion of tempdir failed - this is not a problem, likely a Windows issue" ) # Change back to the original working directory in case parsing changed the working dir os.chdir(current_working_dir) log.info("Found {0} dependencies for the current project".format( len(all_deps))) return all_deps
def _train(directory: Text, model_name: Text, cuda: bool, num_threads: int, num_workers: int, spawn_method: Text, output: Optional[BinaryIO], cache_file: Optional[Text], **kwargs) -> None: # configure PyTorch's OpenMP log.info(f'Number of threads: {num_threads}') _GLOBAL_INITARGS.num_threads = num_threads _GLOBAL_INITARGS.cache_file = cache_file global_initialize(_GLOBAL_INITARGS) data_folder = Path(directory) assert data_folder.is_dir(), 'Invalid data folder' if cuda and cache_file is not None: log.warn('Disk cache file may not be compatible with CUDA tensors.') model, model_info = get_model(model_name, cuda, kwargs) # preload data into memcache full_data_path = data_folder / 'train.csv' if full_data_path.exists(): full_data = util.load_csv(full_data_path) log.info('Preparing data...') for smiles in full_data.keys(): _ = model.process(smiles, **kwargs) # process folds folds = [] roc_record = [] prc_record = [] for fold in sorted(data_folder.iterdir()): if not fold.is_dir(): log.debug(f'Ignored "{fold}".') continue folds.append(fold) if output is not None and num_workers > 1: log.fatal('Attempt to save multiple instances of model.') # multiprocessing configuration if num_workers == 1: model = build_model(model_info) for fold in folds: roc_auc, prc_auc = process_fold(model, fold, **kwargs) roc_record.append(roc_auc) prc_record.append(prc_auc) # saving model parameter if output is not None: log.info('Saving model...') model.save_model(output) else: log.info(f'Number of workers: {num_workers}') # PyTorch's OpenMP seems to be not compatible with the "fork" method if spawn_method == 'fork' and num_threads > 1: log.warn( 'The "fork" method may result in deadlock with multithreaded training.' ) if spawn_method == 'spawn': log.warn( 'The "spawn" method will invalidate memory molecule caches.') ctx = mp.get_context(spawn_method) with ctx.Pool(num_workers, global_initialize, (_GLOBAL_INITARGS, )) as workers: args_iterator = zip(folds, repeat(model_info), repeat(kwargs)) results = workers.imap_unordered(task_wrapper, args_iterator) for roc_auc, prc_auc in results: roc_record.append(roc_auc) prc_record.append(prc_auc) # basic statistics roc = torch.tensor(roc_record) prc = torch.tensor(prc_record) float_fmt = lambda x: float('%.4f' % x) log.info(f'roc = {list(map(float_fmt, roc_record))}') log.info(f'prc = {list(map(float_fmt, prc_record))}') log.info('All folds: ROC-AUC = %.3f±%.3f, PRC-AUC = %.3f±%.3f' % (roc.mean(), roc.std(), prc.mean(), prc.std()))
class AggressiveQueryProvider(QueryProvider): def __init__(self, ns_list, timeout, max_retries, stats=None, query_interval=None, num_threads=1): super(AggressiveQueryProvider,self).__init__( ns_list, timeout, max_retries, stats, query_interval) self._current_queryid = 0 self._active_queries = {} self._results = {} self._query_queue = Queue.Queue() self._result_queue = Queue.Queue() self._querythreads = [] self._start_query_threads(num_threads) def _start_query_threads(self,num=1): for i in xrange(num): qt = QueryThread(self._query_queue, self._result_queue) self._querythreads.append(qt) qt.start() def stop(self): for i in xrange(len(self._querythreads)): self._query_queue.put(None) for qt in self._querythreads: qt.join() def _gen_query_id(self): self._current_queryid += 1 return self._current_queryid def _sendquery(self, query): self.stats['queries'] += 1 log.debug2('query: ', query.query_dn, '; ns = ', query.ns, '; rrtype = ', query.rrtype) self._active_queries[query.id] = query self._query_queue.put(query) return query.id def _checkresult(self, qid, res): q = self._active_queries[qid] if not isinstance(res, N3MapError): q.ns.retries = 0 self._results[qid] = res del self._active_queries[qid] return try: raise res except TimeOutError: try: q.ns.add_timeouterror(self.max_retries) q.ns = self.ns_cycle.next() self._sendquery(q) except TimeOutError, e: del self._active_queries[qid] raise e except QueryError: log.fatal("received bad response from server: ", str(q.ns))
def gen_txt_file(sTxtTemplateFilename, sTxtOutputFilename, dContext=None, output_encoding=None): """ Генерация текстового файла по шаблону. @param sTxtTemplateFilename: Шаблон - текстовый файл. @param sTxtOutputFilename: Наименование выходного текстового файла. @param dContext. Контекст. В качестве контекста может выступать любая словарная структура. По умолчанию контекст - локальное пространство имен модуля config. @param output_encoding: Кодовая страница результирующего файла. Если не определена, то кодовая страница остается такая же как и у шаблона. @return: True - генерация прошла успешно, False - ошибка генерации. """ template_file = None output_file = None template_filename = os.path.abspath(sTxtTemplateFilename) if not os.path.exists(template_filename): log.warning(u'Файл шаблона для генерации текстового файла <%s> не найден' % template_filename) return False # Чтение шаблона из файла try: template_file = open(template_filename, 'r') template_txt = template_file.read() template_file.close() except: if template_file: template_file.close() log.fatal(u'Ошибка чтения шаблона из файла <%s>' % template_filename) return False try: # Определить кодовую страницу текста template_encoding = strfunc.get_codepage(template_txt) log.debug(u'Кодовая страница шаблона <%s>' % template_encoding) # Шаблон необходимо проебразовать в юникод перед заполнением template_txt = unicode(template_txt, template_encoding) except: log.fatal(u'Ошибка преобразования текста шаблона в Unicode') return False # Генерация текста по шаблону gen_txt = gen(template_txt, dContext) if isinstance(gen_txt, unicode): # Перед записью необходимо обратно перекодировать текст if output_encoding is None: gen_txt = gen_txt.encode(template_encoding) else: gen_txt = gen_txt.encode(output_encoding) # Запись текста в выходной результирующий файл output_filename = os.path.abspath(sTxtOutputFilename) try: output_path = os.path.dirname(output_filename) if not os.path.exists(output_path): log.info(u'Создание папки <%s>' % output_path) os.makedirs(output_path) output_file = open(output_filename, 'w+') output_file.write(gen_txt) output_file.close() # Дополнительная проверка на существующий выходной файл return os.path.exists(output_filename) except: if output_file: output_file.close() log.fatal(u'Ошибка генерации текстового файла <%s> по шаблону.' % output_filename) return False