def get_logger(self, key='console'): from lib.logger import get_logger logger = None try: logger = get_logger(key) except KeyError: logger = get_logger('file') return logger
def __init__(self, *args, **kwargs): Thread.__init__(self) self._logger = get_logger(kwargs.get('name', 'snoopy_thread')) self._logger.info('Initializing...') self._is_running = kwargs['is_running']
def __init__(self, *args, **kwargs): Process.__init__(self) log_name = kwargs.get('name', 'snoopy_no_name') if isinstance(kwargs.get('proc_num'), int): log_name = '%s[%0.2d]' % (log_name, kwargs.get('proc_num')) self._logger = get_logger(log_name) self._logger.info('Initializing...') self._is_running = kwargs['is_running']
def __init__(self, cfg): self._cfg = cfg Daemon.__init__(self, self._cfg['pid_path']) self._logger = get_logger('snoopy-collector')
from app.flask_app import celery from env import QuerybookSettings from lib.logger import get_logger from .run_query import run_query_task from .run_sample_query import run_sample_query from .dummy_task import dummy_task from .update_metastore import update_metastore from .sync_elasticsearch import sync_elasticsearch from .run_datadoc import run_datadoc from .delete_mysql_cache import delete_mysql_cache from .poll_engine_status import poll_engine_status from .presto_hive_function_scrapper import presto_hive_function_scrapper from .db_clean_up_jobs import run_all_db_clean_up_jobs LOG = get_logger(__file__) try: tasks_module = import_module("tasks_plugin") except (ImportError, ModuleNotFoundError) as err: LOG.info("Cannot import %s for tasks due to: %s", "task_plugin", err) # Linter celery run_query_task dummy_task update_metastore sync_elasticsearch run_datadoc delete_mysql_cache poll_engine_status
import lib.logger as logging from lib.functions import wait_until from lib.game import ui from lib.game.battle_bot import ManualBattleBot from lib.game.missions.missions import Missions logger = logging.get_logger(__name__) class Story(Missions): """Class for working with Story missions.""" class STORY_MISSION: DIMENSIONAL_CLASH_NORMAL = "STORY_MISSION_DIMENSIONAL_CLASH_NORMAL" DIMENSIONAL_CLASH_ULTIMATE = "STORY_MISSION_DIMENSIONAL_CLASH_ULTIMATE" class STORY_STAGE: DIMENSIONAL_CLASH_1_1 = "STORY_MISSION_DIMENSIONAL_CLASH_1_1" def __init__(self, game): """Class initialization. :param lib.game.game.Game game: instance of the game. """ super().__init__(game, mode_name='STORY') @property def battle_over_conditions(self): def rewards(): return self.emulator.is_ui_element_on_screen( ui.STORY_BATTLE_REWARDS)
import os import os.path import string import random import mlflow import pickle from lib.logger import get_logger from Player import config from Model.model import AZero from Azts.config import GAMEDIR, PLAYERDIR from Azts import player from Azts import stockfish_player from Azts import mock_model log = get_logger("Utility") GAME = "game" STATS = "stats" MOVES = "moves" # from https://pynative.com/python-generate-random-string/ def random_string(length=8): ''' generate random string as an id stamp for self plays ''' letters = string.ascii_lowercase return ''.join(random.choice(letters) for i in range(length))
from datetime import datetime, timedelta from telegram import InlineKeyboardButton from telegram.ext import Updater, Filters from telegram.ext import MessageHandler, CallbackQueryHandler from lib.logger import get_logger logger = get_logger(__name__) class View: def __init__(self, updater: Updater, handler): updater.dispatcher.add_handler( MessageHandler(Filters.chat_type, handler)) updater.dispatcher.add_handler(CallbackQueryHandler(handler)) updater.start_polling() logger.info('TelegramBot started') @staticmethod def welcome(first_name: str) -> str: message = 'Ciao {} 👋'.format(first_name) return message @staticmethod def menu(devices: list) -> tuple: message = 'Seleziona un dispositivo' inline_keyboard = [[ InlineKeyboardButton(text='{ico} {name} {ico}'.format( ico=('❌' if r.boolDisable else ''), name=r.strName), callback_data='device|{}|{}'.format( r.intIdDevice, 0))
import Queue as queue import re import StringIO as stringio import sgmllib import sys import threading import time import urllib import urllib2 import urlparse from lib import htmlparse from lib import config from lib import logger LOG = logger.get_logger(__name__) class Spider(object): """ Spider主程序 Attributes: module_name: 业务模块名称 cfg: 配置文件解析类对象 url_queue: 待抓取的url队列,包括url和当前抓取深度 handled_set: 已处理过的url的set集合,抓取操作完成后将url加入 lock: 线程锁 fetch: 具体抓取操作执行函数 max_depth: 最大抓取深度(种子为0级) crawl_interval: 抓取间隔. 单位: 秒
def parse_column(subcategory, args): """ "Column" handler :param subcategory: :param args: :return: """ log_tag = "parse_column" log = logger.get_logger(log_tag) if subcategory == SubCategories.add: if len(args) != 5: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(5, len(args))) column_view.create_format() else: log.info( "Trying to create a column with the name - {} and description - {}" .format(args[3], args[4])) ColumnController.create_columm(args[0], args[1], args[2], args[3], args[4]) column_view.success_create() log.info("Column {} was successfully created".format(args[3])) if subcategory == SubCategories.delete: if len(args) != 4: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(3, len(args))) column_view.delete_format() else: log.info( "Trying to delete a column with the name - {} and description - {}" .format(args[3], args[4])) ColumnController.delete_column(args[0], args[1], args[2], args[3]) column_view.success_delete() log.info("Column {} was successfully deleted".format(args[3])) if subcategory == SubCategories.edit: if len(args) != 6: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(6, len(args))) else: log.info( "Trying to edit a column with the name - {} and description - {}" .format(args[3], args[4])) if args[0] == 'name': ColumnController.edit_name(args[1], args[2], args[3], args[4], args[5]) column_view.success_edit() log.info( "Column {} is successfully edited. New name is {}".format( args[4], args[5])) elif args[0] == 'description' or 'desc': ColumnController.edit_desc(args[1], args[2], args[3], args[4], args[5]) column_view.success_edit() log.info( "Column {} is successfully edited. New description is {}". format(args[4], args[5])) else: log.error("Incorrect format") if subcategory == SubCategories.show: if len(args) != 4: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(4, len(args))) column_view.show_format() else: if args[3] == 'all': log.info( "Trying to show a column with the name - {} and description - {}" .format(args[3], args[4])) cols = ColumnController.show_all(args[0], args[1], args[2]) column_view.show_all(cols) log.info("All columns was shown")
@timing def function_to_be_timed(): ... OR directly apply the wrapper: timing(function_to_be_timed)() report for accumulated run-times: runtime_summary() """ import time import functools from lib.logger import get_logger log = get_logger("Timing", level="DEBUG") _RT_START = time.perf_counter() _RT_BINS = dict() _RT_FUNCS = dict() def timing(_func=None, *, bin=None): """ decorator for timing functions Usage: @timing def function_to_be_timed(): ... OR directly apply the wrapper:
''' this instantiates an analysis match which tracks metrics on the mlflow server ''' import time import argparse import mlflow import pickle import os from lib.logger import get_logger from Azts import utility from Azts.config import TO_STRING, WHITE, BLACK from Matches import match log = get_logger("AnalysisMatch") class AnalysisMatch(match.Match): match_moves = [] gamestats = [] def simulate(self): ''' simulate a game. this starts a loop of taking turns and making moves between the players while storing each game position and corresponding move distributions in data collection. loop ends with
def worker_run_job(loadtool, user, script_name, script, mission_id, agent_list=list()): logger = get_logger("job", "INFO", "logs/job_%s.log" % mission_id) def create_script(): folder = os.path.join( Config.SCRIPT_FOLDER if loadtool == "gatling" else Config.JMETER_SCRIPT_FOLDER, user) if loadtool == "jmeter": report_folder = os.path.join( Config.REPORT_FOLDER, "report-{mission_id}-jmeter".format(mission_id=mission_id)) os.makedirs(report_folder) if not os.path.exists(folder): os.makedirs(folder) filename = os.path.join( folder, "%s.%s" % ("%s-%s" % (script_name, mission_id) if loadtool != "gatling" else script_name, "scala" if loadtool == "gatling" else "jmx")) with open(filename, "wb") as f: f.write(script) def save_to_remote(js_dir, mission_id, save_list): try: for file in save_list: with open(os.path.join(js_dir, file), "rb") as f: cache.setex("%s-%s" % (file.replace(".", "_"), mission_id), 60, f.read()) except Exception as e: print e try: create_script() exec_cmd = "" if loadtool == "gatling": gatling_bin = os.path.join(Config.GATLING_HOME, "bin") exec_cmd = "sh {gatling_bin}/gatling.sh -s {user}.{script_name} -rf {report_folder} -on {result_dir_name}".format( gatling_bin=gatling_bin, user=user, script_name=script_name, mission_id=mission_id, report_folder=Config.REPORT_FOLDER, result_dir_name="report-%s" % mission_id) elif loadtool == "jmeter": if len(agent_list): exec_cmd = "sh {jmeter_home}/bin/jmeter.sh -n -t {jmeter_script_folder}/{user}/{script_name}-{mission_id}.jmx -R {agent_list} -l {jmeter_script_folder}/{user}/{script_name}.{mission_id}.jtl -e -o {report_folder}/report-{mission_id}-jmeter".format( debug_detail_folder=Config.DEBUG_DETAIL_FOLDER, jmeter_home=Config.JMETER_HOME, jmeter_script_folder=Config.JMETER_SCRIPT_FOLDER, user=user, script_name=script_name, report_folder=Config.REPORT_FOLDER, mission_id=mission_id, agent_list=",".join(agent_list)) else: exec_cmd = "sh {jmeter_home}/bin/jmeter.sh -n -t {jmeter_script_folder}/{user}/{script_name}-{mission_id}.jmx -l {jmeter_script_folder}/{user}/{script_name}.{mission_id}.jtl -e -o {report_folder}/report-{mission_id}-jmeter".format( debug_detail_folder=Config.DEBUG_DETAIL_FOLDER, jmeter_home=Config.JMETER_HOME, jmeter_script_folder=Config.JMETER_SCRIPT_FOLDER, user=user, script_name=script_name, report_folder=Config.REPORT_FOLDER, mission_id=mission_id) cache.append("mission_%s_log" % mission_id, exec_cmd) try: p = pexpect.spawn(exec_cmd, timeout=120) except Exception as e: cache.set("mission_%s_error" % mission_id, str(e)) logger.error(str(e)) while p.isalive(): p.expect(['\n', pexpect.EOF, pexpect.TIMEOUT]) info = p.before cache.append("mission_%s_log" % mission_id, "<br>" + info) else: if loadtool == "jmeter": save_to_remote( os.path.join(Config.REPORT_FOLDER, "report-%s-jmeter" % mission_id, "content", "js"), mission_id, ["dashboard.js", "graph.js"]) elif loadtool == "gatling": report_dir = get_gatling_report_dir(mission_id) source_dir = os.path.join(Config.REPORT_FOLDER, report_dir) report_file = os.path.join(Config.REPORT_FOLDER, "report-gatling", "report-%s.tar.gz" % mission_id) with open( os.path.join(Config.REPORT_FOLDER, report_dir, "js", "stats.json")) as f: cache.setex("mission_%s_stats" % mission_id, 60, f.read()) with tarfile.open(report_file, "w:gz") as tar: tar.add(source_dir, arcname=os.path.basename(source_dir)) Domino.save(report_file, "report-%s.tar.gz" % mission_id) cache.setex("mission_%s_finish" % mission_id, 60, 1) cache.delete("mission_%s_log" % mission_id) send_wechat_notice(mission_id, user) except Exception as e: logger.error(traceback.format_exc()) finally: return "ok"
""" import os.path import time import pickle from lib.logger import get_logger from Player import config from Azts import player from Azts import mock_model from Azts import utility from Azts.config import GAMEDIR, \ ROLLOUTS_PER_MOVE, SHOW_GAME from Matches import match log = get_logger("Contest") class Contest(): ''' selfplay is initialized with the number of rollouts that the matching ai player are using per move. the number of game simulations is determined by the parameter in function start() which actually starts the series of matches. After each match, the match data is written to a separate file which facilitates parallelisation of creating data for many matches. '''
from twisted.words.protocols import irc from twisted.internet import reactor, protocol import random import string import custom_exceptions import lib.logger as logger log = logger.get_logger('irc') # Reference to open IRC connection _connection = None def get_connection(): if _connection: return _connection raise custom_exceptions.IrcClientException("IRC not connected") class IrcLurker(irc.IRCClient): def connectionMade(self): irc.IRCClient.connectionMade(self) self.peers = {} global _connection _connection = self def get_peers(self): return self.peers.values()
from twisted.web.server import Request, Session, NOT_DONE_YET from twisted.internet import defer from twisted.python.failure import Failure import hashlib import json import string import helpers import semaphore #from storage import Storage from protocol import Protocol, RequestCounter from event_handler import GenericEventHandler from lib.settings import s as settings import lib.logger as logger log = logger.get_logger('http_transport') class Transport(object): def __init__(self, session_id, lock): self.buffer = [] self.session_id = session_id self.lock = lock self.push_url = None # None or full URL for HTTP Push self.peer = None # For compatibility with generic transport, not used in HTTP transport self.disconnecting = False def getPeer(self): return self.peer
import time import socket from twisted.protocols.basic import LineOnlyReceiver from twisted.internet import defer, reactor, error from twisted.python.failure import Failure #import services import stats import signature import custom_exceptions import connection_registry from lib.settings import s as settings import lib.logger as logger log = logger.get_logger('protocol') class RequestCounter(object): def __init__(self): self.on_finish = defer.Deferred() self.counter = 0 def set_count(self, cnt): self.counter = cnt def decrease(self): self.counter -= 1 if self.counter <= 0: self.finish() def finish(self):
def initialize(self): self.logger = get_logger('routers') self.http_bridge = HttpBridge() self.db = DB() self.expected_network_magic = Network().network_magic
def parse_project(subcategory, args): """ "Project" handler :param subcategory: :param args: :return: """ log_tag = "parse_project" log = logger.get_logger(log_tag) if subcategory == SubCategories.add: if len(args) != 4: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(4, len(args))) project_view.add_format() else: log.info( "Trying to create a project with the name - {} and description - {}" .format(args[2], args[3])) project = ProjectController.create(args[0], args[1], args[2], args[3]) project_view.success_create(project) log.info("Project {} was successfully created".format(args[2])) elif subcategory == SubCategories.delete: if len(args) != 3: log.error( "Incorrect number of arguments. Expected {}, but {} was recieved" .format(3, len(args))) delete_format() else: log.info("Trying to delete a project {}".format(args[2])) ProjectController.delete(args[0], args[1], args[2]) project_view.success_delete() log.info("Project {} is successfully deleted".format(args[2])) elif subcategory == SubCategories.show: if len(args) != 3: incorrent_args_len() log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(4, len(args))) else: if args[0] == 'all': log.info("Trying to show all projects of this user") projects = ProjectController.show_all(args[1], args[2]) project_view.show_info(projects) log.info("All project was shown") elif subcategory == SubCategories.edit: if len(args) != 5: project_view.edit_format() log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(4, len(args))) else: if args[0] == 'name': log.info("Trying to change the name of '{}' project".format( args[3])) ProjectController.edit_name(args[1], args[2], args[3], args[4]) project_view.success_edit() log.info("Project is successfully edited") if args[0] == 'description' or 'desc': log.info( "Trying to change the description of '{}' project".format( args[3])) ProjectController.edit_description(args[1], args[2], args[3], args[4]) project_view.success_edit() log.info("Project is successfully edited") else: print(project_view.edit_format()) log.error("Incorrect format") elif subcategory == SubCategories.members: if len(args) != 5: members_add_format() log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(4, len(args))) else: if args[0] == 'add': log.info("Trying to add {} to '{}' project".format( args[4], args[3])) ProjectController.add_person_to_project( args[1], args[2], args[3], args[4]) log.info("User is successfully added to the project") project_view.user_added() elif args[0] == 'delete': ProjectController.delete_person_from_project( args[1], args[2], args[3], args[4]) log.info(("Trying to delete {} from '{}' project".format( args[3], args[4]))) log.info("User is successfully deleted from project") project_view.user_deleted() else: members_add_format()
delete_schema, create_table, delete_table, create_table_information, create_column, delete_column, iterate_data_schema, get_table_by_schema_id, get_column_by_table_id, get_schema_by_name, get_table_by_schema_id_and_name, ) from .utils import MetastoreTableACLChecker LOG = get_logger(__name__) class DataSchema(NamedTuple): name: str class DataTable(NamedTuple): name: str # The type of table, it can be an arbitrary string type: str = None owner: str = None # Expected in UTC seconds table_created_at: int = None
def parse_task(subcategory, args): """ "Task" handler :param subcategory: :param args: :return: """ log_tag = "parse_task" log = logger.get_logger(log_tag) if subcategory == SubCategories.add: if len(args) != 10: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(10, len(args))) task_view.create_format() else: log.info("Trying to add task with the name - {}".format(args[4])) task = TaskController.add_task(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], args[8], args[9]) task_view.success_create(task) log.info("Task is successfully added") if subcategory == SubCategories.show: if args[0] == 'all': log.info("Trying to show all tasks in this column of project") tasks = TaskController.show_tasks(args[1], args[2], args[3], args[4]) task_view.show_tasks(tasks) log.info("All task was shown") if subcategory == SubCategories.delete: if len(args) != 5: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(5, len(args))) incorrent_args_len() else: log.info("Trying to delete task") TaskController.delete_task(args[0], args[1], args[2], args[3], args[4]) log.info("Task is successfully deleted") if subcategory == SubCategories.edit: if len(args) != 7: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(7, len(args))) task_view.edit_format() else: log.info("Tring to edit a column") TaskController.edit(args[0], args[1], args[2], args[3], args[4], args[5], args[6]) task_view.success_edit() log.info("Task is successfully edited") if subcategory == SubCategories.subtask: if args[0] == 'add': if len(args) != 7: log.error( "Incorrect number of arguments. Expected {} , but {} was recieved" .format(4, len(args))) task_view.add_subtask_format() else: log.info("Trying to set subtask to task") TaskController.set_subtask(args[1], args[2], args[3], args[4], args[5], args[6]) log.info("Task is successfully set as subtask")
from twisted.internet import defer from twisted.internet import reactor from twisted.names import client import random import time from services import GenericService, signature, synchronous import pubsub import lib.logger as logger log = logger.get_logger('example') class ExampleService(GenericService): service_type = 'example' service_vendor = 'Stratum' is_default = True def hello_world(self): return "Hello world!" hello_world.help_text = "Returns string 'Hello world!'" hello_world.params = [] @signature def ping(self, payload): return payload ping.help_text = "Returns signed message with the payload given by the client." ping.params = [ ('payload', 'mixed', 'This payload will be sent back to the client.'),
""" handles configuration parsing from yaml combines default settings, warning for unknown settings and type checking """ import os import sys import yaml import mlflow ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(ROOTDIR) from lib.logger import get_logger log = get_logger("Config") CONFIGDIR = "Player" class Options(object): """ Options Class able to parse yaml config settings while checking for correct data types and unknown settings (that aren't contained in the derived class' attributes) """ def __init__(self, d, default): self.load_options_safe(d, default) def load_options_safe(self, d, default_options): """ safely set member attributes of Options class while using default attributes; type-assertion for overridden values,
from random import randint import uuid import os import re from lib.utils import mask_line from lib.logger import get_logger logger = get_logger(LOG_NAME='noplp') class NoPLPSong(object): def __init__(self, title, artist, decade, basepath, tab): super(NoPLPSong, self).__init__() self.id = self.id = uuid.uuid1().hex self.title = title self.artist = artist self.decade = decade self.basepath = basepath self.tab = tab self.current_load_level = None self.lines = [] self.levels = {} # {"50": [...], "40": [...] } self.load_song() def __str__(self): return "{0} - {1}".format(self.title, self.artist)
def test_get_logger(self): """test function _get_logger""" self.assertTrue(logger.get_logger("testlog"))
from twisted.internet.protocol import ServerFactory from twisted.internet.protocol import ReconnectingClientFactory from twisted.internet import reactor, defer, endpoints import socksclient import custom_exceptions from protocol import Protocol, ClientProtocol from event_handler import GenericEventHandler import lib.logger as logger log = logger.get_logger('socket_transport') def sockswrapper(proxy, dest): endpoint = endpoints.TCP4ClientEndpoint(reactor, dest[0], dest[1]) return socksclient.SOCKSWrapper(reactor, proxy[0], proxy[1], endpoint) class SocketTransportFactory(ServerFactory): def __init__(self, debug=False, signing_key=None, signing_id=None, event_handler=GenericEventHandler, tcp_proxy_protocol_enable=False): self.debug = debug self.signing_key = signing_key self.signing_id = signing_id self.event_handler = event_handler self.protocol = Protocol
#!/usr/bin/python import argparse import os from lib.logger import get_logger from lib.iocReader import IocReader from lib.md5FileSystemScanner import Md5FileSystemScanner from lib.fileSystemListGeneratorProvider import FileSystemListGeneratorProvider from lib.md5Generator import Md5Generator from lib.resultsWriter import ResultsWriter, Results from lib.iocBundleDownloader import IocBundleDownloader logger = get_logger() class Main(object): ioc_file_path = 'openioc/' def __init__(self): self.parser = argparse.ArgumentParser(description='Sentinel scans for and reports the presence of malware-related and malware-modified files.') md5Generator = Md5Generator() fileSystemListGeneratorProvider = FileSystemListGeneratorProvider() iocReader = IocReader(Main.ioc_file_path) self.fileSystemScanner = Md5FileSystemScanner(md5Generator, fileSystemListGeneratorProvider, iocReader, logger) self.resultsWriter = ResultsWriter(logger) def execute(self): app.process_arguments() app.print_logo()
def __init__(self, adj_mx, **kwargs): self._kwargs = kwargs self._data_kwargs = kwargs.get('data') self._model_kwargs = kwargs.get('model') self._train_kwargs = kwargs.get('train') # logging. self._log_dir = self._get_log_dir(kwargs) log_level = self._kwargs.get('log_level', 'INFO') self._logger = get_logger(self._log_dir, __name__, 'info.log', level=log_level) self._writer = tf.summary.FileWriter(self._log_dir) self._logger.info(kwargs) # Data preparation self.ds = utils.load_dataset(**self._data_kwargs) self._data = self.ds.data for k, v in self._data.items(): if hasattr(v, 'shape'): self._logger.info((k, v.shape)) # Build models. scaler = self._data['scaler'] with tf.name_scope('Train'): with tf.variable_scope('DCRNN', reuse=False): self._train_model = DCRNNModel( is_training=True, scaler=scaler, batch_size=self._data_kwargs['batch_size'], adj_mx=adj_mx, **self._model_kwargs) with tf.name_scope('Test'): with tf.variable_scope('DCRNN', reuse=True): self._test_model = DCRNNModel( is_training=False, scaler=scaler, batch_size=self._data_kwargs['test_batch_size'], adj_mx=adj_mx, **self._model_kwargs) # Learning rate. self._lr = tf.get_variable('learning_rate', shape=(), initializer=tf.constant_initializer(0.01), trainable=False) self._new_lr = tf.placeholder(tf.float32, shape=(), name='new_learning_rate') self._lr_update = tf.assign(self._lr, self._new_lr, name='lr_update') # Configure optimizer optimizer_name = self._train_kwargs.get('optimizer', 'adam').lower() epsilon = float(self._train_kwargs.get('epsilon', 1e-3)) optimizer = tf.train.AdamOptimizer(self._lr, epsilon=epsilon) if optimizer_name == 'sgd': optimizer = tf.train.GradientDescentOptimizer(self._lr, ) elif optimizer_name == 'amsgrad': optimizer = AMSGrad(self._lr, epsilon=epsilon) # Calculate loss output_dim = self._model_kwargs.get('output_dim') preds = self._train_model.outputs labels = self._train_model.labels[..., :output_dim] null_val = 0. self._loss_fn = masked_mae_loss(scaler, null_val) self._train_loss = self._loss_fn(preds=preds, labels=labels) tvars = tf.trainable_variables() grads = tf.gradients(self._train_loss, tvars) max_grad_norm = kwargs['train'].get('max_grad_norm', 1.) grads, _ = tf.clip_by_global_norm(grads, max_grad_norm) global_step = tf.train.get_or_create_global_step() self._train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=global_step, name='train_op') max_to_keep = self._train_kwargs.get('max_to_keep', 100) self._epoch = 0 self._saver = tf.train.Saver(tf.global_variables(), max_to_keep=max_to_keep) # Log model statistics. total_trainable_parameter = utils.get_total_trainable_parameter_size() self._logger.info('Total number of trainable parameters: {:d}'.format( total_trainable_parameter)) for var in tf.global_variables(): self._logger.debug('{}, {}'.format(var.name, var.get_shape()))
import asyncio from db import DB from lib.logger import get_logger from models.http_bridge import HttpBridge from models.genesis import Genesis from models.scheduler import Scheduler from routers import Routers from tornado.web import Application from tornado.ioloop import IOLoop from tornado.options import define, options from tornado.log import enable_pretty_logging enable_pretty_logging() logger = get_logger('server') async def main(): database = DB() http_bridge = HttpBridge() is_loaded = await database.is_genesis_loaded() if not is_loaded: logger.info('start to load genesis.') genesis = Genesis() genesis_file = await http_bridge.get_genesis(genesis.genesis_hash) if genesis_file.get('nonAvvmBalances'): utxos = genesis.non_avvm_balances_to_utxos( genesis_file['nonAvvmBalances']) await database.save_utxos(utxos) if genesis_file.get('avvmDistr'): utxos = genesis.avvm_distr_to_utxos(genesis_file['avvmDistr'],
import random import string import multiprocessing import os.path import pickle import argparse from lib.logger import get_logger from Player import config from Azts import mock_model from Azts import player from Azts import utility from Azts.config import GAMEDIR, DATASETDIR from Matches import contest log = get_logger("create_dataset") # @timing def create_dataset(yamlpaths, rollouts_per_move, num_of_parallel_processes, num_of_games_per_process, fork_method="spawn"): ''' starts parallel training which creates many different game_[...].pkl files in GAMEDIR and then calls assemble_dataset which creates a dataset in DATASETDIR from all created games in GAMESDIR '''
SCHEMA = 'schema' TABLE_ID = 'tableid' SIMPLIFICATION = 'simplification' GEOM_FIELD = 'geomfield' FACTOR = 'factor' MAX_MEMORY = 'maxmemory' SKIP_FAILURES = 'skipfailures' SIMPLIFICATION_MAPSHAPER = 'mapshaper' SIMPLIFICATION_POSTGIS = 'postgis' DEFAULT_SIMPLIFICATION = SIMPLIFICATION_MAPSHAPER DEFAULT_MAXMEMORY = '8192' DEFAULT_SKIPFAILURES = 'no' LOGGER = get_logger(__name__) def get_simplification_params(table_id): with open(os.path.join(os.path.dirname(__file__), 'simplifications.json')) as file: return json.load(file).get(table_id) class Simplify(WrapperTask): schema = Parameter() table = Parameter() table_id = Parameter(default='') suffix = Parameter(default=SIMPLIFIED_SUFFIX) def __init__(self, *args, **kwargs):
# Import in respect to the path of script # http://stackoverflow.com/q/50499 current_frame = inspect.currentframe() script_path = os.path.abspath(inspect.getfile(current_frame)) # Insert external_deps path to import the correct external modules external_deps = os.path.join(os.path.dirname(script_path), "external_deps") sys.path.insert(1, external_deps) import lib.utils as utils from lib import logger from lib.pantri import Pantri if __name__ == "__main__": logger = logger.get_logger() def retrieve(options): # check if git repo was update before retrieving files. pantri = Pantri(options) if pantri.nothing_to_retrieve(): logger.info('it-bin repo already up-to-date. Use -f/--force to override') return # In order to selectively choose which shelves to retrieve and have # different options per shelf, need to call "pantri.retrieve()" for each # shelf. if 'shelf' in options: for shelf in options['shelf']: options['shelf'] = shelf
from keras.optimizers import Adam from keras.callbacks import ReduceLROnPlateau from keras.utils.vis_utils import plot_model sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from Model.utility import * from lib.timing import timing from Azts.config import DATASETDIR from Player.config import Config from Model.resnet import resnet_model, inference_model, transfer_update from lib.logger import get_logger log = get_logger("Model") DEBUG = False class AZero: """ The AlphaZero Class Attributes: model (Keras Model): The ResNet Model with two output heads initial_epoch Functions: train: starts the training process restore_latest_model: restores newest weights from model directory remember_model_architecture: makes sure the architecture along with config is saved once
def __init__(self): self.logger = get_logger('genesis') self.genesis_hash = Network().genesis_hash
#!/usr/bin/python import argparse import os from lib.logger import get_logger from lib.iocReader import IocReader from lib.md5FileSystemScanner import Md5FileSystemScanner from lib.fileSystemListGeneratorProvider import FileSystemListGeneratorProvider from lib.md5Generator import Md5Generator from lib.resultsWriter import ResultsWriter, Results from lib.iocBundleDownloader import IocBundleDownloader logger = get_logger() class Main(object): ioc_hashes_file = 'ioc_files/hacking_team_dll_hashes.ioc' def __init__(self): self.parser = argparse.ArgumentParser(description='Sentinel scans for and reports the presence of malware-related and malware-modified files.') md5Generator = Md5Generator() fileSystemListGeneratorProvider = FileSystemListGeneratorProvider() iocReader = IocReader(Main.ioc_hashes_file) self.fileSystemScanner = Md5FileSystemScanner(md5Generator, fileSystemListGeneratorProvider, iocReader, logger) self.resultsWriter = ResultsWriter(logger) def execute(self):
# -*- coding:utf-8 -*- import argparse from model import get_recognizer from lib.logger import get_logger logger = get_logger('train') if __name__ == '__main__': parser = argparse.ArgumentParser(description='模型训练脚本') parser.add_argument('rec_type', help='识别类别') args = parser.parse_args() logger.info('---- %s train ----' % args.rec_type) recognizer = get_recognizer(args.rec_type, model_config={'train': True}, use_cache=False) recognizer.train() logger.info('---- %s end ----' % args.rec_type)