def main(args): coco_dataloader = CocoDataLoader(args) trainloader = coco_dataloader.get_trainloader(args) yolov2 = DarkNet19() optimizer = Optimizer(yolov2.parameters(), args.lr) criterion = Loss() logger = Logger() logger.info('----- Starting training -----') for epoch in range(args.epochs): for i,data in enumerate(trainloader): images, targets = data optimizer.zero_grad() outputs = yolov2(images) total_loss = criterion.get_total_loss(outputs, targets, args) total_loss.backward() optimizer.step() logger.info(f'Epoch: {epoch+1}/{args.epochs}, Step: {i+1}, Loss: {loss.data}') logger.info('----- Training done! -----')
def __init__(self, name, filename): self.logger = Logger(filename) self.name = name self.memory = {} self.localmemory = {} self.functionargs = {} self.func = None self.r_value = None self.igmemory = {} self.local = {} self.igfunctions = {} self.igfunc = None self.igfuncinfo = None self.igloops = {} self.usedvars = set() self.tempvars = set() self.prefixes = [] self.loop = [] self.break_var = [] self.loops = 0 self.tags = 0
def run(self): self.env_check() if not self.dir_make(): Logger(self.file_name).get_logger().info("mysql dir make failed") return 20002, 'mysql dir make failed' if not self.cnf_make(): Logger(self.file_name).get_logger().info("my.cnf update failed") return 20001, 'my.cnf update failed' if not self.mysql_install(): Logger(self.file_name).get_logger().info("mysql install failed") return 20003, 'mysql install failed' time.sleep(15) if not self.mysql_user_grant(): Logger(self.file_name).get_logger().info("grants failed") return 20010, 'grants failed' return True
def group_check(self): mgroup = os.popen( "cat /etc/group |grep -w mysql|awk -F':' '{print $1}'").read( ).strip('\n') if not mgroup == '': return True Logger(self.file_name).get_logger().error("group check fail2") return False
def user_check(self): muser = os.popen( "cat /etc/passwd |grep -w mysql|awk -F':' '{print $1}'").read( ).strip('\n') if not muser == '': return True Logger(self.file_name).get_logger().info("user check fial2") return False
def __init__(self): self.limitation = int(FREQUENCY) * int(INTERVAL) self.lg = Logger("/var/log/nginx/block_log.log", level="info") self.nl = NginxLog() pool = redis.ConnectionPool(host=REDIS['host'], port=REDIS['port'], password=REDIS['password'], db=REDIS['db']) self.con = redis.Redis(connection_pool=pool)
def env_check(self): mbase = os.path.isdir(self.mysql_base) mdatap = os.path.isdir(self.mysql_data_path) mpak = os.path.exists(self.mysql_package) if not self.group_check(): os.system('groupadd mysql') if not self.user_check(): os.system('useradd -r -g mysql -s /sbin/nologin mysql') if mbase == False: if mdatap == False: if mpak == False: Logger(self.file_name).get_logger().info( "mysql package not exist") return False else: Logger( self.file_name).get_logger().info("mysql data path exist") return False else: Logger(self.file_name).get_logger().info("mysql base path exist") return False return True
def mysql_install(self): Logger( self.file_name).get_logger().info("start unzip mysql package...") t = tarfile.open(self.mysql_package, "r:gz") t.extractall(path=self.mysql_data_path) t.close() # path os.renames(self.mysql_document, self.mysql_base) cmd_own_confirm = "chown -R mysql.mysql {}".format(self.mysql_base) if not subprocess.call(cmd_own_confirm, shell=True) == 0: Logger(self.file_name).get_logger().info("chown mysql base fail") return False cmd_mod_green = "chmod -R g+rw {}".format(self.mysql_data_path) if not subprocess.call(cmd_mod_green, shell=True) == 0: Logger(self.file_name).get_logger().info("chow mysql data fail") return False cmd_mod = [ 'chmod -R 755 %s' % self.mysql_base, 'chmod -R 750 %s/bin' % self.mysql_base ] for cmd in cmd_mod: subprocess.call(cmd, shell=True) Logger(self.file_name).get_logger().info("start init mysql...") result = self.mysql_base + '/bin/mysqld ' + ' --defaults-file=' + self.mysql_data_path + '/my.cnf.' \ + self.port + ' --initialize-insecure --user=mysql >>/dev/null 2>&1' if not subprocess.call(result, shell=True) == 0: Logger(self.file_name).get_logger().info("mysql init fail") return False shutil.copy(self.mysqlserver, self.service) cmds = [ "sed -i 's/PORT/%s/g' %s" % (self.port, self.service), "sed -i 's/PATH_ROOT/\%s/g' %s" % ('/data', self.service), "chmod 755 %s" % self.service, "rm -rf /etc/my.cnf", "rm -rf %s/my.cnf" % self.mysql_base ] for cmd in cmds: subprocess.call(cmd, shell=True) Logger(self.file_name).get_logger().info("mysql starting......") result3 = '{0} restart'.format(self.service) if not subprocess.call(result3, shell=True) == 0: Logger(self.file_name).get_logger().info("mysql start fail") return False return True
.. code-block:: python :linenos: from demo_utils.shell import Shell sh = Shell() sh.run('ls') sh.set_cwd('~') sh.set_cwd('/') sh.run('ls') ''' import os, subprocess from logs import Logger logger = Logger('Shell').getLogger() # The shell class defines just a few functions which can make executing commands easier # run # This command has two possible ways to be called # (command) which is just a single line with all arguments # (command, args) which simply just joins a string with the command and arguments # # It is also possible to change the current working directory (cwd) # for commands which are sensitive to file locations # (Unfortunately 'cd' doesn't work) # If no arguments are passed for args, then it is assumed to be a string of length 0 # If no arguments are passed to the constructor we assume default cwd class Shell:
# Script which installs Zeppelin as an Ambari Service import config, sys, platform, json, time, os from shell import Shell from curl_client import CurlClient from logs import Logger logger = Logger('service_installer').getLogger() def install_hdp_select(): '''Installs the hdp-select package. This is used to help install the Ambari Service for NiFi/Zeppelin. Retrieves the currently installed hadoop version. Typically the Sandbox has hdp-select already, but clusters won't always have it, which is why we provide this method. ''' logger.info('Determining whether system is compatible with hdp-select') dist_info = platform.linux_distribution() if (len(dist_info[0]) == 0): logger.critical('Non Linux System. Could not install hdp-select') raise EnvironmentError( 'You must be running a linux distribution to install hdp-select') # Only want to get distro name # Determine first distro name # Then determine the version (first char char for centos ) distro = dist_info[0].lower() fullname = distro
"ServiceInfo" : { "cluster_name" : "Sandbox", "maintenance_state" : "OFF", "service_name" : "YARN", "state" : "INSTALLED" } } ''' import json, time from curl_client import CurlClient from logs import Logger logger = Logger('Ambari').getLogger() class Ambari: '''Initalize the Ambari client Args: username (str, optional): username to use for authentication (should have admin access) password (str, optional): password to use for authentication (should have admin access) proto (str, optional): Must be one of 'http' or 'https'. Defines which protocol to use. Defaults to 'http' server (str, optional): The hostname (or IP) of the Ambari server. Defaults to 127.0.0.1. port (int, optional): The port that ambari server is running on. Defaults to 8080 service_wait_time (int, optional): The time (in seconds) we should before we decide a service has failed changing states. config (dict, optional): This is a dictionary object which should contain the any of the keys 'username', 'password', 'proto', 'server', 'port', or 'service_wait_time'. Given the config here you can set any of the client's parameters through this object. However, when using this, the config object will override any of the specific arguments passed. Returns:
import json try: from datetime import datetime from logs import Logger, LoggerMode except BaseException as e: print("Import Error: ", e) # Global Var output_file = "logs_{}.log".format(datetime.now().strftime("%d%m%Y_%H%M%S")) logs = Logger(output_file=output_file, log_level=LoggerMode.DEPLOY) config_data = {} def import_config(file_path): global config_data with open(file_path, "r") as data: config_data = json.load(data) def get_config_for(module): if len(config_data) == 0: import_config("config/conf.json") return config_data[module]
import os.path import struct from logs import Logger import importlib.util from utils import AsyncResponse import asyncio import fcgiVariables logger = Logger('./') class Handler: def handleRequest(self, req, res): pass class StaticHandler(Handler): def handleRequest(self, req, res): if os.path.isfile(f'.{req.url}'): f = open(f'.{req.url}', 'r') res.set_body(f.read()) if (req.url.split('.')[1] == 'html'): res.set_headers({ 'content-type': 'text/html' }) f.close() res.send() else: logger.logError(404) res.status_code = 404 res.send()
'''A client for executing HTTP requests via cURL. It does not use any external libraries. The only requirement is that you have the curl executable in your PATH or bin. Used to make arbitrary http(s) requests. ''' import json, logging from shell import Shell from logs import Logger logger = Logger('CurlClient').getLogger() class CurlClient: '''The CurlClient object. Houses the parameters used when making requests Args: username (str, optional): Username for basic authentication. Default empty string password (str, optional): Password for basic authentication. Default empty string proto (str, optional): protocol used in requests. Must be one of http or https server (str, optional): The server or hostname to which we will make the request. Defaults 127.0.0.1 port (int, optional): The integer number of the port that the server runs on. Defaults 8080 ''' username = '' '''Username for basic authentication''' password = '' '''Password for basic authentication''' server = '' '''Server or hostname to connect to'''
def __init__(self, name, filename): self.logger = Logger(filename) self.igfunctions = {} self.igmemory = {} self.igfunc = None
run_dir = os.path.join('..', 'runs', now) os.makedirs(os.path.join(run_dir, 'checkpoints'), exist_ok=True) os.makedirs(os.path.join(run_dir, 'logs'), exist_ok=True) early_stopping_score = None early_stopping_counter = 0 step = 0 # Initialize EarlyStopping early_stopping = EarlyStopping(step=step, run_dir=run_dir, best_score=early_stopping_score, counter=early_stopping_counter, verbose=False) # Initializing logger logger = Logger(os.path.join(run_dir, 'logs')) # Initialize data loaders dataset = ClusterDataset(files=parse_data_structure(hp.files), feature_mean=np.load('mu.npy'), feature_std=np.load('std.npy')) loader = DataLoader(dataset=dataset, batch_size=hp.training.batch_size, num_workers=hp.training.num_workers, pin_memory=True) # Auto select best algorithm to maximize GPU utilization cudnn.benchmark = True # Execute train loop # try:
self.con.set(route_ip, latest_frequency) expire_time = int(block_time) - block_delta self.con.expire(route_ip, expire_time) # 检查防火墙禁用该ip策略是否还在, 不在则加上 if self.check_firewall(ip): lg.logger.info('IP仍在禁用中且已存在防火墙策略,无需重复添加') else: self.add_firewall(ip) lg.logger.info('未过期但无防火墙策略,已重新添加') def block_ip(self): for k, v in self.nl.get_ip_frequency().items(): exist_history = self.get_block_ip_history(k) ip = k.split(':')[1] if exist_history: self.history_ip_add_firewall(k, v, ip) else: self.new_ip_add_firewall(k, v, ip) if __name__ == '__main__': lg = Logger('/var/log/nginx/blockip.log', level='info') nl = NginxLog() # 先清除临时生成的log文件 nl.clear_tmp_log() bi = BlockIp() # 先清除已有过期防火墙策略 bi.clear_expire_firewall() # 后将超过访问限制的IP加入防火墙 bi.block_ip()
checkpoint['optimizer_generator_state_dict']) optimizer['discriminator'].load_state_dict( checkpoint['optimizer_discriminator_state_dict']) scaler['generator'].load_state_dict( checkpoint['scaler_generator_state_dict']) scaler['discriminator'].load_state_dict( checkpoint['scaler_discriminator_state_dict']) phase = checkpoint['phase'] step = checkpoint['step'] else: phase = 0 step = 0 run_dir = utils.core.get_run_dir(process_group, args.local_rank) # Initializing logger logger = Logger(os.path.join(run_dir, 'logs')) if args.local_rank == 0 else None # Auto select best algorithm to maximize GPU utilization cudnn.benchmark = True # Start main loop try: training(model, optimizer, criterion, scaler, logger, process_group, run_dir) except KeyboardInterrupt: pass finally: if args.local_rank == 0: utils.core.save_checkpoint(run_dir, utils.core.ddp(model), optimizer, scaler, phase, step) print('Saved checkpoint.')
gen = DataGenerator(rand_schema) # Generate a single 'row' of data print(gen.generate()) ''' #!/usr/bin/python # -*- coding: utf-8 -*- import json import config import random from abc import abstractmethod from logs import Logger logger = Logger('Generator').getLogger() class DataGenerator: '''The generator object. Pass the configuration here. use the generate() method to get a random object Args: schema (str): The schema file or JSON string which defines the data to be generated. seed (str, optional): The seed value for the generator ''' def __init__(self, schema, seed=''): self.rand = random.Random() self.data_fields = [] self.field_names = []