def recv_fiber(self): while self._running: try: msg = self.receive_next_msg() if msg == None: raise engine.EngineError('Received nothing') if isinstance(msg, Query): raise engine.EngineError('Recieved unexpected Query') elif isinstance(msg, Close): self.close() elif isinstance(msg, Answer): if self._query_list.get(msg.qid): self._query_list[msg.qid].set(msg) del self._query_list[msg.qid] else: logger.get_logger().error('proxy %s get unexpected answer %s. qid may be removed for timeout', self.end_point, msg) else: raise engine.EngineError('Unknown message') except ProxyError as ex: if self._query_list.get(ex.qid): msg = Answer(ex.qid, ex.status, ex.params) self._query_list[ex.qid].set(msg) del self._query_list[ex.qid] except Exception as ex: self.close() break
def receive_next_msg(self): if self.protocol == 'tcp': message = Messager.receive_msg(self._socket) elif self.protocol == 'udp': pkt, address = self.unitive_recvfrom(65535) message = Messager.message_from_data(pkt) if self.debug and logger.is_debug(): if isinstance(message, Query): logger.get_logger().debug('SQ:%d method=%s', message.qid, message.method) elif isinstance(message, Answer): logger.get_logger().debug('SA:%d status=%d', message.qid, message.status) if not message: raise Exception('Recieved Unknown packet') if isinstance(message, Answer): qid = message.qid if self.pending.get(qid): del self.pending[qid] status = message.status if status: raise ProxyError(qid, status, message.data) return message;
def __init__(self, setting): self.setting = setting self.debug = setting.get('debug', False) self._running = True self._adapters = {} self._proxies = {} if self.debug and logger.is_debug(): logger.get_logger().debug("application setting %s", setting)
def send_quest(self, qid, method, params, twoway=True): if qid: self.pending[qid] = time.time() if self.debug and logger.is_debug(): logger.get_logger().debug('CQ:%d service=%s method=%s', qid, self.service, method) self.unitive_send(Messager.data_for_query(qid, self.service, method, params)) return qid
def respond(self, rpc_controller, echo_string, callback): """ called by RpcChannel.receive when a complete request reached. """ logger.get_logger('EchoClient').debug('EchoClient.respond') if self.streamout: self.streamout.write(echo_string.message) if callback: callback()
def echo(self, rpc_controller, echo_string, callback): """ called by RpcChannel.receive when a complete request reached. """ logger.get_logger('EchoService').info('echo service is called') echo_string.message = echo_string.message client_stub = IEchoClient_Stub(rpc_controller.rpc_channel) client_stub.respond(rpc_controller, echo_string, callback=None) if callback: callback()
def servant_worker(self): while self._running: query = self._query_queue.get() if not isinstance(query, Query): logger.get_logger().error('invalid query %s', query) continue if not self._wildcard_servant: self._handle_normal_servant(query) else: self._handle_wildcard_servant(query)
def __init__(self, args): self.fs = FileSystem() get_logger('app', 'logs/app.log', level=logging.DEBUG) self.storage_dir = args.storage_option logging.getLogger('app').debug('App started') self.fs.apply_callback_to_file_iterator(self.storage_dir, self.indexer) logging.getLogger('app').debug('App stoped normaly')
def _fail_quests(self): exinfo = self._build_except_info() import traceback traceback.print_stack() logger.get_logger().error('proxy %s get error %s, fail current querys', self.end_point, exinfo) curr_req_items = self._query_list.items() self._query_list = {} for qid, result in curr_req_items: msg = Answer(qid, 1, exinfo) #AsyncResult set should cause context switch result.set(msg)
def close(self): logger.get_logger().info('close') if self._recv_thread: gevent.kill(self._recv_thread) self._recv_thread = None if self._socket: self._socket.close() self._socket = None self._connected = False self._fail_quests()
def serve_forever(self): for (name, adapter) in self._adapters.items(): adapter.activate() self.start_time = time.strftime('%Y-%m-%d %H:%M:%S') while self._running: # periodic log info that engine is running p = {} p['start_time'] = self.start_time p['adapters'] = self._adapters logger.get_logger().critical("THROB %s", str(p)) gevent.sleep(60)
def snapshot(prices): try: conn = sqlite3.connect("intraday.db") cursor = conn.cursor() cursor.execute("PRAGMA synchronous = OFF") cursor.execute("PRAGMA journal_mode = OFF") get_logger().debug('Database 연결 성공') except: get_logger().warn('Database 연결 실패') raise URL = "http://finance.daum.net/xml/xmlallpanel.daum?stype=P&type=S" global is_market_finished # parse(r.headers['Date']) 할 수 있으나, 다음 웹서버 시간이 부정확 time = datetime.datetime.now() r = requests.get(URL) r.encoding = 'UTF-8' # 정규표현식으로 파싱(키값에 따옴표가 없어서 JSON 파싱 불가) # example : , {code:"095570",name :"AJ네트웍스",cost :"34,650",updn :"▲100",rate :"+0.29%"} pattern = "code:\"(.+)\",name :\"(.+)\",cost :\"(.+)\",updn" rep = re.compile(pattern) for line in r.text.splitlines(): if '장종료' in line: is_market_finished = True if 'code' not in line: continue # print(line) match = rep.search(line) symbol = match.group(1) name = match.group(2) price = match.group(3).replace(',', '') # 직전 가격과 같으면 INSERT 생략 if(prices[symbol] == price): continue prices[symbol] = price row = cursor.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='KS_{}'".format(symbol)).fetchone() if row is None: cursor.execute("CREATE TABLE KS_{} (datetime INTEGER, price INTEGER)".format(symbol)) sql = "INSERT INTO KS_{} (datetime, price) VALUES (CURRENT_TIMESTAMP, '{}')" cursor.execute(sql.format(symbol, price)) conn.close()
def _handle_wildcard_servant(self, query): if self.debug and logger.is_debug(): logger.get_logger().debug('handle_wildcard_servant: %s', query) try: result = self._wildcard_servant(query.service, query.method, query.params) if query.qid: query.inbox.put(Answer(query.qid, 0, result)) except proxy.ProxyError as ex: if query.qid: query.inbox.put(Answer(ex.qid, ex.status, ex.params)) except: if query.qid: query.inbox.put(Answer(query.qid, 1, self._make_exception_params(query)))
def snapshot(prices): with open('config.json') as file: config = json.load(file)['mysql'] try: conn = pymysql.connect(host=config['host'], port=config['port'], user=config['user'], password=config['password'], db=config['db']) cursor = conn.cursor() get_logger().debug('Database 연결 성공') except pymysql.err.OperationalError: get_logger().warn('Database 연결 실패') raise URL = "http://finance.daum.net/xml/xmlallpanel.daum?stype=P&type=S" sql = "INSERT INTO `price_intraday` (`symbol`, `price`) VALUES (%s, %s)" # parse(r.headers['Date']) 할 수 있으나, 다음 웹서버 시간이 부정확 time = datetime.datetime.now() r = requests.get(URL) r.encoding = 'UTF-8' # 정규표현식으로 파싱(키값에 따옴표가 없어서 JSON 파싱 불가) # example : , {code:"095570",name :"AJ네트웍스",cost :"34,650",updn :"▲100",rate :"+0.29%"} pattern = "code:\"(.+)\",name :\"(.+)\",cost :\"(.+)\",updn" rep = re.compile(pattern) for line in r.text.splitlines(): if 'code' not in line: continue # print(line) match = rep.search(line) symbol = match.group(1) name = match.group(2) price = match.group(3).replace(',', '') # 직전 가격과 같으면 INSERT 생략 if(prices[symbol] == price): continue prices[symbol] = price cursor.execute(sql, (symbol, price)) conn.commit() conn.close()
def _method_not_found(self, called_method, query): logger.get_logger().error("method %s not found", called_method) """ query method not found in servant """ exdict = {} exdict['exception'] = 'MethodNotFound' exdict['code'] = 1 exdict['message'] = "servant %s do no have method %s in adapter %s" % (query.service, called_method, self._endpoint) exdict['raiser'] = self._endpoint if query.qid: query.inbox.put(Answer(query.qid, 100, exdict)) else: logger.get_logger().warning("%s.%s %s", query.service, called_method, exdict) return
def main(): global is_market_finished prices = defaultdict(list) interval = 5 if len(sys.argv) == 1 else int(sys.argv[1]) #for i in range(2): while True: #if datetime.datetime.now().hour < 9: continue snapshot(prices) if(is_market_finished): get_logger().info("장이 종료되었습니다.") break time.sleep(interval)
def get_pid(process_name): global pids log_handler = logger.get_logger("pid.log") a = os.popen('ps -ef | grep {}'.format(process_name)).readlines() filter_1 = "grep" filter_2 = "python" for x in a: try: if (x.find(filter_1) != -1) or (x.find(filter_2) != -1) : #log_handler.info("escaped") pass else: #log_handler.info(x) add_to_dict(x) #pids.append(int(x[10:15])) #process[int(x[10:15])] = x[49:] except Exception as inst: #print inst pass #return the list if not null if not pids: return -1 else: return pids
def __init__(self, args): self.field_types = { "sys_uptime": int, "us_atten": float, "update_time": int, "ds_speed": int, "ds_atten": float, "wan_boot_time": int, "ds_margin": float, "us_margin": float, "wan_uptime": int, "sys_boot_time": int, "us_speed": int, } # Get the logger instance self.logger = logger.get_logger() self.logger.info("Router database logger starting up") # Connect to the database self.logger.info( "Connecting to influxDB database %s at host %s port %d", args.influx_db, args.influx_ip, args.influx_port ) self.influx_client = InfluxDBClient(host=args.influx_ip, port=args.influx_port, database=args.influx_db) # Create an MQTT client instance, register connect and message callbacks and connect self.mqtt_client = paho.mqtt.client.Client() self.mqtt_client.on_connect = self.on_connect self.mqtt_client.on_message = self.on_message self.logger.info("Connecting to MQTT server at address %s:%d", args.mqtt_ip, args.mqtt_port) self.mqtt_client.connect(args.mqtt_ip, args.mqtt_port, args.mqtt_timeout)
def main(): with open('config.json') as file: config = json.load(file)['rabbitmq'] credentials = pika.PlainCredentials(config['user'], config['passwd']) parameters = pika.ConnectionParameters(config['host'], config['port'], '/', credentials) try: #get_logger().debug("Connecting to RabbitMQ ({}:{})".format(config['host'], config['port'])) connection = pika.BlockingConnection(parameters) except pika.exceptions.ConnectionClosed: get_logger().warn("RabbitMQ Connection Error ({}:{})".format(config['host'], config['port'])) sys.exit(1) channel = connection.channel() channel.exchange_declare(exchange="price", type="topic") stocks = defaultdict(list) URL = "http://finance.daum.net/xml/xmlallpanel.daum?stype=P&type=S" # parse(r.headers['Date']) 할 수 있으나, 다음 웹서버 시간이 부정확 time = datetime.datetime.now() r = requests.get(URL) r.encoding = 'UTF-8' # 정규표현식으로 파싱(키값에 따옴표가 없어서 JSON 파싱 불가) # example : , {code:"095570",name :"AJ네트웍스",cost :"34,650",updn :"▲100",rate :"+0.29%"} pattern = "code:\"(.+)\",name :\"(.+)\",cost :\"(.+)\",updn" rep = re.compile(pattern) for line in r.text.splitlines(): if 'code' not in line: continue # print(line) match = rep.search(line) symbol = match.group(1) name = match.group(2) price = match.group(3).replace(',', '') channel.basic_publish(exchange='price', routing_key=symbol, body=price) # 헤더에 시간 추가 connection.close()
def __init__( self, host="127.0.0.1", port=8000, server_cls=None, handler_cls=WebTestRequestHandler, use_ssl=False, key_file=None, certificate=None, encrypt_after_connect=False, router_cls=Router, doc_root=os.curdir, routes=None, rewriter_cls=RequestRewriter, bind_hostname=True, rewrites=None, latency=None, config=None, ): if routes is None: routes = default_routes.routes self.host = host self.router = router_cls(doc_root, routes) self.rewriter = rewriter_cls(rewrites if rewrites is not None else []) self.use_ssl = use_ssl self.logger = get_logger() if server_cls is None: server_cls = WebTestServer if use_ssl: if key_file is not None: assert os.path.exists(key_file) assert certificate is not None and os.path.exists(certificate) try: self.httpd = server_cls( (host, port), handler_cls, self.router, self.rewriter, config=config, bind_hostname=bind_hostname, use_ssl=use_ssl, key_file=key_file, certificate=certificate, encrypt_after_connect=encrypt_after_connect, latency=latency, ) self.started = False _host, self.port = self.httpd.socket.getsockname() except Exception: self.logger.error("Init failed! You may need to modify your hosts file. Refer to README.md.") raise
def init_service(): global mac_address, process_id, mongo_db, mongo_userprofile, mylogger mac_address = getHwAddr('eth0') process_id = os.getpid() mylogger = logger.get_logger('/opt/research/NginxUwsgi/log/dmp_%d.log' %(process_id)) mongo_db = Connection(host="192.168.86.18", port=27020).dmp mongo_userprofile = mongo_db.user_profile
def __init__(self): self.logger = logger.get_logger('virtshell_dispatcher') self.listeners_tasks = {} self.tasks_folder = "./tasks" self.special_files = ["__init__.py", "__pycache__"] self.task_execute_method = "main" self.load_plugin_tasks() self.logger.info("service started...")
def init_service(): global mylogger, mongo_client, mongo_db, weibo_redis_clis process_id = os.getpid() mylogger = logger.get_logger("/opt/research/NginxUwsgi/log/ovs_proxy_%d.log" % (process_id)) mongo_client = MongoClient("192.168.86.67", 27020) mongo_db = mongo_client.ovs weibo_redis_clis = init_redis("nginx02", 6379, 0)
def activate(self): for i in range(self._servant_worker_num): gevent.spawn(self.servant_worker) self._pool = Pool(self._accept_pool_num) endpoint = self._endpoint try: service, proto, host, port = proxy.parse_endpoint(endpoint) if proto != 'tcp': raise proxy.ProxyError(1, 'only tcp server supported now') server = StreamServer((host, int(port)), self.sokect_handler, spawn=self._pool) if self.debug and logger.is_debug(): logger.get_logger().debug('adapter start %s', endpoint) self._servers.append(server) server.start() except: logger.get_logger().error('start adapter fail %s', endpoint, exc_info = 1)
def __init__(self, get_stream_api_url, read_token, monitor_cdn_in_api, analyze_thread, extra_streams): threading.Thread.__init__(self) self.api_server_model = ApiServerModel(get_stream_api_url, read_token, monitor_cdn_in_api) self.analyze_thread = analyze_thread self.stream_db = StreamModel() self.logger = logger.get_logger() self.extra_streams = extra_streams self.api_return_streams = None self.retrieve_interval = 60
def __init__(self): threading.Thread.__init__(self) self.logger = logger.get_logger() self.obs = None self.sleep_time = 3 self.analyzer = Analyzer() self.obs_lock = threading.Lock() self.stream_model = StreamModel() self.sample_model = SampleModel()
def write(filename, data): """ Write to a file. """ logger = get_logger(__name__) logger.debug('Writing to %s' % filename) f = open(filename, 'w') f.write(data) f.close()
def init_service(): global mylogger process_id = os.getpid() mylogger = logger.get_logger('/opt/research/NginxUwsgi/log/pyuwsgi_monitor_%d.log' % (process_id)) mylogger.info("pid is: %d" % (process_id)) test_mysql()
def __init__(self, id, host, port, name, password, db_name): self.id = id self.logger = logger.get_logger() self._is_used = False connections.Connection.__init__(self, host = host, user = name, passwd = password, db = db_name, port = port, connect_timeout = DbConnection.timeout, \ charset = "utf8", use_unicode = False) self._is_alive = True
def connect(self): if self.protocol == 'tcp': self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) elif self.protocol == 'udp': self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._socket.connect((self.host, self.port)) if self.debug and logger.is_debug(): ip, port = self._socket.getsockname() logger.get_logger().debug('address = %s:%d' % (ip, port)) if self.welcome: type = Messager.receive_msg_type(self._socket) if type != MESSAGE_TYPE_WELCOME: raise Exception('Imcomplete message') if self.debug and logger.is_debug(): logger.get_logger().debug('S.WELCOME')
from geopy import distance from logger import get_logger from src.schemas.car_status import CarStatusTypes log = get_logger(__file__) MILLISECS = 1000.0 SECS_IN_HOUR = 3600.0 class Car(object): def __init__(self, car_coordinates): self._cur_timestamp = car_coordinates['timestamp'] self._cur_loc = (car_coordinates['location']['long'], car_coordinates['location']['lat']) self._index = car_coordinates['carIndex'] self._distance_travelled = 0 self._cur_speed = 0 self._cur_position = None @property def current_position(self): return self._cur_position @current_position.setter def current_position(self, value): self._cur_position = value @property def distance_travelled(self):
from logger import get_logger import math import pathlib import fire import googlemaps import pandas as pd from tqdm import tqdm logging = get_logger("pincode_generation_logs") logging.info("Logger Setup Complete") def send_reverese_geocoding_request(lat, lng, gmaps): """Responsible for sending the request to GMaps API to reverser geocode Args: lat (float): Latitude lng (float): Longitude gmaps (Object): gmaps object with declared API key Returns: dict: Response recieved from API, bool: Whether we could connect to API """ valid_data_received = True try: reverse_geocode_result = gmaps.reverse_geocode((lat, lng)) except: logging.warning(f"Failure in reaching the API - {(lat,lng)}")
# -*- coding: utf-8 -*- """HTTP/2 client.""" import ssl from hyper import HTTPConnection from logger import get_logger __author__ = 'Ishwarachandra Gowtham' __email__ = '*****@*****.**' __version__ = '1.0' LOGGER = get_logger() class Http2Client: """HTTP/2 client class.""" def __init__(self, host, port, certificate, cert_key, cert_password=None): """ Initialization method. :param host: str Host URI to connect to. :param certificate: str Name along with path of the client certificate file. :param cert_key: str Name along with path of the client certificate key. :param cert_password: str Password for the client certificate, if any. """ self._host = host self._port = port
def __init__(self) -> None: super().__init__(name="Transmit") self.log = logger.get_logger("bus.transmit")
#!/usr/bin/env python import os import logger LOGGER = logger.get_logger() BASEDIR = os.path.dirname(os.path.realpath(__file__)) LOGGER.debug("BASEDIR: %s" % BASEDIR) # GO_REPLAY_PATH = os.getenv("GO_REPLAY_PATH", "/usr/local/bin") GO_REPLAY_PATH = os.getenv("GO_REPLAY_PATH", "../../local/bin") GO_REPLAY_FILENAME = "goreplay" GO_REPLAY_FULLPATH = os.path.join(GO_REPLAY_PATH, GO_REPLAY_FILENAME) LOGGER.debug("GO_REPLAY_PATH: %s", GO_REPLAY_PATH) LOGGER.debug("GO_REPLAY_FILENAME: %s", GO_REPLAY_FILENAME) LOGGER.debug("GO_REPLAY_FULLPATH: %s", GO_REPLAY_FULLPATH) CONFIGURATION_PATH = os.getenv("CONFIGURATION_PATH", "/etc/cloner") CONFIGURATION_CLONER_FILENAME = "cloner.json" CONFIGURATION_CLONER_FULLPATH = os.path.join(CONFIGURATION_PATH, CONFIGURATION_CLONER_FILENAME) CONFIGURATION_CLONER_SERVICE_FILENAME = "cloner-service.json" CONFIGURATION_CLONER_SERVICE_FULLPATH = os.path.join( CONFIGURATION_PATH, CONFIGURATION_CLONER_SERVICE_FILENAME) LOGGER.debug("CONFIGURATION_PATH: %s", CONFIGURATION_PATH) LOGGER.debug("CONFIGURATION_CLONER_FILENAME: %s", CONFIGURATION_CLONER_FILENAME)
import sys import boto3 import fiona from logger import get_logger LOGGER = get_logger(__name__) s3_uri = sys.argv[1] zipped = sys.argv[2] s3 = boto3.client("s3", region_name="us-east-1") if zipped: s3_uri = f"zip+{s3_uri}" with fiona.open(s3_uri) as src: driver = src.driver LOGGER.debug(driver)
import sqlite3 import pandas as pd import os cur_dir = os.path.dirname(os.path.abspath(__file__)) import logger logging = logger.get_logger('logger', 'log.log') class DataBaseConnection(): """ @class Establishes a Database connection to a database. If the database does not exist a new database will be created. Attributes: ----------- database_name: str Name of the database that needs to be created or connected to conn: sqlite connect object Internal variable auto initialized cursor: sqlite cursor object Internal variable auto initialized Methods: -------- execute_query(query, return_results=False)
import requests import string import telebot.types as types import argparse import logger import json import unidecode import random from time import sleep from persistence import * from persistence import tools import os import PrettyUptime import webhook LOG = logger.get_logger('LaVidaModerna_Bot') REMOVE_CHARS = string.punctuation + string.whitespace TELEGRAM_INLINE_MAX_RESULTS = 48 _ENV_TELEGRAM_BOT_TOKEN = "TELEGRAM_BOT_TOKEN" _ENV_TELEGRAM_USER_ALIAS = "TELEGRAM_USER_ALIAS" _ENV_SQLITE_FILE = 'SQLITE_FILE' _ENV_MYSQL_HOST = 'MYSQL_HOST' _ENV_MYSQL_PORT = 'MYSQL_PORT' _ENV_DATA_JSON = 'DATA_JSON' _ENV_LOGGING_FILE = 'LOGFILE' _ENV_WEBHOOK_HOST = 'WEBHOOK_HOST' _ENV_WEBHOOK_PORT = 'WEBHOOK_PORT' _ENV_WEBHOOK_LISTEN = 'WEBHOOK_LISTEN' _ENV_WEBHOOK_LISTEN_PORT = 'WEBHOOK_LISTEN_PORT'
# import datetime, re from logger import get_logger from telebot import types import common from common import bot from icons import * from statistics import * from commands import COMMANDS import keyboards as kb import callbacks as cb import helpers as hlp log = get_logger("bot." + __name__) # # Battle check # (war chat keyboard action) # @bot.callback_query_handler(func=lambda call: call.data in kb.CHECK_OPTIONS) def battle_check_user(call): # print("battle_check_user") # print(call) message_id = call.inline_message_id user = User(call.from_user.id, call.from_user.first_name, call.from_user.username) userChoice = call.data log.debug("%s is trying to vote for battle (%s)" %
momentum = args.momentum out_path = args.out_path label_type = args.label_type use_crf = args.use_crf use_lm = args.use_lm use_elmo = args.use_elmo use_crf = io_utils.parse_bool(use_crf) use_lm = io_utils.parse_bool(use_lm) use_elmo = io_utils.parse_bool(use_elmo) lm_loss = args.lm_loss lm_mode = args.lm_mode bucket_auxiliary = args.bucket_auxiliary bucket_main = args.bucket_main label_bucket = dict(zip(label_type, [bucket_auxiliary, bucket_main])) logger = logger.get_logger("Hierarchical-Shared Model") logger.info("Use Language Model: %s" % use_lm) if use_lm: logger.info("Language Model mode: %s" % lm_mode) logger.info("Use CRF: %s" % use_crf) logger.info("Use ELMo: %s" % use_elmo) embedd_dict, embedd_dim = embedding.load_embedding_dict(embedding_path) logger.info("Creating Word2Indexs") word_word2index, char_word2index, label_word2index_list, = \ io_utils.create_word2indexs(word2index_path, train_path, label_type, test_paths=[[dev, test] for dev, test in zip(dev_path, test_path)], embedd_dict=embedd_dict, max_vocabulary_size=60000) logger.info("Word Word2Index Size: %d" % word_word2index.size()) logger.info("Character Word2Index Size: %d" % char_word2index.size()) for i in range(len(label_word2index_list)):
#!/bin/python3 import os.path import json import logger log = logger.get_logger() class bot_config: def __init__(self, filename): self.config_file = filename self._init_vars() if not os.path.exists(filename): log.error(f'could not find congif file: [{filename}]') return self.load_config() def _init_vars(self): self.token = None self.name = None self.polling_period = None def save_config(self, name='<Bot Name Here>', token='<Bot Token Here>', polling_period=0.5): config = {'name' : name, \ 'token' : token, \ 'polling_period_sec' : str(polling_period)}
import re import os.path as osp import matplotlib.pyplot as plt import logging from logger import get_logger log = get_logger(level=logging.DEBUG) def get_image_regex_pattern(extension=()): ''' Returns a regex object useful for grabbing files with image filename extensions ''' if extension == (): return re.compile(r".*\.(tif|tiff|jpg|jpeg|png)") patter_str = r".*\.(" + extension[0] for ext in extension[1:]: pattern_str += "|" + ext patter_str += ")" return re.compile(patter_str) def create_name_from_path(file_path, pre_ext=[], out_dir=False): ''' Inserts custom tags seperated by dots between filename and extension ''' if type(pre_ext) != list: pre_ext = [pre_ext]
#!/usr/bin/env python """Represents a results of a check against a target system.""" import logger LOGGER = logger.get_logger(__name__) class CheckResult: """Represents a results of a check against a target system.""" def __init__(self, check_obj, result): self.node_id = check_obj.node_id self.metric_id = check_obj.metric_id self.target = check_obj.target self.check_module = check_obj.check_module self.check_type = check_obj.check_type self.credential = check_obj.credential self.parameters = check_obj.parameters self.response = result return
Jie Li, [email protected] """ import time import pytz import utils import dump import slurm import parse import logger import psycopg2 import schedule from datetime import datetime log = logger.get_logger(__name__) def monitor_slurm(): """monitor_slurm Monitor Slurm Monitor Slurm Metrics """ connection = utils.init_tsdb_connection() node_id_mapping = utils.get_node_id_mapping(connection) os_idrac_hostname_mapping = utils.get_os_idrac_hostname_mapping() slurm_config = utils.get_config('slurm_rest_api') #Schedule fetch slurm schedule.every().minutes.at(":00").do(fetch_slurm, slurm_config, connection, node_id_mapping,
"""Faucet-specific topology module""" import copy import os import yaml from base_gateway import BaseGateway import logger from env import DAQ_RUN_DIR, DAQ_LIB_DIR LOGGER = logger.get_logger('topology') class FaucetTopology: """Topology manager specific to FAUCET configs""" MAC_PREFIX = "@mac:" DNS_PREFIX = "@dns:" CTL_PREFIX = "@ctrl:" INST_FILE_PREFIX = DAQ_RUN_DIR BROADCAST_MAC = "ff:ff:ff:ff:ff:ff" IPV4_DL_TYPE = "0x0800" ARP_DL_TYPE = "0x0806" LLDP_DL_TYPE = "0x88cc" PORT_ACL_NAME_FORMAT = "dp_%s_port_%d_acl" DP_ACL_FILE_FORMAT = "dp_port_acls.yaml" PORT_ACL_FILE_FORMAT = os.path.join("port_acls", "dp_%s_port_%d_acl.yaml") TEMPLATE_FILE_FORMAT = os.path.join(INST_FILE_PREFIX, "acl_templates", "template_%s_acl.yaml") FROM_ACL_KEY_FORMAT = "@from:template_%s_acl" TO_ACL_KEY_FORMAT = "@to:template_%s_acl"
parser.add_argument("--proxy_hostname", type=str) parser.add_argument("--proxy_port", type=str) parser.add_argument("--proxy_username", type=str) parser.add_argument("--proxy_password", type=str) parser.add_argument("--aad_client_id", help="AAD client id") parser.add_argument("--aad_secret", help="AAD secret") parser.add_argument("--aad_tenant_id", help="AAD tenant id") parser.add_argument("--storage_conn_str", help="conn str of storage blob to store ownership and checkpoint data") parser.add_argument("--storage_container_name", help="storage container name to store ownership and checkpoint data") parser.add_argument("--uamqp_logging_enable", help="uamqp logging enable", action="store_true") parser.add_argument("--print_console", help="print to console", action="store_true") parser.add_argument("--log_filename", help="log file name", type=str) args = parser.parse_args() starting_position = parse_starting_position(args) LOGGER = get_logger(args.log_filename, "stress_receive_async", level=logging.INFO, print_console=args.print_console) LOG_PER_COUNT = args.output_interval start_time = time.perf_counter() recv_cnt_map = defaultdict(int) recv_cnt_iteration_map = defaultdict(int) recv_time_map = dict() azure_metric_monitor = AzureMonitorMetric("Async EventHubConsumerClient") class EventHubConsumerClientTest(EventHubConsumerClient): async def get_partition_ids(self): if args.partitions != 0: return [str(i) for i in range(args.partitions)] else:
# coding: utf-8 """ some methods about salt """ # need super privileges import salt.config import salt.key import salt.client import salt.wheel import salt.runner from salt.exceptions import SaltClientError from logger import get_logger salt_logger = get_logger('salt') def get_minion_status(salt_client, master_opts): """Return the status of all known salt minions""" ret = {} try: minions = salt_client.cmd('*', 'test.ping', timeout=master_opts.get('timeout')) except SaltClientError as client_error: salt_logger.exception('salt-master is down, Traceback info:\n %s', client_error) return ret keys = get_all_keys(master_opts) # keys == {'minions_rejected': [], 'minions_denied': [], 'minions_pre': [], 'minions': []} # ret['all_minions'] = sorted(keys.get(minions)) ret['up'] = sorted(minions)
import smtplib import os import sys import logger from datetime import datetime from bs4 import BeautifulSoup from email.mime.text import MIMEText # General globals GAME = 'BL3' # BL2, BL3 SHIFT_CODES_URL = 'https://shift.orcicorn.com/shift-code/' REWARDS_URL = 'https://shift.gearboxsoftware.com/rewards' FILE = '{}_shift_codes.txt'.format(GAME) LOGGER = logger.get_logger(os.path.basename(__file__)) # SMTP globals SMTP_PORT = 465 SMTP_SERVER = os.environ['SMTP_SERVER'] SMTP_USER = os.environ['SMTP_USER'] SMTP_PASSWORD = os.environ['SMTP_PASSWORD'] RECIPIENTS = os.environ['SHIFT_CODE_RECIPIENTS'] # Email addresses, semicolon delimited def get_new_codes(): """Scrapes SHIFT_CODES_URL to see if new codes are available. Returns: list: A list of new codes. Empty if no codes are found. """
import time import numpy as np import datetime from logger import get_logger from train_dssm import train, build_estimator, predict date = datetime.datetime.today() now_dir = 'datepart=' + date.strftime('%Y%m%d') log_file_name = os.path.basename(__file__).split('.', 1)[0] + '.log' # 当日志文件大小小于1M时,则以 if os.path.exists(log_file_name) is False or os.path.getsize( log_file_name) / 1024 / 1024 < 1: logger = get_logger(log_file_name, mode='a') else: # 否则删除以前的日志 logger = get_logger(log_file_name) LOCAL_TRAIN_DIR = '/dssm_data/train/' LOCAL_VALID_DIR = '/dssm_data/valid/' LOCAL_PREDICT_DIR = '/dssm_data/predict/' LOCAL_DICT_DIR = '/dssm_data/vocab/' LOCAL_TRAIN_NUM_DIR = '/dssm_data/train_num/' LOCAL_MODEL_DIR = '/cdssm/model/' LOCAL_DSSM_RESULT_DIR = '/cdssm/dssm_result/' if __name__ == '__main__': all_start = time.time()
import asyncio import os from pyppeteer import launch from pyppeteer.page import Page from logger import get_logger logger = get_logger(__name__) path_to_images = os.path.join("/var", "tmp", "screenshots") async def make_screenshot(img_id: str, url: str, actions: list): args = { "headless": True, } browser = await launch(args) page = await browser.newPage() logger.info("Browser have been loaded with args %s." % str(args)) await page.goto(url) logger.info("Page %s have been loaded." % url) screenshot_path = os.path.join(path_to_images, f"{img_id}.png") asyncio.create_task( do_actions(actions=actions, page=page, img_path=screenshot_path)) async def do_actions(actions: list, page: Page, img_path: str): for a in actions: # === configuring page === if a["action"] == "set_resolution": await page.setViewport({
import sys import datetime import traceback from machine.db.MongoDB import MongoDB # For Database Processing from machine.okcoin import OKCoin # For Order Processing from pusher import telegram from logger import get_logger from strategy.base_strategy import Strategy logging = get_logger("step_trade") class StepTrade(Strategy): def __init__(self, machine=None, db_handler=None, strategy=None, currency_type=None, pusher=None): if machine is None or db_handler is None or currency_type is None or strategy is None or pusher is None: raise Exception("Need to machine, db, currency type, strategy") #if isinstance(machine, KorbitMachine): # logging.info("Korbit Machine") # self.currency_type = currency_type+"_krw" #elif isinstance(machine, CoinoneMachine): # logging.info("Coinone Machine") if isinstance(machine, OKCoin): logging.info("OKCoin Machine") self.currency_type = currency_type + "_krw" self.machine = machine self.pusher = pusher self.db_handler = db_handler
import typing as t from queue import Empty as QueueEmptyError import logger from common import BusMessage, BusNode, BusProperty, PublishMessage, SubscribeMessage from modules import ( ATTR_NODES, ATTR_QCORE, ATTR_QPUBLISH, ATTR_QRECEIVE, ATTR_QRESEND, ATTR_QSUBSCRIBE, ATTR_QTRANSMIT, ) log = logger.get_logger("bus") # Communication module queues receive_queue: "mproc.Queue[BusMessage]" = mproc.Queue() transmit_queues: t.Dict[str, "mproc.Queue[BusMessage]"] = {} # Interface module queues publish_queue: "mproc.Queue[BusMessage]" = mproc.Queue() subscribe_queues: t.Dict[str, "mproc.Queue[BusMessage]"] = {} # Data tree data_tree: t.Dict[str, BusNode] = {} # Message Bus threads _thread_receive: threading.Thread = None # type: ignore _thread_transmit: threading.Thread = None # type: ignore
"""Gateway module for device testing""" import datetime import os import shutil from abc import ABC, abstractmethod import logger from env import DAQ_RUN_DIR LOGGER = logger.get_logger('gateway') class BaseGateway(ABC): """Gateway collection class for managing testing services""" GATEWAY_OFFSET = 0 DUMMY_OFFSET = 1 TEST_OFFSET_START = 2 NUM_SET_PORTS = 6 SET_SPACING = 10 _PING_RETRY_COUNT = 5 TEST_IP_FORMAT = '192.168.84.%d' def __init__(self, runner, name, port_set): self.name = name self.runner = runner assert port_set > 0, 'port_set %d, must be > 0' % port_set self.port_set = port_set self.fake_target = None
# -*- coding: utf-8 -*- import os import sys import unittest from proxy_web_api import get_resp_field, call_proxy from utils import generate_random_card_data, generate_random_number import logger log = logger.get_logger('/tmp/web_api_tests-%s.log' % os.environ['USER']) SERVER_URI = 'http://localhost:17117/' def log_func_context(func): def inner(*args, **kwargs): log.debug('---- Start [%s] ----', func.func_name) result = func(*args, **kwargs) log.debug('---- Start [%s] ----', func.func_name) return result return inner class TestBaseWebApi(unittest.TestCase): ''' tokenize_card, detokenize_card, remove_card, etc. ''' # TODO add error test
from pony import orm import logger from models import User from settings import DEFAULT_TOPIC log = logger.get_logger('User CRUD') @orm.db_session def get(user_id: int) -> User: return User[user_id] @orm.db_session def create(user_id: int, topic: str = DEFAULT_TOPIC) -> User: user = User(id=user_id, current_topic=topic) log.debug(f'User was created[{user.id}]') return user @orm.db_session def get_or_create(user_id: int) -> User: try: return get(user_id) except orm.ObjectNotFound: return create(user_id) @orm.db_session def update(user_id: int, topic: str) -> User:
# encoding: utf-8 from __future__ import unicode_literals from selenium import webdriver from bs4 import BeautifulSoup from logger import get_logger logger = get_logger() class ScraperExeception(Exception): pass class Scraper(): def __init__(self): # Initialize a selenium browser when the class is created. self.browser = webdriver.PhantomJS() # Set the cookies by visiting base url first. print self.base_url if self.base_url: self.browser.get(self.base_url) def get_page(self, url, max_retries=3): # Attempts to try to get the page (retries 3 times max) # and turn it into a (beautiful) soup for easy parsing. retries = 0 while retries < max_retries: try: logger.debug("Get -> {}".format(url)) self.browser.get(url)
from pyspark import SparkConf, SparkContext, SQLContext import logger TEST_NAME = 'spk_single_df_single_file_cnt' LOGGER = logger.get_logger(TEST_NAME) # Specify some constants URLPATH = "s3a://dask-avro-data/application-data/app-100096.avro" # Start LOGGER.info('START: Creating spark conf') Sconf = SparkConf() sc = SparkContext(appName="my_test", conf=Sconf) sqlContext = SQLContext(sparkContext=sc) LOGGER.info('FINISH: Finished creating spark conf') LOGGER.info('START: Creating spark dataframe') df = sqlContext.read.format("com.databricks.spark.avro").load(URLPATH) LOGGER.info('FINISH: Spark dataframe created') LOGGER.info('START: Starting count') cnt = df.count() LOGGER.info('FINISH: Count is %s', cnt) sc.stop()
from bot import ShrimpBot from utils import Timer from const import Settings from logger import get_logger timer = Timer() logger = get_logger("shrimp_bot") ShrimpBot(admin=Settings.Admins, logger=logger).run(Settings.token) logger.info("Run time - %dh : %dm : %ds" % timer.end())
def __init__(self) -> None: super().__init__(name="Receive") self.log = logger.get_logger("bus.receive")
def get_logger(self): """ :returns: The python logger root for the framework. """ return logger.get_logger()