def recv_fiber(self):
while self._running:
try:
msg = self.receive_next_msg()
if msg == None:
raise engine.EngineError('Received nothing')
if isinstance(msg, Query):
raise engine.EngineError('Recieved unexpected Query')
elif isinstance(msg, Close):
self.close()
elif isinstance(msg, Answer):
if self._query_list.get(msg.qid):
self._query_list[msg.qid].set(msg)
del self._query_list[msg.qid]
else:
logger.get_logger().error('proxy %s get unexpected answer %s. qid may be removed for timeout', self.end_point, msg)
else:
raise engine.EngineError('Unknown message')
except ProxyError as ex:
if self._query_list.get(ex.qid):
msg = Answer(ex.qid, ex.status, ex.params)
self._query_list[ex.qid].set(msg)
del self._query_list[ex.qid]
except Exception as ex:
self.close()
break
def receive_next_msg(self):
if self.protocol == 'tcp':
message = Messager.receive_msg(self._socket)
elif self.protocol == 'udp':
pkt, address = self.unitive_recvfrom(65535)
message = Messager.message_from_data(pkt)
if self.debug and logger.is_debug():
if isinstance(message, Query):
logger.get_logger().debug('SQ:%d method=%s', message.qid, message.method)
elif isinstance(message, Answer):
logger.get_logger().debug('SA:%d status=%d', message.qid, message.status)
if not message:
raise Exception('Recieved Unknown packet')
if isinstance(message, Answer):
qid = message.qid
if self.pending.get(qid):
del self.pending[qid]
status = message.status
if status:
raise ProxyError(qid, status, message.data)
return message;
def __init__(self, setting):
self.setting = setting
self.debug = setting.get('debug', False)
self._running = True
self._adapters = {}
self._proxies = {}
if self.debug and logger.is_debug():
logger.get_logger().debug("application setting %s", setting)
def send_quest(self, qid, method, params, twoway=True):
if qid:
self.pending[qid] = time.time()
if self.debug and logger.is_debug():
logger.get_logger().debug('CQ:%d service=%s method=%s', qid, self.service, method)
self.unitive_send(Messager.data_for_query(qid, self.service, method, params))
return qid
def respond(self, rpc_controller, echo_string, callback):
""" called by RpcChannel.receive when a complete request reached.
"""
logger.get_logger('EchoClient').debug('EchoClient.respond')
if self.streamout:
self.streamout.write(echo_string.message)
if callback:
callback()
def echo(self, rpc_controller, echo_string, callback):
""" called by RpcChannel.receive when a complete request reached.
"""
logger.get_logger('EchoService').info('echo service is called')
echo_string.message = echo_string.message
client_stub = IEchoClient_Stub(rpc_controller.rpc_channel)
client_stub.respond(rpc_controller, echo_string, callback=None)
if callback:
callback()
def servant_worker(self):
while self._running:
query = self._query_queue.get()
if not isinstance(query, Query):
logger.get_logger().error('invalid query %s', query)
continue
if not self._wildcard_servant:
self._handle_normal_servant(query)
else:
self._handle_wildcard_servant(query)
def __init__(self, args):
self.fs = FileSystem()
get_logger('app', 'logs/app.log', level=logging.DEBUG)
self.storage_dir = args.storage_option
logging.getLogger('app').debug('App started')
self.fs.apply_callback_to_file_iterator(self.storage_dir, self.indexer)
logging.getLogger('app').debug('App stoped normaly')
def _fail_quests(self):
exinfo = self._build_except_info()
import traceback
traceback.print_stack()
logger.get_logger().error('proxy %s get error %s, fail current querys', self.end_point, exinfo)
curr_req_items = self._query_list.items()
self._query_list = {}
for qid, result in curr_req_items:
msg = Answer(qid, 1, exinfo)
#AsyncResult set should cause context switch
result.set(msg)
def close(self):
logger.get_logger().info('close')
if self._recv_thread:
gevent.kill(self._recv_thread)
self._recv_thread = None
if self._socket:
self._socket.close()
self._socket = None
self._connected = False
self._fail_quests()
def serve_forever(self):
for (name, adapter) in self._adapters.items():
adapter.activate()
self.start_time = time.strftime('%Y-%m-%d %H:%M:%S')
while self._running:
# periodic log info that engine is running
p = {}
p['start_time'] = self.start_time
p['adapters'] = self._adapters
logger.get_logger().critical("THROB %s", str(p))
gevent.sleep(60)
def snapshot(prices):
try:
conn = sqlite3.connect("intraday.db")
cursor = conn.cursor()
cursor.execute("PRAGMA synchronous = OFF")
cursor.execute("PRAGMA journal_mode = OFF")
get_logger().debug('Database 연결 성공')
except:
get_logger().warn('Database 연결 실패')
raise
URL = "http://finance.daum.net/xml/xmlallpanel.daum?stype=P&type=S"
global is_market_finished
# parse(r.headers['Date']) 할 수 있으나, 다음 웹서버 시간이 부정확
time = datetime.datetime.now()
r = requests.get(URL)
r.encoding = 'UTF-8'
# 정규표현식으로 파싱(키값에 따옴표가 없어서 JSON 파싱 불가)
# example : , {code:"095570",name :"AJ네트웍스",cost :"34,650",updn :"▲100",rate :"+0.29%"}
pattern = "code:\"(.+)\",name :\"(.+)\",cost :\"(.+)\",updn"
rep = re.compile(pattern)
for line in r.text.splitlines():
if '장종료' in line:
is_market_finished = True
if 'code' not in line:
continue
# print(line)
match = rep.search(line)
symbol = match.group(1)
name = match.group(2)
price = match.group(3).replace(',', '')
# 직전 가격과 같으면 INSERT 생략
if(prices[symbol] == price):
continue
prices[symbol] = price
row = cursor.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='KS_{}'".format(symbol)).fetchone()
if row is None:
cursor.execute("CREATE TABLE KS_{} (datetime INTEGER, price INTEGER)".format(symbol))
sql = "INSERT INTO KS_{} (datetime, price) VALUES (CURRENT_TIMESTAMP, '{}')"
cursor.execute(sql.format(symbol, price))
conn.close()
def _handle_wildcard_servant(self, query):
if self.debug and logger.is_debug():
logger.get_logger().debug('handle_wildcard_servant: %s', query)
try:
result = self._wildcard_servant(query.service, query.method, query.params)
if query.qid:
query.inbox.put(Answer(query.qid, 0, result))
except proxy.ProxyError as ex:
if query.qid:
query.inbox.put(Answer(ex.qid, ex.status, ex.params))
except:
if query.qid:
query.inbox.put(Answer(query.qid, 1, self._make_exception_params(query)))
def snapshot(prices):
with open('config.json') as file:
config = json.load(file)['mysql']
try:
conn = pymysql.connect(host=config['host'],
port=config['port'],
user=config['user'],
password=config['password'],
db=config['db'])
cursor = conn.cursor()
get_logger().debug('Database 연결 성공')
except pymysql.err.OperationalError:
get_logger().warn('Database 연결 실패')
raise
URL = "http://finance.daum.net/xml/xmlallpanel.daum?stype=P&type=S"
sql = "INSERT INTO `price_intraday` (`symbol`, `price`) VALUES (%s, %s)"
# parse(r.headers['Date']) 할 수 있으나, 다음 웹서버 시간이 부정확
time = datetime.datetime.now()
r = requests.get(URL)
r.encoding = 'UTF-8'
# 정규표현식으로 파싱(키값에 따옴표가 없어서 JSON 파싱 불가)
# example : , {code:"095570",name :"AJ네트웍스",cost :"34,650",updn :"▲100",rate :"+0.29%"}
pattern = "code:\"(.+)\",name :\"(.+)\",cost :\"(.+)\",updn"
rep = re.compile(pattern)
for line in r.text.splitlines():
if 'code' not in line: continue
# print(line)
match = rep.search(line)
symbol = match.group(1)
name = match.group(2)
price = match.group(3).replace(',', '')
# 직전 가격과 같으면 INSERT 생략
if(prices[symbol] == price):
continue
prices[symbol] = price
cursor.execute(sql, (symbol, price))
conn.commit()
conn.close()
def _method_not_found(self, called_method, query):
logger.get_logger().error("method %s not found", called_method)
"""
query method not found in servant
"""
exdict = {}
exdict['exception'] = 'MethodNotFound'
exdict['code'] = 1
exdict['message'] = "servant %s do no have method %s in adapter %s" % (query.service, called_method, self._endpoint)
exdict['raiser'] = self._endpoint
if query.qid:
query.inbox.put(Answer(query.qid, 100, exdict))
else:
logger.get_logger().warning("%s.%s %s", query.service, called_method, exdict)
return
def main():
global is_market_finished
prices = defaultdict(list)
interval = 5 if len(sys.argv) == 1 else int(sys.argv[1])
#for i in range(2):
while True:
#if datetime.datetime.now().hour < 9: continue
snapshot(prices)
if(is_market_finished):
get_logger().info("장이 종료되었습니다.")
break
time.sleep(interval)
def get_pid(process_name):
global pids
log_handler = logger.get_logger("pid.log")
a = os.popen('ps -ef | grep {}'.format(process_name)).readlines()
filter_1 = "grep"
filter_2 = "python"
for x in a:
try:
if (x.find(filter_1) != -1) or (x.find(filter_2) != -1) :
#log_handler.info("escaped")
pass
else:
#log_handler.info(x)
add_to_dict(x)
#pids.append(int(x[10:15]))
#process[int(x[10:15])] = x[49:]
except Exception as inst:
#print inst
pass
#return the list if not null
if not pids:
return -1
else:
return pids
def __init__(self, args):
self.field_types = {
"sys_uptime": int,
"us_atten": float,
"update_time": int,
"ds_speed": int,
"ds_atten": float,
"wan_boot_time": int,
"ds_margin": float,
"us_margin": float,
"wan_uptime": int,
"sys_boot_time": int,
"us_speed": int,
}
# Get the logger instance
self.logger = logger.get_logger()
self.logger.info("Router database logger starting up")
# Connect to the database
self.logger.info(
"Connecting to influxDB database %s at host %s port %d", args.influx_db, args.influx_ip, args.influx_port
)
self.influx_client = InfluxDBClient(host=args.influx_ip, port=args.influx_port, database=args.influx_db)
# Create an MQTT client instance, register connect and message callbacks and connect
self.mqtt_client = paho.mqtt.client.Client()
self.mqtt_client.on_connect = self.on_connect
self.mqtt_client.on_message = self.on_message
self.logger.info("Connecting to MQTT server at address %s:%d", args.mqtt_ip, args.mqtt_port)
self.mqtt_client.connect(args.mqtt_ip, args.mqtt_port, args.mqtt_timeout)
def main():
with open('config.json') as file:
config = json.load(file)['rabbitmq']
credentials = pika.PlainCredentials(config['user'], config['passwd'])
parameters = pika.ConnectionParameters(config['host'], config['port'], '/', credentials)
try:
#get_logger().debug("Connecting to RabbitMQ ({}:{})".format(config['host'], config['port']))
connection = pika.BlockingConnection(parameters)
except pika.exceptions.ConnectionClosed:
get_logger().warn("RabbitMQ Connection Error ({}:{})".format(config['host'], config['port']))
sys.exit(1)
channel = connection.channel()
channel.exchange_declare(exchange="price", type="topic")
stocks = defaultdict(list)
URL = "http://finance.daum.net/xml/xmlallpanel.daum?stype=P&type=S"
# parse(r.headers['Date']) 할 수 있으나, 다음 웹서버 시간이 부정확
time = datetime.datetime.now()
r = requests.get(URL)
r.encoding = 'UTF-8'
# 정규표현식으로 파싱(키값에 따옴표가 없어서 JSON 파싱 불가)
# example : , {code:"095570",name :"AJ네트웍스",cost :"34,650",updn :"▲100",rate :"+0.29%"}
pattern = "code:\"(.+)\",name :\"(.+)\",cost :\"(.+)\",updn"
rep = re.compile(pattern)
for line in r.text.splitlines():
if 'code' not in line: continue
# print(line)
match = rep.search(line)
symbol = match.group(1)
name = match.group(2)
price = match.group(3).replace(',', '')
channel.basic_publish(exchange='price',
routing_key=symbol,
body=price)
# 헤더에 시간 추가
connection.close()
def __init__(
self,
host="127.0.0.1",
port=8000,
server_cls=None,
handler_cls=WebTestRequestHandler,
use_ssl=False,
key_file=None,
certificate=None,
encrypt_after_connect=False,
router_cls=Router,
doc_root=os.curdir,
routes=None,
rewriter_cls=RequestRewriter,
bind_hostname=True,
rewrites=None,
latency=None,
config=None,
):
if routes is None:
routes = default_routes.routes
self.host = host
self.router = router_cls(doc_root, routes)
self.rewriter = rewriter_cls(rewrites if rewrites is not None else [])
self.use_ssl = use_ssl
self.logger = get_logger()
if server_cls is None:
server_cls = WebTestServer
if use_ssl:
if key_file is not None:
assert os.path.exists(key_file)
assert certificate is not None and os.path.exists(certificate)
try:
self.httpd = server_cls(
(host, port),
handler_cls,
self.router,
self.rewriter,
config=config,
bind_hostname=bind_hostname,
use_ssl=use_ssl,
key_file=key_file,
certificate=certificate,
encrypt_after_connect=encrypt_after_connect,
latency=latency,
)
self.started = False
_host, self.port = self.httpd.socket.getsockname()
except Exception:
self.logger.error("Init failed! You may need to modify your hosts file. Refer to README.md.")
raise
def init_service():
global mac_address, process_id, mongo_db, mongo_userprofile, mylogger
mac_address = getHwAddr('eth0')
process_id = os.getpid()
mylogger = logger.get_logger('/opt/research/NginxUwsgi/log/dmp_%d.log' %(process_id))
mongo_db = Connection(host="192.168.86.18", port=27020).dmp
mongo_userprofile = mongo_db.user_profile
def __init__(self):
self.logger = logger.get_logger('virtshell_dispatcher')
self.listeners_tasks = {}
self.tasks_folder = "./tasks"
self.special_files = ["__init__.py", "__pycache__"]
self.task_execute_method = "main"
self.load_plugin_tasks()
self.logger.info("service started...")
def init_service():
global mylogger, mongo_client, mongo_db, weibo_redis_clis
process_id = os.getpid()
mylogger = logger.get_logger("/opt/research/NginxUwsgi/log/ovs_proxy_%d.log" % (process_id))
mongo_client = MongoClient("192.168.86.67", 27020)
mongo_db = mongo_client.ovs
weibo_redis_clis = init_redis("nginx02", 6379, 0)
def activate(self):
for i in range(self._servant_worker_num):
gevent.spawn(self.servant_worker)
self._pool = Pool(self._accept_pool_num)
endpoint = self._endpoint
try:
service, proto, host, port = proxy.parse_endpoint(endpoint)
if proto != 'tcp':
raise proxy.ProxyError(1, 'only tcp server supported now')
server = StreamServer((host, int(port)), self.sokect_handler, spawn=self._pool)
if self.debug and logger.is_debug():
logger.get_logger().debug('adapter start %s', endpoint)
self._servers.append(server)
server.start()
except:
logger.get_logger().error('start adapter fail %s', endpoint, exc_info = 1)
def __init__(self, get_stream_api_url, read_token, monitor_cdn_in_api, analyze_thread, extra_streams):
threading.Thread.__init__(self)
self.api_server_model = ApiServerModel(get_stream_api_url, read_token, monitor_cdn_in_api)
self.analyze_thread = analyze_thread
self.stream_db = StreamModel()
self.logger = logger.get_logger()
self.extra_streams = extra_streams
self.api_return_streams = None
self.retrieve_interval = 60
def __init__(self):
threading.Thread.__init__(self)
self.logger = logger.get_logger()
self.obs = None
self.sleep_time = 3
self.analyzer = Analyzer()
self.obs_lock = threading.Lock()
self.stream_model = StreamModel()
self.sample_model = SampleModel()
def write(filename, data):
"""
Write to a file.
"""
logger = get_logger(__name__)
logger.debug('Writing to %s' % filename)
f = open(filename, 'w')
f.write(data)
f.close()
def init_service():
global mylogger
process_id = os.getpid()
mylogger = logger.get_logger('/opt/research/NginxUwsgi/log/pyuwsgi_monitor_%d.log' % (process_id))
mylogger.info("pid is: %d" % (process_id))
test_mysql()
def __init__(self, id, host, port, name, password, db_name):
self.id = id
self.logger = logger.get_logger()
self._is_used = False
connections.Connection.__init__(self, host = host, user = name, passwd = password, db = db_name, port = port, connect_timeout = DbConnection.timeout, \
charset = "utf8", use_unicode = False)
self._is_alive = True
def connect(self):
if self.protocol == 'tcp':
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
elif self.protocol == 'udp':
self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self._socket.connect((self.host, self.port))
if self.debug and logger.is_debug():
ip, port = self._socket.getsockname()
logger.get_logger().debug('address = %s:%d' % (ip, port))
if self.welcome:
type = Messager.receive_msg_type(self._socket)
if type != MESSAGE_TYPE_WELCOME:
raise Exception('Imcomplete message')
if self.debug and logger.is_debug():
logger.get_logger().debug('S.WELCOME')
from geopy import distance
from logger import get_logger
from src.schemas.car_status import CarStatusTypes
log = get_logger(__file__)
MILLISECS = 1000.0
SECS_IN_HOUR = 3600.0
class Car(object):
def __init__(self, car_coordinates):
self._cur_timestamp = car_coordinates['timestamp']
self._cur_loc = (car_coordinates['location']['long'], car_coordinates['location']['lat'])
self._index = car_coordinates['carIndex']
self._distance_travelled = 0
self._cur_speed = 0
self._cur_position = None
@property
def current_position(self):
return self._cur_position
@current_position.setter
def current_position(self, value):
self._cur_position = value
@property
def distance_travelled(self):
from logger import get_logger
import math
import pathlib
import fire
import googlemaps
import pandas as pd
from tqdm import tqdm
logging = get_logger("pincode_generation_logs")
logging.info("Logger Setup Complete")
def send_reverese_geocoding_request(lat, lng, gmaps):
"""Responsible for sending the request to GMaps API to reverser geocode
Args:
lat (float): Latitude
lng (float): Longitude
gmaps (Object): gmaps object with declared API key
Returns:
dict: Response recieved from API,
bool: Whether we could connect to API
"""
valid_data_received = True
try:
reverse_geocode_result = gmaps.reverse_geocode((lat, lng))
except:
logging.warning(f"Failure in reaching the API - {(lat,lng)}")
# -*- coding: utf-8 -*-
"""HTTP/2 client."""
import ssl
from hyper import HTTPConnection
from logger import get_logger
__author__ = 'Ishwarachandra Gowtham'
__email__ = '*****@*****.**'
__version__ = '1.0'
LOGGER = get_logger()
class Http2Client:
"""HTTP/2 client class."""
def __init__(self, host, port, certificate, cert_key, cert_password=None):
"""
Initialization method.
:param host: str
Host URI to connect to.
:param certificate: str
Name along with path of the client certificate file.
:param cert_key: str
Name along with path of the client certificate key.
:param cert_password: str
Password for the client certificate, if any.
"""
self._host = host
self._port = port
def __init__(self) -> None:
super().__init__(name="Transmit")
self.log = logger.get_logger("bus.transmit")
#!/usr/bin/env python
import os
import logger
LOGGER = logger.get_logger()
BASEDIR = os.path.dirname(os.path.realpath(__file__))
LOGGER.debug("BASEDIR: %s" % BASEDIR)
# GO_REPLAY_PATH = os.getenv("GO_REPLAY_PATH", "/usr/local/bin")
GO_REPLAY_PATH = os.getenv("GO_REPLAY_PATH", "../../local/bin")
GO_REPLAY_FILENAME = "goreplay"
GO_REPLAY_FULLPATH = os.path.join(GO_REPLAY_PATH, GO_REPLAY_FILENAME)
LOGGER.debug("GO_REPLAY_PATH: %s", GO_REPLAY_PATH)
LOGGER.debug("GO_REPLAY_FILENAME: %s", GO_REPLAY_FILENAME)
LOGGER.debug("GO_REPLAY_FULLPATH: %s", GO_REPLAY_FULLPATH)
CONFIGURATION_PATH = os.getenv("CONFIGURATION_PATH", "/etc/cloner")
CONFIGURATION_CLONER_FILENAME = "cloner.json"
CONFIGURATION_CLONER_FULLPATH = os.path.join(CONFIGURATION_PATH,
CONFIGURATION_CLONER_FILENAME)
CONFIGURATION_CLONER_SERVICE_FILENAME = "cloner-service.json"
CONFIGURATION_CLONER_SERVICE_FULLPATH = os.path.join(
CONFIGURATION_PATH, CONFIGURATION_CLONER_SERVICE_FILENAME)
LOGGER.debug("CONFIGURATION_PATH: %s", CONFIGURATION_PATH)
LOGGER.debug("CONFIGURATION_CLONER_FILENAME: %s",
CONFIGURATION_CLONER_FILENAME)
import sys
import boto3
import fiona
from logger import get_logger
LOGGER = get_logger(__name__)
s3_uri = sys.argv[1]
zipped = sys.argv[2]
s3 = boto3.client("s3", region_name="us-east-1")
if zipped:
s3_uri = f"zip+{s3_uri}"
with fiona.open(s3_uri) as src:
driver = src.driver
LOGGER.debug(driver)
import sqlite3
import pandas as pd
import os
cur_dir = os.path.dirname(os.path.abspath(__file__))
import logger
logging = logger.get_logger('logger', 'log.log')
class DataBaseConnection():
"""
@class
Establishes a Database connection to a database. If the database does not exist a new database will be created.
Attributes:
-----------
database_name: str
Name of the database that needs to be created or connected to
conn: sqlite connect object
Internal variable auto initialized
cursor: sqlite cursor object
Internal variable auto initialized
Methods:
--------
execute_query(query, return_results=False)
import requests
import string
import telebot.types as types
import argparse
import logger
import json
import unidecode
import random
from time import sleep
from persistence import *
from persistence import tools
import os
import PrettyUptime
import webhook
LOG = logger.get_logger('LaVidaModerna_Bot')
REMOVE_CHARS = string.punctuation + string.whitespace
TELEGRAM_INLINE_MAX_RESULTS = 48
_ENV_TELEGRAM_BOT_TOKEN = "TELEGRAM_BOT_TOKEN"
_ENV_TELEGRAM_USER_ALIAS = "TELEGRAM_USER_ALIAS"
_ENV_SQLITE_FILE = 'SQLITE_FILE'
_ENV_MYSQL_HOST = 'MYSQL_HOST'
_ENV_MYSQL_PORT = 'MYSQL_PORT'
_ENV_DATA_JSON = 'DATA_JSON'
_ENV_LOGGING_FILE = 'LOGFILE'
_ENV_WEBHOOK_HOST = 'WEBHOOK_HOST'
_ENV_WEBHOOK_PORT = 'WEBHOOK_PORT'
_ENV_WEBHOOK_LISTEN = 'WEBHOOK_LISTEN'
_ENV_WEBHOOK_LISTEN_PORT = 'WEBHOOK_LISTEN_PORT'
#
import datetime, re
from logger import get_logger
from telebot import types
import common
from common import bot
from icons import *
from statistics import *
from commands import COMMANDS
import keyboards as kb
import callbacks as cb
import helpers as hlp
log = get_logger("bot." + __name__)
#
# Battle check
# (war chat keyboard action)
#
@bot.callback_query_handler(func=lambda call: call.data in kb.CHECK_OPTIONS)
def battle_check_user(call):
# print("battle_check_user")
# print(call)
message_id = call.inline_message_id
user = User(call.from_user.id, call.from_user.first_name,
call.from_user.username)
userChoice = call.data
log.debug("%s is trying to vote for battle (%s)" %
momentum = args.momentum
out_path = args.out_path
label_type = args.label_type
use_crf = args.use_crf
use_lm = args.use_lm
use_elmo = args.use_elmo
use_crf = io_utils.parse_bool(use_crf)
use_lm = io_utils.parse_bool(use_lm)
use_elmo = io_utils.parse_bool(use_elmo)
lm_loss = args.lm_loss
lm_mode = args.lm_mode
bucket_auxiliary = args.bucket_auxiliary
bucket_main = args.bucket_main
label_bucket = dict(zip(label_type, [bucket_auxiliary, bucket_main]))
logger = logger.get_logger("Hierarchical-Shared Model")
logger.info("Use Language Model: %s" % use_lm)
if use_lm:
logger.info("Language Model mode: %s" % lm_mode)
logger.info("Use CRF: %s" % use_crf)
logger.info("Use ELMo: %s" % use_elmo)
embedd_dict, embedd_dim = embedding.load_embedding_dict(embedding_path)
logger.info("Creating Word2Indexs")
word_word2index, char_word2index, label_word2index_list, = \
io_utils.create_word2indexs(word2index_path, train_path, label_type,
test_paths=[[dev, test] for dev, test in zip(dev_path, test_path)],
embedd_dict=embedd_dict, max_vocabulary_size=60000)
logger.info("Word Word2Index Size: %d" % word_word2index.size())
logger.info("Character Word2Index Size: %d" % char_word2index.size())
for i in range(len(label_word2index_list)):
#!/bin/python3
import os.path
import json
import logger
log = logger.get_logger()
class bot_config:
def __init__(self, filename):
self.config_file = filename
self._init_vars()
if not os.path.exists(filename):
log.error(f'could not find congif file: [{filename}]')
return
self.load_config()
def _init_vars(self):
self.token = None
self.name = None
self.polling_period = None
def save_config(self,
name='<Bot Name Here>',
token='<Bot Token Here>',
polling_period=0.5):
config = {'name' : name, \
'token' : token, \
'polling_period_sec' : str(polling_period)}
import re
import os.path as osp
import matplotlib.pyplot as plt
import logging
from logger import get_logger
log = get_logger(level=logging.DEBUG)
def get_image_regex_pattern(extension=()):
'''
Returns a regex object useful for grabbing files with image filename extensions
'''
if extension == ():
return re.compile(r".*\.(tif|tiff|jpg|jpeg|png)")
patter_str = r".*\.(" + extension[0]
for ext in extension[1:]:
pattern_str += "|" + ext
patter_str += ")"
return re.compile(patter_str)
def create_name_from_path(file_path, pre_ext=[], out_dir=False):
'''
Inserts custom tags seperated by dots between filename and extension
'''
if type(pre_ext) != list:
pre_ext = [pre_ext]
#!/usr/bin/env python
"""Represents a results of a check against a target system."""
import logger
LOGGER = logger.get_logger(__name__)
class CheckResult:
"""Represents a results of a check against a target system."""
def __init__(self, check_obj, result):
self.node_id = check_obj.node_id
self.metric_id = check_obj.metric_id
self.target = check_obj.target
self.check_module = check_obj.check_module
self.check_type = check_obj.check_type
self.credential = check_obj.credential
self.parameters = check_obj.parameters
self.response = result
return
Jie Li, [email protected] """ import time import pytz import utils import dump import slurm import parse import logger import psycopg2 import schedule from datetime import datetime log = logger.get_logger(__name__) def monitor_slurm(): """monitor_slurm Monitor Slurm Monitor Slurm Metrics """ connection = utils.init_tsdb_connection() node_id_mapping = utils.get_node_id_mapping(connection) os_idrac_hostname_mapping = utils.get_os_idrac_hostname_mapping() slurm_config = utils.get_config('slurm_rest_api') #Schedule fetch slurm schedule.every().minutes.at(":00").do(fetch_slurm, slurm_config, connection, node_id_mapping,
"""Faucet-specific topology module"""
import copy
import os
import yaml
from base_gateway import BaseGateway
import logger
from env import DAQ_RUN_DIR, DAQ_LIB_DIR
LOGGER = logger.get_logger('topology')
class FaucetTopology:
"""Topology manager specific to FAUCET configs"""
MAC_PREFIX = "@mac:"
DNS_PREFIX = "@dns:"
CTL_PREFIX = "@ctrl:"
INST_FILE_PREFIX = DAQ_RUN_DIR
BROADCAST_MAC = "ff:ff:ff:ff:ff:ff"
IPV4_DL_TYPE = "0x0800"
ARP_DL_TYPE = "0x0806"
LLDP_DL_TYPE = "0x88cc"
PORT_ACL_NAME_FORMAT = "dp_%s_port_%d_acl"
DP_ACL_FILE_FORMAT = "dp_port_acls.yaml"
PORT_ACL_FILE_FORMAT = os.path.join("port_acls", "dp_%s_port_%d_acl.yaml")
TEMPLATE_FILE_FORMAT = os.path.join(INST_FILE_PREFIX, "acl_templates", "template_%s_acl.yaml")
FROM_ACL_KEY_FORMAT = "@from:template_%s_acl"
TO_ACL_KEY_FORMAT = "@to:template_%s_acl"
parser.add_argument("--proxy_hostname", type=str)
parser.add_argument("--proxy_port", type=str)
parser.add_argument("--proxy_username", type=str)
parser.add_argument("--proxy_password", type=str)
parser.add_argument("--aad_client_id", help="AAD client id")
parser.add_argument("--aad_secret", help="AAD secret")
parser.add_argument("--aad_tenant_id", help="AAD tenant id")
parser.add_argument("--storage_conn_str", help="conn str of storage blob to store ownership and checkpoint data")
parser.add_argument("--storage_container_name", help="storage container name to store ownership and checkpoint data")
parser.add_argument("--uamqp_logging_enable", help="uamqp logging enable", action="store_true")
parser.add_argument("--print_console", help="print to console", action="store_true")
parser.add_argument("--log_filename", help="log file name", type=str)
args = parser.parse_args()
starting_position = parse_starting_position(args)
LOGGER = get_logger(args.log_filename, "stress_receive_async", level=logging.INFO, print_console=args.print_console)
LOG_PER_COUNT = args.output_interval
start_time = time.perf_counter()
recv_cnt_map = defaultdict(int)
recv_cnt_iteration_map = defaultdict(int)
recv_time_map = dict()
azure_metric_monitor = AzureMonitorMetric("Async EventHubConsumerClient")
class EventHubConsumerClientTest(EventHubConsumerClient):
async def get_partition_ids(self):
if args.partitions != 0:
return [str(i) for i in range(args.partitions)]
else:
# coding: utf-8
"""
some methods about salt
"""
# need super privileges
import salt.config
import salt.key
import salt.client
import salt.wheel
import salt.runner
from salt.exceptions import SaltClientError
from logger import get_logger
salt_logger = get_logger('salt')
def get_minion_status(salt_client, master_opts):
"""Return the status of all known salt minions"""
ret = {}
try:
minions = salt_client.cmd('*',
'test.ping',
timeout=master_opts.get('timeout'))
except SaltClientError as client_error:
salt_logger.exception('salt-master is down, Traceback info:\n %s',
client_error)
return ret
keys = get_all_keys(master_opts)
# keys == {'minions_rejected': [], 'minions_denied': [], 'minions_pre': [], 'minions': []}
# ret['all_minions'] = sorted(keys.get(minions))
ret['up'] = sorted(minions)
import smtplib
import os
import sys
import logger
from datetime import datetime
from bs4 import BeautifulSoup
from email.mime.text import MIMEText
# General globals
GAME = 'BL3' # BL2, BL3
SHIFT_CODES_URL = 'https://shift.orcicorn.com/shift-code/'
REWARDS_URL = 'https://shift.gearboxsoftware.com/rewards'
FILE = '{}_shift_codes.txt'.format(GAME)
LOGGER = logger.get_logger(os.path.basename(__file__))
# SMTP globals
SMTP_PORT = 465
SMTP_SERVER = os.environ['SMTP_SERVER']
SMTP_USER = os.environ['SMTP_USER']
SMTP_PASSWORD = os.environ['SMTP_PASSWORD']
RECIPIENTS = os.environ['SHIFT_CODE_RECIPIENTS'] # Email addresses, semicolon delimited
def get_new_codes():
"""Scrapes SHIFT_CODES_URL to see if new codes are available.
Returns:
list: A list of new codes. Empty if no codes are found.
"""
import time
import numpy as np
import datetime
from logger import get_logger
from train_dssm import train, build_estimator, predict
date = datetime.datetime.today()
now_dir = 'datepart=' + date.strftime('%Y%m%d')
log_file_name = os.path.basename(__file__).split('.', 1)[0] + '.log'
# 当日志文件大小小于1M时，则以
if os.path.exists(log_file_name) is False or os.path.getsize(
log_file_name) / 1024 / 1024 < 1:
logger = get_logger(log_file_name, mode='a')
else:
# 否则删除以前的日志
logger = get_logger(log_file_name)
LOCAL_TRAIN_DIR = '/dssm_data/train/'
LOCAL_VALID_DIR = '/dssm_data/valid/'
LOCAL_PREDICT_DIR = '/dssm_data/predict/'
LOCAL_DICT_DIR = '/dssm_data/vocab/'
LOCAL_TRAIN_NUM_DIR = '/dssm_data/train_num/'
LOCAL_MODEL_DIR = '/cdssm/model/'
LOCAL_DSSM_RESULT_DIR = '/cdssm/dssm_result/'
if __name__ == '__main__':
all_start = time.time()
import asyncio
import os
from pyppeteer import launch
from pyppeteer.page import Page
from logger import get_logger
logger = get_logger(__name__)
path_to_images = os.path.join("/var", "tmp", "screenshots")
async def make_screenshot(img_id: str, url: str, actions: list):
args = {
"headless": True,
}
browser = await launch(args)
page = await browser.newPage()
logger.info("Browser have been loaded with args %s." % str(args))
await page.goto(url)
logger.info("Page %s have been loaded." % url)
screenshot_path = os.path.join(path_to_images, f"{img_id}.png")
asyncio.create_task(
do_actions(actions=actions, page=page, img_path=screenshot_path))
async def do_actions(actions: list, page: Page, img_path: str):
for a in actions:
# === configuring page ===
if a["action"] == "set_resolution":
await page.setViewport({
import sys
import datetime
import traceback
from machine.db.MongoDB import MongoDB # For Database Processing
from machine.okcoin import OKCoin # For Order Processing
from pusher import telegram
from logger import get_logger
from strategy.base_strategy import Strategy
logging = get_logger("step_trade")
class StepTrade(Strategy):
def __init__(self,
machine=None,
db_handler=None,
strategy=None,
currency_type=None,
pusher=None):
if machine is None or db_handler is None or currency_type is None or strategy is None or pusher is None:
raise Exception("Need to machine, db, currency type, strategy")
#if isinstance(machine, KorbitMachine):
# logging.info("Korbit Machine")
# self.currency_type = currency_type+"_krw"
#elif isinstance(machine, CoinoneMachine):
# logging.info("Coinone Machine")
if isinstance(machine, OKCoin):
logging.info("OKCoin Machine")
self.currency_type = currency_type + "_krw"
self.machine = machine
self.pusher = pusher
self.db_handler = db_handler
import typing as t
from queue import Empty as QueueEmptyError
import logger
from common import BusMessage, BusNode, BusProperty, PublishMessage, SubscribeMessage
from modules import (
ATTR_NODES,
ATTR_QCORE,
ATTR_QPUBLISH,
ATTR_QRECEIVE,
ATTR_QRESEND,
ATTR_QSUBSCRIBE,
ATTR_QTRANSMIT,
)
log = logger.get_logger("bus")
# Communication module queues
receive_queue: "mproc.Queue[BusMessage]" = mproc.Queue()
transmit_queues: t.Dict[str, "mproc.Queue[BusMessage]"] = {}
# Interface module queues
publish_queue: "mproc.Queue[BusMessage]" = mproc.Queue()
subscribe_queues: t.Dict[str, "mproc.Queue[BusMessage]"] = {}
# Data tree
data_tree: t.Dict[str, BusNode] = {}
# Message Bus threads
_thread_receive: threading.Thread = None # type: ignore
_thread_transmit: threading.Thread = None # type: ignore
"""Gateway module for device testing"""
import datetime
import os
import shutil
from abc import ABC, abstractmethod
import logger
from env import DAQ_RUN_DIR
LOGGER = logger.get_logger('gateway')
class BaseGateway(ABC):
"""Gateway collection class for managing testing services"""
GATEWAY_OFFSET = 0
DUMMY_OFFSET = 1
TEST_OFFSET_START = 2
NUM_SET_PORTS = 6
SET_SPACING = 10
_PING_RETRY_COUNT = 5
TEST_IP_FORMAT = '192.168.84.%d'
def __init__(self, runner, name, port_set):
self.name = name
self.runner = runner
assert port_set > 0, 'port_set %d, must be > 0' % port_set
self.port_set = port_set
self.fake_target = None
# -*- coding: utf-8 -*-
import os
import sys
import unittest
from proxy_web_api import get_resp_field, call_proxy
from utils import generate_random_card_data, generate_random_number
import logger
log = logger.get_logger('/tmp/web_api_tests-%s.log' % os.environ['USER'])
SERVER_URI = 'http://localhost:17117/'
def log_func_context(func):
def inner(*args, **kwargs):
log.debug('---- Start [%s] ----', func.func_name)
result = func(*args, **kwargs)
log.debug('---- Start [%s] ----', func.func_name)
return result
return inner
class TestBaseWebApi(unittest.TestCase):
'''
tokenize_card, detokenize_card, remove_card, etc.
'''
# TODO add error test
from pony import orm
import logger
from models import User
from settings import DEFAULT_TOPIC
log = logger.get_logger('User CRUD')
@orm.db_session
def get(user_id: int) -> User:
return User[user_id]
@orm.db_session
def create(user_id: int, topic: str = DEFAULT_TOPIC) -> User:
user = User(id=user_id, current_topic=topic)
log.debug(f'User was created[{user.id}]')
return user
@orm.db_session
def get_or_create(user_id: int) -> User:
try:
return get(user_id)
except orm.ObjectNotFound:
return create(user_id)
@orm.db_session
def update(user_id: int, topic: str) -> User:
# encoding: utf-8
from __future__ import unicode_literals
from selenium import webdriver
from bs4 import BeautifulSoup
from logger import get_logger
logger = get_logger()
class ScraperExeception(Exception):
pass
class Scraper():
def __init__(self):
# Initialize a selenium browser when the class is created.
self.browser = webdriver.PhantomJS()
# Set the cookies by visiting base url first.
print self.base_url
if self.base_url:
self.browser.get(self.base_url)
def get_page(self, url, max_retries=3):
# Attempts to try to get the page (retries 3 times max)
# and turn it into a (beautiful) soup for easy parsing.
retries = 0
while retries < max_retries:
try:
logger.debug("Get -> {}".format(url))
self.browser.get(url)
from pyspark import SparkConf, SparkContext, SQLContext
import logger
TEST_NAME = 'spk_single_df_single_file_cnt'
LOGGER = logger.get_logger(TEST_NAME)
# Specify some constants
URLPATH = "s3a://dask-avro-data/application-data/app-100096.avro"
# Start
LOGGER.info('START: Creating spark conf')
Sconf = SparkConf()
sc = SparkContext(appName="my_test", conf=Sconf)
sqlContext = SQLContext(sparkContext=sc)
LOGGER.info('FINISH: Finished creating spark conf')
LOGGER.info('START: Creating spark dataframe')
df = sqlContext.read.format("com.databricks.spark.avro").load(URLPATH)
LOGGER.info('FINISH: Spark dataframe created')
LOGGER.info('START: Starting count')
cnt = df.count()
LOGGER.info('FINISH: Count is %s', cnt)
sc.stop()
from bot import ShrimpBot
from utils import Timer
from const import Settings
from logger import get_logger
timer = Timer()
logger = get_logger("shrimp_bot")
ShrimpBot(admin=Settings.Admins, logger=logger).run(Settings.token)
logger.info("Run time - %dh : %dm : %ds" % timer.end())
def __init__(self) -> None:
super().__init__(name="Receive")
self.log = logger.get_logger("bus.receive")
def get_logger(self):
"""
:returns: The python logger root for the framework.
"""
return logger.get_logger()