def shell(obj, username):
gh = obj['githome']
try:
# get user
user = gh.get_user_by_name(username)
log = Logger('githome-shell [{}]'.format(user.name))
# we've got our user, now authorize him or not
shell_cmd = shlex.split(os.environ.get('SSH_ORIGINAL_COMMAND', ''))
log.debug('SSH_ORIGINAL_COMMAND {!r}'.format(shell_cmd))
if not shell_cmd:
log.critical('No shell command given')
abort(1)
cmd = gh.authorize_command(user, shell_cmd)
log.debug('Executing {!r}', cmd)
binary = cmd[0] # we use path through execlp
except Exception as e:
log.error(str(e))
abort(1)
else:
os.execlp(binary, *cmd)
class ProjectManager(object):
""" Class for controlling project's settings.
A project is a collection of setting
for some data transformation.
Each project is yaml file with settings.
Project PID is a yaml file name.
"""
config_path = None
def __init__(self, settings_class, config_path=None):
""" Init projects_folder.
Manager loads settings from default location in PySatDNA root or from given config_path.
Then create work and projects folders if they were not found.
And set config values for settings class.
"""
self.manager_logger = Logger('Manager logger')
if config_path:
self.load_config(config_path)
else:
self.load_config(self.config_path)
self.projects_folder = self.config["projects_folder"]
self.work_folder = self.config["path_work_folder"]
self.settings_class = settings_class
self.settings_class.config = self.config
self.force_folder_creation = False
if self.projects_folder and not os.path.isdir(self.projects_folder):
os.makedirs(self.projects_folder)
if self.work_folder and not os.path.isdir(self.work_folder):
os.makedirs(self.work_folder)
def load_config(self, config_path):
''' Load OS-specific configs.'''
if config_path:
file_path = config_path
if not os.path.isfile(file_path):
message = "ERROR with open config file: %s" % file_path
self.manager_logger.error(message)
raise ProjectManagerException(message)
else:
self.os = platform.system()
if self.os == "Windows":
file_path = "../config.win.yaml"
elif self.os == "Darwin":
file_path = "../config.mac.yaml"
else:
file_path = os.path.expanduser("~/Dropbox/workspace/PySatDNA/config.yaml")
if not os.path.isfile(file_path):
file_path = os.path.expanduser("~/Dropbox/PySatDNA/config.dobi.yaml")
try:
with open(file_path) as fh:
self.config = yaml.load(fh)
except Exception, e:
self.manager_logger.error("ERROR with open config file: %s" % file_path)
self.manager_logger.warning("Loading default settings")
self.config = {
'path_work_folder': 'data',
'path_workspace_folder': '../..',
'projects_folder': 'projects',
}
class LoggableError(Exception):
"""
Parent class for customized exception classes which provides interface to the logging functions
Attributes:
log_file: The file in which to keep logs
logger: Instance of logbook.Handler which handles logging
"""
def __init__(self):
self.orig_message = self.message
self.UPATH = os.getenv("HOME")
self.log_file = '%s/Desktop/tweetlog.rtf' % self.UPATH
self.initialize_logger()
def initialize_logger(self):
try:
if self.logger is not None:
pass
except:
self.logger = Logger()
# self.logger = FileHandler(self.log_file)
# self.logger.push_application() #Pushes handler onto stack of log handlers
def log_error(self, error_message):
self.logger.error(error_message)
def log_warning(self, warning_message):
self.logger.warn(warning_message)
def error(self, id_=None, error_code=None, error_msg=None):
if isinstance(id_, Exception):
# XXX: for an unknown reason 'log' is None in this branch,
# therefore it needs to be instantiated before use
global log
if not log:
log = Logger('IB Broker')
log.exception(id_)
if isinstance(error_code, EClientErrors.CodeMsgPair):
error_msg = error_code.msg()
error_code = error_code.code()
if isinstance(error_code, int):
if error_code in (502, 503, 326):
# 502: Couldn't connect to TWS.
# 503: The TWS is out of date and must be upgraded.
# 326: Unable connect as the client id is already in use.
self.unrecoverable_error = True
if error_code < 1000:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.info("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
def error(self, id_=None, error_code=None, error_msg=None):
if isinstance(id_, Exception):
# XXX: for an unknown reason 'log' is None in this branch,
# therefore it needs to be instantiated before use
global log
if not log:
log = Logger('IB Broker')
log.exception(id_)
if isinstance(error_code, EClientErrors.CodeMsgPair):
error_msg = error_code.msg()
error_code = error_code.code()
if isinstance(error_code, int):
if error_code in (502, 503, 326):
# 502: Couldn't connect to TWS.
# 503: The TWS is out of date and must be upgraded.
# 326: Unable connect as the client id is already in use.
self.unrecoverable_error = True
if error_code < 1000:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.info("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
class actionThread(stopableThread):
def __init__(self,actList, inputParam=None,*args,**kargs):
super(actionThread,self).__init__(*args,**kargs)
self.actList=actList
self.param = inputParam
self.log = Logger(u"%s actionThread"%self.name)
def setParam(self,data):
self.param = data
def changeActList(self,newList):
if type(newList) != type(()) and type(newList) != type(()):
self.log.error(u"changeActList:invalid parameter")
return
self.actList = newList
def process(self):
"""
调用actList中的所有action plugin.Only run once.
"""
actManager = absManager.actionManager()
for i in self.actList:
self.log.info(u"do action %s:%s" % (i,actManager.id2filename(i)))
actManager.call(i,self.param)
self.pause()
class serialDataSource(dataSource):
"""
从串口发送或者接收数据
"""
def __init__(self):
self.log = Logger(u"serialDataSource")
self._ser = None
def open(self,**config):
"""
打开串口
config:串口配置参数，字典类型。
config["port"]:端口名
config["baudrate"]
config["bytesize"]：数据位，默认8
config["parity"]:校验方式：奇校验，偶校验，无。默认为无校验
config["stopbits"]
config["xonxoff"]
config["rtscts"]
config["dsrdtr"]
成功返回True，失败返回False
"""
# if type(config) != type({}):
# self.log.error("open: invalid param")
# return False
try:
self._ser = serial.Serial(**config)
except serial.serialutil.SerialException,ValueError:
self.log.error("open:config port failed %s:%s" % (serial.serialutil.SerialException,ValueError.message))
return False
if not self._ser.isOpen():
self._ser.open()
return True
class dataThread(stopableThread):
"""
feature:
1.定时从数据源获取数据。
2.将数据通过传递给TrigThread
3.订阅PORT_SETTING_CHANGED消息，并作相应处理
"""
def __init__(self,filename="iSerial.ini",section="serial",objsrc=None,qlist=None,*args,**kwargs):
super(dataThread,self).__init__(*args,**kwargs)
self.log = Logger(u"dataThread")
self.config = {}
if objsrc:
self._datasrc = objsrc
else:
self._datasrc = dataSource.serialDataSource()
if os.path.isfile(filename) and section is not None:
self.filename = filename
self.section = section
self.loadConfig()
self._datasrc.open(**self.config)
pub.subscribe(self.notifySettingChanged,topics.PORT_SETTING_CHANGED)
pub.subscribe(self.onSend, topics.USER_INPUT_DATA)
self.queueList = qlist
def setQueueList(self,qlist):
if qlist is not None and type(qlist) == type([]):
self.queueList = qlist
def loadConfig(self):
"""
从配置文件中，读取数据源配置信息。
"""
self.config.clear()
cf = ConfigParser.ConfigParser()
cf.read(self.filename)
for opt in cf.options(self.section):
try :
self.config[opt] = ast.literal_eval(cf.get(self.section,opt))
except:
self.log.error("loadconfig:invalid config in %s"%self.section)
pub.sendMessage(topics.NOTIFY_TO_USER,type="error",
msg="invalid settings in iSerial.ini % section"%self.section)
def notifySettingChanged(self,newconfig,**kwargs):
del self.config
self.config = newconfig
self.pause()
self._datasrc.close()
self._datasrc.open(**self.config)
self.start()
def process(self):
size = self._datasrc.dataAvail()
if size:
msg = self._datasrc.recvData(size)
for q in self.queueList:
q.put(msg)
def onSend(self,strData):
self._datasrc.sendData(strData)
class iSerial(QtGui.QMainWindow):
def __init__(self, parent=None,confname="iSerial.ini"):
super(iSerial,self).__init__(parent)
self._threadDict = {}
self.log = Logger(u"controller")
self.confname = confname
pub.subscribe(self.notifytouser,topics.NOTIFY_TO_USER)
self.ui = ui_mainwindow.Ui_MainWindow()
self.ui.setupUi(self)
self.queuelist = self.createQueues()
self.defconf = self.loadDefaultMap()
self._threadDict[DEFAULT_TRIG_THREAD] = self.createDefaultTrigThread(self.defconf)
self.confManager = configManager()
self.trigManager = triggerManager("plugins/Triggers")
self.actManager = actionManager("plugins/Actions")
self.modMonitor = fileMonitor(os.path.abspath("./plugins"))
self.modMonitor.start()
def createQueues(self):
retQueue = []
for i in range(2):
retQueue.append(Queue.Queue())
return retQueue
def string2class(self,mod,str):
try:
m = importlib.import_module(mod)
c = getattr(m,str)
except AttributeError,KeyError:
self.log.error(u"iSerial::string2class : create class:%s.%s failed" % (mod,str))
QtGui.QMessageBox.critical(None,"Error","iSerial::string2class : create class:%s.%s failed"%(mod,str))
return None
return c
class Client():
def __init__(self, channel="test",host='127.0.0.1'):
self.redis = redis.Redis(host=host)
self.channel = channel
self.timeout = 10
self.query_delay = 0.1
self.idn = 'Client %d' % id(self)
self.Log = Logger('Client')
def __del__(self):
self.Log.debug('__del__()')
self.redis.delete(self.idn)
def str(self):
print self.__unicode__()
def __unicode__(self):
msg = 'Client:'
msg += '\n idn : %s' % str(self.idn)
msg += '\n channel : %s' % str(self.channel)
msg += '\n imeout : %s' % str(self.timeout)
msg += '\n query_delay : %s' % str(self.query_delay)
return msg
def read(self):
self.Log.debug('read()')
data_read = self.redis.get(self.idn)
self.Log.debug(' data_read=%s' % data_read)
if data_read is not None:
self.redis.delete(self.idn)
return data_read
def send(self, cmd="\n", **kwargs):
self.Log.debug('send(cmd=%s)' % cmd)
timeout = kwargs.pop('timeout',0)
query = kwargs.pop('query',1)
try:
if timeout == 0:
timeout = self.timeout
msg = sjson.dumps({'from': self.idn, 'cmd': [cmd, kwargs, query], 'timeout': timeout , 'timestamp': str(datetime.now())})
self.Log.debug(' full msg=%s)' % msg)
self.redis.publish(self.channel, msg)
except Exception as E:
self.Log.error(E.message)
def query(self, cmd, **kwargs):
self.Log.debug('query(cmd=%s, kwargs=%s)' % (cmd, str(kwargs)))
self.send(cmd, **kwargs)
to = time.time()
while time.time() - to < self.timeout and (not self.redis.exists(self.idn)):
time.sleep(0.1)
try:
return sjson.loads(self.read())
except:
return [1, self.read()]
def test_win32_logger(self):
from logbook import NTEventLogHandler, Logger
logger = Logger('MyLogger')
handler = NTEventLogHandler('My Application')
with handler.applicationbound():
logger.error('Testing')
def test2():
log = Logger('Logbook-test-2')
log.critical("critical")
log.error("error")
log.warn("warn")
log.notice("notice")
log.info("test")
log.debug("debug")
def send_msg(update, context):
tele_id = int(context.args[0])
message = " ".join(context.args[1:])
try:
context.bot.send_message(tele_id, message)
except Exception as e:
log = Logger()
log.error(e)
update.effective_message.reply_text(DEV_TELE_ID, "Failed to send message")
def compress_pdf(update, context):
if not check_user_data(update, context, PDF_INFO):
return ConversationHandler.END
_ = set_lang(update, context)
update.effective_message.reply_text(
_("Compressing your PDF file"),
reply_markup=ReplyKeyboardRemove(),
)
with tempfile.NamedTemporaryFile() as tf:
user_data = context.user_data
file_id, file_name = user_data[PDF_INFO]
pdf_file = context.bot.get_file(file_id)
pdf_file.download(custom_path=tf.name)
with tempfile.TemporaryDirectory() as dir_name:
out_fn = os.path.join(
dir_name, f"Compressed_{os.path.splitext(file_name)[0]}.pdf")
cmd = "gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/default \
-dNOPAUSE -dQUIET -dBATCH -sOutputFile={} {}".format(
out_fn, tf.name)
proc = Popen(shlex.split(cmd),
stdout=PIPE,
stderr=PIPE,
shell=False)
out, err = proc.communicate()
if proc.returncode != 0:
log = Logger()
log.error(
f'Stdout:\n{out.decode("utf-8")}\n\nStderr:\n{err.decode("utf-8")}'
)
update.effective_message.reply_text(
_("Something went wrong, try again"))
else:
old_size = os.path.getsize(tf.name)
new_size = os.path.getsize(out_fn)
update.effective_message.reply_text(
_("File size reduced by <b>{:.0%}</b>, "
"from <b>{}</b> to <b>{}</b>".format(
(1 - new_size / old_size),
humanize.naturalsize(old_size),
humanize.naturalsize(new_size),
)),
parse_mode=ParseMode.HTML,
)
send_result_file(update, context, out_fn, "compress")
# Clean up memory
if user_data[PDF_INFO] == file_id:
del user_data[PDF_INFO]
return ConversationHandler.END
def send_msg(update: Update, context: CallbackContext):
tele_id = int(context.args[0])
message = " ".join(context.args[1:])
try:
context.bot.send_message(tele_id, message)
update.effective_message.reply_text("Message sent")
except Exception as e:
log = Logger()
log.error(e)
update.effective_message.reply_text("Failed to send message")
def error_callback(update, context):
"""
Log errors
Args:
update: the update object
context: the context object
Returns:
None
"""
log = Logger()
log.error(f'Update "{update}" caused error "{context.error}"')
def run_cmd(cmd: str) -> bool:
is_success = True
proc = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, shell=False)
out, err = proc.communicate()
if proc.returncode != 0:
is_success = False
log = Logger()
log.error(f"Command:\n{cmd}\n\n"
f'Stdout:\n{out.decode("utf-8")}\n\n'
f'Stderr:\n{err.decode("utf-8")}')
return is_success
class StrategyTemplate:
def __init__(self, user):
self.user = user
self.log = Logger(os.path.basename(__file__))
StreamHandler(sys.stdout).push_application()
def strategy(self, event):
""":param event event.data 为所有股票的信息，结构如下
{'162411':
{'ask1': '0.493',
'ask1_volume': '75500',
'ask2': '0.494',
'ask2_volume': '7699281',
'ask3': '0.495',
'ask3_volume': '2262666',
'ask4': '0.496',
'ask4_volume': '1579300',
'ask5': '0.497',
'ask5_volume': '901600',
'bid1': '0.492',
'bid1_volume': '10765200',
'bid2': '0.491',
'bid2_volume': '9031600',
'bid3': '0.490',
'bid3_volume': '16784100',
'bid4': '0.489',
'bid4_volume': '10049000',
'bid5': '0.488',
'bid5_volume': '3572800',
'buy': '0.492',
'close': '0.499',
'high': '0.494',
'low': '0.489',
'name': '华宝油气',
'now': '0.493',
'open': '0.490',
'sell': '0.493',
'turnover': '420004912',
'volume': '206390073.351'}}
"""
pass
def run(self, event):
try:
self.strategy(event)
except Exception as e:
self.log.error(e)
def clock(self, event):
pass
class ILogger(object):
def __init__(self, **kwargs):
self.logger = Logger(self.name)
def _process_kwargs(self, kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def log(self, msg):
self.logger.notice(msg)
def log_error(self, msg):
self.logger.error(msg)
class PyPLogger(object):
def __init__(self, clazz):
logbook.set_datetime_format("local")
self.serverName = clazz.__name__[clazz.__name__.rfind('.') + 1:]
if not os.path.exists(log_dir):
os.makedirs(log_dir)
self.log_file = TimedRotatingFileHandler(os.path.join(
log_dir, '%s.log' % self.serverName),
date_format='%Y-%m-%d',
bubble=True,
encoding='utf-8')
self.log_std = ColorizedStderrHandler(bubble=True)
# self.log_std = StderrHandler(bubble=True)
self.log = Logger(self.serverName)
self.__init_logger()
self.__setting()
def log_type(self, record, handler):
# log = "[{date}]-[{level}]-[" + self.serverName + "] - {msg}".format(
log = "[" + self.serverName + "]" + "-[{date}]-[{level}] - {msg}".format(
date=record.time,
level=record.level_name,
# filename = os.path.split(record.filename)[-1],
# func_name = record.func_name,
# lineno = record.lineno,
msg=record.message)
return log
def __init_logger(self):
logbook.set_datetime_format("local")
self.log.handlers = []
self.log.handlers.append(self.log_file)
self.log.handlers.append(self.log_std)
def __setting(self):
self.log_std.formatter = self.log_type
self.log_file.formatter = self.log_type
def info(self, *args, **kwargs):
self.log.info(*args, **kwargs)
def warn(self, *args, **kwargs):
self.log.warn(*args, **kwargs)
def error(self, *args, **kwargs):
self.log.error(*args, **kwargs)
def write_photos_in_pdf(input_fn, dir_name, file_name):
is_success = True
root_file_name = os.path.splitext(file_name)[0]
image_prefix = os.path.join(dir_name, root_file_name)
cmd = f"pdfimages -png {input_fn} {image_prefix}"
proc = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE, shell=False)
out, err = proc.communicate()
if proc.returncode != 0:
is_success = False
log = Logger()
log.error(
f'Stdout:\n{out.decode("utf-8")}\n\nStderr:\n{err.decode("utf-8")}'
)
return is_success
class DeleteFiles:
def __init__(self):
handler = TimedRotatingFileHandler('../logs/delete_files.log')
handler.push_application()
self.logger = Logger(name='delete files')
self.path_lists = self.get_conf()
@staticmethod
def get_conf():
##从配置文件获取配置
cp = ConfigParser.ConfigParser()
with codecs.open(os.path.join(BASE_DIR, './config/config.ini'),
'r',
encoding='utf-8') as f:
cp.readfp(f)
path_lists = eval(cp.get('filesystem', 'del_paths').strip())
return path_lists
def file_handler(self):
for file_path in self.path_lists:
file_name = os.path.join('/', file_path)
if os.path.isdir(file_name) or os.path.isfile(file_name):
if os.path.isdir(file_name):
try:
os.removedirs(file_name)
#os.remove(file_name)
except Exception, e:
self.logger.error('Delete dir %s failed!%s' %
(file_name, e))
else:
self.logger.info('Delete dir %s success!' % file_name)
if os.path.isfile(file_name):
try:
#os.removedirs(file_name)
os.remove(file_name)
except Exception, e:
self.logger.error('Delete file %s failed!%s' %
(file_name, e))
else:
self.logger.info('Delete file %s success!' % file_name)
else:
class ILogger(object):
def __init__(self, **kwargs):
self.logger = Logger(self.name)
def _process_kwargs(self, kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def log(self, msg):
"""Records record at notice level"""
self.logger.notice(msg)
def log_error(self, msg):
"""Records record at error level"""
self.logger.error(msg)
def write(self, msg):
"""Writes to a file"""
raise NotImplementedError
def send(update, context):
"""
Send a message to a user
Args:
update: the update object
context: the context object
Returns:
None
"""
tele_id = int(context.args[0])
message = " ".join(context.args[1:])
try:
context.bot.send_message(tele_id, message)
except Exception as e:
log = Logger()
log.error(e)
update.message.reply_text("Failed to send message")
def check_file(update, context, file_id, file_name, file_type):
"""
Check if the file is safe or not
Args:
update: the update object
context: the context object
file_id: the int of the file ID
file_name: the string of the file name
file_type: the string of the file type
Returns:
None
"""
message = update.effective_message
message.chat.send_action(ChatAction.TYPING)
is_safe, status, matches = scan_file(file_name)
chat_type = message.chat.type
if not is_safe:
threat_type = "contains" if status == FOUND else "may contain"
if chat_type in (Chat.GROUP, Chat.SUPERGROUP):
text = (
f"I've deleted a {file_type} that {threat_type} a virus or malware "
f"(sent by @{message.from_user.username})."
)
filter_msg(update, context, file_id, file_type, text)
else:
message.reply_text(
f"I think it {threat_type} a virus or malware, don't download or open it.",
quote=True,
)
else:
if chat_type == Chat.PRIVATE:
if status == OK:
message.reply_text(
"I think it doesn't contain any virus or malware.", quote=True
)
else:
log = Logger()
log.error(matches)
message.reply_text("Something went wrong, try again.", quote=True)
def crop_pdf(update, context, percent=None, offset=None):
_ = set_lang(update, context)
update.effective_message.reply_text(_("Cropping your PDF file"),
reply_markup=ReplyKeyboardRemove())
with tempfile.NamedTemporaryFile(suffix=".pdf") as tf:
user_data = context.user_data
file_id, file_name = user_data[PDF_INFO]
pdf_file = context.bot.get_file(file_id)
pdf_file.download(custom_path=tf.name)
with tempfile.TemporaryDirectory() as dir_name:
out_fn = os.path.join(dir_name, f"Cropped_{file_name}")
if percent is not None:
cmd = f"pdf-crop-margins -p {percent} -o {out_fn} {tf.name}"
else:
cmd = f"pdf-crop-margins -a {offset} -o {out_fn} {tf.name}"
proc = Popen(shlex.split(cmd),
stdout=PIPE,
stderr=PIPE,
shell=False)
out, err = proc.communicate()
if proc.returncode != 0:
log = Logger()
log.error(
f'Stdout:\n{out.decode("utf-8")}\n\nStderr:\n{err.decode("utf-8")}'
)
update.effective_message.reply_text(
_("Something went wrong, try again"))
else:
send_result_file(update, context, out_fn, "crop")
# Clean up memory
if user_data[PDF_INFO] == file_id:
del user_data[PDF_INFO]
return ConversationHandler.END
def run():
raven_client = Client()
logger = Logger("ardegra")
spider_name = " ".join(sys.argv[1:])
client = pymongo.MongoClient("mongodb://{}/ardegra".format(
Config.DATABASE_ADDRESS))
logger.debug("Running: {}".format(spider_name))
try:
db = client["ardegra"]
document = db.spiders.find_one({"name": spider_name})
if document["type"]["name"] == "Forum Spider 1":
spider = ForumSpider1(name=spider_name)
elif document["type"]["name"] == "News Spider 1":
spider = NewsSpider1(name=spider_name)
elif document["type"]["name"] == "News Spider 2":
spider = NewsSpider2(name=spider_name)
spider.run()
except Exception as err:
raven_client.captureException()
logger.error(str(err))
finally:
client.close()
class PmLogHandler(log.CementLogHandler):
"""
PmLogHandler - override CementLogHandler to use logbook.
This class uses the same configuration options as
:ref:`LoggingLogHandler <cement.ext.ext_logging>`
"""
class Meta:
interface = log.ILog
"""The interface that this class implements."""
label = 'pmlog'
"""The string identifier of this handler."""
namespace = "pm"
"""
The logging namespace.
Note: Although Meta.namespace defaults to None, Cement will set this
to the application label (CementApp.Meta.label) if not set during
setup.
"""
file_format = "{record.time} ({record.level_name}) {record.channel} : {record.message}"
"""The logging format for the file logger."""
console_format = "{record.time:%Y-%m-%d %H:%M} ({record.level_name}): {record.message}"
"""The logging format for the consoler logger."""
debug_format = "{record.time} ({record.level_name}) {record.channel} : {record.message}"
"""The logging format for both file and console if ``debug==True``."""
log_setup = None
"""Nested log setup placeholder"""
level = 0
"""Global level for handlers"""
clear_loggers = True
"""Whether of not to clear previous loggers first."""
# These are the default config values, overridden by any '[log]'
# section in parsed config files.
config_section = 'log'
"""
The section of the application configuration that holds this handlers
configuration.
"""
config_defaults = dict(
file=None,
level='INFO',
to_console=True,
rotate=False,
max_bytes=512000,
max_files=4,
)
"""
The default configuration dictionary to populate the ``log`` section.
"""
levels = ['INFO', 'WARN', 'ERROR', 'DEBUG', 'FATAL']
def __init__(self, *args, **kw):
super(PmLogHandler, self).__init__(*args, **kw)
self.app = None
def _setup(self, app_obj):
super(PmLogHandler, self)._setup(app_obj)
if self._meta.namespace is None:
self._meta.namespace = self.app._meta.label
self.backend = Logger(self._meta.namespace)
# hack for application debugging
if is_true(self.app._meta.debug):
self.app.config.set('log', 'level', 'DEBUG')
# Mainly for backwards compatibility since Logger level should
# be NOTSET (level 0). Output level is controlled by handlers
self.set_level(self.app.config.get('log', 'level'))
# clear loggers?
if is_true(self._meta.clear_loggers):
self.clear_loggers()
# console
if is_true(self.app.config.get('log', 'to_console')):
self._setup_console_log()
# file
if self.app.config.get('log', 'file'):
self._setup_file_log()
# nested setup
self.backend.handlers.append(logbook.NullHandler(bubble=False))
self.log_setup = logbook.NestedSetup(self.backend.handlers)
with self._console_handler.applicationbound():
self.debug("logging initialized for '%s' using PmLogHandler" % \
self._meta.namespace)
def set_level(self, level):
"""
Set the log level. Must be one of the log levels configured in
self.levels which are ``['INFO', 'WARN', 'ERROR', 'DEBUG', 'FATAL']``.
:param level: The log level to set.
"""
level = level.upper()
if level not in self.levels:
level = 'INFO'
level = logbook.lookup_level(level.upper())
self.level = level
def get_level(self):
"""Returns a string representation of the current log level."""
return logbook.get_level_name(self.level)
def _setup_console_log(self):
"""Add a console log handler."""
if logbook.lookup_level(self.get_level()) == logbook.DEBUG:
fmt_string = self._meta.debug_format
else:
fmt_string = self._meta.console_format
console_handler = logbook.StderrHandler(
format_string=fmt_string,
level = logbook.lookup_level(self.get_level()),
bubble = True)
self._console_handler = console_handler
self.backend.handlers.append(console_handler)
def _setup_file_log(self):
"""Add a file log handler."""
file_path = os.path.expandvars(fs.abspath(self.app.config.get('log', 'file')))
log_dir = os.path.dirname(file_path)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if logbook.lookup_level(self.get_level()) == logbook.DEBUG:
fmt_string = self._meta.debug_format
else:
fmt_string = self._meta.file_format
if self.app.config.get('log', 'rotate'):
from logbook import RotatingFileHandler
file_handler = RotatingFileHandler(
file_path,
max_size=int(self.app.config.get('log', 'max_bytes')),
backup_count=int(self.app.config.get('log', 'max_files')),
format_string=fmt_string,
level = logbook.lookup_level(self.get_level()),
bubble = True,
)
else:
from logbook import FileHandler
file_handler = FileHandler(file_path,
format_string=fmt_string,
level = logbook.lookup_level(self.get_level()),
bubble = True,
)
self._file_handler = file_handler
self.backend.handlers.append(file_handler)
def _get_logging_kwargs(self, namespace, **kw):
if namespace is None:
namespace = self._meta.namespace
if 'extra' in kw.keys() and 'namespace' in kw['extra'].keys():
pass
elif 'extra' in kw.keys() and 'namespace' not in kw['extra'].keys():
kw['extra']['namespace'] = namespace
else:
kw['extra'] = dict(namespace=namespace)
return kw
def info(self, msg, namespace=None, **kw):
"""
Log to the INFO facility.
:param msg: The message the log.
:param namespace: A log prefix, generally the module ``__name__`` that
the log is coming from. Will default to self._meta.namespace if
None is passed.
:keyword kw: Keyword arguments are passed on to the backend logging
system.
"""
kwargs = self._get_logging_kwargs(namespace, **kw)
self.backend.info(msg, **kwargs)
def debug(self, msg, namespace=None, **kw):
"""
Log to the DEBUG facility.
:param msg: The message the log.
:param namespace: A log prefix, generally the module ``__name__`` that
the log is coming from. Will default to self._meta.namespace if
None is passed. For debugging, it can be useful to set this to
``__file__``, though ``__name__`` is much less verbose.
:keyword kw: Keyword arguments are passed on to the backend logging
system.
"""
kwargs = self._get_logging_kwargs(namespace, **kw)
self.backend.debug(msg, **kwargs)
def warn(self, msg, namespace=None, **kw):
"""
Log to the WARN facility.
:param msg: The message the log.
:param namespace: A log prefix, generally the module ``__name__`` that
the log is coming from. Will default to self._meta.namespace if
None is passed. For debugging, it can be useful to set this to
``__file__``, though ``__name__`` is much less verbose.
:keyword kw: Keyword arguments are passed on to the backend logging
system.
"""
kwargs = self._get_logging_kwargs(namespace, **kw)
self.backend.warn(msg, **kwargs)
def critical(self, msg, namespace=None, **kw):
"""
Log to the CRITICAL facility.
:param msg: The message the log.
:param namespace: A log prefix, generally the module ``__name__`` that
the log is coming from. Will default to self._meta.namespace if
None is passed. For debugging, it can be useful to set this to
``__file__``, though ``__name__`` is much less verbose.
:keyword kw: Keyword arguments are passed on to the backend logging
system.
"""
kwargs = self._get_logging_kwargs(namespace, **kw)
self.backend.critical(msg, **kwargs)
def fatal(self, msg, namespace=None, **kw):
"""
Log to the FATAL facility.
:param msg: The message the log.
:param namespace: A log prefix, generally the module ``__name__`` that
the log is coming from. Will default to self._meta.namespace if
None is passed. For debugging, it can be useful to set this to
``__file__``, though ``__name__`` is much less verbose.
:keyword kw: Keyword arguments are passed on to the backend logging
system.
"""
kwargs = self._get_logging_kwargs(namespace, **kw)
self.backend.fatal(msg, **kwargs)
def error(self, msg, namespace=None, **kw):
"""
Log to the ERROR facility.
:param msg: The message the log.
:param namespace: A log prefix, generally the module ``__name__`` that
the log is coming from. Will default to self._meta.namespace if
None is passed. For debugging, it can be useful to set this to
``__file__``, though ``__name__`` is much less verbose.
:keyword kw: Keyword arguments are passed on to the backend logging
system.
"""
kwargs = self._get_logging_kwargs(namespace, **kw)
self.backend.error(msg, **kwargs)
## NOTE: do we even need this for logbook?
def clear_loggers(self):
"""Clear any previously configured logging namespaces.
"""
if not self._meta.namespace:
# _setup() probably wasn't run
return
self.handlers = []
class LockFactory(ClientFactory):
protocol = LockProtocol
def __init__(self, config):
self.log = Logger('lockfactory')
self.interface = config.LOCK_INTERFACE or '127.0.0.1'
self.port = config.LOCK_PORT or 4001
def inject_node(rec):
rec.extra['node'] = self.port
self._logger_group = LoggerGroup(processor=inject_node)
self._logger_group.add_logger(self.log)
self.log.debug('creating lock factory')
self.paxos = Paxos(
self,
on_learn=self.on_learn,
on_prepare=self.on_prepare,
on_stale=lambda last_accepted_id: self.set_stale(True),
logger_group=self._logger_group,
)
self.neighbours = config.NODES
self._first_connect_delay = config.FIRST_CONNECT_DELAY or 0
self.master = None
self._stale = False
self._delayed_reconnect = None
# used to prevent server from reconnecting when it was
# closed in a unittest
self._closed = False
# this flag is used to prevent recursion in _reconnect method
self._reconnecting = False
self.connections = {}
self._all_connections = []
# list of deferreds to be called when
# connections with all other nodes will be established
self._connection_waiters = []
# same for sync complete event
self._sync_completion_waiters = []
# state
self._epoch = 0 # received commands counter
self._log = []
# a buffer for learn commands if they come out of order.
self._learn_queue = []
# actual data storage
self._keys = {}
# last successful Paxos ID
self.last_accepted_iteration = 0
self.state = []
self.callbacks = []
# map paxos-id -> proposer
self._proposers = {}
# this buffer is used to keep messages
# when node is stale
self._paxos_messages_buffer = deque()
self.add_callback('^paxos_.*$', self._process_paxos_messages)
self.syncronizer = Syncronizer(self)
self.add_callback('^sync_subscribe$', self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_unsubscribe$', self.syncronizer.on_sync_subscribe)
self.add_callback('^sync_snapshot .*$', self.syncronizer.on_sync_snapshot)
self.log.debug('Opening the port %s:%s' % (self.interface, self.port))
self._port_listener = reactor.listenTCP(self.port, self, interface=self.interface)
self.web_server = server.Site(Root(self))
self.http_interface = config.HTTP_INTERFACE or '127.0.0.1'
self.http_port = config.HTTP_PORT or 9001
self._webport_listener = reactor.listenTCP(
self.http_port,
self.web_server,
interface = self.http_interface,
)
def close(self):
self._closed = True
d1 = self._port_listener.stopListening()
d2 = self._webport_listener.stopListening()
if self._delayed_reconnect is not None:
stop_waiting(self._delayed_reconnect)
self.disconnect()
def add_callback(self, regex, callback):
self.callbacks.append((re.compile(regex), callback))
def remove_callback(self, callback):
self.callbacks = filter(lambda x: x[1] != callback, self.callbacks)
def find_callback(self, line):
for regex, callback in self.callbacks:
if regex.match(line) != None:
return callback
def get_status(self):
"""Returns a list of tuples (param_name, value)."""
if self.master is not None:
master = ':'.join(map(str, self.master.http))
else:
master = 'None'
return [
('bind', '%s:%s' % (self.interface, self.port)),
('master', master),
('stale', self._stale),
('current_id', self.paxos.id),
('max_seen_id', self.paxos.max_seen_id),
('last_accepted_id', self.paxos.last_accepted_id),
('num_connections', len(self.connections)),
('quorum_size', self.quorum_size),
('log_size', len(self._log)),
('epoch', self._epoch),
]
def get_key(self, key):
d = Deferred()
def cb():
if key not in self._keys:
raise KeyNotFound('Key "%s" not found' % key)
return self._keys[key]
d.addCallback(cb)
return d
def set_key(self, key, value):
value = 'set-key %s "%s"' % (key, escape(value))
return self.paxos.propose(value)
def del_key(self, key):
value = 'del-key %s' % key
return self.paxos.propose(value)
def add_connection(self, conn):
self.connections[conn.other_side] = conn
self._notify_waiters_if_needed()
def _notify_waiters_if_needed(self):
num_disconnected = len(self.neighbours) - len(self.connections)
if num_disconnected == 0:
for waiter in self._connection_waiters:
waiter.callback(True)
self._connection_waiters = []
def remove_connection(self, conn):
for key, value in self.connections.items():
if value == conn:
self.log.info(
'Connection to the %s:%s (%s) lost.' % (
conn.other_side[0],
conn.other_side[1],
conn.transport.getPeer()
)
)
del self.connections[key]
break
def when_connected(self):
d = Deferred()
self._connection_waiters.append(d)
self._notify_waiters_if_needed()
return d
def when_sync_completed(self):
d = Deferred()
if self.get_stale():
# sync in progress
self._sync_completion_waiters.append(d)
else:
# we are not a stale node
d.callback(True)
return d
def disconnect(self):
for conn in self._all_connections:
if conn.connected:
conn.transport.loseConnection()
def startFactory(self):
self.log.info('factory started at %s:%s' % (self.interface, self.port))
if self._first_connect_delay > 0:
def delay_connect():
# delay connection to other server
# this is needed to start few test servers
# on the same machine without errors
self._delayed_reconnect = reactor.callLater(self._first_connect_delay, self._reconnect)
reactor.callWhenRunning(delay_connect)
else:
reactor.callWhenRunning(self._reconnect)
def _reconnect(self):
if not self._closed and not self._reconnecting:
try:
self._reconnecting = True
for host, port in self.neighbours:
if (host, port) not in self.connections:
self.log.info('reconnecting to %s:%s' % (host, port))
reactor.connectTCP(host, port, self)
self._delayed_reconnect = reactor.callLater(RECONNECT_INTERVAL, self._reconnect)
finally:
self._reconnecting = False
def startedConnecting(self, connector):
self.log.info('Started to connect to another server: %s:%s' % (
connector.host,
connector.port
))
def buildProtocol(self, addr):
conn = addr.host, addr.port
result = ClientFactory.buildProtocol(self, addr)
result.other_side = conn
def on_connect():
"""This callback will be called when actual connection happened."""
self._all_connections.append(result)
if addr.port in map(itemgetter(1), self.neighbours):
self.log.info('Connected to another server: %s:%s' % conn)
self.add_connection(result)
else:
self.log.info('Connection from another server accepted: %s:%s' % conn)
def on_disconnect():
self.remove_connection(result)
result.on_connect = on_connect
result.on_disconnect = on_disconnect
return result
def clientConnectionFailed(self, connector, reason):
self.log.info('Connection to %s:%s failed. Reason: %s' % (
connector.host,
connector.port,
reason
))
# BEGIN Paxos Transport methods
def broadcast(self, line):
for connection in self.connections.values():
connection.sendLine(line)
return len(self.connections)
@property
def quorum_size(self):
return max(2, len(self.connections)/ 2 + 1)
def on_learn(self, num, value, client):
"""First callback in the paxos result accepting chain."""
self.log.info('factory.on_learn %s %s' % (len(self._log) + 1, value))
self._log.append(value)
self._epoch += 1
splitted = shlex.split(value)
command_name, args = splitted[0], splitted[1:]
command = '_log_cmd_' + command_name.replace('-', '_')
cmd = getattr(self, command)
try:
return cmd(*args)
except Exception, e:
self.log.error('command "%s" failed: %s' % (command_name, e))
raise
class BaseDataGenerator:
"""
Base synthetic data provider class.
"""
def __init__(self,
episode_duration=None,
timeframe=1,
generator_fn=base_random_generator_fn,
generator_parameters_fn=base_generator_parameters_fn,
generator_parameters_config=None,
spread_generator_fn=None,
spread_generator_parameters=None,
name='BaseSyntheticDataGenerator',
data_names=('default_asset', ),
parsing_params=None,
target_period=-1,
global_time=None,
task=0,
log_level=WARNING,
_nested_class_ref=None,
_nested_params=None,
**kwargs):
"""
Args:
episode_duration: dict, duration of episode in days/hours/mins
generator_fn callabale, should return generated data as 1D np.array
generator_parameters_fn: callable, should return dictionary of generator_fn kwargs
generator_parameters_config: dict, generator_parameters_fn args
spread_generator_fn: callable, should return values of spread to form {High, Low}
spread_generator_parameters: dict, spread_generator_fn args
timeframe: int, data periodicity in minutes
name: str
data_names: iterable of str
target_period: int or dict, if set to -1 - disables `test` sampling
global_time: dict {y, m, d} to set custom global time (only for plotting)
task: int
log_level: logbook.Logger level
**kwargs:
"""
# Logging:
self.log_level = log_level
self.task = task
self.name = name
self.filename = self.name + '_sample'
self.target_period = target_period
self.data_names = data_names
self.data_name = self.data_names[0]
self.sample_instance = None
self.metadata = {
'sample_num': 0,
'type': None,
'parent_sample_type': None
}
self.data = None
self.data_stat = None
self.sample_num = 0
self.is_ready = False
if _nested_class_ref is None:
self.nested_class_ref = BaseDataGenerator
else:
self.nested_class_ref = _nested_class_ref
if _nested_params is None:
self.nested_params = dict(
episode_duration=episode_duration,
timeframe=timeframe,
generator_fn=generator_fn,
generator_parameters_fn=generator_parameters_fn,
generator_parameters_config=generator_parameters_config,
name=name,
data_names=data_names,
task=task,
log_level=log_level,
_nested_class_ref=_nested_class_ref,
_nested_params=_nested_params,
)
else:
self.nested_params = _nested_params
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(self.name, self.task),
level=self.log_level)
# Default sample time duration:
if episode_duration is None:
self.episode_duration = dict(
days=0,
hours=23,
minutes=55,
)
else:
self.episode_duration = episode_duration
# Btfeed parsing setup:
if parsing_params is None:
self.parsing_params = dict(names=['ask', 'bid', 'mid'],
datetime=0,
timeframe=1,
open='mid',
high='ask',
low='bid',
close='mid',
volume=-1,
openinterest=-1)
else:
self.parsing_params = parsing_params
self.columns_map = {
'open': 'mean',
'high': 'maximum',
'low': 'minimum',
'close': 'mean',
'bid': 'minimum',
'ask': 'maximum',
'mid': 'mean',
'volume': 'nothing',
}
self.nested_params['parsing_params'] = self.parsing_params
for key, value in self.parsing_params.items():
setattr(self, key, value)
# base data feed related:
self.params = {}
if global_time is None:
self.global_time = datetime.datetime(year=2018, month=1, day=1)
else:
self.global_time = datetime.datetime(**global_time)
self.global_timestamp = self.global_time.timestamp()
# Infer time indexes and sample number of records:
self.train_index = pd.timedelta_range(
start=datetime.timedelta(days=0, hours=0, minutes=0),
end=datetime.timedelta(**self.episode_duration),
freq='{}min'.format(self.timeframe))
self.test_index = pd.timedelta_range(
start=self.train_index[-1] +
datetime.timedelta(minutes=self.timeframe),
periods=len(self.train_index),
freq='{}min'.format(self.timeframe))
self.train_index += self.global_time
self.test_index += self.global_time
self.episode_num_records = len(self.train_index)
self.generator_fn = generator_fn
self.generator_parameters_fn = generator_parameters_fn
if generator_parameters_config is not None:
self.generator_parameters_config = generator_parameters_config
else:
self.generator_parameters_config = {}
self.spread_generator_fn = spread_generator_fn
if spread_generator_parameters is not None:
self.spread_generator_parameters = spread_generator_parameters
else:
self.spread_generator_parameters = {}
def set_logger(self, level=None, task=None):
"""
Sets logbook logger.
Args:
level: logbook.level, int
task: task id, int
"""
if task is not None:
self.task = task
if level is not None:
self.log = Logger('{}_{}'.format(self.name, self.task),
level=level)
def reset(self, **kwargs):
self.read_csv()
self.sample_num = 0
self.is_ready = True
def read_csv(self, **kwargs):
self.data = self.generate_data(
self.generator_parameters_fn(**self.generator_parameters_config))
def generate_data(self, generator_params, sample_type=0):
"""
Generates data trajectory, performs base consistency checks.
Args:
generator_params: dict, data_generating_function parameters
sample_type: 0 - generate train data | 1 - generate test data
Returns:
data as pandas dataframe
"""
assert sample_type in [0, 1],\
'Expected sample type be either 0 (train), or 1 (test) got: {}'.format(sample_type)
# Generate data points:
data_array = self.generator_fn(num_points=self.episode_num_records,
**generator_params)
assert len(data_array.shape) == 1 and data_array.shape[0] == self.episode_num_records,\
'Expected generated data to be 1D array of length {}, got data shape: {}'.format(
self.episode_num_records,
data_array.shape
)
if self.spread_generator_fn is not None:
spread_array = self.spread_generator_fn(
num_points=self.episode_num_records,
**self.spread_generator_parameters)
assert len(spread_array.shape) == 1 and spread_array.shape[0] == self.episode_num_records, \
'Expected generated spread to be 1D array of length {}, got data shape: {}'.format(
self.episode_num_records,
spread_array.shape
)
else:
spread_array = np.zeros(self.episode_num_records)
data_dict = {
'mean': data_array,
'maximum': data_array + .5 * spread_array,
'minimum': data_array - .5 * spread_array,
'nothing': data_array * 0.0,
}
# negs = data_dict['minimum'] < 0
# if negs.any():
# self.log.warning('{} negative generated values detected'.format(negs.shape[0]))
# Make dataframe:
if sample_type:
index = self.test_index
else:
index = self.train_index
# Map dictionary of data to dataframe columns:
df = pd.DataFrame(data={
name: data_dict[self.columns_map[name]]
for name in self.names
},
index=index)
# df = df.set_index('hh:mm:ss')
return df
def sample(self, get_new=True, sample_type=0, **kwargs):
"""
Samples continuous subset of data.
Args:
get_new (bool): not used;
sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets
respectively.
Returns:
Dataset instance with number of records ~ max_episode_len.
"""
try:
assert sample_type in [0, 1]
except AssertionError:
msg = 'Sampling attempt: expected sample type be in {}, got: {}'.format(
[0, 1], sample_type)
self.log.error(msg)
raise ValueError(msg)
if self.target_period == -1 and sample_type:
msg = 'Attempt to sample type {} given disabled target_period'.format(
sample_type)
self.log.error(msg)
raise ValueError(msg)
if self.metadata['type'] is not None:
if self.metadata['type'] != sample_type:
self.log.warning(
'Attempt to sample type {} given current sample type {}, overriden.'
.format(sample_type, self.metadata['type']))
sample_type = self.metadata['type']
# Get sample:
self.sample_instance = self.sample_synthetic(sample_type)
self.sample_instance.metadata['type'] = sample_type
self.sample_instance.metadata['sample_num'] = self.sample_num
self.sample_instance.metadata['parent_sample_num'] = self.metadata[
'sample_num']
self.sample_instance.metadata['parent_sample_type'] = self.metadata[
'type']
self.sample_num += 1
return self.sample_instance
def sample_synthetic(self, sample_type=0):
"""
Get data_generator instance containing synthetic data.
Args:
sample_type (int or bool): 0 (train) or 1 (test) - get sample with train or test time periods
respectively.
Returns:
nested_class_ref instance
"""
# Generate data:
generator_params = self.generator_parameters_fn(
**self.generator_parameters_config)
data = self.generate_data(generator_params, sample_type=sample_type)
# Make data_class instance:
sample_instance = self.nested_class_ref(**self.nested_params)
sample_instance.filename += '_{}'.format(self.sample_num)
self.log.info('New sample id: <{}>.'.format(sample_instance.filename))
# Add data and metadata:
sample_instance.data = data
sample_instance.metadata['generator'] = generator_params
sample_instance.metadata['first_row'] = 0
sample_instance.metadata['last_row'] = self.episode_num_records
return sample_instance
def describe(self):
"""
Returns summary dataset statistic as pandas dataframe:
- records count,
- data mean,
- data std dev,
- min value,
- 25% percentile,
- 50% percentile,
- 75% percentile,
- max value
for every data column.
"""
# Pretty straightforward, using standard pandas utility.
# The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again,
# thus avoiding passing big files to BT server:
flush_data = False
try:
assert not self.data.empty
pass
except (AssertionError, AttributeError) as e:
self.read_csv()
flush_data = True
self.data_stat = self.data.describe()
self.log.info('Data summary:\n{}'.format(self.data_stat.to_string()))
if flush_data:
self.data = None
self.log.info('Flushed data.')
return self.data_stat
def to_btfeed(self):
"""
Performs BTgymData-->bt.feed conversion.
Returns:
dict of type: {data_line_name: bt.datafeed instance}.
"""
try:
assert not self.data.empty
btfeed = btfeeds.PandasDirectData(dataname=self.data,
timeframe=self.timeframe,
datetime=self.datetime,
open=self.open,
high=self.high,
low=self.low,
close=self.close,
volume=self.volume,
openinterest=self.openinterest)
btfeed.numrecords = self.data.shape[0]
return {self.data_name: btfeed}
except (AssertionError, AttributeError) as e:
msg = 'Instance holds no data. Hint: forgot to call .read_csv()?'
self.log.error(msg)
raise AssertionError(msg)
def set_global_timestamp(self, timestamp):
pass
class Fibratus():
"""Fibratus entrypoint.
Setup the core components including the kernel
event stream collector and the tracing controller.
At this point the system handles are also being
enumerated.
"""
def __init__(self, filament):
self.logger = Logger(Fibratus.__name__)
self.file_handler = FileHandler(os.path.join(os.path.abspath(__file__), '..', '..', '..', 'fibratus.log'),
mode='w+')
self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of kernel trace
with self.file_handler.applicationbound():
self.logger.info('Starting fibratus...')
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository, self._handles)
self.kevent = KEvent(self.thread_registry)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.requires_render = {}
self.filters_count = 0
def run(self):
@atexit.register
def _exit():
self.stop_ktrace()
self.kcontroller.start_ktrace(etw.KERNEL_LOGGER_NAME, self.ktrace_props)
def on_kstream_open():
if self._filament is None:
IO.write_console('Done! ')
self.kevt_streamc.set_kstream_open_callback(on_kstream_open)
self._open_kstream()
def _open_kstream(self):
try:
self.kevt_streamc.open_kstream(self._on_next_kevent)
except Exception as e:
with self.file_handler.applicationbound():
self.logger.error(e)
except KeyboardInterrupt:
self.stop_ktrace()
def stop_ktrace(self):
IO.write_console('Stopping fibratus...')
if self._filament:
self._filament.close()
self.kcontroller.stop_ktrace(self.ktrace_props)
self.kevt_streamc.close_kstream()
def add_filters(self, kevent_filters):
if len(kevent_filters) > 0:
self.filters_count = len(kevent_filters)
# include the basic filters
# that are essential to the
# rest of kernel events
self.kevt_streamc.add_kevent_filter(ENUM_PROCESS)
self.kevt_streamc.add_kevent_filter(ENUM_THREAD)
self.kevt_streamc.add_kevent_filter(ENUM_IMAGE)
self.kevt_streamc.add_kevent_filter(REG_CREATE_KCB)
self.kevt_streamc.add_kevent_filter(REG_DELETE_KCB)
# these kevents are necessary for consistent state
# of the trace. If the user doesn't include them
# in a filter list, then we do the job but set the
# kernel event type as not eligible for rendering
if not KEvents.CREATE_PROCESS in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_PROCESS)
self.requires_render[CREATE_PROCESS] = False
else:
self.requires_render[CREATE_PROCESS] = True
if not KEvents.CREATE_THREAD in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_THREAD)
self.requires_render[CREATE_THREAD] = False
else:
self.requires_render[CREATE_THREAD] = True
if not KEvents.CREATE_FILE in kevent_filters:
self.kevt_streamc.add_kevent_filter(CREATE_FILE)
self.requires_render[CREATE_FILE] = False
else:
self.requires_render[CREATE_FILE] = True
for kevent_filter in kevent_filters:
ktuple = kname_to_tuple(kevent_filter)
if isinstance(ktuple, list):
for kt in ktuple:
self.kevt_streamc.add_kevent_filter(kt)
if not kt in self.requires_render:
self.requires_render[kt] = True
else:
self.kevt_streamc.add_kevent_filter(ktuple)
if not ktuple in self.requires_render:
self.requires_render[ktuple] = True
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS,
CREATE_THREAD,
ENUM_PROCESS,
ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [CREATE_FILE,
DELETE_FILE,
CLOSE_FILE,
READ_FILE,
WRITE_FILE]:
self.fsio.parse_fsio(ktype, kparams)
self._render(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._render(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._render(ktype)
# registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [REG_CREATE_KEY,
REG_DELETE_KEY,
REG_OPEN_KEY,
REG_QUERY_KEY,
REG_SET_VALUE,
REG_DELETE_VALUE,
REG_QUERY_VALUE]:
self.hive_parser.parse_hive(ktype, kparams)
self._render(ktype)
# network kernel events
elif ktype in [SEND_SOCKET_TCPV4,
SEND_SOCKET_UDPV4,
RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4,
ACCEPT_SOCKET_TCPV4,
CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4,
RECONNECT_SOCKET_TCPV4]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._render(ktype)
if self._filament:
# call filament method
# to process the next
# kernel event from the stream
if ktype not in [ENUM_PROCESS,
ENUM_THREAD, ENUM_IMAGE]:
if self.kevent.name:
self._filament.process(self.kevent)
def _render(self, ktype):
"""Renders the kevent to the standard output stream.
Parameters
----------
ktype: tuple
Identifier of the kernel event
"""
if not self._filament:
if ktype in self.requires_render:
rr = self.requires_render[ktype]
if rr:
self.kevent.render()
elif self.filters_count == 0:
self.kevent.render()
class InsteonPLM(object):
"""
Class used to connect to Insteon serial power line modem (PLM)
"""
debug = 1
# Controls verbosity output
Interface = None
def __init__(self, port="/dev/insteon_plm", redislog=False):
"""
"""
if redislog:
self.Log = logger.RedisLogger("insteon.py:InsteonPLM")
self.Log.addHandler(handlers.RedisHandler.to("log", host="localhost", port=6379))
else:
self.Log = Logger("InsteonPLM")
self.channel = "cmd:insteon"
self.redis = redis.Redis()
self.pubsub = self.redis.pubsub()
if os.name == "nt":
port = port - 1
try:
self.uart = serial.Serial(port=port, baudrate=19200, timeout=3)
self.uart.timeout = 1
self.Log.debug("__init__(port={0}, channel={1})".format(port, self.channel))
except:
self.uart = None
self.Log.error("Error occured while oppening serial Interface")
self._start_listner()
def __del__(self):
if self.is_open():
self._stop_thread()
self.uart.close()
def __unicode__(self):
if self.uart is not None:
name = "PLM(%s)" % self.uart.port
else:
name = "PLM(None)"
return name
def is_open(self):
"""return open if port is open"""
return self.uart.isOpen()
def __open__(self):
if not (self.uart.isOpen()):
try:
self.uart.open()
self.Log.debug("Port {} is now open".format(str(self.uart.port)))
return PORT_OPEN
except uartial.uartialException:
self.Log.error("uart Exception Cannot open port: {} ".format(str(self.uart.port)))
return PORT_ERROR
except:
self.Log.error("OtherException Cannot open port: {}".format(str(self.uart.port)))
return PORT_ERROR
else:
self.Log.debug("Port {} is already open".format(str(self.uart.port)))
return PORT_OPEN
def read(self):
if self.uart is not None:
data = self.uart.read(self.uart.inWaiting())
self.Log.debug("read(): = {}".format(str(data)))
else:
self.Log.debug("No uartial connection")
data = ""
return data
def send(self, cmd):
self.Log.debug("send(cmd={}):".format(str(cmd)))
if not isinstance(cmd, list):
raise TypeError("cmd must be a list")
if len(cmd) < 2:
raise ValueError("every insteon command must be at least 2 bytes long")
if cmd[0] != 2:
raise ValueError("All insteon commands must start with 0x02 (2 dec)")
cmd_meta = cmd_dict.get(cmd[1], 0)
if isinstance(cmd_meta, int) and cmd_meta == 0:
raise ValueError("Command 0x{0:x} ({0}) was not found in command dictionary".format(cmd[1]))
cmd_length = cmd_meta[1]
if len(cmd) != cmd_length:
raise ValueError('Command 0x{0:x} ({0}) - "{1}", must be {2} bytes'.format(cmd[1], cmd_meta[0], cmd_length))
if self.is_open():
out = self.uart.write(cmd)
else:
self.Log.debug("Serial port is not open")
out = 0
if out != cmd_length:
self.Log.debug("Warning serial port sent {0} bytes while {1} were expected".format(out, cmd_length))
return out
def query(self, cmd, expected_response_length=None):
# read data which could have arrived
leftover_buffer = self.read()
if not cmd_dict.has_key(cmd[1]):
self.Log.debug("Command not found")
return [None, leftover_buffer]
cmd_details = cmd_dict.get(cmd[1])
if expected_response_length is None:
expected_response_length = cmd_details[2]
# TODO this is bit of a hack and will not work for extended commands
if cmd[1] == 0x62:
expected_response_length += 11
# print "Sending {0}, expecting response length of {1}".format(cmd_details[0], expected_response_length)
bytes_sent = self.send(cmd)
timeout = 1
to = time.time()
tn = to
response = ""
n = 0
while tn - to < timeout and n < expected_response_length:
n = self.uart.inWaiting()
tn = time.time()
if n >= expected_response_length:
# response += self.Ser.read(cmd_details[2])
response += self.read()
data = [ord(x) for x in response]
dbg_msg = "cmd succesful: = {0}".format(data)
return_data = data
else:
dbg_msg = "did not recive expected number of bytes within the time out period"
response += self.read()
data = [ord(x) for x in response]
return_data = data
self.Log.debug(dbg_msg)
return [return_data, leftover_buffer]
def send_sd_cmd(self, *args):
cmd = build_sd_message(args[0], 15, args[1], args[2])
res = self.query(cmd)
parsed = parse(res[0])[0]
if len(parsed) > 1:
ack_received = parsed[1][2]["flag"][1][0] == 1
if ack_received:
return [parsed[1][2]["cmd2"], parsed]
return None
###########################################################################################################
# REDIS Thread
#
def _start_listner(self):
self.Log.debug("Start redis sub channel and listen for commands send via redis")
self._redis_subscriber_alive = True
self.redis_subscriber_thread = threading.Thread(target=self.cmd_via_redis_subscriber)
self.redis_subscriber_thread.setDaemon(True)
self.redis_subscriber_thread.start()
def _stop_thread(self):
self.Log.debug("_stop_thread()")
self._redis_subscriber_alive = False
self.redis.publish(self.channel, "unsubscribe")
self.redis_subscriber_thread.join(3)
if self.redis_subscriber_thread.is_alive():
self.Log.debug("thread stopped()")
else:
self.Log.debug("thread still alive after 3 delay")
# publish cmd:insteon "{\"cmd\": \"SetSwOff\", \"addr\" : \"20.1f.11\"}"
def cmd_via_redis_subscriber(self):
self.Log.debug("cmd_via_redis_subscriber()")
self.pubsub.subscribe(self.channel)
while self._redis_subscriber_alive:
try:
for item in self.pubsub.listen():
self.Log.debug(item)
if item["data"] == "unsubscribe":
self.pubsub.unsubscribe()
self.Log.info("unsubscribed and finished")
break
else:
cmd = item["data"]
if isinstance(cmd, str):
self.Log.debug("cmd_via_redis_subscriber() cmd = {}".format(cmd))
try:
cmd_obj = deserialize(cmd)
if isinstance(cmd_obj, dict):
self.Log.debug(cmd_obj)
cmd_str = cmd_obj["cmd"]
if cmd_str == "GetStatusOfAllDevices":
res = self.GetStatusOfAllDevices()
final_data = dict()
timestamp = datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
final_data["timestamp"] = timestamp
final_data["raw"] = res
final_data["data"] = res
M = Message("InsteonPLM")
M.msg = final_data
self.redis.publish("data", M.as_json())
if cmd_str == "GetLightLevel":
addr_str = cmd_obj["addr"]
res = self.GetLightLevel(addr_str)
if cmd_str == "SetLightLevel":
addr_str = cmd_obj["addr"]
val = cmd_obj["val"]
res = self.SetLightLevel(addr_str, val)
if cmd_str == "SetSwOn":
addr_str = cmd_obj["addr"]
res = self.SetSwOn(addr_str)
if cmd_str == "SetSwOff":
addr_str = cmd_obj["addr"]
res = self.SetSwOff(addr_str)
else:
addr = str2hex(cmd_obj[0])
insteon_cmd = cmd_obj[1]
val = cmd_obj[2]
res = self.send_sd_cmd(addr, insteon_cmd, val)
self.Log.debug(res)
self.redis.publish(self.channel + "_res", serialize(res))
except Exception as E:
error_msg = {
"source": "serinsteon:cmd_via_redis_subscriber",
"function": "def run(self):",
"error": E.message,
}
self.Log.error(error_msg)
else:
self.Log.debug(cmd)
except Exception as E:
error_msg = {"source": "InsteonSub", "function": "def run(self):", "error": E.message}
self.Log.error(error_msg)
self.Log.debug("end of cmd_via_redis_subscriber()")
###########################################################################################################
# HIGH LEVEL FUNCTIONS
#
def Ping(self, addr):
out = self.send_sd_cmd(addr, 0x0F, 0)
if out is None:
return False
if out[0] == 0:
return True
else:
return False
def GetLightLevel(self, addr):
self.Log.debug("GetLightLevel")
out = self.send_sd_cmd(addr, 25, 0)
if out is None:
return -1
return round(float(out[0]) / 2.55)
def SetLightLevel(self, addr, level):
self.Log.debug("SetLightLevel")
if level < 0 or level > 100:
raise ValueError("Light level must be between 0-100%")
level = int(round(255 * float(level) / 100.0))
out = self.send_sd_cmd(addr, 17, level)
return round(float(out[0]) / 2.55)
def SetSwOff(self, addr):
self.Log.debug("SetSwOff({})".format(addr))
out = self.send_sd_cmd(addr, 19, 0)
return out
def SetSwOn(self, addr):
self.Log.debug("SetSwOn({})".format(addr))
out = self.send_sd_cmd(addr, 17, 255)
return out
def GetStatusOfAllDevices(self, devices=devices):
self.Log.debug("GetStatusOfAllDevices")
data = []
for k, v in devices.items():
for i in range(2):
stat = self.GetLightLevel(k)
if stat == -1:
sleep(0.1)
else:
break
data.append([k, stat, v])
return data
if not statuses:
break
for status in statuses:
tweet = status.AsDict()
tweet_id = str(tweet['id'])
if tweet_id not in existing_tweets_id:
log.debug(u"Processing tweet: %s" % tweet_id)
if tweet['user']['screen_name']==SCREEN_NAME and not tweet['retweeted']:
created_at = datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y")
retweet_count = 'retweet_count' in tweet and tweet['retweet_count'] or u'0'
log.debug(u"Urlizing content: %s" % tweet['text'])
try:
content = __urlize(tweet['text'])
except socket.gaierror:
content = tweet['text']
log.error(u"Error urlizing: %s" % content)
log.error(u"Verify the tweet online: https://twitter.com/%s/status/%s" % (SCREEN_NAME, tweet_id))
except socket.error: # couldn't figure out the difference easily
content = tweet['text']
log.error(u"Alternative error urlizing: %s" % content)
log.error(u"Verify the tweet online: https://twitter.com/%s/status/%s" % (SCREEN_NAME, tweet_id))
text = u'%s$%s$%s\n%s' % (tweet_id, created_at, retweet_count, content)
open(TWEETS_PATH+tweet_id+'.txt', 'w').write(text.encode('utf-8'))
time.sleep(0.1) # throttled api + urlize
else:
log.info(u"Ownership concern, check: https://twitter.com/%s/status/%s" % (SCREEN_NAME, tweet_id))
else:
log.debug(u"Already fetched tweet: %s" % tweet_id)
if option_delete:
try:
api.DestroyStatus(tweet_id)
class IrqSubmit(threading.Thread):
def __init__(self, channel='irq', host='127.0.0.1', submit_to='127.0.0.1'):
threading.Thread.__init__(self)
self.redis = Redis(host=host)
self.submit_to = submit_to
self.msg_count = 0
self.busy = 0;
self.last = []
self.Q = Queue(connection=Redis())
self.last_q = self.Q.enqueue(submit,([[0,'0',0]]))
self.channel = channel
self.pubsub = self.redis.pubsub()
self.Log = Logger('IrqSubmit')
self.pubsub.subscribe(self.channel)
self.start()
self.setName('IrqSubmit-Thread')
def __del__(self):
self.Log.info('__del__()')
self.stop()
def stop(self):
self.Log.info('stop()')
self.busy = False
self.redis.publish(self.channel,'KILL')
time.sleep(1)
self.Log.info(' stopped')
def run(self):
self.Log.debug('run()')
for item in self.pubsub.listen():
if item['data'] == "KILL":
self.pubsub.unsubscribe()
self.Log.info("unsubscribed and finished")
break
if item['data'] == "ERROR_TEST":
self.redis.publish('error', __name__ + ": ERROR_TEST")
else:
if item['type'] == 'message':
self.process_message(item)
#self.process_message(item)
self.Log.debug('end of run()')
def process_message(self, item):
#self.Log.debug('process_message()')
#print(item)
try:
#print(" item = {0}".format(item))
msg = sjson.loads(item['data'])
print(" msg = {0}".format(msg))
device_data = msg['MSG']['data']
timestamp = msg['MSG']['timestamp']
print(" msg = {0}".format(msg))
for data in device_data:
sn = data[0]
processing_function = ProcessingFunctions.get(sn,process_default)
val = processing_function(data)
threshold = 0
submit_data = [[sn, val]]
if len(sn):
print("Final dataset : sn = {0}, val = {1}".format(sn, val))
self.last_enqueue = self.Q.enqueue(submit, submit_data,\
timestamp=timestamp,\
submit_to=self.submit_to,\
threshold=threshold)
except Exception as E:
self.Log.error("process_message(): " + E.message)
self.Log.error(item)
class BaseSynchroRunner():
"""
Data provider class. Interacts with environment and outputs data in form of rollouts augmented with
relevant summaries and metadata. This runner is `synchronous` in sense that data collection is `in-process'
and every rollout is collected by explicit call to respective `get_data()` method [this is unlike 'async-`
thread-runner version found earlier in this this package which, once being started,
runs on its own and can not be moderated].
Makes precise control on policy being executed possible.
Does not support 'atari' mode.
"""
def __init__(self,
env,
task,
rollout_length,
episode_summary_freq,
env_render_freq,
ep_summary,
test=False,
policy=None,
data_sample_config=None,
memory_config=None,
test_conditions=None,
slowdown_steps=0,
global_step_op=None,
aux_render_modes=None,
_implemented_aux_render_modes=None,
name='synchro',
log_level=WARNING,
**kwargs):
"""
Args:
env: BTgym environment instance
task: int, runner task id
rollout_length: int
episode_summary_freq: int
env_render_freq: int
test: legacy, not used
ep_summary: legacy, not used
policy: policy instance to execute
data_sample_config: dict, data sampling configuration dictionary
memory_config: dict, replay memory configuration
test_conditions: dict or None,
dictionary of single experience conditions to check to mark it as test one.
slowdown_time: time to sleep between steps
aux_render_modes: iterable of str, additional summaries to compute
_implemented_aux_render_modes iterable of str, implemented additional summaries
name: str, name scope
log_level: int, logbook.level
"""
self.env = env
self.task = task
self.name = name
self.rollout_length = rollout_length
self.episode_summary_freq = episode_summary_freq
self.env_render_freq = env_render_freq
self.memory_config = memory_config
self.policy = policy
self.data_sample_config = data_sample_config
self.log_level = log_level
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_Runner_{}'.format(self.name, self.task),
level=self.log_level)
# Aux rendering setup:
if _implemented_aux_render_modes is None:
self.implemented_aux_render_modes = []
else:
self.implemented_aux_render_modes = _implemented_aux_render_modes
self.aux_render_modes = []
if aux_render_modes is not None:
for mode in aux_render_modes:
if mode in self.implemented_aux_render_modes:
self.aux_render_modes.append(mode)
else:
msg = 'Render mode `{}` is not implemented.'.format(mode)
self.log.error(msg)
raise NotImplementedError(msg)
self.log.debug('self.render modes: {}'.format(self.aux_render_modes))
self.sess = None
self.summary_writer = None
self.global_step_op = global_step_op
if self.task == 0 and slowdown_steps > 0 and self.global_step_op is not None:
self.log.notice(
'is slowed down by {} global_iterations/step'.format(
slowdown_steps))
self.slowdown_steps = slowdown_steps
else:
self.slowdown_steps = 0
if test_conditions is None:
# Default test conditions are: experience comes from test episode, from target domain:
self.test_conditions = {
'state': {
'metadata': {
'type': 1,
'trial_type': 1
}
}
}
else:
self.test_conditions = test_conditions
# Make replay memory:
if self.memory_config is not None:
self.memory = self.memory_config['class_ref'](
**self.memory_config['kwargs'])
else:
self.memory = _DummyMemory()
self.length = 0
self.local_episode = 0
self.reward_sum = 0
self.terminal_end = True
# Summary averages accumulators:
self.total_r = []
self.cpu_time = []
self.final_value = []
self.total_steps = []
self.total_steps_atari = []
self.info = [None]
self.pre_experience = None
self.state = None
self.context = None
self.last_action = None
self.last_reward = None
# Episode accumulators:
self.ep_accum = None
self.log.debug('__init__() done.')
def sleep(self):
if self.slowdown_steps > 0:
start_global_step = self.sess.run(self.global_step_op)
while start_global_step + self.slowdown_steps > self.sess.run(
self.global_step_op):
time.sleep(0.05)
def start_runner(self, sess, summary_writer, **kwargs):
"""
Legacy wrapper.
"""
self.start(sess, summary_writer, **kwargs)
def start(self,
sess,
summary_writer,
init_context=None,
data_sample_config=None):
"""
Executes initial sequence; fills initial replay memory if any.
"""
assert self.policy is not None, 'Initial policy not specified'
self.sess = sess
self.summary_writer = summary_writer
# # Hacky but we need env.renderer methods ready: NOT HERE, went to VerboseRunner
# self.env.renderer.initialize_pyplot()
self.pre_experience, self.state, self.context, self.last_action, self.last_reward = self.get_init_experience(
policy=self.policy,
init_context=init_context,
data_sample_config=data_sample_config)
if self.memory_config is not None:
while not self.memory.is_full():
# collect some rollouts to fill memory:
_ = self.get_data()
self.log.notice('Memory filled')
self.log.notice('started collecting data.')
def get_init_experience(self,
policy,
policy_sync_op=None,
init_context=None,
data_sample_config=None):
"""
Starts new environment episode.
Args:
policy: policy to execute.
policy_sync_op: operation copying local behavioural policy params from global one
init_context: initial policy context for new episode.
data_sample_config: configuration dictionary of type `btgym.datafeed.base.EnvResetConfig`
Returns:
incomplete initial experience of episode as dictionary (misses bootstrapped R value),
next_state,
next, policy RNN context
action_reward
"""
self.length = 0
self.reward_sum = 0
# Increment global and local episode counters:
self.sess.run(self.policy.inc_episode)
self.local_episode += 1
# self.log.warning('get_init_exp() data_sample_config: {}'.format(data_sample_config))
if data_sample_config is None:
data_sample_config = policy.get_sample_config()
# Pass sample config to environment (.get_sample_config() is actually aac framework method):
init_state = self.env.reset(**data_sample_config)
# Master worker always resets context at the episode beginning:
# TODO: !
# if not self.data_sample_config['mode']:
init_context = None
# self.log.warning('init_context_passed: {}'.format(init_context))
# self.log.warning('state_metadata: {}'.format(state['metadata']))
init_action = self.env.action_space.encode(
self.env.get_initial_action())
init_reward = np.asarray(0.0)
# Update policy:
if policy_sync_op is not None:
self.sess.run(policy_sync_op)
init_context = policy.get_initial_features(state=init_state,
context=init_context)
action, logits, value, next_context = policy.act(
init_state,
init_context,
init_action[None, ...],
init_reward[None, ...],
)
next_state, reward, terminal, self.info = self.env.step(
action['environment'])
experience = {
'position': {
'episode': self.local_episode,
'step': self.length
},
'state': init_state,
'action': action['one_hot'],
'reward': reward,
'value': value,
'terminal': terminal,
'context': init_context,
'last_action': init_action,
'last_reward': init_reward,
'r': None, # to be updated
'info': self.info[-1],
}
# Execute user-defined callbacks to policy, if any:
for key, callback in policy.callback.items():
experience[key] = callback(**locals())
# reset per-episode counters and accumulators:
self.ep_accum = {
'logits': [logits],
'value': [value],
'context': [init_context]
}
self.terminal_end = terminal
#self.log.warning('init_experience_context: {}'.format(context))
# Take a nap:
self.sleep()
return experience, next_state, next_context, action['encoded'], reward
def get_experience(self,
policy,
state,
context,
action,
reward,
policy_sync_op=None):
"""
Get single experience (possibly terminal).
Returns:
incomplete experience as dictionary (misses bootstrapped R value),
next_state,
next, policy RNN context
action_reward
"""
# Update policy:
# TODO: for contd. enable for meta-learning
# if policy_sync_op is not None:
# self.sess.run(policy_sync_op)
# TODO: meta-learning related; TEMPORAL, refract:
# if hasattr(policy, 'meta') and self.task == 0:
# wait = 1
# i = 0
# while wait:
# wait = policy.meta.act()
# # self.log.warning('policy_meta_action: {}'.format(wait))
# time.sleep(4)
# # i += 1
# self.sess.run([policy_sync_op, policy.meta.sync_slot_op])
#
# policy.meta.reset()
# policy.meta.global_reset()
#
# # self.log.notice('waited: {}'.format(i))
# Continue adding experiences to rollout:
next_action, logits, value, next_context = policy.act(
state,
context,
action[None, ...],
reward[None, ...],
)
self.ep_accum['logits'].append(logits)
self.ep_accum['value'].append(value)
self.ep_accum['context'].append(next_context)
# self.log.notice('context: {}'.format(context))
next_state, next_reward, terminal, self.info = self.env.step(
next_action['environment'])
# Partially compose experience:
experience = {
'position': {
'episode': self.local_episode,
'step': self.length
},
'state': state,
'action': next_action['one_hot'],
'reward': next_reward,
'value': value,
'terminal': terminal,
'context': context,
'last_action': action,
'last_reward': reward,
'r': None,
'info': self.info[-1],
}
for key, callback in policy.callback.items():
experience[key] = callback(**locals())
# Housekeeping:
self.length += 1
# Take a nap:
# self.sleep()
return experience, next_state, next_context, next_action[
'encoded'], next_reward
def get_train_stat(self, is_test=False):
"""
Updates and computes average statistics for train episodes.
Args:
is_test: bool, current episode type
Returns:
dict of stats
"""
ep_stat = {}
if not is_test:
self.total_r += [self.reward_sum]
episode_stat = self.env.get_stat() # get episode statistic
last_i = self.info[-1] # pull most recent info
self.cpu_time += [episode_stat['runtime'].total_seconds()]
self.final_value += [last_i['broker_value']]
self.total_steps += [episode_stat['length']]
if self.local_episode % self.episode_summary_freq == 0:
ep_stat = dict(total_r=np.average(self.total_r),
cpu_time=np.average(self.cpu_time),
final_value=np.average(self.final_value),
steps=np.average(self.total_steps))
self.total_r = []
self.cpu_time = []
self.final_value = []
self.total_steps = []
self.total_steps_atari = []
return ep_stat
def get_test_stat(self, is_test=False):
"""
Updates and computes statistics for single test episode.
Args:
is_test: bool, current episode type
Returns:
dict of stats
"""
ep_stat = {}
if is_test:
episode_stat = self.env.get_stat() # get episode statistic
last_i = self.info[-1] # pull most recent info
ep_stat = dict(total_r=self.reward_sum,
final_value=last_i['broker_value'],
steps=episode_stat['length'])
return ep_stat
def get_ep_render(self, is_test=False):
"""
Collects environment renderings. Relies on environment renderer class methods,
so it is only valid when environment rendering is enabled (typically it is true for master runner).
Returns:
dictionary of images as rgb arrays
"""
# Only render chief worker and test (slave) environment:
# if self.task < 1 and (
# is_test or(
# self.local_episode % self.env_render_freq == 0 and not self.data_sample_config['mode']
# )
# ):
if self.task < 1 and self.local_episode % self.env_render_freq == 0:
# Render environment (chief worker only):
render_stat = {
mode: self.env.render(mode)[None, :]
for mode in self.env.render_modes
}
else:
render_stat = None
return render_stat
def get_data(self,
policy=None,
policy_sync_op=None,
init_context=None,
data_sample_config=None,
rollout_length=None,
force_new_episode=False):
"""
Collects single trajectory rollout and bunch of summaries using specified policy.
Updates episode statistics and replay memory.
Args:
policy: policy to execute
policy_sync_op: operation copying local behavioural policy params from global one
init_context: if specified, overrides initial episode context provided bu self.context
(valid only if new episode is started within this rollout).
data_sample_config: environment configuration parameters for next episode to sample:
configuration dictionary of type `btgym.datafeed.base.EnvResetConfig
rollout_length: length of rollout to collect, if specified - overrides self.rollout_length attr
force_new_episode: bool, if True - resets the environment
Returns:
data dictionary
"""
if policy is None:
policy = self.policy
if init_context is None:
init_context = self.context
if rollout_length is None:
rollout_length = self.rollout_length
rollout = Rollout()
is_test = False
train_ep_summary = None
test_ep_summary = None
render_ep_summary = None
if self.terminal_end or force_new_episode:
# Start new episode:
self.pre_experience, self.state, self.context, self.last_action, self.last_reward = self.get_init_experience(
policy=policy,
policy_sync_op=policy_sync_op,
init_context=init_context,
data_sample_config=data_sample_config)
# self.log.warning(
# 'started new episode with:\ndata_sample_config: {}\nforce_new_episode: {}'.
# format(data_sample_config, force_new_episode)
# )
# self.log.warning('pre_experience_metadata: {}'.format(self.pre_experience['state']['metadata']))
# NOTE: self.terminal_end is set actual via get_init_experience() method
# Collect single rollout:
while rollout.size < rollout_length - 1 and not self.terminal_end:
if self.pre_experience['terminal']:
# Episode has been just finished,
# need to complete and push last experience and update all episode summaries
self.pre_experience['r'] = np.asarray([0.0])
experience = None
self.state = None
self.context = None
self.last_action = None
self.last_reward = None
self.terminal_end = True
train_ep_summary = self.get_train_stat(is_test)
test_ep_summary = self.get_test_stat(is_test)
render_ep_summary = self.get_ep_render(is_test)
else:
experience, self.state, self.context, self.last_action, self.last_reward = self.get_experience(
policy=policy,
policy_sync_op=policy_sync_op,
state=self.state,
context=self.context,
action=self.last_action,
reward=self.last_reward)
# Complete previous experience by bootstrapping V from next one:
self.pre_experience['r'] = experience['value']
# Push:
rollout.add(self.pre_experience)
# Where are you coming from?
is_test = is_subdict(self.test_conditions, self.pre_experience)
# try:
# # Was it test (i.e. test episode from traget domain)?
# if self.pre_experience['state']['metadata']['type']\
# and self.pre_experience['state']['metadata']['trial_type']:
# is_test = True
#
# except KeyError:
# pass
# Only training rollouts are added to replay memory:
if not is_test:
self.memory.add(self.pre_experience)
self.reward_sum += self.pre_experience['reward']
# Move one step froward:
self.pre_experience = experience
# Done collecting rollout, either got termination of episode or not:
if not self.terminal_end:
# Bootstrap:
self.pre_experience['r'] = np.asarray([
policy.get_value(
self.pre_experience['state'],
self.pre_experience['context'],
self.pre_experience['last_action'][None, ...],
self.pre_experience['last_reward'][None, ...],
)
])
rollout.add(self.pre_experience)
if not is_test:
self.memory.add(self.pre_experience)
# self.log.warning('rollout.terminal: {}'.format(self.terminal_end))
# self.log.warning('rollout.size: {}'.format(rollout.size))
data = dict(
on_policy=rollout,
terminal=self.terminal_end,
off_policy=self.memory.sample_uniform(
sequence_size=rollout_length),
off_policy_rp=self.memory.sample_priority(exact_size=True),
ep_summary=train_ep_summary,
test_ep_summary=test_ep_summary,
render_summary=render_ep_summary,
is_test=is_test,
)
return data
def get_episode(self,
policy=None,
policy_sync_op=None,
init_context=None,
data_sample_config=None):
"""
== WRONG DO NOT USE ===
Collects entire episode trajectory as single rollout and bunch of summaries using specified policy.
Updates episode statistics and replay memory.
Args:
policy: policy to execute
policy_sync_op: operation copying local behavioural policy params from global one
init_context: if specified, overrides initial episode context provided bu self.context
data_sample_config: environment configuration parameters for next episode to sample:
configuration dictionary of type `btgym.datafeed.base.EnvResetConfig
Returns:
data dictionary
"""
if policy is None:
policy = self.policy
if init_context is None:
init_context = self.context
elif init_context == 0: # mmm... TODO: fix this shame
init_context = None
rollout = Rollout()
train_ep_summary = None
test_ep_summary = None
render_ep_summary = None
# Start new episode:
self.pre_experience, self.state, self.context, self.last_action, self.last_reward = self.get_init_experience(
policy=policy,
policy_sync_op=policy_sync_op,
init_context=
init_context, # None (initially) or final context of previous episode
data_sample_config=data_sample_config)
is_test = is_subdict(self.test_conditions, self.pre_experience)
# try:
# # Was it test (`type` in metadata is not zero)?
# # TODO: change to source/target?
# if self.pre_experience['state']['metadata']['type']:
# is_test = True
#
# except KeyError:
# pass
# Collect data until episode is over:
while not self.terminal_end:
experience, self.state, self.context, self.last_action = self.get_experience(
policy=policy,
policy_sync_op=policy_sync_op,
state=self.state,
context=self.context,
action=self.last_action,
reward=self.last_reward,
)
# Complete previous experience by bootstrapping V from next one:
self.pre_experience['r'] = experience['value']
# Push:
rollout.add(self.pre_experience)
if not is_test:
self.memory.add(self.pre_experience)
# Move one step froward:
self.pre_experience = experience
self.reward_sum += experience['reward']
if self.pre_experience['terminal']:
# Episode has been just finished,
# need to complete and push last experience and update all episode summaries:
self.terminal_end = True
# Bootstrap:
# TODO: should be zero here
self.pre_experience['r'] = np.asarray([
policy.get_value(
self.pre_experience['state'],
self.pre_experience['context'],
self.pre_experience['last_reward'][None, ...],
self.pre_experience['last_reward'][None, ...],
)
])
rollout.add(self.pre_experience)
if not is_test:
self.memory.add(self.pre_experience)
train_ep_summary = self.get_train_stat(is_test)
test_ep_summary = self.get_test_stat(is_test)
render_ep_summary = self.get_ep_render(is_test)
# self.log.warning('episodic_rollout.size: {}'.format(rollout.size))
data = dict(
on_policy=rollout,
terminal=self.terminal_end,
off_policy=self.memory.sample_uniform(
sequence_size=self.rollout_length),
off_policy_rp=self.memory.sample_priority(exact_size=True),
ep_summary=train_ep_summary,
test_ep_summary=test_ep_summary,
render_summary=render_ep_summary,
is_test=is_test,
)
return data
def get_batch(self,
size,
policy=None,
policy_sync_op=None,
require_terminal=True,
same_trial=True,
init_context=None,
data_sample_config=None):
"""
Returns batch as list of 'size' or more rollouts collected under specified policy.
Rollouts can be collected from several episodes consequently; there is may be more rollouts than set 'size' if
it is necessary to collect at least one terminal rollout.
Args:
size: int, number of rollouts to collect
policy: policy to use
policy_sync_op: operation copying local behavioural policy params from global one
require_terminal: bool, if True - require at least one terminal rollout to be present.
same_trial: bool, if True - all episodes are sampled from same trial
init_context: if specified, overrides initial episode context provided bu self.context
data_sample_config: environment configuration parameters for all episodes in batch:
configuration dictionary of type `btgym.datafeed.base.EnvResetConfig
Returns:
dict containing:
'data'key holding list of data dictionaries;
'terminal_context' key holding list of terminal output contexts.
If 'require_terminal = True, this list is guarantied to hold at least one element.
"""
batch = []
terminal_context = []
if require_terminal:
got_terminal = False
else:
got_terminal = True
if same_trial:
assert isinstance(data_sample_config, dict),\
'get_batch(same_trial=True) expected `data_sample_config` dict., got: {}'.format(data_sample_config)
# Collect first rollout:
batch = [
self.get_data(policy=policy,
policy_sync_op=policy_sync_op,
init_context=init_context,
data_sample_config=data_sample_config,
force_new_episode=True)
]
if same_trial:
# sample new episodes from same trial only:
data_sample_config['trial_config']['get_new'] = False
collected_size = 1
if batch[0]['terminal']:
terminal_context.append(self.context)
got_terminal = True
# Collect others:
while not (collected_size >= size and got_terminal):
rollout_data = self.get_data(policy=policy,
policy_sync_op=policy_sync_op,
init_context=init_context,
data_sample_config=data_sample_config)
batch.append(rollout_data)
if rollout_data['terminal']:
terminal_context.append(self.context)
got_terminal = True
collected_size += 1
data = dict(
data=batch,
terminal_context=terminal_context,
)
return data
class Admin(object):
"""Administrative task object."""
def __init__(self, conf):
"""conf: dict, yaml parameters."""
self.conf = conf
handler = TimedRotatingFileHandler(conf.log_file, date_format="%Y-%m-%d")
handler.push_application()
self.logger = Logger("Firetower-admin")
self.queue = redis_util.get_redis_conn(host=conf.redis_host, port=conf.redis_port, redis_db=conf.redis_db)
self.classifier = classifier.Levenshtein()
self.last_archive_run = None
# Wrote this to avoid numpy dependencies until we absolutely require them.
def mean_stdev(self, items):
"""Return mean and stdev of numerical list of items.
Args:
items: list, list of numbers (int or float) to perform calculations upon.
Returns:
tuble of (mean between items, and stdeviation of items and mean).
"""
n, mean, std = 0, 0, 0
if items:
n = len(items)
else:
return (0.0, 0.0)
for item in items:
mean = mean + item
mean = mean / float(n)
for item in items:
std = std + (item + mean) ** 2
std = sqrt(std / float(n)) # Avoid DivideByZero by not subtracting one.
return mean, std
def calc_stats(self):
"""Calculate and save mean and standard deviation."""
categories = category.Category.get_all_categories(self.queue)
for cat in categories:
all_events = cat.events.range(0, -1)
ratios = []
for event in all_events:
event = json.loads(event)
ratios.append(self.classifier.str_ratio(cat.signature, event["sig"]))
cat.mean, cat.stdev = self.mean_stdev(ratios)
def archive_events(self):
"""Run the timeseries archiving for all categories.
This code moves counts from the atomically incrementable HASHes
to Sorted Sets (which can be sliced by date)."""
now = datetime.datetime.utcnow()
if self.last_archive_run is None:
self.last_archive_run = datetime.datetime.utcnow()
return
delta = datetime.timedelta(seconds=self.conf.archive_time)
if self.last_archive_run < (now - delta):
self.logger.debug("Archiving counts older than %s seconds" % (self.conf.archive_time,))
for c in category.Category.get_all_categories(self.queue):
self.logger.debug("Archiving for %s category" % (c.cat_id))
c.timeseries.archive_cat_counts(self.last_archive_run)
def run(self, args):
"""Run set of jobs specified on commandline or config."""
self.logger.info("Running with tasks: %s" % (",".join(args)))
for arg in args:
if arg not in TASKS:
self.logger.error("Specified unknown task: %s" % (arg,))
sys.exit(1)
if arg == "calc_stats":
self.logger.info("Calculating stats for each category")
self.calc_stats()
if arg == "archive_events":
self.archive_events()
self.logger.info("Archiving old data from each category")
class Worker(object):
redis_worker_namespace_prefix = 'rq:worker:'
redis_workers_keys = 'rq:workers'
@classmethod
def all(cls, connection=None):
"""Returns an iterable of all Workers.
"""
if connection is None:
connection = get_current_connection()
reported_working = connection.smembers(cls.redis_workers_keys)
workers = [cls.find_by_key(key, connection) for key in
reported_working]
return compact(workers)
@classmethod
def find_by_key(cls, worker_key, connection=None):
"""Returns a Worker instance, based on the naming conventions for
naming the internal Redis keys. Can be used to reverse-lookup Workers
by their Redis keys.
"""
prefix = cls.redis_worker_namespace_prefix
name = worker_key[len(prefix):]
if not worker_key.startswith(prefix):
raise ValueError('Not a valid RQ worker key: %s' % (worker_key,))
if connection is None:
connection = get_current_connection()
if not connection.exists(worker_key):
return None
name = worker_key[len(prefix):]
worker = cls([], name, connection=connection)
queues = connection.hget(worker.key, 'queues')
worker._state = connection.hget(worker.key, 'state') or '?'
if queues:
worker.queues = [Queue(queue, connection=connection)
for queue in queues.split(',')]
return worker
def __init__(self, queues, name=None, default_result_ttl=DEFAULT_RESULT_TTL,
connection=None, exc_handler=None): # noqa
if connection is None:
connection = get_current_connection()
self.connection = connection
if isinstance(queues, Queue):
queues = [queues]
self._name = name
self.queues = queues
self.validate_queues()
self._exc_handlers = []
self.default_result_ttl = default_result_ttl
self._state = 'starting'
self._is_horse = False
self._horse_pid = 0
self._stopped = False
self.log = Logger('worker')
self.failed_queue = get_failed_queue(connection=self.connection)
# By default, push the "move-to-failed-queue" exception handler onto
# the stack
self.push_exc_handler(self.move_to_failed_queue)
if exc_handler is not None:
self.push_exc_handler(exc_handler)
def validate_queues(self): # noqa
"""Sanity check for the given queues."""
if not iterable(self.queues):
raise ValueError('Argument queues not iterable.')
for queue in self.queues:
if not isinstance(queue, Queue):
raise NoQueueError('Give each worker at least one Queue.')
def queue_names(self):
"""Returns the queue names of this worker's queues."""
return map(lambda q: q.name, self.queues)
def queue_keys(self):
"""Returns the Redis keys representing this worker's queues."""
return map(lambda q: q.key, self.queues)
@property # noqa
def name(self):
"""Returns the name of the worker, under which it is registered to the
monitoring system.
By default, the name of the worker is constructed from the current
(short) host name and the current PID.
"""
if self._name is None:
hostname = socket.gethostname()
shortname, _, _ = hostname.partition('.')
self._name = '%s.%s' % (shortname, self.pid)
return self._name
@property
def key(self):
"""Returns the worker's Redis hash key."""
return self.redis_worker_namespace_prefix + self.name
@property
def pid(self):
"""The current process ID."""
return os.getpid()
@property
def horse_pid(self):
"""The horse's process ID. Only available in the worker. Will return
0 in the horse part of the fork.
"""
return self._horse_pid
@property
def is_horse(self):
"""Returns whether or not this is the worker or the work horse."""
return self._is_horse
def procline(self, message):
"""Changes the current procname for the process.
This can be used to make `ps -ef` output more readable.
"""
setprocname('rq: %s' % (message,))
def register_birth(self): # noqa
"""Registers its own birth."""
self.log.debug('Registering birth of worker %s' % (self.name,))
if self.connection.exists(self.key) and \
not self.connection.hexists(self.key, 'death'):
raise ValueError(
'There exists an active worker named \'%s\' '
'already.' % (self.name,))
key = self.key
now = time.time()
queues = ','.join(self.queue_names())
with self.connection.pipeline() as p:
p.delete(key)
p.hset(key, 'birth', now)
p.hset(key, 'queues', queues)
p.sadd(self.redis_workers_keys, key)
p.execute()
def register_death(self):
"""Registers its own death."""
self.log.debug('Registering death')
with self.connection.pipeline() as p:
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
p.srem(self.redis_workers_keys, self.key)
p.hset(self.key, 'death', time.time())
p.expire(self.key, 60)
p.execute()
def set_state(self, new_state):
self._state = new_state
self.connection.hset(self.key, 'state', new_state)
def get_state(self):
return self._state
state = property(get_state, set_state)
@property
def stopped(self):
return self._stopped
def _install_signal_handlers(self):
"""Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
def request_force_stop(signum, frame):
"""Terminates the application (cold shutdown).
"""
self.log.warning('Cold shut down.')
# Take down the horse with the worker
if self.horse_pid:
msg = 'Taking down horse %d with me.' % self.horse_pid
self.log.debug(msg)
try:
os.kill(self.horse_pid, signal.SIGKILL)
except OSError as e:
# ESRCH ("No such process") is fine with us
if e.errno != errno.ESRCH:
self.log.debug('Horse already down.')
raise
raise SystemExit()
def request_stop(signum, frame):
"""Stops the current worker loop but waits for child processes to
end gracefully (warm shutdown).
"""
self.log.debug('Got signal %s.' % signal_name(signum))
signal.signal(signal.SIGINT, request_force_stop)
signal.signal(signal.SIGTERM, request_force_stop)
msg = 'Warm shut down requested.'
self.log.warning(msg)
# If shutdown is requested in the middle of a job, wait until
# finish before shutting down
if self.state == 'busy':
self._stopped = True
self.log.debug('Stopping after current horse is finished. '
'Press Ctrl+C again for a cold shutdown.')
else:
raise StopRequested()
signal.signal(signal.SIGINT, request_stop)
signal.signal(signal.SIGTERM, request_stop)
def work(self, burst=False): # noqa
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
self._install_signal_handlers()
did_perform_work = False
self.register_birth()
self.log.info('RQ worker started, version %s' % VERSION)
self.state = 'starting'
try:
while True:
if self.stopped:
self.log.info('Stopping on request.')
break
self.state = 'idle'
qnames = self.queue_names()
self.procline('Listening on %s' % ','.join(qnames))
self.log.info('')
self.log.info('*** Listening on %s...' % \
green(', '.join(qnames)))
wait_for_job = not burst
try:
result = Queue.dequeue_any(self.queues, wait_for_job, \
connection=self.connection)
if result is None:
break
except StopRequested:
break
except UnpickleError as e:
msg = '*** Ignoring unpickleable data on %s.' % \
green(e.queue.name)
self.log.warning(msg)
self.log.debug('Data follows:')
self.log.debug(e.raw_data)
self.log.debug('End of unreadable data.')
self.failed_queue.push_job_id(e.job_id)
continue
self.state = 'busy'
job, queue = result
# Use the public setter here, to immediately update Redis
job.status = Status.STARTED
self.log.info('%s: %s (%s)' % (green(queue.name),
blue(job.description), job.id))
self.fork_and_perform_job(job)
did_perform_work = True
finally:
if not self.is_horse:
self.register_death()
return did_perform_work
def fork_and_perform_job(self, job):
"""Spawns a work horse to perform the actual work and passes it a job.
The worker will wait for the work horse and make sure it executes
within the given timeout bounds, or will end the work horse with
SIGALRM.
"""
child_pid = os.fork()
if child_pid == 0:
self.main_work_horse(job)
else:
self._horse_pid = child_pid
self.procline('Forked %d at %d' % (child_pid, time.time()))
while True:
try:
os.waitpid(child_pid, 0)
break
except OSError as e:
# In case we encountered an OSError due to EINTR (which is
# caused by a SIGINT or SIGTERM signal during
# os.waitpid()), we simply ignore it and enter the next
# iteration of the loop, waiting for the child to end. In
# any other case, this is some other unexpected OS error,
# which we don't want to catch, so we re-raise those ones.
if e.errno != errno.EINTR:
raise
def main_work_horse(self, job):
"""This is the entry point of the newly spawned work horse."""
# After fork()'ing, always assure we are generating random sequences
# that are different from the worker.
random.seed()
# Always ignore Ctrl+C in the work horse, as it might abort the
# currently running job.
# The main worker catches the Ctrl+C and requests graceful shutdown
# after the current work is done. When cold shutdown is requested, it
# kills the current job anyway.
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self._is_horse = True
self.log = Logger('horse')
success = self.perform_job(job)
# os._exit() is the way to exit from childs after a fork(), in
# constrast to the regular sys.exit()
os._exit(int(not success))
def perform_job(self, job):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.procline('Processing %s from %s since %s' % (
job.func_name,
job.origin, time.time()))
try:
with death_penalty_after(job.timeout or 180):
rv = job.perform()
# Pickle the result in the same try-except block since we need to
# use the same exc handling when pickling fails
pickled_rv = dumps(rv)
job._status = Status.FINISHED
except:
# Use the public setter here, to immediately update Redis
job.status = Status.FAILED
self.handle_exception(job, *sys.exc_info())
return False
if rv is None:
self.log.info('Job OK')
else:
self.log.info('Job OK, result = %s' % (yellow(unicode(rv)),))
# How long we persist the job result depends on the value of
# result_ttl:
# - If result_ttl is 0, cleanup the job immediately.
# - If it's a positive number, set the job to expire in X seconds.
# - If result_ttl is negative, don't set an expiry to it (persist
# forever)
result_ttl = self.default_result_ttl if job.result_ttl is None else job.result_ttl # noqa
if result_ttl == 0:
job.delete()
self.log.info('Result discarded immediately.')
else:
p = self.connection.pipeline()
p.hset(job.key, 'result', pickled_rv)
p.hset(job.key, 'status', job._status)
if result_ttl > 0:
p.expire(job.key, result_ttl)
self.log.info('Result is kept for %d seconds.' % result_ttl)
else:
self.log.warning('Result will never expire, clean up result key manually.')
p.execute()
return True
def handle_exception(self, job, *exc_info):
"""Walks the exception handler stack to delegate exception handling."""
exc_string = ''.join(
traceback.format_exception_only(*exc_info[:2]) +
traceback.format_exception(*exc_info))
self.log.error(exc_string)
for handler in reversed(self._exc_handlers):
self.log.debug('Invoking exception handler %s' % (handler,))
fallthrough = handler(job, *exc_info)
# Only handlers with explicit return values should disable further
# exc handling, so interpret a None return value as True.
if fallthrough is None:
fallthrough = True
if not fallthrough:
break
def move_to_failed_queue(self, job, *exc_info):
"""Default exception handler: move the job to the failed queue."""
exc_string = ''.join(traceback.format_exception(*exc_info))
self.log.warning('Moving job to %s queue.' % self.failed_queue.name)
self.failed_queue.quarantine(job, exc_info=exc_string)
def push_exc_handler(self, handler_func):
"""Pushes an exception handler onto the exc handler stack."""
self._exc_handlers.append(handler_func)
def pop_exc_handler(self):
"""Pops the latest exception handler off of the exc handler stack."""
return self._exc_handlers.pop()
class Admin(object):
"""Administrative task object."""
def __init__(self, conf):
"""conf: dict, yaml parameters."""
self.conf = conf
handler = TimedRotatingFileHandler(
conf.log_file, date_format='%Y-%m-%d')
handler.push_application()
self.logger = Logger('Firetower-admin')
self.queue = redis_util.get_redis_conn(
host=conf.redis_host, port=conf.redis_port, redis_db=conf.redis_db
)
self.classifier = classifier.Levenshtein()
# Wrote this to avoid numpy dependencies until we absolutely require them.
def mean_stdev(self, items):
"""Return mean and stdev of numerical list of items.
Args:
items: list, list of numbers (int or float) to perform calculations upon.
Returns:
tuble of (mean between items, and stdeviation of items and mean).
"""
n, mean, std = len(items), 0, 0
for item in items:
mean = mean + item
mean = mean / float(n)
for item in items:
std = std + (item + mean)**2
std = sqrt(std / float(n)) # Avoid DivideByZero by not subtracting one.
return mean, std
def calc_stats(self):
"""Calculate and save mean and standard deviation."""
categories = category.Category.get_all_categories(self.queue)
for cat in categories:
all_events = cat.events.range(0, -1)
ratios = []
for event in all_events:
event = json.loads(event)
ratios.append(
self.classifier.str_ratio(cat.signature, event['sig']))
cat.mean, cat.stdev = self.mean_stdev(ratios)
def run(self, args):
"""Run set of jobs specified on commandline or config."""
self.logger.info('Running with tasks: %s' % (','.join(args)))
for arg in args:
if arg not in TASKS:
self.logger.error('Specified unknown task: %s' % (arg,))
sys.exit(1)
if arg == 'cat_stats':
self.logger.info('Calculating stats for each category')
self.calc_stats()
if arg == 'archive_events':
self.logger.info('Archiving old data from each category')
pass
sound.read(SAMPLES_TO_DROP * 2)
s = 0
for i in range(SAMPLES_PER_FRAME):
s += getMsbFromAudio(sound) ** 2
x = sqrt(s / SAMPLES_PER_FRAME)
self.upload_anim('B', [int(1.0446300614125956 ** x) - 1])
if __name__ == "__main__":
# log_handler = FileHandler('daemon.log')
# log_handler.push_application()
logger = Logger('Main')
logger.info('Starting')
try:
a = AmbianceDaemon(logger=logger, boot_time=2)
logger.info('Got Ambianceduino version ' + a.version)
a.run()
except KeyboardInterrupt:
a.stop()
logger.info('Stopping')
exit()
except Exception as err:
logger.error("%s: %s" % (err.__class__.__name__, err.message))
print traceback.format_exc()
try:
a.stop()
except NameError:
pass
class AmbianceDaemon(Ambianceduino):
def __init__(self, *args, **kwargs):
self.logger = Logger('Daemon')
super(AmbianceDaemon, self).__init__(*args, **kwargs)
self.current_delay = None
self.powered = None
self.meteo = []
self.__init_puka_client()
self.anims_uploaded = 0
def delay_red(self, milliseconds):
self.delay('R', milliseconds)
def delay_blue(self, milliseconds):
self.delay('B', milliseconds)
def __spacestatus(self):
try:
statuspage = urlopen(STATUS_URL)
payload = json.loads(statuspage.read())
if 'state' in payload:
if payload['state'] == 'open' and not self.powered:
self.on()
elif payload['state'] == 'closed' and self.powered:
self.off()
except Exception as err:
self.logger.error("%s: %s" % (err.__class__.__name__, err.message))
def __peoplecount(self):
try:
pamelapage = urlopen('http://pamela.urlab.be/mac.json')
payload = json.loads(pamelapage.read())
people = len(payload['color']) + len(payload['grey'])
d = int(25 / log(people + 2))
if d != self.current_delay:
self.delay_red(d)
except Exception as err:
self.logger.error("%s: %s" % (err.__class__.__name__, err.message))
def __init_puka_client(self):
try:
self.puka_client = puka.Client(AMQ_SERVER)
promise = self.puka_client.connect()
self.puka_client.wait(promise)
self.logger.info("Connected AMQ to %s" % (AMQ_SERVER))
promise = self.puka_client.queue_declare(queue=METEO_QUEUE)
self.puka_client.wait(promise)
self.logger.info("Got queue %s" % (METEO_QUEUE))
promise = self.puka_client.queue_declare(queue=EVENTS_QUEUE)
self.puka_client.wait(promise)
self.logger.info("Got queue %s" % (EVENTS_QUEUE))
except:
self.logger.error("Unable to connect to %s#%s" % (AMQ_SERVER, METEO_QUEUE))
self.puka_client = None
def __send_message(self, queue, json_dumpable):
if self.puka_client:
msg = json.dumps(json_dumpable)
promise = self.puka_client.basic_publish(
exchange='',
routing_key=queue,
body=msg
)
self.puka_client.wait(promise)
self.logger.info('Sent [%s] << %s' % (queue, msg[:50]))
def __http_requests(self):
i = 0
while self.running:
self.analogs()
if i == 0:
self.__spacestatus()
self.__peoplecount()
i = (i + 1) % 20
sleep(1)
def stop(self):
with open("/tmp/mpd.fifo", "w") as fifo:
fifo.write(chr(0) * (80200 / 14))
super(Ambianceduino, self).stop()
def run(self):
super(AmbianceDaemon, self).run()
self.requestsThread = Thread(target=self.__http_requests, name="http-request")
self.threads.append(self.requestsThread)
self.requestsThread.start()
self.mpdThread = Thread(target=self.__mpd_loop, name="mpd")
self.threads.append(self.mpdThread)
self.mpdThread.start()
while True:
for t in self.threads:
if not t.is_alive():
self.logger.error("One of my children is dead, suicide.")
self.stop()
exit()
sleep(0.5)
def default_handler(self, *args):
self.logger.info(' '.join(map(str, args)))
def when_on(self):
self.powered = True
self.logger.info('Powered on')
def when_off(self):
self.powered = False
self.logger.info('Powered off')
def when_delay(self, output, delay):
if output == 'R':
self.current_delay = delay
self.logger.info('Delay RED set to %d' % (delay))
elif output == 'B':
self.logger.info('Delay BLUE set to %d' % (delay))
def when_bell(self):
self.__send_message(EVENTS_QUEUE, {
'trigger': 'bell', 'time': str(datetime.now())
})
def when_door(self):
if not self.powered:
self.__send_message(EVENTS_QUEUE, {
'trigger': 'door', 'time': str(datetime.now())
})
def when_radiator(self):
self.__send_message(EVENTS_QUEUE, {
'trigger': 'radiator', 'time': str(datetime.now())
})
def when_anim(self, anim_name, anim_length):
self.anims_uploaded += 1
def when_analogs(self, analogs):
self.meteo.append(analogs)
if len(self.meteo) == METEO_LEN:
self.__send_message(METEO_QUEUE, {
'time': str(datetime.now()),
'light': {
'inside': avg_for_key(self.meteo, 'light_in') / 10.23,
'outside': avg_for_key(self.meteo, 'light_out') / 10.23
},
'temperature': {
'radiator': convertThermistor(avg_for_key(self.meteo, 'temp_radia'), 4700, 3950),
'ambiant': convertThermistor(avg_for_key(self.meteo, 'temp_amb'), 2200, 3950)
}
})
self.meteo = []
def __mpd_loop(self):
SAMPLES_PER_FRAME = 25
AUDIO_SAMPLE_RATE = 44100
FRAMES_PER_SECOND = 14
SAMPLES_TO_DROP = (AUDIO_SAMPLE_RATE / FRAMES_PER_SECOND) - SAMPLES_PER_FRAME
self.delay('B', int(1000 / FRAMES_PER_SECOND))
#with nbopen("/tmp/mpd.fifo", os.O_RDONLY | os.O_NONBLOCK) as sound:
with open("/tmp/mpd.fifo", "rb") as sound:
while self.running:
if not self.powered:
sleep(1)
continue
sound.read(SAMPLES_TO_DROP * 2)
s = 0
for i in range(SAMPLES_PER_FRAME):
s += getMsbFromAudio(sound) ** 2
x = sqrt(s / SAMPLES_PER_FRAME)
self.upload_anim('B', [int(1.0446300614125956 ** x) - 1])
class HwRedisInterface(threading.Thread):
def __init__(self, interface=InterfaceTemplate(), channel='', host='127.0.0.1'):
threading.Thread.__init__(self)
self.timeout = 1
self.interface = interface
self.redis = redis.Redis(host=host)
self.msg_count = 0
self.busy = 0;
if channel=='':
self.channel = str(interface)
else:
self.channel = channel
self.pubsub = self.redis.pubsub()
self.Log = Logger('HwRedisInterface')
self.Log.debug('__init__(channel=%s)' % self.channel)
self.pubsub.subscribe(self.channel)
self.start()
self.setName('HwRedisInterface-Thread')
def __del__(self):
self.Log.info('__del__()')
self.stop()
def stop(self):
self.Log.info('stop()')
self.busy = False
self.redis.publish(self.channel,'KILL')
time.sleep(1)
self.Log.info(' stopped')
def send(self, cmd, **kwargs):
self.Log.debug("send(%s,%s)" % (cmd,str(kwargs)))
to = time.time()
try:
self.interface.send(cmd, kwargs)
self.Log.debug(" send = %.3f" % (time.time() - to))
self.redis.set('%s_send_last' % self.channel, cmd)
return True
except Exception as E:
self.Log.error(E.message)
return False
def read(self, **kwargs):
self.Log.debug("read(%s)" % str(kwargs))
to = time.time()
try:
out = self.interface.read(**kwargs)
self.Log.debug(" interface.read() time = %.3f" % (time.time() - to))
self.redis.set('%s_read_last' % self.channel, out[1])
return out[1]
except Exception as E:
self.Log.error(E.message)
return None
def query(self, cmd, **kwargs):
self.Log.debug("query(%s,%s)" % (cmd,str(kwargs)))
to = time.time()
self.Log.debug(" query(%s, %s)" % (cmd, str(kwargs)))
try:
out = self.interface.query(cmd, **kwargs)
self.Log.debug(" interface.query() time = %.3f" % (time.time() - to))
return out
except Exception as E:
self.Log.error(E.message)
return E
def run(self):
self.Log.debug('run()')
for item in self.pubsub.listen():
if item['data'] == "KILL":
self.pubsub.unsubscribe()
self.Log.info("unsubscribed and finished")
break
else:
# self.Log.debug('run() - incoming message')
if not self.busy:
self.process_message(item)
self.Log.debug('end of run()')
def process_message(self, item):
self.Log.debug('process_message(type=%s)' % item['type'])
to = time.time()
self.busy = True
self.msg_count = self.msg_count + 1
if item['type'] == 'message':
try:
msg = sjson.loads(item['data'])
self.Log.debug(' msg=%s, from=%s' % (msg['cmd'], msg['from']))
if isinstance(msg['cmd'],list):
cmd = msg['cmd'][0]
kwargs = msg['cmd'][1]
is_query = msg['cmd'][2]
else:
cmd = msg['cmd']
is_query = True
except Exception as E:
self.Log.error(E.message)
return None
self.Log.debug(' is_query=%d' % is_query)
if is_query:
out = self.query(cmd,**kwargs)
self.redis.publish('res', sjson.dumps(out))
timeout = msg['timeout']
timeout = self.timeout
self.redis.set(msg['from'],sjson.dumps(out))
self.redis.expire(msg['from'], timeout)
self.Log.debug(' query(cmd=%s) = %s' % (cmd, str(out)))
else:
self.Log.debug(' send(cmd)')
out = self.interface.send(cmd)
self.busy = False
return out
else:
self.busy = False
return None
class NodesMetrics:
def __init__(self):
handler = TimedRotatingFileHandler('../logs/nodes_metrics.log')
handler.push_application()
self.logger = Logger(name='nodes metrics', level='info')
self.nodes_filename, self.nodes_file_backup_name, self.exclude_file, self.url, self.query_all_ip, self.query_current_ip = self.get_conf()
self.nodes = {}
self.ips = {}
self.nodes_list = []
self.ips_list = []
@staticmethod
def get_conf():
##从配置文件获取配置
cp = ConfigParser.ConfigParser()
with codecs.open(os.path.join(BASE_DIR, './config/config.ini'), 'r', encoding='utf-8') as f:
cp.readfp(f)
nodes_filename = cp.get('file_ds', 'file_sd_filename').strip()
nodes_file_backup_name = cp.get('file_ds', 'nodes_file_backup_name').strip()
exclude_file = cp.get('file_ds', 'agent_exclude_file').strip()
url = cp.get('prome', 'url').strip()
query_all_ip = cp.get('prome', 'query_all_ip').strip()
query_current_ip = cp.get('prome', 'query_current_ip').strip()
return os.path.join(BASE_DIR, nodes_filename), os.path.join(BASE_DIR, nodes_file_backup_name), os.path.join(BASE_DIR, exclude_file), url, query_all_ip, query_current_ip
#@staticmethod
def get_ips(self, res):
ips = []
if res.status_code == 200:
data = res.json()['data']['result']
for metric in data:
# print(metric)
ip = metric['metric']['instance_ip']
ips.append(ip)
# print(ip)
#print('All ips in prome: %s' % ips)
#self.logger.info('All ips in prome: %s' % ips)
return ips
# res = requests.get(url + query_current_ip)
#
# current_ips = get_ips(res)
#@staticmethod
def get_hosts(self, file_name):
all_ips = []
with open(file_name, 'r') as f:
ips = f.read()
ip_list = ips.split('\n')
if file_name != self.exclude_file:
for ip in ip_list:
if ip.strip().endswith(':9100",'):
nodes_str = ip.split('"')[1]
ip_str = nodes_str.split(':')[0]
#print(nodes_str)
all_ips.append(ip_str)
else:
all_ips = ip_list
print('All nodes from file %s is: %s' % (file_name, all_ips))
self.logger.info('All nodes from file %s is: %s' % (file_name, all_ips))
return all_ips
def get_nodes_discovery(self):
new_ips = self.get_hosts(self.nodes_filename)
nodes_hosts = list(set(new_ips))
current_nodes = len(nodes_hosts)
print(current_nodes)
if current_nodes:
self.logger.info('************************************')
self.logger.info(
'There are %s nodes after nodes discovery finished: %s' % \
(current_nodes, nodes_hosts))
self.logger.info('************************************')
else:
self.logger.info('************************************')
self.logger.error(
'There are 0 nodes founded by nodes discovery: %s')
self.logger.info('************************************')
all_nodes = ' '.join(nodes_hosts)
return current_nodes, all_nodes
def get_stop_nodes(self):
new_ips = self.get_hosts(self.nodes_filename)
print(len(new_ips))
res = requests.get(url + query_all_ip)
all_ips = self.get_ips(res)
self.logger.info(all_ips)
print('All nodes are:')
print(all_ips)
old_ips = self.get_hosts(self.exclude_file)
failed_hosts = list(set(new_ips) - set(all_ips) - set(old_ips))
stop_nodes = len(failed_hosts)
#failed_hosts = list(set(new_ips) - set(all_ips))
self.logger.info('************************************')
self.logger.warning(
'There are %s nodes stopped now' % stop_nodes)
self.logger.info('************************************')
return stop_nodes, failed_hosts
from logbook import Logger, StreamHandler
from logbook.notifiers import PushoverHandler
import sys
StreamHandler(sys.stdout).push_application()
# PushoverHandler(application_name='Dog Bark', apikey='aydx41drtskjxbj14egvyunchi64ka', userkey='uofggy6u4pw9exq3ikmmwe5iwfarxm').push_application()
log = Logger('Logbook')
log.info('Hello world!')
log.error('oh shit!!!')
class BTgymMultiData:
"""
Multiply data streams wrapper.
"""
def __init__(self,
data_class_ref=None,
data_config=None,
name='multi_data',
data_names=None,
task=0,
log_level=WARNING,
**kwargs):
"""
Args:
data_class_ref: one of BTgym single-stream datafeed classes
data_config: nested dictionary of individual data streams sources, see notes below.
kwargs: shared parameters for all data streams, see base dataclass
Notes:
`Data_config` specifies all data sources consumed by strategy::
data_config = {
data_line_name_0: {
filename: [source csv filename string or list of strings],
[config: {optional dict of individual stream config. params},]
},
...,
data_line_name_n : {...}
}
Example::
data_config = {
'usd': {'filename': '.../DAT_ASCII_EURUSD_M1_2017.csv'},
'gbp': {'filename': '.../DAT_ASCII_EURGBP_M1_2017.csv'},
'jpy': {'filename': '.../DAT_ASCII_EURJPY_M1_2017.csv'},
'chf': {'filename': '.../DAT_ASCII_EURCHF_M1_2017.csv'},
}
It is user responsibility to correctly choose historic data conversion rates wrt cash currency (here - EUR).
"""
self.data_class_ref = data_class_ref
if data_config is None:
self.data_config = {}
else:
self.data_config = data_config
self.master_data = None
self.name = name
self.task = task
self.metadata = {'sample_num': 0, 'type': None}
self.filename = None
self.is_ready = False
self.global_timestamp = 0
self.log_level = log_level
self.params = {}
self.names = []
self.sample_num = 0
# Logging:
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(self.name, self.task),
level=self.log_level)
if data_names is None:
# Infer from data configuration (at top-level):
self.data_names = list(self.data_config.keys())
else:
self.data_names = data_names
try:
assert len(self.data_names
) > 0, 'At least one data_line should be provided'
except AssertionError:
self.log.error('At least one data_line should be provided')
raise ValueError
# Make dictionary of single-stream datasets:
self.data = OrderedDict()
for key, stream in self.data_config.items():
try:
stream['config'].update(kwargs)
except KeyError:
stream['config'] = kwargs
try:
if stream['dataframe'] is None:
pass
except KeyError:
stream['dataframe'] = None
self.data[key] = self.data_class_ref(filename=stream['filename'],
dataframe=stream['dataframe'],
data_names=(key, ),
task=task,
name='{}_{}'.format(
name, key),
log_level=log_level,
**stream['config'])
try:
# If master-data has been pointed explicitly by 'base' kwarg:
if stream['base']:
self.master_data = self.data[key]
except KeyError:
pass
def set_logger(self, level=None, task=None):
"""
Sets logbook logger.
Args:
level: logbook.level, int
task: task id, int
"""
if task is not None:
self.task = task
if level is not None:
for stream in self.data.values():
stream.log = Logger('{}_{}'.format(stream.name, stream.task),
level=level)
self.log = Logger('{}_{}'.format(self.name, self.task),
level=level)
def set_params(self, params_dict):
"""
Batch attribute setter.
Args:
params_dict: dictionary of parameters to be set as instance attributes.
"""
for key, value in params_dict.items():
for stream in self.data.values():
setattr(stream, key, value)
def read_csv(self, data_filename=None, force_reload=False):
# Load:
indexes = []
for stream in self.data.values():
stream.read_csv(force_reload=force_reload)
indexes.append(stream.data.index)
# Get indexes intersection:
if len(indexes) > 1:
idx_intersected = indexes[0]
for i in range(1, len(indexes)):
idx_intersected = idx_intersected.intersection(indexes[i])
# Truncate data to common index:
for stream in self.data.values():
stream.data = stream.data.loc[idx_intersected]
def reset(self, **kwargs):
indexes = []
for stream in self.data.values():
stream.reset(**kwargs)
indexes.append(stream.data.index)
# Get indexes intersection:
if len(indexes) > 1:
idx_intersected = indexes[0]
for i in range(1, len(indexes)):
idx_intersected = idx_intersected.intersection(indexes[i])
idx_intersected.drop_duplicates()
self.log.info('shared num. records: {}'.format(
len(idx_intersected)))
# Truncate data to common index:
for stream in self.data.values():
stream.data = stream.data.loc[idx_intersected]
# Choose master_data
if self.master_data is None:
# Just choose first key:
all_keys = list(self.data.keys())
if len(all_keys) > 0:
self.master_data = self.data[all_keys[0]]
self.global_timestamp = self.master_data.global_timestamp
self.names = self.master_data.names
self.sample_num = 0
self.is_ready = True
def set_global_timestamp(self, timestamp):
for stream in self.data.values():
stream.set_global_timestamp(timestamp)
self.global_timestamp = self.master_data.global_timestamp
def describe(self):
return {key: stream.describe() for key, stream in self.data.items()}
def sample(self, **kwargs):
# Get sample to infer exact interval:
self.log.debug('Making master sample...')
master_sample = self.master_data.sample(**kwargs)
self.log.debug('Making master ok.')
# Prepare empty instance of multistream data:
sample = BTgymMultiData(
data_names=self.data_names,
task=self.task,
log_level=self.log_level,
name='sub_' + self.name,
)
sample.metadata = copy.deepcopy(master_sample.metadata)
interval = [
master_sample.metadata['first_row'],
master_sample.metadata['last_row']
]
# Populate sample with data:
for key, stream in self.data.items():
self.log.debug(
'Sampling <{}> with interval: {}, kwargs: {}'.format(
key, interval, kwargs))
sample.data[key] = stream.sample(interval=interval,
force_interval=True,
**kwargs)
sample.filename = {
key: stream.filename
for key, stream in self.data.items()
}
self.sample_num += 1
return sample
def to_btfeed(self):
feed = OrderedDict()
for key, stream in self.data.items():
# Get single-dataline btfeed dict:
feed_dict = stream.to_btfeed()
assert len(list(feed_dict.keys())) == 1, \
'Expected base datafeed dictionary contain single data_line, got: {}'.format(feed_dict)
# Rename every base btfeed according to data_config keys:
feed[key] = feed_dict[list(feed_dict.keys())[0]]
return feed
from logbook import Logger, StreamHandler
import sys
import inspect
import sqlite3
import asyncio
from utils import checks
StreamHandler(sys.stdout).push_application()
log = Logger("GAF Bot")
log.notice("Loading Configuration File")
try:
with open("config/config.json") as f:
_config = json.load(f)
except FileNotFoundError:
log.error("Config file not found, loading defaults")
df = open("config/defaults/default.config.json")
_config = json.load(df)
with open("config/config.json", "w") as f:
log.info("Saved new config to file")
f.write(json.dumps(_config))
try:
with open("config/modules.json") as f:
_modules = json.load(f)
with open("config/defaults/default.modules.json") as df:
_df_modules = json.load(df)
for x in list(_df_modules):
if x not in list(_modules):
_modules[x] = _df_modules[x]
data = json.dumps(_modules)
class Schema(object):
url = None
version = None
homepage = None
deps = []
dirs_to_symlink = ['bin', 'sbin', 'etc', 'lib', 'include', 'var']
def __init__(self, env):
self.env = env.copy()
self.log = Logger()
if self.version is None:
self.version = _figureout_version(self.url)
if self.url.startswith('git://'):
self.deps.append('git')
self.retriver = self.git
else:
self.retriver = self.wget
self.env['prefix'] = os.path.join(
env['osbench_root'],
'workbench',
env['schema'],
self.version
)
self.env['version'] = self.version
# Methods to override
def install(self):
pass
def is_installed(self):
"""Right now we'll simple check if program
was built and installed into workbench."""
return os.path.exists(self.env['prefix'])
# Internals
def _get_source(self):
if not self.url:
return
self.log.info('Getting source code')
with higher_log_indent():
self.retriver(self.url)
patch_names = sorted(name for name in dir(self) if name.startswith('patch_'))
for name in patch_names:
filename = name.replace('_', '-') + '.diff'
with open(filename, 'w') as f:
f.write(self._get_patch(name))
self.call('patch -p1 < ' + filename)
def _get_patch(self, name):
data = getattr(self, name)
if len(data[:200].split('\n')) == 1:
# then probably this is a file or URL
patch_filename = os.path.join(
os.path.dirname(self.env['schema_filename']),
data
)
if os.path.exists(patch_filename):
data = open(patch_filename).read()
data = self._substitute_vars(data)
return data
def _get_environment_vars(self):
""" Returns dict with variables to set in shell and to replace in the templates.
"""
return dict(
OSBENCH_ROOT=self.env['osbench_root'],
OSBENCH_PREFIX=self.env['prefix'],
)
def _substitute_vars(self, text):
for name, value in self._get_environment_vars().items():
text = text.replace(name, value)
return text
def wget(self, url):
self.call('wget "{0}"'.format(self.url))
files = os.listdir('.')
assert len(files) == 1
filename = files[0]
# remove url params if they was added to the filename
stripped_filename = filename.split('?', 1)[0]
if stripped_filename.endswith('.gz') or \
stripped_filename.endswith('.tgz'):
tar_options = '-zxvf'
elif stripped_filename.endswith('.bz2'):
tar_options = '-jxvf'
else:
raise RuntimeError('Unknown archive format.')
self.call('tar {0} "{1}"'.format(tar_options, filename))
os.unlink(filename)
dirs = os.listdir('.')
assert len(dirs) == 1
os.chdir(dirs[0])
def git(self, url):
self.call("git clone '{0}'".format(url))
dirs = os.listdir('.')
assert len(dirs) == 1
os.chdir(dirs[0])
def _install_deps(self):
if self.deps:
self.log.info('Installing dependencies')
with higher_log_indent():
for dep in self.deps:
if dep.startswith('system.'):
self.call('sudo apt-get --yes install %s' % dep[7:])
def _install(self, interactive=False):
self.log.info('Installing "{schema}"'.format(**self.env))
with higher_log_indent():
self._install_deps()
root = tempfile.mkdtemp(prefix='diy-')
if not os.path.exists(self.env['prefix']):
os.makedirs(self.env['prefix'])
try:
os.chdir(root)
self._get_source()
if interactive:
self.log.info('Entering into the interactive mode')
with higher_log_indent():
shell = os.environ['SHELL']
self.call('git init')
self.call('git add -A')
for name, value in self._get_environment_vars().items():
os.environ[name] = value
self.call(shell, pass_output=True)
else:
self.log.info('Running schema\'s install method')
with higher_log_indent():
self.install()
self._link()
finally:
self.call('rm -fr "{0}"'.format(root))
def _uninstall(self):
"""Uninstalls the schema from the prefix.
It removes symlinks and deletes installed files from workbenches.
"""
self.log.info('Uninstalling "{schema}"'.format(**self.env))
with higher_log_indent():
self._unlink()
self._delete()
def _delete(self):
shutil.rmtree(self.env['prefix'])
def _join_path(self, *args):
return os.path.normpath(os.path.join(*args))
def _link(self):
self.log.info('Making symlinks')
with higher_log_indent():
for dir_name in self.dirs_to_symlink:
s_dir = self._join_path(self.env['prefix'], dir_name)
t_dir = self._join_path(self.env['osbench_root'], dir_name)
if not os.path.exists(t_dir):
self.log.debug('Creating directory "{0}"', t_dir)
os.makedirs(t_dir)
if os.path.exists(s_dir):
for root, dirs, files in os.walk(s_dir):
# making root, relative
root = os.path.relpath(root, s_dir)
for dir_name in dirs:
full_dir_name = self._join_path(
t_dir, root, dir_name
)
if not os.path.exists(full_dir_name):
self.log.debug('Creating directory "{0}"', full_dir_name)
os.makedirs(full_dir_name)
for filename in files:
source = self._join_path(s_dir, root, filename)
target = self._join_path(t_dir, root, filename)
if os.path.exists(target):
if os.path.islink(target):
if os.readlink(target) == source:
self.log.debug('Symlink {target} already exists', target=target)
continue
else:
self.log.warning('Unlinking file {target}, pointing to {source}',
target=target, source=source)
os.unlink(target)
else:
self.log.warning('File {target} already exists and it is not a link',
target=target)
if not os.path.exists(target):
self.log.debug('Creating symlink from "{source}" to {target}',
source=source, target=target)
os.symlink(source, target)
def _unlink(self):
self.log.info('Removing symlinks')
for ftype, name in self.get_files_to_unlink():
if ftype == 'file':
os.unlink(name)
else:
try:
os.rmdir(name)
except OSError:
pass
def get_files_to_unlink(self):
with higher_log_indent():
for dir_name in self.dirs_to_symlink:
s_dir = self._join_path(self.env['prefix'], dir_name)
t_dir = self._join_path(self.env['osbench_root'], dir_name)
if os.path.exists(s_dir):
for root, dirs, files in os.walk(s_dir, topdown=False):
# making root, relative
root = os.path.relpath(root, s_dir)
for filename in files:
source = self._join_path(s_dir, root, filename)
target = self._join_path(t_dir, root, filename)
if os.path.islink(target) and \
os.path.realpath(target) == source:
yield ('file', target)
for dir_name in dirs:
full_dir_name = self._join_path(
t_dir, root, dir_name
)
yield ('dir', full_dir_name)
# Utilities
def call(self, command, pass_output=False):
command = command.format(**self.env)
self.log.info('Running "{0}"'.format(command))
with higher_log_indent():
options = dict(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
)
if pass_output:
del options['stdout']
del options['stderr']
proc = subprocess.Popen(
command,
**options
)
full_output = []
if not pass_output:
for line in proc.stdout:
line = line.decode('utf-8').strip(u'\n')
full_output.append(line)
self.log.debug(line)
return_code = proc.wait()
if return_code != 0:
for line in full_output:
self.log.error(line)
raise BadExitCode('subprogram exit with non zero exit code')
def makedirs(self, *dirs):
"""Makes dirs inside the prefix.
Use this command inside your `install` method.
"""
for d in dirs:
fullname = os.path.join(self.env['prefix'], d)
if not os.path.exists(fullname):
self.log.info('Creating directory "{0}".'.format(fullname))
os.makedirs(fullname)
def create_file_with_content(self, filename, content, mode=None):
"""Creates a file inside 'prefix'.
Use this command inside your `install` method.
Note: Source and target directory should exists.
Warning: if there is some file already, it will be overwritten.
"""
filename = os.path.join(self.env['prefix'], filename)
self.log.info('Creating file "{0}"'.format(filename))
with open(filename, 'w') as f:
f.write(self._substitute_vars(content))
if mode is not None:
self.call('chmod "{0}" "{1}"'.format(mode, filename))
def copy_file(self, from_filename, to_filename, mode=None):
"""Copies file, to a directory inside 'prefix'.
from_filename could be relative to the current directory, or
use variables to be expanded to self.env.
Use this command inside your `install` method.
Note: Source and target directory should exists.
Warning: if there is some file already, it will be overwritten.
"""
with open(from_filename.format(**self.env), 'r') as f:
self.create_file_with_content(to_filename, f.read(), mode=mode)
class BTgymEnv(gym.Env):
"""
OpenAI Gym API shell for Backtrader backtesting/trading library.
"""
# Datafeed Server management:
data_master = True
data_network_address = 'tcp://127.0.0.1:' # using localhost.
data_port = 4999
data_server = None
data_server_pid = None
data_context = None
data_socket = None
data_server_response = None
# Dataset:
dataset = None # BTgymDataset instance.
dataset_stat = None
# Backtrader engine:
engine = None # bt.Cerbro subclass for server to execute.
# Strategy:
strategy = None # strategy to use if no <engine> class been passed.
# Server and network:
server = None # Server process.
context = None # ZMQ context.
socket = None # ZMQ socket, client side.
port = 5500 # network port to use.
network_address = 'tcp://127.0.0.1:' # using localhost.
ctrl_actions = ('_done', '_reset', '_stop', '_getstat', '_render'
) # server control messages.
server_response = None
# Connection timeout:
connect_timeout = 60 # server connection timeout in seconds.
#connect_timeout_step = 0.01 # time between retries in seconds.
# Rendering:
render_enabled = True
render_modes = [
'human',
'episode',
]
# `episode` - plotted episode results.
# `human` - raw_state observation in conventional human-readable format.
# <obs_space_key> - rendering of arbitrary state presented in observation_space with same key.
renderer = None # Rendering support.
rendered_rgb = dict() # Keep last rendered images for each mode.
# Logging and id:
log = None
log_level = None # logbook level: NOTICE, WARNING, INFO, DEBUG etc. or its integer equivalent;
verbose = 0 # verbosity mode, valid only if no `log_level` arg has been provided:
# 0 - WARNING, 1 - INFO, 2 - DEBUG.
task = 0
closed = True
def __init__(self, **kwargs):
"""
Keyword Args:
filename=None (str, list): csv data file.
**datafeed_args (any): any datafeed-related args, passed through to
default btgym.datafeed class.
dataset=None (btgym.datafeed): BTgymDataDomain instance,
overrides `filename` or any other datafeed-related args.
strategy=None (btgym.startegy): strategy to be used by `engine`, any subclass of
btgym.strategy.base.BTgymBaseStrateg
engine=None (bt.Cerebro): environment simulation engine, any bt.Cerebro subclass,
overrides `strategy` arg.
network_address=`tcp://127.0.0.1:` (str): BTGym_server address.
port=5500 (int): network port to use for server - API_shell communication.
data_master=True (bool): let this environment control over data_server;
data_network_address=`tcp://127.0.0.1:` (str): data_server address.
data_port=4999 (int): network port to use for server -- data_server communication.
connect_timeout=60 (int): server connection timeout in seconds.
render_enabled=True (bool): enable rendering for this environment;
render_modes=['human', 'episode'] (list): `episode` - plotted episode results;
`human` - raw_state observation.
**render_args (any): any render-related args, passed through to renderer class.
verbose=0 (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG}
log_level=None (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13},
overrides `verbose` arg;
log=None (logbook.Logger): external logbook logger,
overrides `log_level` and `verbose` args.
task=0 (int): environment id
Environment kwargs applying logic::
if <engine> kwarg is given:
do not use default engine and strategy parameters;
ignore <strategy> kwarg and all strategy and engine-related kwargs.
else (no <engine>):
use default engine parameters;
if any engine-related kwarg is given:
override corresponding default parameter;
if <strategy> is given:
do not use default strategy parameters;
if any strategy related kwarg is given:
override corresponding strategy parameter;
else (no <strategy>):
use default strategy parameters;
if any strategy related kwarg is given:
override corresponding strategy parameter;
if <dataset> kwarg is given:
do not use default dataset parameters;
ignore dataset related kwargs;
else (no <dataset>):
use default dataset parameters;
if any dataset related kwarg is given:
override corresponding dataset parameter;
If any <other> kwarg is given:
override corresponding default parameter.
"""
# Parameters and default values:
self.params = dict(
# Backtrader engine mandatory parameters:
engine=dict(
start_cash=10.0, # initial trading capital.
broker_commission=
0.001, # trade execution commission, default is 0.1% of operation value.
fixed_stake=10, # single trade stake is fixed type by def.
),
# Dataset mandatory parameters:
dataset=dict(filename=None, ),
strategy=dict(state_shape=dict(), ),
render=dict(),
)
p2 = dict( # IS HERE FOR REFERENCE ONLY
# Strategy related parameters:
# Observation state shape is dictionary of Gym spaces,
# at least should contain `raw_state` field.
# By convention first dimension of every Gym Box space is time embedding one;
# one can define any shape; should match env.observation_space.shape.
# observation space state min/max values,
# For `raw_state' - absolute min/max values from BTgymDataset will be used.
state_shape=dict(raw_state=spaces.Box(
shape=(10, 4), low=-100, high=100, dtype=np.float32)),
drawdown_call=
None, # episode maximum drawdown threshold, default is 90% of initial value.
portfolio_actions=None,
# agent actions,
# should consist with BTgymStrategy order execution logic;
# defaults are: 0 - 'do nothing', 1 - 'buy', 2 - 'sell', 3 - 'close position'.
skip_frame=None,
# Number of environment steps to skip before returning next response,
# e.g. if set to 10 -- agent will interact with environment every 10th episode step;
# Every other step agent's action is assumed to be 'hold'.
# Note: INFO part of environment response is a list of all skipped frame's info's,
# i.e. [info[-9], info[-8], ..., info[0].
)
# Update self attributes, remove used kwargs:
for key in dir(self):
if key in kwargs.keys():
setattr(self, key, kwargs.pop(key))
self.metadata = {'render.modes': self.render_modes}
# Logging and verbosity control:
if self.log is None:
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
log_levels = [(0, NOTICE), (1, INFO), (2, DEBUG)]
self.log_level = WARNING
for key, value in log_levels:
if key == self.verbose:
self.log_level = value
self.log = Logger('BTgymAPIshell_{}'.format(self.task),
level=self.log_level)
# Network parameters:
self.network_address += str(self.port)
self.data_network_address += str(self.data_port)
# Set server rendering:
if self.render_enabled:
self.renderer = BTgymRendering(self.metadata['render.modes'],
log_level=self.log_level,
**kwargs)
else:
self.renderer = BTgymNullRendering()
self.log.info(
'Rendering disabled. Call to render() will return null-plug image.'
)
# Append logging:
self.renderer.log = self.log
# Update params -1: pull from renderer, remove used kwargs:
self.params['render'].update(self.renderer.params)
for key in self.params['render'].keys():
if key in kwargs.keys():
_ = kwargs.pop(key)
if self.data_master:
# DATASET preparation, only data_master executes this:
#
if self.dataset is not None:
# If BTgymDataset instance has been passed:
# do nothing.
msg = 'Custom Dataset class used.'
else:
# If no BTgymDataset has been passed,
# Make default dataset with given CSV file:
try:
os.path.isfile(str(self.params['dataset']['filename']))
except:
raise FileNotFoundError(
'Dataset source data file not specified/not found')
# Use kwargs to instantiate dataset:
self.dataset = BTgymDataset(**kwargs)
msg = 'Base Dataset class used.'
# Append logging:
self.dataset.set_logger(self.log_level, self.task)
# Update params -2: pull from dataset, remove used kwargs:
self.params['dataset'].update(self.dataset.params)
for key in self.params['dataset'].keys():
if key in kwargs.keys():
_ = kwargs.pop(key)
self.log.info(msg)
# Connect/Start data server (and get dataset statistic):
self.log.info('Connecting data_server...')
self._start_data_server()
self.log.info('...done.')
# ENGINE preparation:
# Update params -3: pull engine-related kwargs, remove used:
for key in self.params['engine'].keys():
if key in kwargs.keys():
self.params['engine'][key] = kwargs.pop(key)
if self.engine is not None:
# If full-blown bt.Cerebro() subclass has been passed:
# Update info:
msg = 'Custom Cerebro class used.'
self.strategy = msg
for key in self.params['engine'].keys():
self.params['engine'][key] = msg
# Note: either way, bt.observers.DrawDown observer [and logger] will be added to any BTgymStrategy instance
# by BTgymServer process at runtime.
else:
# Default configuration for Backtrader computational engine (Cerebro),
# if no bt.Cerebro() custom subclass has been passed,
# get base class Cerebro(), using kwargs on top of defaults:
self.engine = bt.Cerebro()
msg = 'Base Cerebro class used.'
# First, set STRATEGY configuration:
if self.strategy is not None:
# If custom strategy has been passed:
msg2 = 'Custom Strategy class used.'
else:
# Base class strategy :
self.strategy = BTgymBaseStrategy
msg2 = 'Base Strategy class used.'
# Add, using kwargs on top of defaults:
#self.log.debug('kwargs for strategy: {}'.format(kwargs))
strat_idx = self.engine.addstrategy(self.strategy, **kwargs)
msg += ' ' + msg2
# Second, set Cerebro-level configuration:
self.engine.broker.setcash(self.params['engine']['start_cash'])
self.engine.broker.setcommission(
self.params['engine']['broker_commission'])
self.engine.addsizer(bt.sizers.SizerFix,
stake=self.params['engine']['fixed_stake'])
self.log.info(msg)
# Define observation space shape, minimum / maximum values and agent action space.
# Retrieve values from configured engine or...
# ...Update params -4:
# Pull strategy defaults to environment params dict :
for t_key, t_value in self.engine.strats[0][0][0].params._gettuple():
self.params['strategy'][t_key] = t_value
# Update it with values from strategy 'passed-to params':
for key, value in self.engine.strats[0][0][2].items():
self.params['strategy'][key] = value
# ... Push it all back (don't ask):
for key, value in self.params['strategy'].items():
self.engine.strats[0][0][2][key] = value
# For 'raw_state' min/max values,
# the only way is to infer from raw Dataset price values (we already got those from data_server):
if 'raw_state' in self.params['strategy']['state_shape'].keys():
# Exclude 'volume' from columns we count:
self.dataset_columns.remove('volume')
#print(self.params['strategy'])
#print('self.engine.strats[0][0][2]:', self.engine.strats[0][0][2])
#print('self.engine.strats[0][0][0].params:', self.engine.strats[0][0][0].params._gettuple())
# Override with absolute price min and max values:
self.params['strategy']['state_shape']['raw_state'].low =\
self.engine.strats[0][0][2]['state_shape']['raw_state'].low =\
np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) +\
self.dataset_stat.loc['min', self.dataset_columns].min()
self.params['strategy']['state_shape']['raw_state'].high = \
self.engine.strats[0][0][2]['state_shape']['raw_state'].high = \
np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) + \
self.dataset_stat.loc['max', self.dataset_columns].max()
self.log.info(
'Inferring `state_raw` high/low values form dataset: {:.6f} / {:.6f}.'
.format(
self.dataset_stat.loc['min', self.dataset_columns].min(),
self.dataset_stat.loc['max', self.dataset_columns].max()))
# Set observation space shape from engine/strategy parameters:
self.observation_space = DictSpace(
self.params['strategy']['state_shape'])
self.log.debug('Obs. shape: {}'.format(self.observation_space.spaces))
#self.log.debug('Obs. min:\n{}\nmax:\n{}'.format(self.observation_space.low, self.observation_space.high))
# Set action space and corresponding server messages:
self.action_space = spaces.Discrete(
len(self.params['strategy']['portfolio_actions']))
self.server_actions = self.params['strategy']['portfolio_actions']
# Finally:
self.server_response = None
self.env_response = None
#if not self.data_master:
self._start_server()
self.closed = False
self.log.info('Environment is ready.')
def _seed(self, seed=None):
"""
Sets env. random seed.
Args:
seed: int or None
"""
np.random.seed(seed)
@staticmethod
def _comm_with_timeout(
socket,
message,
):
"""
Exchanges messages via socket, timeout sensitive.
Args:
socket: zmq connected socket to communicate via;
message: message to send;
Note:
socket zmq.RCVTIMEO and zmq.SNDTIMEO should be set to some finite number of milliseconds.
Returns:
dictionary:
`status`: communication result;
`message`: received message if status == `ok` or None;
`time`: remote side response time.
"""
response = dict(
status='ok',
message=None,
)
try:
socket.send_pyobj(message)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
response['status'] = 'send_failed_due_to_connect_timeout'
else:
response['status'] = 'send_failed_for_unknown_reason'
return response
start = time.time()
try:
response['message'] = socket.recv_pyobj()
response['time'] = time.time() - start
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
response['status'] = 'receive_failed_due_to_connect_timeout'
else:
response['status'] = 'receive_failed_for_unknown_reason'
return response
return response
def _start_server(self):
"""
Configures backtrader REQ/REP server instance and starts server process.
"""
# Ensure network resources:
# 1. Release client-side, if any:
if self.context:
self.context.destroy()
self.socket = None
# 2. Kill any process using server port:
cmd = "kill $( lsof -i:{} -t ) > /dev/null 2>&1".format(self.port)
os.system(cmd)
# Set up client channel:
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.RCVTIMEO, self.connect_timeout * 1000)
self.socket.setsockopt(zmq.SNDTIMEO, self.connect_timeout * 1000)
self.socket.connect(self.network_address)
# Configure and start server:
self.server = BTgymServer(
cerebro=self.engine,
render=self.renderer,
network_address=self.network_address,
data_network_address=self.data_network_address,
connect_timeout=self.connect_timeout,
log_level=self.log_level,
task=self.task,
)
self.server.daemon = False
self.server.start()
# Wait for server to startup:
time.sleep(1)
# Check connection:
self.log.info('Server started, pinging {} ...'.format(
self.network_address))
self.server_response = self._comm_with_timeout(
socket=self.socket, message={'ctrl': 'ping!'})
if self.server_response['status'] in 'ok':
self.log.info('Server seems ready with response: <{}>'.format(
self.server_response['message']))
else:
msg = 'Server unreachable with status: <{}>.'.format(
self.server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
self._closed = False
def _stop_server(self):
"""
Stops BT server process, releases network resources.
"""
if self.server:
if self._force_control_mode():
# In case server is running and client side is ok:
self.socket.send_pyobj({'ctrl': '_stop'})
self.server_response = self.socket.recv_pyobj()
else:
self.server.terminate()
self.server.join()
self.server_response = 'Server process terminated.'
self.log.info('{} Exit code: {}'.format(self.server_response,
self.server.exitcode))
# Release client-side, if any:
if self.context:
self.context.destroy()
self.socket = None
def _force_control_mode(self):
"""Puts BT server to control mode.
"""
# Check is there any faults with server process and connection?
network_error = [
(not self.server or not self.server.is_alive(),
'No running server found. Hint: forgot to call reset()?'),
(not self.context
or self.context.closed, 'No network connection found.'),
]
for (err, msg) in network_error:
if err:
self.log.info(msg)
self.server_response = msg
return False
# If everything works, insist to go 'control':
self.server_response = {}
attempt = 0
while 'ctrl' not in self.server_response:
self.socket.send_pyobj({'ctrl': '_done'})
self.server_response = self.socket.recv_pyobj()
attempt += 1
self.log.debug(
'FORCE CONTROL MODE attempt: {}.\nResponse: {}'.format(
attempt, self.server_response))
return True
def _assert_response(self, response):
"""
Simple watcher:
roughly checks if we really talking to environment (== episode is running).
Rises exception if response given is not as expected.
"""
try:
assert type(response) == tuple and len(response) == 4
except AssertionError:
msg = 'Unexpected environment response: {}\nHint: Forgot to call reset() or reset_data()?'.format(
response)
self.log.exception(msg)
raise AssertionError(msg)
self.log.debug('Response checker received:\n{}\nas type: {}'.format(
response, type(response)))
def _print_space(self, space, _tab=''):
"""
Parses observation space shape or response.
Args:
space: gym observation space or state.
Returns:
description as string.
"""
response = ''
if type(space) in [dict, OrderedDict]:
for key, value in space.items():
response += '\n{}{}:{}\n'.format(
_tab, key, self._print_space(value, ' '))
elif type(space) in [spaces.Dict, DictSpace]:
for s in space.spaces:
response += self._print_space(s, ' ')
elif type(space) in [tuple, list]:
for i in space:
response += self._print_space(i, ' ')
elif type(space) == np.ndarray:
response += '\n{}array of shape: {}, low: {}, high: {}'.format(
_tab, space.shape, space.min(), space.max())
else:
response += '\n{}{}, '.format(_tab, space)
try:
response += 'low: {}, high: {}'.format(space.low.min(),
space.high.max())
except (KeyError, AttributeError, ArithmeticError,
ValueError) as e:
pass
#response += '\n{}'.format(e)
return response
def reset(self, **kwargs):
"""
Implementation of OpenAI Gym env.reset method. Starts new episode. Episode data are sampled
according to data provider class logic, controlled via kwargs. Refer `BTgym_Server` and data provider
classes for details.
Args:
kwargs: any kwargs; this dictionary is passed through to BTgym_server side without any checks and
modifications; currently used for data sampling control;
Returns:
observation space state
Notes:
Current kwargs accepted is::
episode_config=dict(
get_new=True,
sample_type=0,
b_alpha=1,
b_beta=1
),
trial_config=dict(
get_new=True,
sample_type=0,
b_alpha=1,
b_beta=1
)
"""
# Data Server check:
if self.data_master:
if not self.data_server or not self.data_server.is_alive():
self.log.info('No running data_server found, starting...')
self._start_data_server()
# Domain dataset status check:
self.data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': '_get_info'})
if not self.data_server_response['message']['dataset_is_ready']:
self.log.info(
'Data domain `reset()` called prior to `reset_data()` with [possibly inconsistent] defaults.'
)
self.reset_data()
# Server process check:
if not self.server or not self.server.is_alive():
self.log.info('No running server found, starting...')
self._start_server()
if self._force_control_mode():
self.server_response = self._comm_with_timeout(socket=self.socket,
message={
'ctrl':
'_reset',
'kwargs': kwargs
})
# Get initial environment response:
self.env_response = self.step(0)
# Check (once) if it is really (o,r,d,i) tuple:
self._assert_response(self.env_response)
# Check (once) if state_space is as expected:
try:
#assert self.observation_space.contains(self.env_response[0])
pass
except (AssertionError, AttributeError) as e:
msg1 = self._print_space(self.observation_space.spaces)
msg2 = self._print_space(self.env_response[0])
msg3 = ''
for step_info in self.env_response[-1]:
msg3 += '{}\n'.format(step_info)
msg = ('\nState observation shape/range mismatch!\n' +
'Space set by env: \n{}\n' +
'Space returned by server: \n{}\n' +
'Full response:\n{}\n' + 'Reward: {}\n' + 'Done: {}\n' +
'Info:\n{}\n' +
'Hint: Wrong Strategy.get_state() parameters?').format(
msg1,
msg2,
self.env_response[0],
self.env_response[1],
self.env_response[2],
msg3,
)
self.log.exception(msg)
self._stop_server()
raise AssertionError(msg)
return self.env_response[0] #["raw_state"][np.newaxis]
else:
msg = 'Something went wrong. env.reset() can not get response from server.'
self.log.exception(msg)
raise ChildProcessError(msg)
def step(self, action):
"""
Implementation of OpenAI Gym env.step() method.
Makes a step in the environment.
Args:
action: int, number representing action from env.action_space
Returns:
tuple (Observation, Reward, Info, Done)
"""
# Are you in the list, ready to go and all that?
if self.action_space.contains(action)\
and not self._closed\
and (self.socket is not None)\
and not self.socket.closed:
pass
else:
msg = ('\nAt least one of these is true:\n' +
'Action error: (space is {}, action sent is {}): {}\n' +
'Environment closed: {}\n' +
'Network error [socket doesnt exists or closed]: {}\n' +
'Hint: forgot to call reset()?').format(
self.action_space,
action,
not self.action_space.contains(action),
self._closed,
not self.socket or self.socket.closed,
)
self.log.exception(msg)
raise AssertionError(msg)
# Send action to backtrader engine, receive environment response
env_response = self._comm_with_timeout(
socket=self.socket,
message={'action': self.server_actions[action]})
if not env_response['status'] in 'ok':
msg = '.step(): server unreachable with status: <{}>.'.format(
env_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# self.env_response = env_response ['message']
tempNew_state, tempReward, tempDone, tempInfo = env_response['message']
tempNew_state = tempNew_state["raw_state"][np.newaxis]
self.env_response = tempNew_state, tempReward, tempDone, tempInfo
return self.env_response
def close(self):
"""
Implementation of OpenAI Gym env.close method.
Puts BTgym server in Control Mode.
"""
self.log.debug('close.call()')
self._stop_server()
self._stop_data_server()
self.log.info('Environment closed.')
def get_stat(self):
"""
Returns last run episode statistics.
Note:
when invoked, forces running episode to terminate.
"""
if self._force_control_mode():
self.socket.send_pyobj({'ctrl': '_getstat'})
return self.socket.recv_pyobj()
else:
return self.server_response
def render(self, mode='other_mode', close=False):
"""
Implementation of OpenAI Gym env.render method.
Visualises current environment state.
Args:
`mode`: str, any of these::
`human` - current state observation as price lines;
`episode` - plotted results of last completed episode.
[other_key] - corresponding to any custom observation space key
"""
if close:
return None
if not self._closed\
and self.socket\
and not self.socket.closed:
pass
else:
msg = ('\nCan'
't get renderings.'
'\nAt least one of these is true:\n' +
'Environment closed: {}\n' +
'Network error [socket doesnt exists or closed]: {}\n' +
'Hint: forgot to call reset()?').format(
self._closed,
not self.socket or self.socket.closed,
)
self.log.warning(msg)
return None
if mode not in self.render_modes:
raise ValueError('Unexpected render mode {}'.format(mode))
self.socket.send_pyobj({'ctrl': '_render', 'mode': mode})
rgb_array_dict = self.socket.recv_pyobj()
self.rendered_rgb.update(rgb_array_dict)
return self.rendered_rgb[mode]
def _stop(self):
"""
Finishes current episode if any, does nothing otherwise. Leaves server running.
"""
if self._force_control_mode():
self.log.info('Episode stop forced.')
def _restart_server(self):
"""Restarts server.
"""
self._stop_server()
self._start_server()
self.log.info('Server restarted.')
def _start_data_server(self):
"""
For data_master environment:
- configures backtrader REQ/REP server instance and starts server process.
For others:
- establishes network connection to existing data_server.
"""
self.data_server = None
# Ensure network resources:
# 1. Release client-side, if any:
if self.data_context:
self.data_context.destroy()
self.data_socket = None
# Only data_master launches/stops data_server process:
if self.data_master:
# 2. Kill any process using server port:
cmd = "kill $( lsof -i:{} -t ) > /dev/null 2>&1".format(
self.data_port)
os.system(cmd)
# Configure and start server:
self.data_server = BTgymDataFeedServer(
dataset=self.dataset,
network_address=self.data_network_address,
log_level=self.log_level,
task=self.task)
self.data_server.daemon = False
self.data_server.start()
# Wait for server to startup
time.sleep(1)
# Set up client channel:
self.data_context = zmq.Context()
self.data_socket = self.data_context.socket(zmq.REQ)
self.data_socket.setsockopt(zmq.RCVTIMEO, self.connect_timeout * 1000)
self.data_socket.setsockopt(zmq.SNDTIMEO, self.connect_timeout * 1000)
self.data_socket.connect(self.data_network_address)
# Check connection:
self.log.debug('Pinging data_server at: {} ...'.format(
self.data_network_address))
self.data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': 'ping!'})
if self.data_server_response['status'] in 'ok':
self.log.debug(
'Data_server seems ready with response: <{}>'.format(
self.data_server_response['message']))
else:
msg = 'Data_server unreachable with status: <{}>.'.\
format(self.data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# Get info and statistic:
self.dataset_stat, self.dataset_columns, self.data_server_pid = self._get_dataset_info(
)
def _stop_data_server(self):
"""
For data_master:
- stops BT server process, releases network resources.
"""
if self.data_master:
if self.data_server is not None and self.data_server.is_alive():
# In case server is running and is ok:
self.data_socket.send_pyobj({'ctrl': '_stop'})
self.data_server_response = self.data_socket.recv_pyobj()
else:
self.data_server.terminate()
self.data_server.join()
self.data_server_response = 'Data_server process terminated.'
self.log.info('{} Exit code: {}'.format(self.data_server_response,
self.data_server.exitcode))
if self.data_context:
self.data_context.destroy()
self.data_socket = None
def _restart_data_server(self):
"""
Restarts data_server.
"""
if self.data_master:
self._stop_data_server()
self._start_data_server()
def _get_dataset_info(self):
"""
Retrieves dataset descriptive statistic.
"""
self.data_socket.send_pyobj({'ctrl': '_get_info'})
self.data_server_response = self.data_socket.recv_pyobj()
return self.data_server_response['dataset_stat'],\
self.data_server_response['dataset_columns'],\
self.data_server_response['pid']
def reset_data(self, **kwargs):
"""
Resets data provider class used, whatever it means for that class. Gets data_server ready to provide data.
Supposed to be called before first env.reset().
Note:
when invoked, forces running episode to terminate.
Args:
**kwargs: data provider class .reset() method specific.
"""
if self.closed:
self._start_server()
if self.data_master:
self._start_data_server()
self.closed = False
else:
_ = self._force_control_mode()
if self.data_master:
if self.data_server is None or not self.data_server.is_alive():
self._restart_data_server()
self.data_server_response = self._comm_with_timeout(
socket=self.data_socket,
message={
'ctrl': '_reset_data',
'kwargs': kwargs
})
if self.data_server_response['status'] in 'ok':
self.log.debug(
'Dataset seems ready with response: <{}>'.format(
self.data_server_response['message']))
else:
msg = 'Data_server unreachable with status: <{}>.'. \
format(self.data_server_response['status'])
self.log.error(msg)
raise SystemExit(msg)
else:
pass
class triggerThread(stopableThread):
"""
对于每个已经注册的trigger，会有一个actionThread与之对应。
当trigger触发时，启动对应的线程处理action。线程Handle保存在self.threadDict中。
"""
def __init__(self,config=None,dataQueue=None,*args,**kwargs):
super(triggerThread,self).__init__(*args,**kwargs)
self.config = config
self.dataQueue = dataQueue
self.log = Logger(u"%s triggerThread" % self.name)
self.threadDict = {} #Thread 列表
def setDataQueue(self, q):
if q is not None:
self.dataQueue = q
def getDataQueue(self):
return self.dataQueue
def changeConfig(self,newconf):
if type(newconf) != type(()) and type(newconf) != type([]):
self.log.error(u"changeConfig:invalid parameter")
return
for map in newconf:
if len(map) != 2 :
self.log.error(u"changeConfig:invalid map")
return
# stop all thread created before and delete them
for t in self.threadDict:
t.stop()
del self.threadDict[t]
self.threadDict.clear()
self.config = newconf
def process(self):
"""
对于每一个dataQueue中的数据，循环调用self.config列表中的所有trigger，判断是否触发。
若某个触发成功，则创建actionThread类去执行相关的action列表。
"""
#get message from queue. Block if no data
try:
print "before queue get"
msg = self.dataQueue.get(block=True)
print "after queue get:%s" % msg
except AttributeError:
self.log.error(u"process : dataQueue invalid")
self.stop()
return
#Get triggerManager Handle
trigManager = absManager.triggerManager()
for map in self.config:
if type(map) != type([]) and type(map) != type(()):
self.log.error(u"process: invalid config parameter %s" % self.config)
self.stop()
return
triggerID = map[0]
if not trigManager.validID(triggerID):
self.log.error(u"process : invalid plugin ID:%s" % triggerID)
continue
t,ret = trigManager.call(triggerID,msg)
if t:
#triggered,call all actions in another thread
if map[0] not in self.threadDict:
#创建线程
self.threadDict[map[0]] = actionThread(actList=map[1],name=triggerID)
#设置此次action参数
self.threadDict[map[0]].setParam((msg,ret))
self.threadDict[map[0]].start()
return
sChar = Character.getInstance()
try:
activeChar = self.charEditor.entityEditor.getActiveEntity()
list = sChar.apiCharList(activeChar.ID, self.inputID.GetLineText(0), self.inputKey.GetLineText(0))
except AuthenticationError, e:
msg = "Authentication failure. Please check keyID and vCode combination."
pyfalog.info(msg)
self.stStatus.SetLabel(msg)
except TimeoutError, e:
msg = "Request timed out. Please check network connectivity and/or proxy settings."
pyfalog.info(msg)
self.stStatus.SetLabel(msg)
except Exception, e:
pyfalog.error(e)
self.stStatus.SetLabel("Error:\n%s" % e.message)
else:
self.charChoice.Clear()
for charName in list:
self.charChoice.Append(charName)
self.btnFetchSkills.Enable(True)
self.charChoice.Enable(True)
self.Layout()
self.charChoice.SetSelection(0)
def fetchSkills(self, event):
charName = self.charChoice.GetString(self.charChoice.GetSelection())
return True
else:
print response.text
print response.status_code
return False
# Questionnaire
questionnaire = {'name': 'FMR Continuation Desire Questionnaire'}
r = requests.post('{}/questionnaire'.format(BASE_URL),
data=json.dumps(questionnaire),
headers={'content-type': 'application/json'})
if r.status_code == 201:
log.info('Questionnaire {} with status code {}'.format(questionnaire['name'], r.status_code))
else:
log.error('Questionnaire {} with status code {}'.format(questionnaire['name'], r.status_code))
# QuestionSet
question_sets = [
{'name': 'Before the experience', 'sort_nr': '1',
'info_text': 'Questions that need to be filled out before you start playing',
'questionnaire_id': '1'},
{'name': 'During the experience', 'sort_nr': '2',
'info_text': 'Questions regarding playing the game', 'questionnaire_id': '1'},
{'name': 'After the experience', 'sort_nr': '3',
'info_text': 'Questions regarding post-play', 'questionnaire_id': '1'}
]
question_sets_ids = []
class BTgymServer(multiprocessing.Process):
"""Backtrader server class.
Expects to receive dictionary, containing at least 'action' field.
Control mode IN::
dict(action=<control action, type=str>,), where control action is:
'_reset' - rewinds backtrader engine and runs new episode;
'_getstat' - retrieve episode results and statistics;
'_stop' - server shut-down.
Control mode OUT::
<string message> - reports current server status;
<statisic dict> - last run episode statisics. NotImplemented.
Within-episode signals:
Episode mode IN:
dict(action=<agent_action, type=str>,), where agent_action is:
{'buy', 'sell', 'hold', 'close', '_done'} - agent or service actions; '_done' - stops current episode;
Episode mode OUT::
response <tuple>: observation, <array> - observation of the current environment state,
could be any tensor; default is:
[4,m] array of <fl32>, where:
m - num. of last datafeed values,
4 - num. of data features (Lines);
reward, <any> - current portfolio statistics for environment reward estimation;
done, <bool> - episode termination flag;
info, <list> - auxiliary information.
"""
data_server_response = None
def __init__(
self,
cerebro=None,
render=None,
network_address=None,
data_network_address=None,
connect_timeout=90,
log_level=None,
task=0,
):
"""
Args:
cerebro: backtrader.cerebro engine class.
render: render class
network_address: environmnet communication, str
data_network_address: data communication, str
connect_timeout: seconds, int
log_level: int, logbook.level
"""
super(BTgymServer, self).__init__()
self.task = task
self.log_level = log_level
self.log = None
self.process = None
self.cerebro = cerebro
self.network_address = network_address
self.render = render
self.data_network_address = data_network_address
self.connect_timeout = connect_timeout # server connection timeout in seconds.
self.connect_timeout_step = 0.01
self.trial_sample = None
self.trial_stat = None
self.dataset_stat = None
@staticmethod
def _comm_with_timeout(socket, message):
"""
Exchanges messages via socket with timeout.
Note:
socket zmq.RCVTIMEO and zmq.SNDTIMEO should be set to some finite number of milliseconds
Returns:
dictionary:
status: communication result;
message: received message, if any.
"""
response = dict(
status='ok',
message=None,
)
try:
socket.send_pyobj(message)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
response['status'] = 'send_failed_due_to_connect_timeout'
else:
response['status'] = 'send_failed_for_unknown_reason'
return response
start = time.time()
try:
response['message'] = socket.recv_pyobj()
response['time'] = time.time() - start
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
response['status'] = 'receive_failed_due_to_connect_timeout'
else:
response['status'] = 'receive_failed_for_unknown_reason'
return response
return response
def get_dataset_stat(self):
data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': '_get_info'})
if data_server_response['status'] in 'ok':
self.log.debug(
'Data_server @{} responded with dataset statistic in about {} seconds.'
.format(self.data_network_address,
data_server_response['time']))
return data_server_response['message']
else:
msg = 'BtgymServer_sampling_attempt: data_server @{} unreachable with status: <{}>.'. \
format(self.data_network_address, data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
def get_trial(self, **reset_kwargs):
"""
Args:
reset_kwargs: dictionary of args to pass to parent data iterator
Returns:
trial_sample, trial_stat, dataset_stat
"""
wait = 0
while True:
# Get new data subset:
data_server_response = self._comm_with_timeout(
socket=self.data_socket,
message={
'ctrl': '_get_data',
'kwargs': reset_kwargs
})
if data_server_response['status'] in 'ok':
self.log.debug(
'Data_server @{} responded in ~{:1.6f} seconds.'.format(
self.data_network_address,
data_server_response['time']))
else:
msg = 'BtgymServer_sampling_attempt: data_server @{} unreachable with status: <{}>.'. \
format(self.data_network_address, data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# Ready or not?
try:
assert 'Dataset not ready' in data_server_response['message'][
'ctrl']
if wait <= self.wait_for_data_reset:
pause = random.random() * 2
time.sleep(pause)
wait += pause
self.log.info(
'Domain dataset not ready, wait time left: {:4.2f}s.'.
format(self.wait_for_data_reset - wait))
else:
data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': '_stop'})
self.socket.close()
self.context.destroy()
raise RuntimeError(
'Failed to assert Domain dataset is ready. Exiting.')
except (AssertionError, KeyError) as e:
break
# Get trial instance:
trial_sample = data_server_response['message']['sample']
trial_stat = trial_sample.describe()
trial_sample.reset()
dataset_stat = data_server_response['message']['stat']
origin = data_server_response['message']['origin']
timestamp = data_server_response['message']['timestamp']
return trial_sample, trial_stat, dataset_stat, origin, timestamp
def get_trial_message(self):
"""
Prepares message containing current trial instance, mimicking data_server message protocol.
Intended for serving requests from data_slave environment.
Returns:
dict containing trial instance, d_set statistic and origin key; dict containing 'ctrl' response if master
d_set is not ready;
"""
if self.trial_sample is not None:
message = {
'sample': self.trial_sample,
'stat': self.dataset_stat,
'origin': 'master_environment',
'timestamp': self.get_global_time()
}
else:
message = {'ctrl': 'Dataset not ready, hold on...'}
self.log.debug('Sent to slave: ' + str(message))
return message
def get_global_time(self):
"""
Asks dataserver for current dataset global_time.
Returns:
POSIX timestamp
"""
data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': '_get_global_time'})
if data_server_response['status'] in 'ok':
pass
else:
msg = 'BtgymServer_sampling_attempt: data_server @{} unreachable with status: <{}>.'. \
format(self.data_network_address, data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
return data_server_response['message']['timestamp']
def run(self):
"""
Server process runtime body. This method is invoked by env._start_server().
"""
# Logging:
from logbook import Logger, StreamHandler, WARNING
import sys
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
self.log_level = WARNING
self.log = Logger('BTgymServer_{}'.format(self.task),
level=self.log_level)
self.process = multiprocessing.current_process()
self.log.info('PID: {}'.format(self.process.pid))
# Runtime Housekeeping:
cerebro = None
episode_result = dict()
episode_sample = None
# How long to wait for data_master to reset data:
self.wait_for_data_reset = 300 # seconds
connect_timeout = 60 # in seconds
# Set up a comm. channel for server as ZMQ socket
# to carry both service and data signal
# !! Reminder: Since we use REQ/REP - messages do go in pairs !!
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REP)
self.socket.setsockopt(zmq.RCVTIMEO, -1)
self.socket.setsockopt(zmq.SNDTIMEO, connect_timeout * 1000)
self.socket.bind(self.network_address)
self.data_context = zmq.Context()
self.data_socket = self.data_context.socket(zmq.REQ)
self.data_socket.setsockopt(zmq.RCVTIMEO, connect_timeout * 1000)
self.data_socket.setsockopt(zmq.SNDTIMEO, connect_timeout * 1000)
self.data_socket.connect(self.data_network_address)
# Check connection:
self.log.debug('Pinging data_server at: {} ...'.format(
self.data_network_address))
data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': 'ping!'})
if data_server_response['status'] in 'ok':
self.log.debug(
'Data_server seems ready with response: <{}>'.format(
data_server_response['message']))
else:
msg = 'Data_server unreachable with status: <{}>.'.\
format(data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# Init renderer:
self.render.initialize_pyplot()
# Mandatory DrawDown and auxillary plotting observers to add to data-master strategy instance:
# TODO: make plotters optional args
if self.render.enabled:
aux_obsrevers = [bt.observers.DrawDown, Reward, Position, NormPnL]
else:
aux_obsrevers = [bt.observers.DrawDown]
# Server 'Control Mode' loop:
for episode_number in itertools.count(0):
while True:
# Stuck here until '_reset' or '_stop':
service_input = self.socket.recv_pyobj()
msg = 'Control mode: received <{}>'.format(service_input)
self.log.debug(msg)
if 'ctrl' in service_input:
# It's time to exit:
if service_input['ctrl'] == '_stop':
# Server shutdown logic:
# send last run statistic, release comm channel and exit:
message = 'Exiting.'
self.log.info(message)
self.socket.send_pyobj(message)
self.socket.close()
self.context.destroy()
return None
# Start episode:
elif service_input['ctrl'] == '_reset':
message = 'Preparing new episode with kwargs: {}'.format(
service_input['kwargs'])
self.log.debug(message)
self.socket.send_pyobj(message) # pairs '_reset'
break
# Retrieve statistic:
elif service_input['ctrl'] == '_getstat':
self.socket.send_pyobj(episode_result)
self.log.debug('Episode statistic sent.')
# Send episode rendering:
elif service_input[
'ctrl'] == '_render' and 'mode' in service_input.keys(
):
# Just send what we got:
self.socket.send_pyobj(
self.render.render(service_input['mode']))
self.log.debug(
'Episode rendering for [{}] sent.'.format(
service_input['mode']))
# Serve data-dependent environment with trial instance:
elif service_input['ctrl'] == '_get_data':
message = 'Sending trial data to slave'
self.log.debug(message)
self.socket.send_pyobj(self.get_trial_message())
# Serve data-dependent environment with dataset statisitc:
elif service_input['ctrl'] == '_get_info':
message = 'Sending dataset statistic to slave'
self.log.debug(message)
self.socket.send_pyobj(self.get_dataset_stat())
else: # ignore any other input
# NOTE: response string must include 'ctrl' key
# for env.reset(), env.get_stat(), env.close() correct operation.
message = {
'ctrl':
'send control keys: <_reset>, <_getstat>, <_render>, <_stop>.'
}
self.log.debug('Control mode: sent: ' + str(message))
self.socket.send_pyobj(
message) # pairs any other input
else:
message = 'No <ctrl> key received:{}\nHint: forgot to call reset()?'.format(
msg)
self.log.debug(message)
self.socket.send_pyobj(message)
# Got '_reset' signal -> prepare Cerebro subclass and run episode:
start_time = time.time()
cerebro = copy.deepcopy(self.cerebro)
cerebro._socket = self.socket
cerebro._data_socket = self.data_socket
cerebro._log = self.log
cerebro._render = self.render
# Pass methods for serving capabilities:
cerebro._get_data = self.get_trial_message
cerebro._get_info = self.get_dataset_stat
# Add auxillary observers, if not already:
for aux in aux_obsrevers:
is_added = False
for observer in cerebro.observers:
if aux in observer:
is_added = True
if not is_added:
cerebro.addobserver(aux)
# Add communication utility:
cerebro.addanalyzer(
_BTgymAnalyzer,
_name='_env_analyzer',
)
# Data preparation:
# Parse args we got with _reset call:
sample_config = dict(
episode_config=copy.deepcopy(DataSampleConfig),
trial_config=copy.deepcopy(DataSampleConfig))
for key, config in sample_config.items():
try:
config.update(service_input['kwargs'][key])
except KeyError:
self.log.debug(
'_reset <{}> kwarg not found, using default values: {}'
.format(key, config))
# Get new Trial from data_server if requested,
# despite bult-in new/reuse data object sampling option, perform checks here to avoid
# redundant traffic:
if sample_config['trial_config'][
'get_new'] or self.trial_sample is None:
self.log.info(
'Requesting new Trial sample with args: {}'.format(
sample_config['trial_config']))
self.trial_sample, self.trial_stat, self.dataset_stat, origin, current_timestamp =\
self.get_trial(**sample_config['trial_config'])
if origin in 'data_server':
self.trial_sample.set_logger(self.log_level, self.task)
self.log.debug('Got new Trial: <{}>'.format(
self.trial_sample.filename))
else:
self.log.info('Reusing Trial <{}>'.format(
self.trial_sample.filename))
current_timestamp = self.get_global_time()
self.log.debug('current global_time: {}'.format(
datetime.datetime.fromtimestamp(current_timestamp)))
# Get episode:
if sample_config['episode_config']['timestamp'] is None or\
sample_config['episode_config']['timestamp'] < current_timestamp:
sample_config['episode_config'][
'timestamp'] = current_timestamp
self.log.info('Requesting episode from <{}> with args: {}'.format(
self.trial_sample.filename, sample_config['episode_config']))
episode_sample = self.trial_sample.sample(
**sample_config['episode_config'])
self.log.debug('Got new Episode: <{}>'.format(
episode_sample.filename))
# Get episode data statistic and pass it to strategy params:
cerebro.strats[0][0][2]['trial_stat'] = self.trial_stat
cerebro.strats[0][0][2][
'trial_metadata'] = self.trial_sample.metadata
cerebro.strats[0][0][2]['dataset_stat'] = self.dataset_stat
cerebro.strats[0][0][2]['episode_stat'] = episode_sample.describe()
cerebro.strats[0][0][2]['metadata'] = episode_sample.metadata
# Set nice broker cash plotting:
cerebro.broker.set_shortcash(False)
# Convert and add data to engine:
feed = episode_sample.to_btfeed()
if isinstance(feed, dict):
for key, stream in feed.items():
cerebro.adddata(stream, name=key)
else:
cerebro.adddata(feed, name='base_asset')
# Finally:
episode = cerebro.run(stdstats=True,
preload=False,
oldbuysell=True)[0]
# Update episode rendering:
_ = self.render.render('just_render', cerebro=cerebro)
_ = None
# Recover that bloody analytics:
analyzers_list = episode.analyzers.getnames()
analyzers_list.remove('_env_analyzer')
elapsed_time = timedelta(seconds=time.time() - start_time)
self.log.debug('Episode elapsed time: {}.'.format(elapsed_time))
episode_result['episode'] = episode_number
episode_result['runtime'] = elapsed_time
episode_result['length'] = len(episode.data.close)
for name in analyzers_list:
episode_result[name] = episode.analyzers.getbyname(
name).get_analysis()
gc.collect()
# Just in case -- we actually shouldn't get there except by some error:
return None
class Fetcher(object):
''' Web access to data '''
def __init__(self, timezone=pytz.utc, lvl='debug'):
self.tz = timezone
self.log = Logger(self.__class__.__name__)
def getMinutelyQuotes(self, symbol, market, index):
days = abs((index[index.shape[0] - 1] - index[0]).days)
freq = int(index.freqstr[0])
if index.freqstr[1] == 'S':
freq += 1
elif index.freqstr[1] == 'T':
freq *= 61
elif index.freqstr[1] == 'H':
freq *= 3601
else:
self.log.error('** No suitable time frequency: {}'.format(index.freqstr))
return None
url = 'http://www.google.com/finance/getprices?q=%s&x=%s&p=%sd&i=%s' \
% (symbol, market, str(days), str(freq + 1))
self.log.info('On %d days with a precision of %d secs' % (days, freq))
try:
page = urllib2.urlopen(url)
except urllib2.HTTPError:
self.log.error('** Unable to fetch data for stock: %s'.format(symbol))
return None
except urllib2.URLError:
self.log.error('** URL error for stock: %s'.format(symbol))
return None
feed = ''
data = []
while (re.search('^a', feed) is None):
feed = page.readline()
while (feed != ''):
data.append(np.array(map(float, feed[:-1].replace('a', '').split(','))))
feed = page.readline()
dates, open, close, high, low, volume = zip(*data)
adj_close = np.empty(len(close))
adj_close.fill(np.NaN)
data = {
'open' : open,
'close' : close,
'high' : high,
'low' : low,
'volume' : volume,
'adj_close' : adj_close # for compatibility with Fields.QUOTES
}
#NOTE use here index ?
dates = Index(epochToDate(d) for d in dates)
return DataFrame(data, index=dates.tz_localize(self.tz))
def getHistoricalQuotes(self, symbol, index, market=None):
assert (isinstance(index, pd.Index))
source = 'yahoo'
try:
quotes = DataReader(symbol, source, index[0], index[-1])
except:
self.log.error('** Could not get {} quotes'.format(symbol))
return pd.DataFrame()
if index.freq != pd.datetools.BDay() or index.freq != pd.datetools.Day():
#NOTE reIndexDF has a column arg but here not provided
quotes = reIndexDF(quotes, delta=index.freq, reset_hour=False)
if not quotes.index.tzinfo:
quotes.index = quotes.index.tz_localize(self.tz)
quotes.columns = Fields.QUOTES
return quotes
def get_stock_snapshot(self, symbols, markets, light=True):
snapshot = {q: dict() for q in symbols}
if light:
data = self._lightSummary(symbols, markets)
else:
data = self._heavySummary(symbols)
i = 0
if not data:
self.log.error('** No stock informations')
return None
for item in symbols:
snapshot[item] = data[i]
i += 1
return snapshot
def _lightSummary(self, symbols, markets):
#TODO Finir de changer les index et comprendre tous les champs
url = 'http://finance.google.com/finance/info?client=ig&q=%s:%s' \
% (symbols[0], markets[0])
for i in range(1, len(symbols)):
url = url + ',%s:%s' % (symbols[i], markets[i])
self.log.info('Retrieving light Snapshot from %s' % url)
return json.loads(urllib2.urlopen(url).read()[3:], encoding='latin-1')
def _heavySummary(self, symbols):
url = 'http://www.google.com/ig/api?stock=' + symbols[0]
for s in symbols[1:]:
url = url + '&stock=' + s
self.log.info('Retrieving heavy Snapshot from %s' % url)
try:
url_fd = urllib2.urlopen(url)
except IOError:
self.log.error('** Bad url: %s' % url)
return None
try:
xml_doc = minidom.parse(url_fd)
root_node = xml_doc.documentElement
except:
self.log.error('** Parsing xml google response')
return None
i = 0
#snapshot = {q : dict() for q in symbols}
snapshot = list()
ticker_data = dict()
for node in root_node.childNodes:
if (node.nodeName != 'finance'):
continue
ticker_data.clear()
for item_node in node.childNodes:
if (item_node.nodeType != Node.ELEMENT_NODE):
continue
ticker_data[item_node.nodeName] = item_node.getAttribute('data')
i += 1
snapshot.append(ticker_data)
return snapshot
#TODO: a separate class with functions per categories of data
#NOTE: The YQL can fetch this data (http://www.yqlblog.net/blog/2009/06/02/getting-stock-information-with-%60yql%60-and-open-data-tables/)
def getStockInfo(self, symbols, fields):
for f in fields:
#NOTE could just remove field and continue
if f not in yahooCode:
self.log.error('** Invalid stock information request.')
return None
#TODO: remove " from results
#TODO A wrapper interface to have this document through ticker names
#symbols, markets = self.db.getTickersCodes(index, quotes)
fields.append('error')
url = 'http://finance.yahoo.com/d/quotes.csv?s='
url = url + '+'.join(symbols) + '&f='
url += ''.join([yahooCode[item.lower()] for item in fields])
data = urllib2.urlopen(url)
df = dict()
for item in symbols:
#FIXME: ask size return different length arrays !
df[item] = Series(data.readline().strip().strip('"').split(','), index=fields)
return DataFrame(df)
class Fibratus(object):
"""Fibratus entrypoint.
Setup the core components including the kernel
event stream collector and the tracing controller.
At this point the system handles are also being
enumerated.
"""
def __init__(self, filament, **kwargs):
self._start = datetime.now()
try:
log_path = os.path.join(os.path.expanduser('~'), '.fibratus',
'fibratus.log')
FileHandler(log_path, mode='w+').push_application()
StreamHandler(sys.stdout, bubble=True).push_application()
except PermissionError:
panic(
"ERROR - Unable to open log file for writing due to permission error"
)
self.logger = Logger(Fibratus.__name__)
self._config = YamlConfig()
self.logger.info('Starting Fibratus...')
enable_cswitch = kwargs.pop('cswitch', False)
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags(cswitch=enable_cswitch)
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
enum_handles = kwargs.pop('enum_handles', True)
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of the kernel trace
if enum_handles:
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
image_meta_config = self._config.image_meta
self.image_meta_registry = ImageMetaRegistry(
image_meta_config.enabled, image_meta_config.imports,
image_meta_config.file_info)
self.thread_registry = ThreadRegistry(self.handle_repository,
self._handles,
self.image_meta_registry)
self.kevt_streamc = KEventStreamCollector(
etw.KERNEL_LOGGER_NAME.encode())
skips = self._config.skips
image_skips = skips.images if 'images' in skips else []
if len(image_skips) > 0:
self.logger.info("Adding skips for images %s" % image_skips)
for skip in image_skips:
self.kevt_streamc.add_skip(skip)
self.kevent = KEvent(self.thread_registry)
self._output_classes = dict(console=ConsoleOutput,
amqp=AmqpOutput,
smtp=SmtpOutput,
elasticsearch=ElasticsearchOutput)
self._outputs = self._construct_outputs()
self.output_aggregator = OutputAggregator(self._outputs)
self._binding_classes = dict(yara=YaraBinding)
self._bindings = self._construct_bindings()
if filament:
filament.logger = self.logger
filament.do_output_accessors(self._outputs)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.context_switch_registry = ContextSwitchRegistry(
self.thread_registry, self.kevent)
self.output_kevents = {}
self.filters_count = 0
def run(self):
@atexit.register
def _exit():
self.stop_ktrace()
self.kcontroller.start_ktrace(etw.KERNEL_LOGGER_NAME,
self.ktrace_props)
def on_kstream_open():
if self._filament is None:
delta = datetime.now() - self._start
self.logger.info(
'Started in %sm:%02ds.%s' %
(int(delta.total_seconds() / 60), delta.seconds,
int(delta.total_seconds() * 1000)))
else:
self.logger.info('Running [%s] filament...' %
self._filament.name)
self.kevt_streamc.set_kstream_open_callback(on_kstream_open)
self._open_kstream()
def _open_kstream(self):
try:
self.kevt_streamc.open_kstream(self._on_next_kevent)
except Exception as e:
self.logger.error(e)
except KeyboardInterrupt:
self.stop_ktrace()
def _construct_outputs(self):
"""Instantiates output classes.
Builds the dictionary with instances
of the output classes.
"""
outputs = {}
output_configs = self._config.outputs
if not output_configs:
return outputs
for output in output_configs:
name = next(iter(list(output.keys())), None)
if name and \
name in self._output_classes.keys():
# get the output configuration
# and instantiate its class
output_config = output[name]
self.logger.info("Deploying [%s] output - [%s]" % (name, {
k: v
for k, v in output_config.items() if 'password' not in k
}))
output_class = self._output_classes[name]
outputs[name] = output_class(**output_config)
return outputs
def _construct_bindings(self):
"""Builds binding classes.
:return: dict: dictionary with instances of the binding classes
"""
bindings = {}
binding_configs = self._config.bindings
if not binding_configs:
return bindings
for b in binding_configs:
name = next(iter(list(b.keys())), None)
if name and \
name in self._binding_classes.keys():
binding_config = b[name]
self.logger.info("Starting [%s] binding - [%s]" %
(name, binding_config))
binding_class = self._binding_classes[name]
try:
binding = binding_class(self._outputs, self.logger,
**binding_config)
bindings[name] = binding
except BindingError as e:
self.logger.error(
"Couldn't start [%s] binding. Reason: %s" % (name, e))
return bindings
def __find_binding(self, name):
return self._bindings[name] if name in self._bindings else None
def stop_ktrace(self):
self.logger.info('Stopping fibratus...')
if self._filament:
self._filament.close()
self.kcontroller.stop_ktrace(self.ktrace_props)
self.kevt_streamc.close_kstream()
def add_filters(self, kevent_filters, **kwargs):
self.kevt_streamc.add_pid_filter(kwargs.pop('pid', None))
if len(kevent_filters) > 0:
self.filters_count = len(kevent_filters)
# include the basic filters
# that are essential to the
# rest of kernel events
self.kevt_streamc.add_ktuple_filter(ENUM_PROCESS)
self.kevt_streamc.add_ktuple_filter(ENUM_THREAD)
self.kevt_streamc.add_ktuple_filter(ENUM_IMAGE)
self.kevt_streamc.add_ktuple_filter(REG_CREATE_KCB)
self.kevt_streamc.add_ktuple_filter(REG_DELETE_KCB)
# these kevents are necessary for consistent state
# of the trace. If the user doesn't include them
# in a filter list, then we do the job but set the
# kernel event type as not eligible for rendering
if KEvents.CREATE_PROCESS not in kevent_filters:
self.kevt_streamc.add_ktuple_filter(CREATE_PROCESS)
self.output_kevents[CREATE_PROCESS] = False
else:
self.output_kevents[CREATE_PROCESS] = True
if KEvents.CREATE_THREAD not in kevent_filters:
self.kevt_streamc.add_ktuple_filter(CREATE_THREAD)
self.output_kevents[CREATE_THREAD] = False
else:
self.output_kevents[CREATE_THREAD] = True
if KEvents.TERMINATE_PROCESS not in kevent_filters:
self.kevt_streamc.add_ktuple_filter(TERMINATE_PROCESS)
self.output_kevents[TERMINATE_PROCESS] = False
else:
self.output_kevents[TERMINATE_PROCESS] = True
if KEvents.TERMINATE_THREAD not in kevent_filters:
self.kevt_streamc.add_ktuple_filter(TERMINATE_THREAD)
self.output_kevents[TERMINATE_THREAD] = False
else:
self.output_kevents[TERMINATE_THREAD] = True
for kevent_filter in kevent_filters:
ktuple = kname_to_tuple(kevent_filter)
if isinstance(ktuple, list):
for kt in ktuple:
self.kevt_streamc.add_ktuple_filter(kt)
if kt not in self.output_kevents:
self.output_kevents[kt] = True
else:
self.kevt_streamc.add_ktuple_filter(ktuple)
if ktuple not in self.output_kevents:
self.output_kevents[ktuple] = True
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS, CREATE_THREAD, ENUM_PROCESS, ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(
self.kevent, ktype, kparams)
# apply yara binding by matching against the process's image path
if ktype == CREATE_PROCESS:
yara_binding = self.__find_binding('yara')
pid = int(kparams.process_id, 16)
thread = self.thread_registry.get_thread(pid)
if thread and yara_binding:
yara_binding.run(thread_info=thread,
kevent=self.kevent)
self._aggregate(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent, ktype,
kparams)
self._aggregate(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [
CREATE_FILE, DELETE_FILE, CLOSE_FILE, READ_FILE, WRITE_FILE,
RENAME_FILE, SET_FILE_INFORMATION
]:
self.fsio.parse_fsio(ktype, kparams)
self._aggregate(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._aggregate(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._aggregate(ktype)
#
# # registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [
REG_CREATE_KEY, REG_DELETE_KEY, REG_OPEN_KEY, REG_QUERY_KEY,
REG_SET_VALUE, REG_DELETE_VALUE, REG_QUERY_VALUE
]:
self.hive_parser.parse_hive(ktype, kparams)
self._aggregate(ktype)
# network kernel events
elif ktype in [
SEND_SOCKET_TCPV4, SEND_SOCKET_UDPV4, RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4, ACCEPT_SOCKET_TCPV4, CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4, RECONNECT_SOCKET_TCPV4
]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._aggregate(ktype)
# context switch events
elif ktype == CONTEXT_SWITCH:
self.context_switch_registry.next_cswitch(cpuid, ts, kparams)
self._aggregate(ktype)
if self._filament:
if ktype not in [
ENUM_PROCESS, ENUM_THREAD, ENUM_IMAGE, REG_CREATE_KCB,
REG_DELETE_KCB
]:
ok = self.output_kevents[ktype] if ktype in self.output_kevents \
else False
if self.kevent.name and ok:
thread = self.kevent.thread
kevent = {
'params': self.kevent.params,
'name': self.kevent.name,
'pid': self.kevent.pid,
'tid': self.kevent.tid,
'timestamp': self.kevent.ts,
'cpuid': self.kevent.cpuid,
'category': self.kevent.category
}
if thread:
kevent.update({
'thread': {
'name': thread.name,
'exe': thread.exe,
'comm': thread.comm,
'pid': thread.pid,
'ppid': thread.ppid
}
})
self._filament.on_next_kevent(kevent)
def _aggregate(self, ktype):
"""Aggregates the kernel event to the output sink.
Parameters
----------
ktype: tuple
Identifier of the kernel event
"""
if not self._filament:
if ktype in self.output_kevents:
if self.output_kevents[ktype]:
self.kevent.inc_kid()
self.output_aggregator.aggregate(self.kevent)
elif self.filters_count == 0:
self.kevent.inc_kid()
self.output_aggregator.aggregate(self.kevent)
class configManager(Singleton):
def __init__(self):
super(configManager,self).__init__()
self._map = {}
self.log = Logger(u"configManager")
def counts(self):
return len(self._map)
def addMap(self,trigger,actions):
"""
添加一个trigger到Action的映射。如果存在，则替换原来的映射。
trigger 为trigger plugin 的ID
actions为 action plugin 的ID 列表
"""
if trigger is None or actions is None or type(actions) != type([]):
self.log.error(u"[addMap] invalid param")
return False
self._map[trigger] = actions
return True
def delMap(self,trigger):
"""
删除一个映射。trigger为trigger plugin的ID
"""
if trigger in self._map:
del self._map[trigger]
def map(self,trigger):
"""
获取trigger对应的action列表,如不存在，则返回None
"""
if trigger in self._map:
return self._map[trigger]
return None
def reverseMap(self,action):
"""
返回某个action所对应的trigger.
action是需要反映射的plugin的ID列表
返回每个Plugin所对应的trigger的ID的列表
"""
if type(action) != type([]):
self.log.error(u"reverseMap:invalid param type")
return None
keys = self._map.keys()
triggerlist = []
for i in action:
for k in keys:
if i in self._map[k]:
triggerlist.append(k)
return triggerlist
def all(self):
"""
返回一个包含所有映射的元组.
"""
list = []
for key in self._map:
list.append((key,self._map[key]))
return list
sChar = Character.getInstance()
try:
activeChar = self.charEditor.entityEditor.getActiveEntity()
list = sChar.apiCharList(activeChar.ID,
self.inputID.GetLineText(0),
self.inputKey.GetLineText(0))
except AuthenticationError, e:
msg = "Authentication failure. Please check keyID and vCode combination."
pyfalog.info(msg)
self.stStatus.SetLabel(msg)
except TimeoutError, e:
msg = "Request timed out. Please check network connectivity and/or proxy settings."
pyfalog.info(msg)
self.stStatus.SetLabel(msg)
except Exception, e:
pyfalog.error(e)
self.stStatus.SetLabel("Error:\n%s" % e.message)
else:
self.charChoice.Clear()
for charName in list:
self.charChoice.Append(charName)
self.btnFetchSkills.Enable(True)
self.charChoice.Enable(True)
self.Layout()
self.charChoice.SetSelection(0)
def fetchSkills(self, event):
charName = self.charChoice.GetString(self.charChoice.GetSelection())
class RollBot:
CONFIG_LOCATION = "./config.json"
def __init__(self):
self.command_list = {}
self.logger = Logger('RollBot', level=2)
self.logger.info("RollBot started.")
self.last_ping = None
self.registered = False
with open(self.CONFIG_LOCATION) as f:
self.config = json.load(f)
self.nick = self.config['botnick']
self.owner = self.config['owner']['nick']
self.channels = set([x.lower() for x in self.config['channel']])
self.command_prefix = self.config['prefix']
self.command_list = {x: getattr(self, x) for x in commands}
print("Added {} commands: {}".format(len(self.command_list), ", ".join(self.command_list.keys())))
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket_file = self.socket.makefile(encoding="utf-8", errors="ignore")
self.warn_interval = 5 # seconds
self.last_warn = -self.warn_interval # To allow using the warn command instantly.
def send_message(self, channel, message):
message_template = "PRIVMSG {} :{}"
self.send_raw(message_template.format(channel, message))
def send_ping(self, ping_message):
message_template = "PONG : {}"
self.send_raw(message_template.format(ping_message))
self.update_ping_time()
def join_channel(self, channel):
if channel:
message_template = "JOIN {}"
self.send_raw(message_template.format(channel))
def leave_channel(self, channel):
if channel in self.channels:
message_template = "PART {}"
self.send_raw(message_template.format(channel))
self.channels.remove(channel)
def connect(self):
server_information = (self.config['server'], self.config['port'])
self.socket.connect(server_information)
self.send_raw("PASS " + self.config['password'])
self.send_raw("USER {} {} {} :{}".format(self.nick, self.nick, self.nick, "rollbot"))
self.send_raw("NICK " + self.nick)
self.run_loop()
def get_message_from_server(self):
return self.socket_file.readline()
def run_loop(self):
message_regex = r"^(?:[:](?P<prefix>\S+) )" \
r"?(?P<type>\S+)" \
r"(?: (?!:)(?P<destination>.+?))" \
r"?(?: [:](?P<message>.+))?$" # Extracts all appropriate groups from a raw IRC message
compiled_message = re.compile(message_regex)
print(compiled_message)
while True:
try:
message = self.get_message_from_server()
self.logger.debug("Received server message: {}", message)
parsed_message = compiled_message.finditer(message)
message_dict = [m.groupdict() for m in parsed_message][0] # Extract all the named groups into a dict
source_nick = ""
hostmask = ""
ircmsg = message.strip('\n\r') # remove new lines
print(ircmsg.encode("ascii", errors="ignore"))
if message_dict['prefix'] is not None:
if "!" in message_dict['prefix']: # Is the prefix from a nickname?
hostmask = message_dict['prefix'].split("@")[1]
source_nick = message_dict['prefix'].split("!")[0]
if message_dict['type'] == "PING":
self.send_ping(message_dict['message'])
if message_dict['type'] == "PRIVMSG":
self.handle_message(hostmask, source_nick, message_dict['destination'], message_dict['message'])
# if source_nick not in mods:
# mods[source_nick] = {"date":str(arrow.utcnow()), "message":message_dict['message'], "channel":message_dict['destination']}
if source_nick != "TagChatBot":
mods[source_nick] = {"date":str(arrow.utcnow()), "message":message_dict['message'], "channel":message_dict['destination']}
while tell_message.contains(Query().target.test(lambda s: s.lower() == source_nick.lower())) and message_dict['destination'] == "#TagProMods":
name = tell_message.get(Query().target.test(lambda s: s.lower() == source_nick.lower()))
date_remove = name['date']
self.send_message(source_nick,"{}, {} left a message: \"{}\"".format(source_nick, name['source'], name['message']))
tell_message.remove(Query().date.test(lambda s: s.lower() == date_remove.lower()))
time.sleep(.1)
if message_dict['type'] == "001": # Registration confirmation message
self.registered = True
self.logger.info("{} connected to server successfully.", self.nick)
for channel in self.config['channel']:
self.logger.info("Attempting to join {}", channel)
self.join_channel(channel)
except socket.timeout:
self.logger.error("Disconnected. Attempting to reconnect.")
self.socket.close()
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect()
def handle_message(self, hostmask, source, destination, message):
is_command = message.startswith(self.config['prefix'])
if is_command:
self.handle_command(hostmask, source, destination, message)
def handle_command(self, hostmask, source, destination, message):
try:
split_message = message[1:].split()
command_key = split_message[0].lower()
except IndexError:
self.logger.info("No Command")
return
arguments = split_message[1:]
reply_to = destination
try:
if destination == self.nick:
reply_to = source # If it's a private message, reply to the source. Otherwise it's a channel message and reply there.
if command_key in self.command_list:
self.logger.info("Received command '{}' from {}", command_key, source)
command = self.command_list[command_key]
return_message = command(hostmask, source, reply_to, *arguments)
if return_message is not None:
if isinstance(return_message, str): # Is it a string?
self.send_message(reply_to, return_message) # If so, just send it along.
else: # Otherwise it's a list or a tuple
for message in return_message: # So let's loop over them all
self.send_message(reply_to, message) # And send them.
else:
pass
# combined_command = self.command_prefix + command_key
# self.send_message(reply_to, "Sorry, {} isn't a recognized command.".format(combined_command))
except Exception as e:
self.send_message(reply_to, "Sorry, I encountered an error while running that command.")
print("Exception in command {}: {}".format(command_key, e))
def send_raw(self, message):
return self.socket.send((message + "\n").encode("utf-8"))
def update_ping_time(self):
self.last_ping = time.time()
# Commands
@command
def commands(self, hostmask, source, reply_to, *args):
return "Available commands: {}".format(", ".join(sorted(self.command_list.keys())))
@command
def netsplit(self, hostmask, source, reply_to, *args):
return "technically we all netsplit http://pastebin.com/mPanErhR"
@command
def mods(self, hostmask, source, reply_to, *args):
if reply_to != "#TPmods":
if source in ["WOLOWOLO", "justanotheruser", "MRCOW", "LEBRONxJAMES", "defense_bot"]:
return "can you not"
else:
return "Sorry! You must use this command in the channel #TPmods | Double click the channel to join."
else:
if ' '.join(args) == "":
return "{} - Please recall !mods with a reason to notify a moderator.".format(source)
else:
self.send_raw("NAMES #TPmods")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
try:
actualip = "{}".format(re.findall(r'\b(?:\d{1,3}[\.-]){3}\d{1,3}\b', hostmask)[0])
actualipp = actualip.replace("-", ".")
ippfinal = " ( http://tagpro-origin.koalabeast.com/moderate/ips/{} )".format(actualipp)
except IndexError:
ippfinal = ""
if ircmsg.find(' 353 {} '.format(self.nick)) != -1:
namelist = ircmsg.split(":")[2]
modlist = " ".join(x[1:] for x in namelist.split() if x.startswith('+'))
#oplist = " ".join(x[1:] for x in namelist.split() if x.startswith('@'))
oplist = ""
modmsg = "- " + ' '.join(args)
if ' '.join(args) == "":
modmsg = ""
if modlist == "" and oplist == "":
self.send_raw(
"PRIVMSG #TPmods :Sorry {}, all mods are currently AFK. You can stick around or leave your request for one to find later.".format(
source))
else:
self.send_raw("PRIVMSG #TagProMods :Mods - {} {}".format(modlist, oplist))
self.send_raw(
"PRIVMSG #TPmods :{} - the mods have received your request. Please stay patient while waiting. Make sure to state the user/issue to speed up the request process.".format(
source))
self.send_raw(
"PRIVMSG #TagProMods :Mod request from {}{} in {} {}".format(source, ippfinal, reply_to,
modmsg))
@command
def check(self, hostmask, source, reply_to, *args):
ipaddress = ' '.join(args)
if re.match('^[-0-9.]*$', ipaddress):
ipaddress = ipaddress.replace("-", ".")
else:
return "Sorry, that's not an IP address!"
with open('email.txt') as e:
email = e.read().strip()
page = requests.get('http://check.getipintel.net/check.php?ip={}&contact={}'.format(ipaddress, email))
return "{}: chances of naughty IP = {}%".format(source, int(float(re.findall("(\d+(?:.\d+)?)", page.text)[0]) * 100))
@command
def seen(self, hostmask, source, reply_to, *args):
name = ' '.join(args)
if name not in mods:
return "Sorry, haven't seen that weenie"
if name in mods:
timeseen = arrow.get(mods[name]["date"])
formattime = timeseen.format(('YYYY-MM-DD HH:mm:ss ZZ'))
humantime = timeseen.humanize()
return "{} was seen {} ({}) saying {}".format(name,humantime, formattime, mods[name]["message"])
@command
def tell(self, hostmask, source, reply_to, *args):
target = args[0]
message = ' '.join(args[1:])
#mods[source_nick] = {"date":str(arrow.utcnow()), "message":message_dict['message'], "channel":message_dict['destination']}
# tell_message[target] = {"source":source, "message":message, "date":str(arrow.utcnow())}
if tell_message.search(Query().target.test(lambda s: s.lower() == target.lower)):
nick = tell_message.get(Query().target.test(lambda s: s.lower() == target.lower()))
pass
if True:
nick = tell_message.get(Query().target.test(lambda s: s.lower() == target.lower()))
if tell_message.count(Query().target.test(lambda s: s.lower() == target.lower()) & Query().source.test(lambda s: s.lower() == source.lower())) <= 6:
tell_message.insert({"target":target,"message":message, 'date':str(arrow.utcnow()), 'source':source})
return "Ok! I'll pass that on when they become active"
else:
return "U message that person like wayyy too much"
@command
def optin(self, hostmask, source, reply_to, *args):
if reply_to not in ["#TagProMods","#tagprochat"]:
return "Sorry! This command is not authorized here."
if reply_to == "#TagProMods":
self.send_raw("NAMES #TPmods")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
duty = "duty"
if ircmsg.find('+{}'.format(source)) != -1:
print(ircmsg.find('+{}'.format(source)))
return "You are already on {}, {}.".format(duty, source)
elif ircmsg.find('{}'.format(source)) != -1:
print(ircmsg.find('{}'.format(source)))
self.send_raw("PRIVMSG Chanserv :voice #TPmods {}".format(source))
return "You are now on {}, {}.".format(duty, source)
else:
return "You are not in #TPmods, {}!".format(source)
if reply_to == "#tagprochat":
self.send_raw("NAMES #tagprochat")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
duty = "duty"
if ircmsg.find('+{}'.format(source)) != -1:
return "You are already on {}, {}.".format(duty, source)
elif ircmsg.find('{}'.format(source)) != -1:
self.send_raw("PRIVMSG Chanserv :voice #tagprochat {}".format(source))
return "You are now on {}, {}.".format(duty, source)
else:
return "You are not in #tagprochat, {}!".format(source)
@command
def optout(self, hostmask, source, reply_to, *args):
if reply_to not in ["#TagProMods","#tagprochat"]:
return "Sorry! This command is not authorized here."
if reply_to == "#TagProMods":
self.send_raw("NAMES #TPmods")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
duty = "duty"
if source == "Hootie":
duty = "dootie"
if ircmsg.find('+{}'.format(source)) != -1:
self.send_raw("PRIVMSG Chanserv :devoice #TPmods {}".format(source))
if source.lower() in ['cignul9']:
return "Eat my ass {}".format(source)
else: return "You are now off {}, {}.".format(duty, source)
elif ircmsg.find('{}'.format(source)) != -1:
return "You are already off {}, {}.".format(duty, source)
else:
return "You are not in #TPmods, {}!".format(source)
if reply_to == "#tagprochat":
self.send_raw("NAMES #tagprochat")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
duty = "duty"
if source == "Hootie":
duty = "dootie"
if ircmsg.find('+{}'.format(source)) != -1:
self.send_raw("PRIVMSG Chanserv :devoice #tagprochat {}".format(source))
if source.lower() in ['cignul9']:
return "{} is a dink".format(source)
else: return "You are now off {}, {}.".format(duty, source)
elif ircmsg.find('{}'.format(source)) != -1:
return "You are already off {}, {}.".format(duty, source)
else:
return "You are not in #tagprochat, {}!".format(source)
@command
def op(self, hostmask, source, reply_to, *args):
if reply_to != "#TagProMods":
return "Sorry! This command is not authorized here."
else:
self.send_raw("NAMES #TPmods")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
if ircmsg.find('@{}'.format(source)) != -1:
return "You are already an operator, {}.".format(source)
elif ircmsg.find('{}'.format(source)) != -1:
self.send_raw("PRIVMSG Chanserv :op #TPmods {}".format(source))
return "You are now an operator, {}.".format(source)
else:
return "You are not in #TPmods, {}!".format(source)
@command
def deop(self, hostmask, source, reply_to, *args):
if reply_to != "#TagProMods":
return "Sorry! This command is not authorized here."
else:
self.send_raw("NAMES #TPmods")
message = self.get_message_from_server()
ircmsg = message.strip('\n\r')
if ircmsg.find('@{}'.format(source)) != -1:
self.send_raw("PRIVMSG Chanserv :deop #TPmods {}".format(source))
return "You are no longer an operator, {}.".format(source)
elif ircmsg.find('{}'.format(source)) != -1:
return "You are not an operator, {}.".format(source)
else:
return "You are not in #TPmods, {}!".format(source)
@command
def ticket(self, hostmask, source, reply_to, tickett=None, *args):
if tickett is None:
return "http://support.koalabeast.com/#/appeal"
else:
return "http://support.koalabeast.com/#/appeal/{}".format(tickett)
@command
def ip(self, hostmask, source, reply_to, *args):
ipaddress = ' '.join(args)
if re.match('^[-0-9.]*$', ipaddress):
return ipaddress.replace("-", ".")
else:
return "Sorry, that's not an IP address!"
@command
def warn(self, hostmask, source, reply_to, *args):
if reply_to != "#TagProMods":
return "Sorry! This command is not authorized here."
if time.time() - self.last_warn < self.warn_interval:
return "You're using that too much."
self.send_raw("NOTICE #TPmods :Please take off-topic discussion to #tagpro")
self.last_warn = time.time()
@owner_command
def quit(self, hostmask, source, reply_to, *args):
self.logger.warn("Shutting down by request of {}", source)
self.send_raw("QUIT :{}'s out!".format(self.nick))
self.socket.shutdown(1)
self.socket.close()
sys.exit()
@owner_command
def join(self, hostmask, source, reply_to, channel=None, *args):
if channel is None:
return "Please specify a channel you wish me to join."
else:
self.logger.info("Joining {} by request of {}".format(channel, source))
self.join_channel(channel)
@owner_command
def part(self, hostmask, source, reply_to, channel=None, *args):
if reply_to == source and channel is None: # If this was a private message, we have no channel to leave.
return "Sorry, you must run this command in a channel or provide a channel as an argument."
elif channel is not None:
if channel in self.channels:
self.leave_channel(channel)
return "Left channel {}!".format(channel)
else:
return "I don't believe I'm in that channel!"
else: # It was a channel message, so let's leave.
self.leave_channel(reply_to)
@owner_command
def say(self, hostmask, source, reply_to, channel=None, *args):
if reply_to != source:
return "{} {}".format(channel, ' '.join(args))
elif channel is not None:
if channel in self.channels:
self.send_message(channel, ' '.join(args))
else:
return "Whoops! I'm not in the channel {}".format(channel)
else:
return "The format is: |say <channel> <message>"
class Genome:
def __init__(self, genome, assembly_summary=None):
"""
:param genome: Path to genome
:returns: Path to genome and name of the genome
:rtype:
"""
self.path = os.path.abspath(genome)
self.species_dir, self.fasta = os.path.split(self.path)
self.name = os.path.splitext(self.fasta)[0]
self.log = Logger(self.name)
self.qc_dir = os.path.join(self.species_dir, "qc")
self.msh = os.path.join(self.qc_dir, self.name + ".msh")
self.stats_path = os.path.join(self.qc_dir, self.name + '.csv')
if os.path.isfile(self.stats_path):
self.stats = pd.read_csv(self.stats_path, index_col=0)
self.assembly_summary = assembly_summary
self.metadata = defaultdict(lambda: 'missing')
self.xml = defaultdict(lambda: 'missing')
try:
self.accession_id = re.match('GCA_.*\.\d', self.name).group()
self.metadata["accession"] = self.accession_id
except AttributeError:
# Raise custom exception
self.accession_id = "missing"
self.log.error("Invalid accession ID")
self.log.exception()
if isinstance(self.assembly_summary, pd.DataFrame):
try:
biosample = assembly_summary.loc[self.accession_id].biosample
self.metadata["biosample_id"] = biosample
except (AttributeError, KeyError):
self.log.info("Unable to get biosample ID")
self.log.info("Instantiated")
def get_contigs(self):
"""
Return a list of of Bio.Seq.Seq objects for fasta and calculate
the total the number of contigs.
"""
try:
self.contigs = [seq.seq for seq in SeqIO.parse(self.path, "fasta")]
self.count_contigs = len(self.contigs)
self.log.info("Contigs: {}".format(self.count_contigs))
except UnicodeDecodeError:
self.log.exception()
def get_assembly_size(self):
"""Calculate the sum of all contig lengths"""
# TODO: map or reduce might be more elegant here
self.assembly_size = sum((len(str(seq)) for seq in self.contigs))
self.log.info("Assembly Size: {}".format(self.assembly_size))
def get_unknowns(self):
"""Count the number of unknown bases, i.e. not [ATCG]"""
# TODO: Would it be useful to allow the user to define p?
p = re.compile("[^ATCG]")
self.unknowns = sum(
(len(re.findall(p, str(seq))) for seq in self.contigs))
self.log.info("Unknowns: {}".format(self.unknowns))
def get_distance(self, dmx_mean):
self.distance = dmx_mean.loc[self.name]
self.log.info("Distance: {}".format(self.distance))
def sketch(self):
cmd = "mash sketch '{}' -o '{}'".format(self.path, self.msh)
if os.path.isfile(self.msh):
self.log.info("Sketch file already exists")
else:
subprocess.Popen(cmd, shell="True",
stderr=subprocess.DEVNULL).wait()
self.log.info("Sketch file created")
def get_stats(self, dmx_mean):
if not os.path.isfile(self.stats_path):
self.get_contigs()
self.get_assembly_size()
self.get_unknowns()
self.get_distance(dmx_mean)
data = {
"contigs": self.count_contigs,
"assembly_size": self.assembly_size,
"unknowns": self.unknowns,
"distance": self.distance
}
self.stats = pd.DataFrame(data, index=[self.name])
self.stats.to_csv(self.stats_path)
self.log.info("Generated stats and wrote to disk")
one_minute = 60000
# Retry 3 times over a period of 3 minutes max,
# waiting five seconds in between retries
@retry(stop_max_attempt_number=3, stop_max_delay=10000, wait_fixed=100)
def efetch(self, db):
"""
Use NCBI's efetch tools to get xml for genome's biosample id or SRA id
"""
if db == "biosample":
db_id = db + "_id"
elif db == "sra":
db_id = db + "_id"
cmd = ("esearch -db {} -query {} | "
"efetch -format docsum".format(db, self.metadata[db_id]))
# Make efetch timeout and retry after 30 seconds
time_limit = 30
if self.metadata[db_id] is not 'missing':
try:
p = subprocess.run(cmd,
shell="True",
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
timeout=time_limit)
xml = p.stdout
self.xml[db] = xml
self.log.info("{} XML downloaded".format(db))
except subprocess.TimeoutExpired:
self.log.error("Retrying efetch after timeout")
raise subprocess.TimeoutExpired(cmd, time_limit)
except Exception:
self.log.error(db)
self.log.exception()
def parse_biosample(self):
"""
Get what we need to get out of the xml returned by efetch("biosample")
Including the SRA ID and fields of interest as defined in
Metadata.biosample_fields
"""
try:
tree = ET.fromstring(self.xml["biosample"])
sra = tree.find(
'DocumentSummary/SampleData/BioSample/Ids/Id/[@db="SRA"]')
self.log.info("Parsed biosample XML")
try:
self.metadata["sra_id"] = sra.text
except AttributeError:
self.metadata["sra_id"] = "missing"
for name in Metadata.Metadata.biosample_fields:
xp = ('DocumentSummary/SampleData/BioSample/Attributes/'
'Attribute/[@harmonized_name="{}"]'.format(name))
attrib = tree.find(xp)
try:
self.metadata[name] = attrib.text
except AttributeError:
self.metadata[name] = "missing"
except ParseError:
self.log.error("Parse error for biosample XML")
def parse_sra(self):
try:
tree = ET.fromstring(self.xml["sra"])
elements = tree.iterfind("DocumentSummary/Runs/Run/[@acc]")
self.log.info("Parsed SRA XML")
srr_accessions = []
for el in elements:
items = el.items()
acc = [i[1] for i in items if i[0] == 'acc']
acc = acc[0]
srr_accessions.append(acc)
self.metadata["srr_accessions"] = ','.join(srr_accessions)
except ParseError:
self.log.error("Parse error for SRA XML")
def get_metadata(self):
self.efetch("biosample")
self.parse_biosample()
self.efetch("sra")
self.parse_sra()
import jsondate as json
import requests
from logbook import Logger, FileHandler, StderrHandler
BASE_URL = 'http://fmr-api-507.herokuapp.com/api'
log = Logger('Logbook')
log_filehandler = FileHandler('application.log')
log_stderrhandler = StderrHandler()
# Participants
participants = [
{'scenario': '0'},
{'scenario': '1'},
{'scenario': '0'}
]
participants_ids = []
for p in participants:
r = requests.post('{}/participant'.format(BASE_URL),
data=json.dumps(p),
headers={'content-type': 'application/json'})
if r.status_code == 201:
log.info('Participant with scenario {} with status code {}'.format(p['scenario'], r.status_code))
participants_ids.append(json.loads(r.text)['id'])
else:
log.error('Participant with scenario {} with status code {}'.format(p['scenario'], r.status_code))
print r.text
