class CustomLogger(object):
def __init__(
self,
log_level=LogLevel.INFO,
format_str='[{record.time:%Y-%m-%d %H:%M:%S}] - {record.channel} - {record.level_name} '
'- {record.message}'):
self.logger = Logger('WindAdapter')
set_datetime_format('local')
StreamHandler(sys.stdout, format_string=format_str).push_application()
FileHandler('WindAdapter.log', bubble=True,
format_string=format_str).push_application()
self.set_level(log_level)
def set_level(self, log_level):
if log_level.lower() == LogLevel.INFO:
self.logger.level = logbook.INFO
elif log_level.lower() == LogLevel.WARNING:
self.logger.level = logbook.WARNING
elif log_level.lower() == LogLevel.CRITICAL:
self.logger.level = logbook.CRITICAL
elif log_level.lower() == LogLevel.NOTSET:
self.logger.level = logbook.NOTSET
def info(self, msg):
self.logger.info(msg)
def warning(self, msg):
self.logger.warning(msg)
def critical(self, msg):
self.logger.critical(msg)
def rpc_server(socket, protocol, dispatcher):
log = Logger('rpc_server')
log.debug('starting up...')
while True:
try:
message = socket.recv_multipart()
except Exception as e:
log.warning('Failed to receive message from client, ignoring...')
log.exception(e)
continue
log.debug('Received message %s from %r' % (message[-1], message[0]))
# assuming protocol is threadsafe and dispatcher is theadsafe, as long
# as its immutable
def handle_client(message):
try:
request = protocol.parse_request(message[-1])
except RPCError as e:
log.exception(e)
response = e.error_respond()
else:
response = dispatcher.dispatch(request)
log.debug('Response okay: %r' % response)
# send reply
message[-1] = response.serialize()
log.debug('Replying %s to %r' % (message[-1], message[0]))
socket.send_multipart(message)
gevent.spawn(handle_client, message)
def rpc_server(socket, protocol, dispatcher):
log = Logger('rpc_server')
log.debug('starting up...')
while True:
try:
message = socket.recv_multipart()
except Exception as e:
log.warning('Failed to receive message from client, ignoring...')
log.exception(e)
continue
log.debug('Received message %s from %r', message[-1], message[0])
# assuming protocol is threadsafe and dispatcher is theadsafe, as long
# as its immutable
def handle_client(message):
try:
request = protocol.parse_request(message[-1])
except RPCError as e:
log.exception(e)
response = e.error_respond()
else:
response = dispatcher.dispatch(request)
log.debug('Response okay: %r', response)
# send reply
message[-1] = response.serialize()
log.debug('Replying %s to %r', message[-1], message[0])
socket.send_multipart(message)
gevent.spawn(handle_client, message)
class ProjectManager(object):
""" Class for controlling project's settings.
A project is a collection of setting
for some data transformation.
Each project is yaml file with settings.
Project PID is a yaml file name.
"""
config_path = None
def __init__(self, settings_class, config_path=None):
""" Init projects_folder.
Manager loads settings from default location in PySatDNA root or from given config_path.
Then create work and projects folders if they were not found.
And set config values for settings class.
"""
self.manager_logger = Logger('Manager logger')
if config_path:
self.load_config(config_path)
else:
self.load_config(self.config_path)
self.projects_folder = self.config["projects_folder"]
self.work_folder = self.config["path_work_folder"]
self.settings_class = settings_class
self.settings_class.config = self.config
self.force_folder_creation = False
if self.projects_folder and not os.path.isdir(self.projects_folder):
os.makedirs(self.projects_folder)
if self.work_folder and not os.path.isdir(self.work_folder):
os.makedirs(self.work_folder)
def load_config(self, config_path):
''' Load OS-specific configs.'''
if config_path:
file_path = config_path
if not os.path.isfile(file_path):
message = "ERROR with open config file: %s" % file_path
self.manager_logger.error(message)
raise ProjectManagerException(message)
else:
self.os = platform.system()
if self.os == "Windows":
file_path = "../config.win.yaml"
elif self.os == "Darwin":
file_path = "../config.mac.yaml"
else:
file_path = os.path.expanduser("~/Dropbox/workspace/PySatDNA/config.yaml")
if not os.path.isfile(file_path):
file_path = os.path.expanduser("~/Dropbox/PySatDNA/config.dobi.yaml")
try:
with open(file_path) as fh:
self.config = yaml.load(fh)
except Exception, e:
self.manager_logger.error("ERROR with open config file: %s" % file_path)
self.manager_logger.warning("Loading default settings")
self.config = {
'path_work_folder': 'data',
'path_workspace_folder': '../..',
'projects_folder': 'projects',
}
class Plugin(object):
def __init__(self, site, name=None):
self.name = name or remove_suffix('plugin', self.__class__.__name__)
self.log = Logger(self.__class__.__name__.lower())
self.log.debug('{} initialized'.format(self.name))
self.base_dir = Path(inspect.getfile(self.__class__)).parent
# initialize templates
template_path = self.base_dir / 'templates'
if template_path.exists():
self.jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(str(template_path)),
extensions=['jinja2.ext.with_']
)
# load possible default configuration
self.register(site)
@property
def DEFAULTS_FILE(self):
return self.base_dir / 'defaults.cfg'
def register(self, site):
pass
def enable_app(self, app):
pass
def render_template(self, template_name, **kwargs):
if not hasattr(self, 'jinja_env'):
return RuntimeError('Plugin {} has no template path'.format(
self.__class__.__name__
))
tpl = self.jinja_env.get_template(template_name)
return tpl.render(**kwargs)
def output_template(self, template_name, dest, _mode=0o644, **kwargs):
if not dest.parent.exists():
self.log.warning('Path {} did not exist and was created'.format(
dest.parent,
))
dest.parent.mkdir(parents=True)
with new_file(dest, _mode) as out:
self.log.info('Writing {}'.format(dest.resolve()))
out.write(self.render_template(template_name, **kwargs))
class Root(resource.Resource):
isLeaf = True
def __init__(self, lock):
self._lock = lock
self.log = Logger('web')
resource.Resource.__init__(self)
@delayed
def render_GET(self, request):
try:
key = _get_key_from_path(request.path)
if key == 'info/keys':
request.write('%r\n' % (self._lock._keys,))
elif key == 'info/status':
for line in self._lock.get_status():
request.write('%s %s\n' % line)
elif key == 'info/log':
for line in self._lock._log:
request.write('%s\n' % line)
else:
value = yield self._lock.get_key(key)
request.write(value)
except KeyNotFound:
request.setResponseCode(NOT_FOUND)
returnValue('')
@proxy_to_master
@delayed
def render_POST(self, request):
try:
key = _get_key_from_path(request.path)
data = request.args.get('data', [''])[0]
self.log.info('Set key %s=%r' % (key, data))
yield self._lock.set_key(key, data)
except KeyAlreadyExists, e:
self.log.warning(e)
request.setResponseCode(CONFLICT)
except PaxosError, e:
self.log.warning(e)
request.setResponseCode(EXPECTATION_FAILED)
class NodesMetrics:
def __init__(self):
handler = TimedRotatingFileHandler('../logs/nodes_metrics.log')
handler.push_application()
self.logger = Logger(name='nodes metrics', level='info')
self.nodes_filename, self.nodes_file_backup_name, self.exclude_file, self.url, self.query_all_ip, self.query_current_ip = self.get_conf()
self.nodes = {}
self.ips = {}
self.nodes_list = []
self.ips_list = []
@staticmethod
def get_conf():
##从配置文件获取配置
cp = ConfigParser.ConfigParser()
with codecs.open(os.path.join(BASE_DIR, './config/config.ini'), 'r', encoding='utf-8') as f:
cp.readfp(f)
nodes_filename = cp.get('file_ds', 'file_sd_filename').strip()
nodes_file_backup_name = cp.get('file_ds', 'nodes_file_backup_name').strip()
exclude_file = cp.get('file_ds', 'agent_exclude_file').strip()
url = cp.get('prome', 'url').strip()
query_all_ip = cp.get('prome', 'query_all_ip').strip()
query_current_ip = cp.get('prome', 'query_current_ip').strip()
return os.path.join(BASE_DIR, nodes_filename), os.path.join(BASE_DIR, nodes_file_backup_name), os.path.join(BASE_DIR, exclude_file), url, query_all_ip, query_current_ip
#@staticmethod
def get_ips(self, res):
ips = []
if res.status_code == 200:
data = res.json()['data']['result']
for metric in data:
# print(metric)
ip = metric['metric']['instance_ip']
ips.append(ip)
# print(ip)
#print('All ips in prome: %s' % ips)
#self.logger.info('All ips in prome: %s' % ips)
return ips
# res = requests.get(url + query_current_ip)
#
# current_ips = get_ips(res)
#@staticmethod
def get_hosts(self, file_name):
all_ips = []
with open(file_name, 'r') as f:
ips = f.read()
ip_list = ips.split('\n')
if file_name != self.exclude_file:
for ip in ip_list:
if ip.strip().endswith(':9100",'):
nodes_str = ip.split('"')[1]
ip_str = nodes_str.split(':')[0]
#print(nodes_str)
all_ips.append(ip_str)
else:
all_ips = ip_list
print('All nodes from file %s is: %s' % (file_name, all_ips))
self.logger.info('All nodes from file %s is: %s' % (file_name, all_ips))
return all_ips
def get_nodes_discovery(self):
new_ips = self.get_hosts(self.nodes_filename)
nodes_hosts = list(set(new_ips))
current_nodes = len(nodes_hosts)
print(current_nodes)
if current_nodes:
self.logger.info('************************************')
self.logger.info(
'There are %s nodes after nodes discovery finished: %s' % \
(current_nodes, nodes_hosts))
self.logger.info('************************************')
else:
self.logger.info('************************************')
self.logger.error(
'There are 0 nodes founded by nodes discovery: %s')
self.logger.info('************************************')
all_nodes = ' '.join(nodes_hosts)
return current_nodes, all_nodes
def get_stop_nodes(self):
new_ips = self.get_hosts(self.nodes_filename)
print(len(new_ips))
res = requests.get(url + query_all_ip)
all_ips = self.get_ips(res)
self.logger.info(all_ips)
print('All nodes are:')
print(all_ips)
old_ips = self.get_hosts(self.exclude_file)
failed_hosts = list(set(new_ips) - set(all_ips) - set(old_ips))
stop_nodes = len(failed_hosts)
#failed_hosts = list(set(new_ips) - set(all_ips))
self.logger.info('************************************')
self.logger.warning(
'There are %s nodes stopped now' % stop_nodes)
self.logger.info('************************************')
return stop_nodes, failed_hosts
if hasattr(sys, 'frozen'):
pyfalog.info("Running in a frozen state.")
else:
pyfalog.info("Running in a thawed state.")
if not hasattr(sys, 'frozen') and wxversion:
try:
if options.force28 is True:
pyfalog.info("Selecting wx version: 2.8. (Forced)")
wxversion.select('2.8')
else:
pyfalog.info("Selecting wx versions: 3.0, 2.8")
wxversion.select(['3.0', '2.8'])
except:
pyfalog.warning(
"Unable to select wx version. Attempting to import wx without specifying the version."
)
else:
if not wxversion:
pyfalog.warning(
"wxVersion not found. Attempting to import wx without specifying the version."
)
try:
# noinspection PyPackageRequirements
import wx
except:
exit_message = "Cannot import wxPython. You can download wxPython (2.8+) from http://www.wxpython.org/"
raise PreCheckException(exit_message)
pyfalog.info("wxPython version: {0}.", str(wx.VERSION_STRING))
class Worker(object):
redis_worker_namespace_prefix = 'rq:worker:'
redis_workers_keys = 'rq:workers'
@classmethod
def all(cls):
"""Returns an iterable of all Workers.
"""
conn = get_current_connection()
reported_working = conn.smembers(cls.redis_workers_keys)
return compact(map(cls.find_by_key, reported_working))
@classmethod
def find_by_key(cls, worker_key):
"""Returns a Worker instance, based on the naming conventions for
naming the internal Redis keys. Can be used to reverse-lookup Workers
by their Redis keys.
"""
prefix = cls.redis_worker_namespace_prefix
name = worker_key[len(prefix):]
if not worker_key.startswith(prefix):
raise ValueError('Not a valid RQ worker key: %s' % (worker_key, ))
conn = get_current_connection()
if not conn.exists(worker_key):
return None
name = worker_key[len(prefix):]
worker = cls([], name)
queues = conn.hget(worker.key, 'queues')
worker._state = conn.hget(worker.key, 'state') or '?'
if queues:
worker.queues = map(Queue, queues.split(','))
return worker
def __init__(self, queues, name=None, rv_ttl=500, connection=None): # noqa
if connection is None:
connection = get_current_connection()
self.connection = connection
if isinstance(queues, Queue):
queues = [queues]
self._name = name
self.queues = queues
self.validate_queues()
self.rv_ttl = rv_ttl
self._state = 'starting'
self._is_horse = False
self._horse_pid = 0
self._stopped = False
self.log = Logger('worker')
self.failed_queue = get_failed_queue(connection=self.connection)
def validate_queues(self): # noqa
"""Sanity check for the given queues."""
if not iterable(self.queues):
raise ValueError('Argument queues not iterable.')
for queue in self.queues:
if not isinstance(queue, Queue):
raise NoQueueError('Give each worker at least one Queue.')
def queue_names(self):
"""Returns the queue names of this worker's queues."""
return map(lambda q: q.name, self.queues)
def queue_keys(self):
"""Returns the Redis keys representing this worker's queues."""
return map(lambda q: q.key, self.queues)
@property # noqa
def name(self):
"""Returns the name of the worker, under which it is registered to the
monitoring system.
By default, the name of the worker is constructed from the current
(short) host name and the current PID.
"""
if self._name is None:
hostname = socket.gethostname()
shortname, _, _ = hostname.partition('.')
self._name = '%s.%s' % (shortname, self.pid)
return self._name
@property
def key(self):
"""Returns the worker's Redis hash key."""
return self.redis_worker_namespace_prefix + self.name
@property
def pid(self):
"""The current process ID."""
return os.getpid()
@property
def horse_pid(self):
"""The horse's process ID. Only available in the worker. Will return
0 in the horse part of the fork.
"""
return self._horse_pid
@property
def is_horse(self):
"""Returns whether or not this is the worker or the work horse."""
return self._is_horse
def procline(self, message):
"""Changes the current procname for the process.
This can be used to make `ps -ef` output more readable.
"""
procname.setprocname('rq: %s' % (message, ))
def register_birth(self): # noqa
"""Registers its own birth."""
self.log.debug('Registering birth of worker %s' % (self.name, ))
if self.connection.exists(self.key) and \
not self.connection.hexists(self.key, 'death'):
raise ValueError('There exists an active worker named \'%s\' '
'already.' % (self.name, ))
key = self.key
now = time.time()
queues = ','.join(self.queue_names())
with self.connection.pipeline() as p:
p.delete(key)
p.hset(key, 'birth', now)
p.hset(key, 'queues', queues)
p.sadd(self.redis_workers_keys, key)
p.execute()
def register_death(self):
"""Registers its own death."""
self.log.debug('Registering death')
with self.connection.pipeline() as p:
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
p.srem(self.redis_workers_keys, self.key)
p.hset(self.key, 'death', time.time())
p.expire(self.key, 60)
p.execute()
def set_state(self, new_state):
self._state = new_state
self.connection.hset(self.key, 'state', new_state)
def get_state(self):
return self._state
state = property(get_state, set_state)
@property
def stopped(self):
return self._stopped
def _install_signal_handlers(self):
"""Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
def request_force_stop(signum, frame):
"""Terminates the application (cold shutdown).
"""
self.log.warning('Cold shut down.')
# Take down the horse with the worker
if self.horse_pid:
msg = 'Taking down horse %d with me.' % self.horse_pid
self.log.debug(msg)
try:
os.kill(self.horse_pid, signal.SIGKILL)
except OSError as e:
# ESRCH ("No such process") is fine with us
if e.errno != errno.ESRCH:
self.log.debug('Horse already down.')
raise
raise SystemExit()
def request_stop(signum, frame):
"""Stops the current worker loop but waits for child processes to
end gracefully (warm shutdown).
"""
self.log.debug('Got %s signal.' % signal_name(signum))
signal.signal(signal.SIGINT, request_force_stop)
signal.signal(signal.SIGTERM, request_force_stop)
if self.is_horse:
self.log.debug('Ignoring signal %s.' % signal_name(signum))
return
msg = 'Warm shut down. Press Ctrl+C again for a cold shutdown.'
self.log.warning(msg)
self._stopped = True
self.log.debug('Stopping after current horse is finished.')
signal.signal(signal.SIGINT, request_stop)
signal.signal(signal.SIGTERM, request_stop)
def work(self, burst=False): # noqa
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
self._install_signal_handlers()
did_perform_work = False
self.register_birth()
self.state = 'starting'
try:
while True:
if self.stopped:
self.log.info('Stopping on request.')
break
self.state = 'idle'
qnames = self.queue_names()
self.procline('Listening on %s' % ','.join(qnames))
self.log.info('')
self.log.info('*** Listening on %s...' % \
green(', '.join(qnames)))
wait_for_job = not burst
try:
result = Queue.dequeue_any(self.queues, wait_for_job, \
connection=self.connection)
if result is None:
break
except UnpickleError as e:
msg = '*** Ignoring unpickleable data on %s.' % \
green(e.queue.name)
self.log.warning(msg)
self.log.debug('Data follows:')
self.log.debug(e.raw_data)
self.log.debug('End of unreadable data.')
self.failed_queue.push_job_id(e.job_id)
continue
job, queue = result
self.log.info(
'%s: %s (%s)' %
(green(queue.name), blue(job.description), job.id))
self.state = 'busy'
self.fork_and_perform_job(job)
did_perform_work = True
finally:
if not self.is_horse:
self.register_death()
return did_perform_work
def fork_and_perform_job(self, job):
"""Spawns a work horse to perform the actual work and passes it a job.
The worker will wait for the work horse and make sure it executes
within the given timeout bounds, or will end the work horse with
SIGALRM.
"""
child_pid = os.fork()
if child_pid == 0:
self.main_work_horse(job)
else:
self._horse_pid = child_pid
self.procline('Forked %d at %d' % (child_pid, time.time()))
while True:
try:
os.waitpid(child_pid, 0)
break
except OSError as e:
# In case we encountered an OSError due to EINTR (which is
# caused by a SIGINT or SIGTERM signal during
# os.waitpid()), we simply ignore it and enter the next
# iteration of the loop, waiting for the child to end. In
# any other case, this is some other unexpected OS error,
# which we don't want to catch, so we re-raise those ones.
if e.errno != errno.EINTR:
raise
def main_work_horse(self, job):
"""This is the entry point of the newly spawned work horse."""
# After fork()'ing, always assure we are generating random sequences
# that are different from the worker.
random.seed()
self._is_horse = True
self.log = Logger('horse')
success = self.perform_job(job)
# os._exit() is the way to exit from childs after a fork(), in
# constrast to the regular sys.exit()
os._exit(int(not success))
def perform_job(self, job):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.procline('Processing %s from %s since %s' %
(job.func_name, job.origin, time.time()))
try:
with death_pentalty_after(job.timeout or 180):
rv = job.perform()
except Exception as e:
fq = self.failed_queue
self.log.exception(red(str(e)))
self.log.warning('Moving job to %s queue.' % fq.name)
fq.quarantine(job, exc_info=traceback.format_exc())
return False
if rv is None:
self.log.info('Job OK')
else:
self.log.info('Job OK, result = %s' % (yellow(unicode(rv)), ))
if rv is not None:
p = self.connection.pipeline()
p.hset(job.key, 'result', dumps(rv))
p.expire(job.key, self.rv_ttl)
p.execute()
else:
# Cleanup immediately
job.delete()
return True
class BTgymBaseData:
"""
Base BTgym data provider class.
Provides core data loading, sampling, splitting and converting functionality.
Do not use directly.
Enables Pipe::
CSV[source data]-->pandas[for efficient sampling]-->bt.feeds
"""
def __init__(
self,
filename=None,
parsing_params=None,
sampling_params=None,
name='base_data',
data_names=('default_asset',),
task=0,
frozen_time_split=None,
log_level=WARNING,
_config_stack=None,
**kwargs
):
"""
Args:
filename: Str or list of str, should be given either here or when calling read_csv(),
see `Notes`.
specific_params CSV to Pandas parsing
sep: ';'
header: 0
index_col: 0
parse_dates: True
names: ['open', 'high', 'low', 'close', 'volume']
specific_params Pandas to BT.feeds conversion
timeframe=1: 1 minute.
datetime: 0
open: 1
high: 2
low: 3
close: 4
volume: -1
openinterest: -1
specific_params Sampling
sample_class_ref: None - if not None, than sample() method will return instance of specified
class, which itself must be subclass of BaseBTgymDataset,
else returns instance of the base data class.
start_weekdays: [0, 1, 2, 3, ] - Only weekdays from the list will be used for sample start.
start_00: True - sample start time will be set to first record of the day
(usually 00:00).
sample_duration: {'days': 1, 'hours': 23, 'minutes': 55} - Maximum sample time duration
in days, hours, minutes
time_gap: {''days': 0, hours': 5, 'minutes': 0} - Data omittance threshold:
maximum no-data time gap allowed within sample in days, hours.
Thereby, if set to be < 1 day, samples containing weekends and holidays gaps
will be rejected.
test_period: {'days': 0, 'hours': 0, 'minutes': 0} - setting this param to non-zero
duration forces instance.data split to train / test subsets with test
subset duration equal to `test_period` with `time_gap` tolerance. Train data
always precedes test one:
[0_record<-train_data->split_point_record<-test_data->last_record].
sample_expanding: None, reserved for child classes.
Note:
- CSV file can contain duplicate records, checks will be performed and all duplicates will be removed;
- CSV file should be properly sorted by date_time in ascending order, no sorting checks performed.
- When supplying list of file_names, all files should be also listed ascending by their time period,
no correct sampling will be possible otherwise.
- Default parameters are source-specific and made to correctly parse 1 minute Forex generic ASCII
data files from www.HistData.com. Tune according to your data source.
"""
self.filename = filename
if parsing_params is None:
self.parsing_params = dict(
# Default parameters for source-specific CSV datafeed class,
# correctly parses 1 minute Forex generic ASCII
# data files from www.HistData.com:
# CSV to Pandas params.
sep=';',
header=0,
index_col=0,
parse_dates=True,
names=['open', 'high', 'low', 'close', 'volume'],
# Pandas to BT.feeds params:
timeframe=1, # 1 minute.
datetime=0,
open=1,
high=2,
low=3,
close=4,
volume=-1,
openinterest=-1,
)
else:
self.parsing_params = parsing_params
if sampling_params is None:
self.sampling_params = dict(
# Sampling params:
start_weekdays=[], # Only weekdays from the list will be used for episode start.
start_00=False, # Sample start time will be set to first record of the day (usually 00:00).
sample_duration=dict( # Maximum sample time duration in days, hours, minutes:
days=0,
hours=0,
minutes=0
),
time_gap=dict( # Maximum data time gap allowed within sample in days, hours. Thereby,
days=0, # if set to be < 1 day, samples containing weekends and holidays gaps will be rejected.
hours=0,
),
test_period=dict( # Time period to take test samples from, in days, hours, minutes:
days=0,
hours=0,
minutes=0
),
expanding=False,
)
else:
self.sampling_params = sampling_params
self.name = name
# String will be used as key name for bt_feed data-line:
self.task = task
self.log_level = log_level
self.data_names = data_names
self.data_name = self.data_names[0]
self.data = None # Will hold actual data as pandas dataframe
self.is_ready = False
self.global_timestamp = 0
self.start_timestamp = 0
self.final_timestamp = 0
self.data_stat = None # Dataset descriptive statistic as pandas dataframe
self.data_range_delta = None # Dataset total duration timedelta
self.max_time_gap = None
self.time_gap = None
self.max_sample_len_delta = None
self.sample_duration = None
self.sample_num_records = 0
self.start_weekdays = {0, 1, 2, 3, 4, 5, 6}
self.start_00 = False
self.expanding = False
self.sample_instance = None
self.test_range_delta = None
self.train_range_delta = None
self.test_num_records = 0
self.train_num_records = 0
self.total_num_records = 0
self.train_interval = [0, 0]
self.test_interval = [0, 0]
self.test_period = {'days': 0, 'hours': 0, 'minutes': 0}
self.train_period = {'days': 0, 'hours': 0, 'minutes': 0}
self._test_period_backshift_delta = datetime.timedelta(**{'days': 0, 'hours': 0, 'minutes': 0})
self.sample_num = 0
self.task = 0
self.metadata = {'sample_num': 0, 'type': None}
self.set_params(self.parsing_params)
self.set_params(self.sampling_params)
self._config_stack = copy.deepcopy(_config_stack)
try:
nested_config = self._config_stack.pop()
except (IndexError, AttributeError) as e:
# IF stack is empty, sample of this instance itself is not supposed to be sampled.
nested_config = dict(
class_ref=None,
kwargs=dict(
parsing_params=self.parsing_params,
sample_params=None,
name='data_stream',
task=self.task,
log_level=self.log_level,
_config_stack=None,
)
)
# Configure sample instance parameters:
self.nested_class_ref = nested_config['class_ref']
self.nested_params = nested_config['kwargs']
self.sample_name = '{}_w_{}_'.format(self.nested_params['name'], self.task)
self.nested_params['_config_stack'] = self._config_stack
# Logging:
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(self.name, self.task), level=self.log_level)
# Legacy parameter dictionary, left here for BTgym API_shell:
self.params = {}
self.params.update(self.parsing_params)
self.params.update(self.sampling_params)
if frozen_time_split is not None:
self.frozen_time_split = datetime.datetime(**frozen_time_split)
else:
self.frozen_time_split = None
self.frozen_split_timestamp = None
def set_params(self, params_dict):
"""
Batch attribute setter.
Args:
params_dict: dictionary of parameters to be set as instance attributes.
"""
for key, value in params_dict.items():
setattr(self, key, value)
def set_logger(self, level=None, task=None):
"""
Sets logbook logger.
Args:
level: logbook.level, int
task: task id, int
"""
if task is not None:
self.task = task
if level is not None:
self.log = Logger('{}_{}'.format(self.name, self.task), level=level)
def set_global_timestamp(self, timestamp):
if self.data is not None:
self.global_timestamp = self.data.index[0].timestamp()
def reset(self, data_filename=None, **kwargs):
"""
Gets instance ready.
Args:
data_filename: [opt] string or list of strings.
kwargs: not used.
"""
self._reset(data_filename=data_filename, **kwargs)
def _reset(self, data_filename=None, timestamp=None, **kwargs):
self.read_csv(data_filename)
# Add global timepoints:
self.start_timestamp = self.data.index[0].timestamp()
self.final_timestamp = self.data.index[-1].timestamp()
if self.frozen_time_split is not None:
frozen_index = self.data.index.get_loc(self.frozen_time_split, method='ffill')
self.frozen_split_timestamp = self.data.index[frozen_index].timestamp()
self.set_global_timestamp(self.frozen_split_timestamp)
else:
self.frozen_split_timestamp = None
self.set_global_timestamp(timestamp)
self.log.debug(
'time stamps start: {}, current: {} final: {}'.format(
self.start_timestamp,
self.global_timestamp,
self.final_timestamp
)
)
# Maximum data time gap allowed within sample as pydatetimedelta obj:
self.max_time_gap = datetime.timedelta(**self.time_gap)
# Max. gap number of records:
self.max_gap_num_records = int(self.max_time_gap.total_seconds() / (60 * self.timeframe))
# ... maximum episode time duration:
self.max_sample_len_delta = datetime.timedelta(**self.sample_duration)
# Maximum possible number of data records (rows) within episode:
self.sample_num_records = int(self.max_sample_len_delta.total_seconds() / (60 * self.timeframe))
self.backshift_num_records = round(self._test_period_backshift_delta.total_seconds() / (60 * self.timeframe))
# Train/test timedeltas:
if self.train_period is None or self.test_period == -1:
# No train data assumed, test only:
self.train_num_records = 0
self.test_num_records = self.data.shape[0] - self.backshift_num_records
break_point = self.backshift_num_records
self.train_interval = [0, 0]
self.test_interval = [self.backshift_num_records, self.data.shape[0]]
else:
# Train and maybe test data assumed:
if self.test_period is not None:
self.test_range_delta = datetime.timedelta(**self.test_period)
self.test_num_records = round(self.test_range_delta.total_seconds() / (60 * self.timeframe))
self.train_num_records = self.data.shape[0] - self.test_num_records
break_point = self.train_num_records
self.train_interval = [0, break_point]
self.test_interval = [break_point - self.backshift_num_records, self.data.shape[0]]
else:
self.test_num_records = 0
self.train_num_records = self.data.shape[0]
break_point = self.train_num_records
self.train_interval = [0, break_point]
self.test_interval = [0, 0]
if self.train_num_records > 0:
try:
assert self.train_num_records + self.max_gap_num_records >= self.sample_num_records
except AssertionError:
self.log.exception(
'Train subset should contain at least one sample, ' +
'got: train_set size: {} rows, sample_size: {} rows, tolerance: {} rows'.
format(self.train_num_records, self.sample_num_records, self.max_gap_num_records)
)
raise AssertionError
if self.test_num_records > 0:
try:
assert self.test_num_records + self.max_gap_num_records >= self.sample_num_records
except AssertionError:
self.log.exception(
'Test subset should contain at least one sample, ' +
'got: test_set size: {} rows, sample_size: {} rows, tolerance: {} rows'.
format(self.test_num_records, self.sample_num_records, self.max_gap_num_records)
)
raise AssertionError
self.sample_num = 0
self.is_ready = True
def read_csv(self, data_filename=None, force_reload=False):
"""
Populates instance by loading data: CSV file --> pandas dataframe.
Args:
data_filename: [opt] csv data filename as string or list of such strings.
force_reload: ignore loaded data.
"""
if self.data is not None and not force_reload:
data_range = pd.to_datetime(self.data.index)
self.total_num_records = self.data.shape[0]
self.data_range_delta = (data_range[-1] - data_range[0]).to_pytimedelta()
self.log.debug('data has been already loaded. Use `force_reload=True` to reload')
return
if data_filename:
self.filename = data_filename # override data source if one is given
if type(self.filename) == str:
self.filename = [self.filename]
dataframes = []
for filename in self.filename:
try:
assert filename and os.path.isfile(filename)
current_dataframe = pd.read_csv(
filename,
sep=self.sep,
header=self.header,
index_col=self.index_col,
parse_dates=self.parse_dates,
names=self.names,
)
# Check and remove duplicate datetime indexes:
duplicates = current_dataframe.index.duplicated(keep='first')
how_bad = duplicates.sum()
if how_bad > 0:
current_dataframe = current_dataframe[~duplicates]
self.log.warning('Found {} duplicated date_time records in <{}>.\
Removed all but first occurrences.'.format(how_bad, filename))
dataframes += [current_dataframe]
self.log.info('Loaded {} records from <{}>.'.format(dataframes[-1].shape[0], filename))
except:
msg = 'Data file <{}> not specified / not found / parser error.'.format(str(filename))
self.log.error(msg)
raise FileNotFoundError(msg)
self.data = pd.concat(dataframes)
data_range = pd.to_datetime(self.data.index)
self.total_num_records = self.data.shape[0]
self.data_range_delta = (data_range[-1] - data_range[0]).to_pytimedelta()
def describe(self):
"""
Returns summary dataset statistic as pandas dataframe:
- records count,
- data mean,
- data std dev,
- min value,
- 25% percentile,
- 50% percentile,
- 75% percentile,
- max value
for every data column.
"""
# Pretty straightforward, using standard pandas utility.
# The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again,
# thus avoiding passing big files to BT server:
flush_data = False
try:
assert not self.data.empty
pass
except (AssertionError, AttributeError) as e:
self.read_csv()
flush_data = True
self.data_stat = self.data.describe()
self.log.info('Data summary:\n{}'.format(self.data_stat.to_string()))
if flush_data:
self.data = None
self.log.info('Flushed data.')
return self.data_stat
def to_btfeed(self):
"""
Performs BTgymData-->bt.feed conversion.
Returns:
dict of type: {data_line_name: bt.datafeed instance}.
"""
def bt_timeframe(minutes):
timeframe = TimeFrame.Minutes
if minutes / 1440 == 1:
timeframe = TimeFrame.Days
return timeframe
try:
assert not self.data.empty
btfeed = btfeeds.PandasDirectData(
dataname=self.data,
timeframe=bt_timeframe(self.timeframe),
datetime=self.datetime,
open=self.open,
high=self.high,
low=self.low,
close=self.close,
volume=self.volume,
openinterest=self.openinterest
)
btfeed.numrecords = self.data.shape[0]
return {self.data_name: btfeed}
except (AssertionError, AttributeError) as e:
msg = 'Instance holds no data. Hint: forgot to call .read_csv()?'
self.log.error(msg)
raise AssertionError(msg)
def sample(self, **kwargs):
return self._sample(**kwargs)
def _sample(
self,
get_new=True,
sample_type=0,
b_alpha=1.0,
b_beta=1.0,
force_interval=False,
interval=None,
**kwargs
):
"""
Samples continuous subset of data.
Args:
get_new (bool): sample new (True) or reuse (False) last made sample;
sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets
respectively.
b_alpha (float): beta-distribution sampling alpha > 0, valid for train episodes.
b_beta (float): beta-distribution sampling beta > 0, valid for train episodes.
force_interval(bool): use exact sampling interval (should be given)
interval(iterable of int, len2): exact interval to sample from when force_interval=True
Returns:
if no sample_class_ref param been set:
BTgymDataset instance with number of records ~ max_episode_len,
where `~` tolerance is set by `time_gap` param;
else:
`sample_class_ref` instance with same as above number of records.
Note:
Train sample start position within interval is drawn from beta-distribution
with default parameters b_alpha=1, b_beta=1, i.e. uniform one.
Beta-distribution makes skewed sampling possible , e.g.
to give recent episodes higher probability of being sampled, e.g.: b_alpha=10, b_beta=0.8.
Test samples are always uniform one.
"""
try:
assert self.is_ready
except AssertionError:
msg = 'sampling attempt: data not ready. Hint: forgot to call data.reset()?'
self.log.error(msg)
raise RuntimeError(msg)
try:
assert sample_type in [0, 1]
except AssertionError:
msg = 'sampling attempt: expected sample type be in {}, got: {}'.format([0, 1], sample_type)
self.log.error(msg)
raise ValueError(msg)
if force_interval:
try:
assert interval is not None and len(list(interval)) == 2
except AssertionError:
msg = 'sampling attempt: got force_interval=True, expected interval=[a,b], got: <{}>'.format(interval)
self.log.error(msg)
raise ValueError(msg)
if self.sample_instance is None or get_new:
if sample_type == 0:
# Get beta_distributed sample in train interval:
if force_interval:
sample_interval = interval
else:
sample_interval = self.train_interval
self.sample_instance = self._sample_interval(
sample_interval,
force_interval=force_interval,
b_alpha=b_alpha,
b_beta=b_beta,
name='train_' + self.sample_name,
**kwargs
)
else:
# Get uniform sample in test interval:
if force_interval:
sample_interval = interval
else:
sample_interval = self.test_interval
self.sample_instance = self._sample_interval(
sample_interval,
force_interval=force_interval,
b_alpha=1,
b_beta=1,
name='test_' + self.sample_name,
**kwargs
)
self.sample_instance.metadata['type'] = sample_type # TODO: can move inside sample()
self.sample_instance.metadata['sample_num'] = self.sample_num
self.sample_instance.metadata['parent_sample_num'] = copy.deepcopy(self.metadata['sample_num'])
self.sample_instance.metadata['parent_sample_type'] = copy.deepcopy(self.metadata['type'])
self.sample_num += 1
else:
# Do nothing:
self.log.debug('Reusing sample, id: {}'.format(self.sample_instance.filename))
return self.sample_instance
def _sample_random(
self,
sample_type=0,
timestamp=None,
name='random_sample_',
interval=None,
force_interval=False,
**kwargs
):
"""
Randomly samples continuous subset of data.
Args:
name: str, sample filename id
Returns:
BTgymDataset instance with number of records ~ max_episode_len,
where `~` tolerance is set by `time_gap` param.
"""
try:
assert not self.data.empty
except (AssertionError, AttributeError) as e:
self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?')
raise AssertionError
if force_interval:
raise NotImplementedError('Force_interval for random sampling not implemented.')
self.log.debug('Maximum sample time duration set to: {}.'.format(self.max_sample_len_delta))
self.log.debug('Respective number of steps: {}.'.format(self.sample_num_records))
self.log.debug('Maximum allowed data time gap set to: {}.\n'.format(self.max_time_gap))
sampled_data = None
sample_len = 0
# Sanity check param:
max_attempts = 100
attempts = 0
# # Keep sampling random enter points until all conditions are met:
while attempts <= max_attempts:
# Randomly sample record (row) from entire datafeed:
first_row = int((self.data.shape[0] - self.sample_num_records - 1) * random.random())
sample_first_day = self.data[first_row:first_row + 1].index[0]
self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday()))
# Keep sampling until good day:
while not sample_first_day.weekday() in self.start_weekdays and attempts <= max_attempts:
self.log.debug('Not a good day to start, resampling...')
first_row = int((self.data.shape[0] - self.sample_num_records - 1) * random.random())
sample_first_day = self.data[first_row:first_row + 1].index[0]
self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday()))
attempts +=1
# Check if managed to get proper weekday:
assert attempts <= max_attempts, \
'Quitting after {} sampling attempts. Hint: check sampling params / dataset consistency.'. \
format(attempts)
# If 00 option set, get index of first record of that day:
if self.start_00:
adj_timedate = sample_first_day.date()
self.log.debug('Start time adjusted to <00:00>')
else:
adj_timedate = sample_first_day
first_row = self.data.index.get_loc(adj_timedate, method='nearest')
# Easy part:
last_row = first_row + self.sample_num_records # + 1
sampled_data = self.data[first_row: last_row]
sample_len = (sampled_data.index[-1] - sampled_data.index[0]).to_pytimedelta()
self.log.debug('Actual sample duration: {}.'.format(sample_len, ))
self.log.debug('Total sample time gap: {}.'.format(self.max_sample_len_delta - sample_len))
# Perform data gap check:
if self.max_sample_len_delta - sample_len < self.max_time_gap:
self.log.debug('Sample accepted.')
# If sample OK - compose and return sample:
new_instance = self.nested_class_ref(**self.nested_params)
new_instance.filename = name + 'n{}_at_{}'.format(self.sample_num, adj_timedate)
self.log.info('Sample id: <{}>.'.format(new_instance.filename))
new_instance.data = sampled_data
new_instance.metadata['type'] = 'random_sample'
new_instance.metadata['first_row'] = first_row
new_instance.metadata['last_row'] = last_row
return new_instance
else:
self.log.debug('Duration too big, resampling...\n')
attempts += 1
# Got here -> sanity check failed:
msg = (
'\nQuitting after {} sampling attempts.\n' +
'Full sample duration: {}\n' +
'Total sample time gap: {}\n' +
'Sample start time: {}\n' +
'Sample finish time: {}\n' +
'Hint: check sampling params / dataset consistency.'
).format(
attempts,
sample_len,
sample_len - self.max_sample_len_delta,
sampled_data.index[0],
sampled_data.index[-1]
)
self.log.error(msg)
raise RuntimeError(msg)
def _sample_interval(
self,
interval,
b_alpha=1.0,
b_beta=1.0,
name='interval_sample_',
force_interval=False,
**kwargs
):
"""
Samples continuous subset of data,
such as entire episode records lie within positions specified by interval.
Episode start position within interval is drawn from beta-distribution parametrised by `b_alpha, b_beta`.
By default distribution is uniform one.
Args:
interval: tuple, list or 1d-array of integers of length 2: [lower_row_number, upper_row_number];
b_alpha: float > 0, sampling B-distribution alpha param, def=1;
b_beta: float > 0, sampling B-distribution beta param, def=1;
name: str, sample filename id
force_interval: bool, if true: force exact interval sampling
Returns:
- BTgymDataset instance such as:
1. number of records ~ max_episode_len, subj. to `time_gap` param;
2. actual episode start position is sampled from `interval`;
- `False` if it is not possible to sample instance with set args.
"""
try:
assert not self.data.empty
except (AssertionError, AttributeError) as e:
self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?')
raise AssertionError
try:
assert len(interval) == 2
except AssertionError:
self.log.exception(
'Invalid interval arg: expected list or tuple of size 2, got: {}'.format(interval)
)
raise AssertionError
if force_interval:
return self._sample_exact_interval(interval, name)
try:
assert b_alpha > 0 and b_beta > 0
except AssertionError:
self.log.exception(
'Expected positive B-distribution [alpha, beta] params, got: {}'.format([b_alpha, b_beta])
)
raise AssertionError
if interval[-1] - interval[0] + self.max_gap_num_records > self.sample_num_records:
sample_num_records = self.sample_num_records
else:
sample_num_records = interval[-1] - interval[0]
self.log.debug('Sample interval: {}'.format(interval))
self.log.debug('Maximum sample time duration set to: {}.'.format(self.max_sample_len_delta))
self.log.debug('Sample number of steps (adjusted to interval): {}.'.format(sample_num_records))
self.log.debug('Maximum allowed data time gap set to: {}.\n'.format(self.max_time_gap))
sampled_data = None
sample_len = 0
# Sanity check param:
max_attempts = 100
attempts = 0
# # Keep sampling random enter points until all conditions are met:
while attempts <= max_attempts:
first_row = interval[0] + int(
(interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta)
)
#print('_sample_interval_sample_num_records: ', sample_num_records)
#print('_sample_interval_first_row: ', first_row)
sample_first_day = self.data[first_row:first_row + 1].index[0]
self.log.debug(
'Sample start row: {}, day: {}, weekday: {}.'.
format(first_row, sample_first_day, sample_first_day.weekday())
)
# Keep sampling until good day:
while not sample_first_day.weekday() in self.start_weekdays and attempts <= max_attempts:
self.log.debug('Not a good day to start, resampling...')
first_row = interval[0] + round(
(interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta)
)
#print('r_sample_interval_sample_num_records: ', sample_num_records)
#print('r_sample_interval_first_row: ', first_row)
sample_first_day = self.data[first_row:first_row + 1].index[0]
self.log.debug(
'Sample start row: {}, day: {}, weekday: {}.'.
format(first_row, sample_first_day, sample_first_day.weekday())
)
attempts += 1
# Check if managed to get proper weekday:
try:
assert attempts <= max_attempts
except AssertionError:
self.log.exception(
'Quitting after {} sampling attempts. Hint: check sampling params / dataset consistency.'.
format(attempts)
)
raise RuntimeError
# If 00 option set, get index of first record of that day:
if self.start_00:
adj_timedate = sample_first_day.date()
self.log.debug('Start time adjusted to <00:00>')
first_row = self.data.index.get_loc(adj_timedate, method='nearest')
else:
adj_timedate = sample_first_day
# first_row = self.data.index.get_loc(adj_timedate, method='nearest')
# Easy part:
last_row = first_row + sample_num_records # + 1
sampled_data = self.data[first_row: last_row]
self.log.debug(
'first_row: {}, last_row: {}, data_shape: {}'.format(
first_row,
last_row,
sampled_data.shape
)
)
sample_len = (sampled_data.index[-1] - sampled_data.index[0]).to_pytimedelta()
self.log.debug('Actual sample duration: {}.'.format(sample_len))
self.log.debug('Total sample time gap: {}.'.format(self.max_sample_len_delta - sample_len))
# Perform data gap check:
if self.max_sample_len_delta - sample_len < self.max_time_gap:
self.log.debug('Sample accepted.')
# If sample OK - return new dataset:
new_instance = self.nested_class_ref(**self.nested_params)
new_instance.filename = name + 'num_{}_at_{}'.format(self.sample_num, adj_timedate)
self.log.info('New sample id: <{}>.'.format(new_instance.filename))
new_instance.data = sampled_data
new_instance.metadata['type'] = 'interval_sample'
new_instance.metadata['first_row'] = first_row
new_instance.metadata['last_row'] = last_row
return new_instance
else:
self.log.debug('Attempt {}: gap is too big, resampling, ...\n'.format(attempts))
attempts += 1
# Got here -> sanity check failed:
msg = (
'\nQuitting after {} sampling attempts.\n' +
'Full sample duration: {}\n' +
'Total sample time gap: {}\n' +
'Sample start time: {}\n' +
'Sample finish time: {}\n' +
'Hint: check sampling params / dataset consistency.'
).format(
attempts,
sample_len,
sample_len - self.max_sample_len_delta,
sampled_data.index[0],
sampled_data.index[-1]
)
self.log.error(msg)
raise RuntimeError(msg)
def _sample_aligned_interval(
self,
interval,
align_left=False,
b_alpha=1.0,
b_beta=1.0,
name='interval_sample_',
force_interval=False,
**kwargs
):
"""
Samples continuous subset of data,
such as entire episode records lie within positions specified by interval
Episode start position within interval is drawn from beta-distribution parametrised by `b_alpha, b_beta`.
By default distribution is uniform one.
Args:
interval: tuple, list or 1d-array of integers of length 2: [lower_row_number, upper_row_number];
align: if True - try to align sample to beginning of interval;
b_alpha: float > 0, sampling B-distribution alpha param, def=1;
b_beta: float > 0, sampling B-distribution beta param, def=1;
name: str, sample filename id
force_interval: bool, if true: force exact interval sampling
Returns:
- BTgymDataset instance such as:
1. number of records ~ max_episode_len, subj. to `time_gap` param;
2. actual episode start position is sampled from `interval`;
- `False` if it is not possible to sample instance with set args.
"""
try:
assert not self.data.empty
except (AssertionError, AttributeError) as e:
self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?')
raise AssertionError
try:
assert len(interval) == 2
except AssertionError:
self.log.exception(
'Invalid interval arg: expected list or tuple of size 2, got: {}'.format(interval)
)
raise AssertionError
if force_interval:
return self._sample_exact_interval(interval, name)
try:
assert b_alpha > 0 and b_beta > 0
except AssertionError:
self.log.exception(
'Expected positive B-distribution [alpha, beta] params, got: {}'.format([b_alpha, b_beta])
)
raise AssertionError
sample_num_records = self.sample_num_records
self.log.debug('Maximum sample time duration set to: {}.'.format(self.max_sample_len_delta))
self.log.debug('Respective number of steps: {}.'.format(sample_num_records))
self.log.debug('Maximum allowed data time gap set to: {}.\n'.format(self.max_time_gap))
# Sanity check param:
if align_left:
max_attempts = interval[-1] - interval[0]
else:
# Sanity check:
max_attempts = 100
attempts = 0
align_shift = 0
# Sample enter point as close to beginning until all conditions are met:
while attempts <= max_attempts:
if align_left:
first_row = interval[0] + align_shift
else:
first_row = interval[0] + int(
(interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta)
)
#print('_sample_interval_sample_num_records: ', sample_num_records)
self.log.debug('_sample_interval_first_row: {}'.format(first_row))
sample_first_day = self.data[first_row:first_row + 1].index[0]
self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday()))
# Keep sampling until good day:
while not sample_first_day.weekday() in self.start_weekdays and attempts <= max_attempts:
align_shift += 1
self.log.debug('Not a good day to start, resampling...')
if align_left:
first_row = interval[0] + align_shift
else:
first_row = interval[0] + int(
(interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta)
)
#print('r_sample_interval_sample_num_records: ', sample_num_records)
self.log.debug('_sample_interval_first_row: {}'.format(first_row))
sample_first_day = self.data[first_row:first_row + 1].index[0]
self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday()))
attempts += 1
# Check if managed to get proper weekday:
try:
assert attempts <= max_attempts
except AssertionError:
self.log.exception(
'Quitting after {} sampling attempts. Hint: check sampling params / dataset consistency.'.
format(attempts)
)
raise RuntimeError
# If 00 option set, get index of first record of that day:
if self.start_00:
adj_timedate = sample_first_day.date()
self.log.debug('Start time adjusted to <00:00>')
first_row = self.data.index.get_loc(adj_timedate, method='nearest')
else:
adj_timedate = sample_first_day
# first_row = self.data.index.get_loc(adj_timedate, method='nearest')
# Easy part:
last_row = first_row + sample_num_records # + 1
sampled_data = self.data[first_row: last_row]
sample_len = (sampled_data.index[-1] - sampled_data.index[0]).to_pytimedelta()
self.log.debug('Actual sample duration: {}.'.format(sample_len))
self.log.debug('Total sample time gap: {}.'.format(sample_len - self.max_sample_len_delta))
# Perform data gap check:
if sample_len - self.max_sample_len_delta < self.max_time_gap:
self.log.debug('Sample accepted.')
# If sample OK - return new dataset:
new_instance = self.nested_class_ref(**self.nested_params)
new_instance.filename = name + 'num_{}_at_{}'.format(self.sample_num, adj_timedate)
self.log.info('New sample id: <{}>.'.format(new_instance.filename))
new_instance.data = sampled_data
new_instance.metadata['type'] = 'interval_sample'
new_instance.metadata['first_row'] = first_row
new_instance.metadata['last_row'] = last_row
return new_instance
else:
self.log.debug('Attempt {}: duration too big, resampling, ...\n'.format(attempts))
attempts += 1
align_shift += 1
# Got here -> sanity check failed:
msg = ('Quitting after {} sampling attempts.' +
'Hint: check sampling params / dataset consistency.').format(attempts)
self.log.error(msg)
raise RuntimeError(msg)
def _sample_exact_interval(self, interval, name='interval_sample_', **kwargs):
"""
Samples exactly defined interval.
Args:
interval: tuple, list or 1d-array of integers of length 2: [lower_row_number, upper_row_number];
name: str, sample filename id
Returns:
BTgymDataset instance.
"""
try:
assert not self.data.empty
except (AssertionError, AttributeError) as e:
self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?')
raise AssertionError
try:
assert len(interval) == 2
except AssertionError:
self.log.exception(
'Invalid interval arg: expected list or tuple of size 2, got: {}'.format(interval)
)
raise AssertionError
first_row = interval[0]
last_row = interval[-1]
sampled_data = self.data[first_row: last_row]
sample_first_day = self.data[first_row:first_row + 1].index[0]
new_instance = self.nested_class_ref(**self.nested_params)
new_instance.filename = name + 'num_{}_at_{}'.format(self.sample_num, sample_first_day)
self.log.info('New sample id: <{}>.'.format(new_instance.filename))
new_instance.data = sampled_data
new_instance.metadata['type'] = 'interval_sample'
new_instance.metadata['first_row'] = first_row
new_instance.metadata['last_row'] = last_row
return new_instance
class Launcher():
"""
Configures and starts distributed TF training session with workers
running sets of separate instances of BTgym/Atari environment.
"""
def __init__(self,
env_config=None,
cluster_config=None,
policy_config=None,
trainer_config=None,
max_env_steps=None,
root_random_seed=None,
test_mode=False,
purge_previous=0,
log_level=None,
verbose=0):
"""
Args:
env_config (dict): environment class_config_dict, see 'Note' below.
cluster_config (dict): tf cluster configuration, see 'Note' below.
policy_config (dict): policy class_config_dict holding corr. policy class args.
trainer_config (dict): trainer class_config_dict holding corr. trainer class args.
max_env_steps (int): total number of environment steps to run training on.
root_random_seed (int): int or None
test_mode (bool): if True - use Atari gym env., BTGym otherwise.
purge_previous (int): keep or remove previous log files and saved checkpoints from log_dir:
{0 - keep, 1 - ask, 2 - remove}.
verbose (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG}.
log_level (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13},
overrides `verbose` arg.
Note:
class_config_dict: dictionary containing at least two keys:
- `class_ref`: reference to class constructor or function;
- `kwargs`: dictionary of keyword arguments, see corr. environment class args.
cluster_config: dictionary containing at least these keys:
- 'host': cluster host, def: '127.0.0.1'
- 'port': cluster port, def: 12222
- 'num_workers': number of workers to run, def: 1
- 'num_ps': number of parameter servers, def: 1
- 'num_envs': number of environments to run in parallel for each worker, def: 1
- 'log_dir': directory to save model and summaries, def: './tmp/btgym_aac_log'
"""
self.env_config = dict(class_ref=None,
kwargs=dict(
port=5000,
data_port=4999,
gym_id=None,
))
self.cluster_config = dict(
host='127.0.0.1',
port=12222,
num_workers=1,
num_ps=1,
log_dir='./tmp/btgym_aac_log',
num_envs=1,
)
self.policy_config = dict(class_ref=BaseAacPolicy,
kwargs=dict(lstm_layers=(256, )))
self.trainer_config = dict(class_ref=A3C, kwargs={})
self.max_env_steps = 100 * 10**6
self.ports_to_use = []
self.root_random_seed = root_random_seed
self.purge_previous = purge_previous
self.test_mode = test_mode
self.log_level = log_level
self.verbose = verbose
if max_env_steps is not None:
self.max_env_steps = max_env_steps
self.env_config = self._update_config_dict(self.env_config, env_config)
self.cluster_config = self._update_config_dict(self.cluster_config,
cluster_config)
self.policy_config = self._update_config_dict(self.policy_config,
policy_config)
self.trainer_config = self._update_config_dict(self.trainer_config,
trainer_config)
self.trainer_config['kwargs']['test_mode'] = self.test_mode
# Logging config:
# TODO: Warnings --> to Notices
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
log_levels = [(0, WARNING), (1, INFO), (2, DEBUG)]
self.log_level = WARNING
for key, value in log_levels:
if key == self.verbose:
self.log_level = value
self.log = Logger('Launcher_shell', level=self.log_level)
# Seeding:
if self.root_random_seed is not None:
np.random.seed(self.root_random_seed)
self.log.info('Random seed: {}'.format(self.root_random_seed))
# Seeding for workers:
workers_rnd_seeds = list(
np.random.randint(
0, 2**30, self.cluster_config['num_workers'] +
self.cluster_config['num_ps']))
# Log_dir:
if os.path.exists(self.cluster_config['log_dir']):
# Remove previous log files and saved model if opted:
if self.purge_previous > 0:
confirm = 'y'
if self.purge_previous < 2:
confirm = input(
'<{}> already exists. Override[y/n]? '.format(
self.cluster_config['log_dir']))
if confirm in 'y':
files = glob.glob(self.cluster_config['log_dir'] + '/*')
p = psutil.Popen([
'rm',
'-R',
] + files,
stdout=PIPE,
stderr=PIPE)
self.log.warning('Files in <{}> purged.'.format(
self.cluster_config['log_dir']))
else:
self.log.warning('Appending to <{}>.'.format(
self.cluster_config['log_dir']))
else:
os.makedirs(self.cluster_config['log_dir'])
self.log.warning('<{}> created.'.format(
self.cluster_config['log_dir']))
for kwarg in ['port', 'data_port']:
assert kwarg in self.env_config['kwargs'].keys()
assert self.env_config['class_ref'] is not None
# Make cluster specification dict:
self.cluster_spec = self.make_cluster_spec(self.cluster_config)
# Configure workers:
self.workers_config_list = []
env_ports = np.arange(self.cluster_config['num_envs'])
worker_port = self.env_config['kwargs'][
'port'] # start value for BTGym comm. port
#for k, v in self.env_config['kwargs'].items():
# try:
# c = copy.deepcopy(v)
# print('key: {} -- deepcopy ok.'.format(k))
# except:
# print('key: {} -- deepcopy failed!.'.format(k))
# TODO: Hacky, cause dataset is threadlocked; do: pass dataset as class_ref + kwargs_dict:
if self.test_mode:
dataset_instance = None
else:
dataset_instance = self.env_config['kwargs'].pop('dataset')
for key, spec_list in self.cluster_spec.items():
task_index = 0 # referenced farther as worker id
for _id in spec_list:
env_config = copy.deepcopy(self.env_config)
worker_config = {}
if key in 'worker':
# Configure worker BTgym environment:
if task_index == 0:
env_config['kwargs'][
'data_master'] = True # set worker_0 as chief and data_master
env_config['kwargs']['dataset'] = dataset_instance
env_config['kwargs']['render_enabled'] = True
else:
env_config['kwargs']['data_master'] = False
env_config['kwargs'][
'render_enabled'] = False # disable rendering for all but chief
# Add list of connection ports for every parallel env for each worker:
env_config['kwargs']['port'] = list(worker_port +
env_ports)
worker_port += self.cluster_config['num_envs']
worker_config.update({
'env_config': env_config,
'policy_config': self.policy_config,
'trainer_config': self.trainer_config,
'cluster_spec': self.cluster_spec,
'job_name': key,
'task': task_index,
'test_mode': self.test_mode,
'log_dir': self.cluster_config['log_dir'],
'max_env_steps': self.max_env_steps,
#'log': self.log,
'log_level': self.log_level,
'random_seed': workers_rnd_seeds.pop()
})
self.clear_port(env_config['kwargs']['port'])
self.workers_config_list.append(worker_config)
task_index += 1
self.clear_port(self.env_config['kwargs']['data_port'])
self.log.debug('Launcher ready.')
def make_cluster_spec(self, config):
"""
Composes cluster specification dictionary.
"""
cluster = {}
all_ps = []
port = config['port']
for _ in range(config['num_ps']):
self.clear_port(port)
self.ports_to_use.append(port)
all_ps.append('{}:{}'.format(config['host'], port))
port += 1
cluster['ps'] = all_ps
all_workers = []
for _ in range(config['num_workers']):
self.clear_port(port)
self.ports_to_use.append(port)
all_workers.append('{}:{}'.format(config['host'], port))
port += 1
cluster['worker'] = all_workers
return cluster
def clear_port(self, port_list):
"""
Kills process on specified ports list, if any.
"""
if not isinstance(port_list, list):
port_list = [port_list]
for port in port_list:
p = psutil.Popen(['lsof', '-i:{}'.format(port), '-t'],
stdout=PIPE,
stderr=PIPE)
pid = p.communicate()[0].decode()[:-1] # retrieving PID
if pid is not '':
p = psutil.Popen(['kill', pid])
self.log.info('port {} cleared'.format(port))
def _update_config_dict(self, old_dict, new_dict=None):
"""
Service, updates nested dictionary with values from other one of same structure.
Args:
old_dict: dict to update to
new_dict: dict to update from
Returns:
new updated dict
"""
if type(new_dict) is not dict:
new_dict = old_dict # ~identity op
for key, value in new_dict.items():
if type(value) == dict:
old_dict[key] = self._update_config_dict(old_dict[key], value)
else:
old_dict[key] = value
return old_dict
def run(self):
"""
Launches processes:
distributed workers;
parameter_server.
"""
workers_list = []
p_servers_list = []
chief_worker = None
def signal_handler(signal, frame):
nonlocal workers_list
nonlocal chief_worker
nonlocal p_servers_list
def stop_worker(worker_list):
for worker in worker_list:
worker.terminate()
stop_worker(workers_list)
stop_worker([chief_worker])
stop_worker(p_servers_list)
# Start workers:
for worker_config in self.workers_config_list:
# Make:
worker = Worker(**worker_config)
# Launch:
worker.daemon = False
worker.start()
if worker.job_name in 'worker':
# Allow data-master to launch datafeed_server:
if worker_config['env_config']['kwargs']['data_master']:
time.sleep(5)
chief_worker = worker
else:
workers_list.append(worker)
else:
p_servers_list.append(worker)
# TODO: auto-launch tensorboard?
signal.signal(signal.SIGINT, signal_handler)
# Halt here:
msg = 'Press `Ctrl-C` or [Kernel]->[Interrupt] to stop training and close launcher.'
print(msg)
self.log.info(msg)
signal.pause()
# Wait every worker to finish:
for worker in workers_list:
worker.join()
self.log.info('worker_{} has joined.'.format(worker.task))
chief_worker.join()
self.log.info('chief_worker_{} has joined.'.format(chief_worker.task))
for ps in p_servers_list:
ps.join()
self.log.info('parameter_server_{} has joined.'.format(ps.task))
# TODO: close tensorboard
self.log.info('Launcher closed.')
class Paxos(object):
def __init__(self, transport, on_learn, on_prepare=None,
on_stale=None, quorum_timeout=3,
logger_group=None,
):
self._logger = Logger('paxos')
if logger_group is not None:
logger_group.add_logger(self._logger)
self.transport = transport
self.on_learn = on_learn
self.on_prepare = on_prepare
self.on_stale = on_stale
self.quorum_timeout = quorum_timeout
self.id = 0
self.max_seen_id = 0
self.last_accepted_id = 0
self._logger.debug('2 last_accepted_id=%(last_accepted_id)s' % self.__dict__)
self.proposed_value = None
self.deferred = None
self.queue = deque() # queue of (value, deferred) to propose
self._learn_queue = [] # sorted list with learn requests which come out of order
# delayed calls for timeouts
self._accepted_timeout = None
self._acks_timeout = None
self._waiting_to_learn_id = deque()
def recv(self, message, client):
message = shlex.split(message)
command = getattr(self, message[0])
command(client=client, *message[1:])
def propose(self, value):
deferred = Deferred()
if self.proposed_value is None:
self._start_paxos(value, deferred)
else:
self.queue.append((value, deferred))
self._logger.debug('Request for %s was queued, queue size is %s (because we are proposing %s now)' % (
value,
len(self.queue),
self.proposed_value,
)
)
return deferred
def _start_paxos(self, value, deferred):
"""Starts paxos iteration proposing given value."""
self.id = self.max_seen_id + 1
self.proposed_value = value
self.deferred = deferred
self._num_acks_to_wait = self.transport.quorum_size
def _timeout_callback():
self._logger.info('+++ prepare timeout')
# TODO sometimes self.deferred is None when this callbach is called
self.deferred.errback(PrepareTimeout())
self.deferred = None
self.proposed_value = None
self._acks_timeout = reactor.callLater(self.quorum_timeout, _timeout_callback)
self.transport.broadcast('paxos_prepare %s %s' % (self.id, self.last_accepted_id))
def paxos_prepare(self, num, last_accepted_id, client):
num = int(num)
last_accepted_id = int(last_accepted_id)
if last_accepted_id > self.last_accepted_id:
# Move to the "stale" state
self._logger.debug('stale last_accepted_id(%s) < self.last_accepted_id(%s)' % (
last_accepted_id, self.last_accepted_id
))
self.on_stale(last_accepted_id)
else:
if num > self.max_seen_id:
if self.on_prepare is not None:
self.on_prepare(num, client)
self.max_seen_id = num
self._send_to(client, 'paxos_ack %s' % num)
def paxos_ack(self, num, client):
num = int(num)
if self.proposed_value is not None and num == self.id:
self._num_acks_to_wait -= 1
if self._num_acks_to_wait == 0:
_stop_waiting(self._acks_timeout)
self._num_accepts_to_wait = self.transport.quorum_size
def _timeout_callback():
self._logger.info('+++ accept timeout')
self.deferred.errback(AcceptTimeout())
self.deferred = None
self.proposed_value = None
self._accepted_timeout = reactor.callLater(
self.quorum_timeout,
_timeout_callback
)
self.transport.broadcast('paxos_accept %s "%s"' % (self.id, escape(self.proposed_value)))
def paxos_accept(self, num, value, client):
num = int(num)
if num == self.max_seen_id:
if self.id == num:
# we have a deferred to return result in this round
self._waiting_to_learn_id.append((num, self.deferred))
else:
# may be we have deferred but it is for another Paxos round
self._waiting_to_learn_id.append((num, None))
self._send_to(client, 'paxos_accepted %s' % num)
def paxos_accepted(self, num, client):
num = int(num)
if self.proposed_value is not None and num == self.id:
self._num_accepts_to_wait -= 1
if self._num_accepts_to_wait == 0:
_stop_waiting(self._accepted_timeout)
self.transport.broadcast('paxos_learn %s "%s"' % (self.id, escape(self.proposed_value)))
def paxos_learn(self, num, value, client):
self._logger.info('paxos.learn %s' % value)
num = int(num)
if self._waiting_to_learn_id and num == self._waiting_to_learn_id[0][0]:
num, deferred = self._waiting_to_learn_id.popleft()
try:
result = self.on_learn(num, value, client)
except Exception, e:
self._logger.exception('paxos.learn %s' % value)
result = e
self.last_accepted_id = num
self._logger.debug('1 last_accepted_id=%(last_accepted_id)s' % self.__dict__)
if deferred is not None and value == self.proposed_value:
# this works for current round coordinator only
# because it must return result to the client
# and to start a new round for next request
if isinstance(result, Exception):
self._logger.warning('returning error from paxos.learn %s, %s' % (value, result))
deferred.errback(result)
else:
self._logger.warning('returning success from paxos.learn %s' % value)
deferred.callback(result)
self._logger.debug('queue size: %s' % len(self.queue))
if self.queue:
# start new Paxos instance
# for next value from the queue
next_value, deferred = self.queue.pop()
self._logger.debug('next value from the queue: %s' % next_value)
self._start_paxos(next_value, deferred)
else:
self.proposed_value = None
self.deferred = None
if self._learn_queue:
self._logger.debug('relearning remembered values')
# clear queue because it will be filled again if needed
queue, self._learn_queue = self._learn_queue, []
for args in queue:
self.paxos_learn(*args)
else:
class Plug(object):
"""The Plug is the communication pipe between a Driver and Onitu.
The driver should instantiate a new Plug as soon as possible, define
his handlers (see :func:`Plug.handler`) and call :func:`Plug.start`
when it's ready to receive notifications.
"""
def __init__(self):
super(Plug, self).__init__()
self.redis = connect_to_redis()
self.name = None
self.logger = None
self.router = None
self.worker = None
self.options = {}
self._handlers = {}
def start(self, name):
"""This method should be called when the driver is ready to
communicate with Onitu.
Takes a parameter `name` that corresponds to the first
parameter given to the Driver at start.
:func:`start` launches two threads :
- The :class:`worker.Worker`, listening to notifications from the
Referee and handling them by calling the handlers defined by the
Driver
- The :class:`router.Router`, listening to requests by other Drivers
that need a chunk of a file and getting this chunk by calling the
`get_chunk` handler.
"""
self.name = name
self.logger = Logger(self.name)
self.options = self.redis.hgetall('drivers:{}:options'.format(name))
self.logger.info("Started")
self.router = Router(name, self.redis, self._handlers.get('get_chunk'))
self.router.start()
self.worker = Worker(self)
self.worker.start()
self.worker.resume_transfers()
def wait(self):
"""Waits until the :class:`Plug` is killed by another process.
"""
self.router.join()
def handler(self, task=None):
"""Decorator used to bind to a function assigned to a specific
task. Example::
@plug.handler('get_chunk')
def read(filename, offset, size):
with open(filename, 'rb') as f:
f.seek(offset)
return f.read(size)
Currently, the supported tasks are :
- `get_chunk`, which takes the name of the file, the offset and
the size of the chunk in parameter, and should return a
string.
- `start_upload`, which takes a `Metadata` in parameter,
returns nothing, and is called before each transfer of a
complete file.
- `upload_chunk`, which takes the name of the file, the offset
and the chunk to be uploaded and return nothing.
- `end_upload`, which takes a `Metadata` in parameter, returns
nothing, and is called at the end of each transfer of a
complete file.
A Driver can implement any, all or none of the tasks above.
"""
def decorator(handler):
self._handlers[task if task else handler.__name__] = handler
return handler
return decorator
def update_file(self, metadata):
"""This method should be called by the Driver after each update
of a file or after the creation of a file.
It takes a `Metadata` object in parameter that should have been
updated with the new value of the properties.
"""
fid = self.redis.hget('files', metadata.filename)
if not fid:
fid = self.redis.incr('last_id')
self.redis.hset('files', metadata.filename, fid)
self.redis.hset('drivers:{}:files'.format(self.name), fid, "")
metadata.owners = [self.name]
metadata._fid = fid
elif self.redis.sismember('drivers:{}:transfers'.format(self.name),
fid):
# The event has been triggered during a transfer, we
# have to cancel it.
self.logger.warning(
"About to send an event for '{}' when downloading it, "
"aborting the event", metadata.filename)
return
metadata.uptodate = [self.name]
metadata.write()
self.redis.rpush('events', "{}:{}".format(self.name, fid))
def get_metadata(self, filename):
"""Returns a `Metadata` object corresponding to the given
filename.
"""
metadata = Metadata.get_by_filename(self, filename)
if not metadata:
metadata = Metadata(plug=self, filename=filename)
metadata.entry = self.name
return metadata
class BaseDataSet():
"""
Basic data preparation: csv -> pd.dataframe -> sparse matrix
"""
def __init__(self,
filename,
path='data/',
rates_filename='rub_exchange_rates.csv'):
StreamHandler(sys.stdout).push_application()
self.path = path
self.filename = filename
self.rates_filename = rates_filename
self.log = Logger('BaseData/' + self.filename)
self.mcc_codes_table = pd.read_html(requests.get(
'https://mcc-codes.ru/code', headers={
'User-agent': 'Mozilla/5.0'
}).text,
converters={'MCC': str})[0]
self.mcc_map = self.mcc_codes_table[[
u'MCC', u'Группа'
]].set_index('MCC').to_dict()[u'Группа']
self.rates = pd.read_csv(self.path + self.rates_filename)
self.data = None
self.aggregated_data = None
self.sparse_array_data = None
self.mcc_groups = None
self.grouped_by_id = None
@staticmethod
def extract_month(s):
day, month, year = s.split('/')
month = int(month)
month += (int(year) - 2016) * 12
return month
def _to_rur(self, obj):
if obj['currency'] != 810:
# Get currency:
curr = iso4217parse.parse(int(obj['currency']))[0]
if curr is not None:
curr_alpha3 = curr[0]
else:
raise ValueError('Unknown currency code: {}'.format(
obj['currency']))
curr_amount = obj['amount']
# Convert:
try:
obj['amount'] = curr_amount / self.rates.loc[curr_alpha3][
'rate']
# print('Converted {} {} to {} RUR'.format(curr_amount, curr_alpha3, obj['amount']))
except KeyError as e:
# Ex. rate not found, fake 1000 rub on tis transaction:
self.log.warning(
'Rate missing for {} {}, substituted by 1000 RUR'.format(
obj['amount'], curr_alpha3))
obj['amount'] = 1000
obj['currency'] = 643
return obj
@staticmethod
def _day_to_int(obj):
obj['rel_day'] = obj['rel_day'].days
return obj
def load_csv(self, truncate=None, **kwargs):
self.data = pd.read_csv(self.path + self.filename)
self.log.info('Loaded data shape: {}'.format(self.data.shape))
if truncate is not None:
assert truncate < self.data.shape[0],\
'Truncation index {} is bigger than data size {}'.format(truncate, self.data.shape[0])
self.data = self.data[0:truncate]
self.log.info(
'Data truncated down to first {} rows'.format(truncate))
self.data['PERIOD'] = pd.to_datetime(self.data['PERIOD'],
format='%m/%d/%Y')
self.data['TRDATETIME'] = pd.to_datetime(self.data['TRDATETIME'],
format='%d%b%y:%X')
self.data['rel_day'] = self.data.apply(
lambda zero: datetime.timedelta(), axis=1)
self.data['channel_type'] = self.data['channel_type'].fillna(
'0').apply(lambda s: int(s[-1]))
self.data['mcc_group'] = self.data['MCC'].astype(str).map(self.mcc_map)
self.mcc_groups = list(set(self.data.mcc_group.unique()))
self.grouped_by_id = None
def to_relative_days(self, **kwargs):
self.grouped_by_id = self.data.groupby('cl_id')
for cl_id, group in self.grouped_by_id:
start_tstamp = copy.deepcopy(group['TRDATETIME'].min())
idx = copy.deepcopy(group.index)
self.data.loc[idx, 'rel_day'] = (self.data.loc[idx, 'TRDATETIME'] -
start_tstamp)
self.data = self.data.transform(self._day_to_int, axis=1)
def to_rur(self, **kwargs):
self.data = self.data.transform(self._to_rur, axis=1)
def aggregate_by_daily_sums(self, **kwargs):
self.grouped_by_id = self.data.groupby('cl_id')
current_idx = 0
col_names = ['cl_id', 'rel_day', 'sum_POS']
col_names += ['sum_{}'.format(group) for group in self.mcc_groups]
col_names += ['target_flag', 'target_sum']
aggr_data = pd.DataFrame(index=None, columns=col_names).fillna(0)
for cl_id, cl_group in self.grouped_by_id:
id_by_day = cl_group.groupby('rel_day')
for day, ts_group in id_by_day:
day_by_mcc = ts_group.groupby('MCC')
day_sum_pos = 0
s = pd.Series(name=current_idx, index=col_names).fillna(0)
s['cl_id'] = cl_id
s['rel_day'] = day
try:
s['target_flag'] = ts_group.target_flag.all()
s['target_sum'] = ts_group.target_sum.mean()
except AttributeError:
s['target_flag'] = float('NaN')
s['target_sum'] = float('NaN')
for mcc_id, ts in day_by_mcc:
day_sum_pos += ts[ts.trx_category == 'POS']['amount'].sum()
s['sum_{}'.format(
ts.mcc_group.values[0])] += ts['amount'].sum()
s['sum_POS'] = day_sum_pos
aggr_data = aggr_data.append(s)
current_idx += 1
return aggr_data
def save_csv(self, data, file_prefix='_', **kwargs):
data.to_csv(self.path + file_prefix + self.filename,
index=True,
index_label=False)
def save_sparse(self, data, file_prefix='sparse_array_', **kwargs):
assert isinstance(
data, sparse.COO), 'Expected sparse.COO data type, got: {}'.format(
type(data))
sparse.save_npz(self.path + file_prefix + self.filename[:-4] + '.npz',
data)
def to_sparse_array(self,
min_days=90,
max_days=None,
rebase=True,
**kwargs):
if self.aggregated_data is not None:
clients = []
if max_days is None:
max_days = int(self.aggregated_data['rel_day'].max()
) + 1 # infer from data
if max_days < 91:
max_days = 91
self.log.info(
'max_days inferred from data: {}'.format(max_days))
if rebase:
assert min_days < max_days, 'It is not possible to rebase with min_days={}, max_days={}'.format(
min_days, max_days)
self.log.info(
'Rebasing observations with min_days={}'.format(min_days))
num_col = int(self.aggregated_data.shape[-1]) - 1
client_shape = [1, max_days, num_col]
agg_by_id = self.aggregated_data.groupby('cl_id')
for cl_id, cl_group in agg_by_id:
client_array = np.zeros(client_shape)
client_values = cl_group.values
client_index = client_values[:, 1].astype(
int) # second column - rel_day values --> to 0_dim values
if rebase:
# Rebase but allow no less than 90 days observation period:
client_max_day = client_index.max()
if client_max_day < min_days:
client_max_day = min_days
rebase_index = max_days - client_max_day - 1
client_index += rebase_index
client_array[:,
client_index, :] = client_values[:,
1:] # remove cl_id (gone to new dim) and rel_day
# Fill all records for single client:
client_array[..., 0] = int(cl_id) # id
if np.isnan(cl_group.target_sum).any():
client_array[..., -1] = float('NaN')
client_array[..., -2] = float('NaN')
else:
client_array[..., -1] = cl_group.target_sum.mean()
client_array[..., -2] = cl_group.target_flag.all()
# Save as sparse 3d array:
clients.append(sparse.COO(client_array.astype('float32')))
full_array_sparse = sparse.concatenate(clients, axis=0)
else:
self.log.warning(
'No aggregated data found, call .aggregate_by_daily_sums() method first.'
)
full_array_sparse = None
return full_array_sparse
def process(self, level=2, **kwargs):
if self.data is None:
self.log.info('Loading data...')
else:
self.log.info('Reloading data...')
self.load_csv(**kwargs)
self.log.info('Converting to RUB...')
self.to_rur(**kwargs)
self.log.info('Calculating relative days...')
self.to_relative_days(**kwargs)
if level > 0:
self.log.info('Aggregating by daily sums...')
self.aggregated_data = self.aggregate_by_daily_sums(**kwargs)
self.log.info('Saving aggregated csv file...')
self.save_csv(self.aggregated_data, file_prefix='aggr_')
if level > 1:
self.log.info('Making sparse data array...')
self.sparse_array_data = self.to_sparse_array(**kwargs)
self.log.info('Saving sparse array...')
self.save_sparse(self.sparse_array_data)
self.log.info('Done.')
class BTgymDataFeedServer(multiprocessing.Process):
"""
Data provider server class.
Enables efficient data sampling for asynchronous multiply BTgym environments execution.
Manages global back-testing time.
"""
process = None
dataset_stat = None
def __init__(self,
dataset=None,
network_address=None,
log_level=None,
task=0):
"""
Configures data server instance.
Args:
dataset: data domain instance;
network_address: ...to bind to.
log_level: int, logbook.level
task: id
"""
super(BTgymDataFeedServer, self).__init__()
self.log_level = log_level
self.task = task
self.log = None
self.local_step = 0
self.dataset = dataset
self.network_address = network_address
self.default_sample_config = copy.deepcopy(DataSampleConfig)
self.debug_pre_sample_fails = 0
self.debug_pre_sample_attempts = 0
# self.global_timestamp = 0
def get_data(self, sample_config=None):
"""
Get Trial sample according to parameters received.
If no parameters being passed - makes sample with default parameters.
Args:
sample_config: sampling parameters configuration dictionary
Returns:
sample: if `sample_params` arg has been passed and dataset is ready
None: otherwise
"""
if self.dataset.is_ready:
if sample_config is not None:
# We do not allow configuration timestamps which point earlier than current global_timestamp;
# if config timestamp points later - it is ok because global time will be shifted accordingly after
# [traget test] sample will get into work.
if sample_config['timestamp'] is None:
sample_config['timestamp'] = 0
# If config timestamp is outdated - refresh with latest:
if sample_config['timestamp'] < self.dataset.global_timestamp:
sample_config['timestamp'] = copy.deepcopy(
self.dataset.global_timestamp)
self.log.debug(
'Sampling with params: {}'.format(sample_config))
sample = self.dataset.sample(**sample_config)
else:
self.default_sample_config['timestamp'] = copy.deepcopy(
self.dataset.global_timestamp)
self.log.debug('Sampling with default params: {}'.format(
self.default_sample_config))
sample = self.dataset.sample(**self.default_sample_config)
self.local_step += 1
else:
# Dataset not ready, make dummy:
sample = None
return sample
def run(self):
"""
Server process runtime body.
"""
# Logging:
from logbook import Logger, StreamHandler, WARNING
import sys
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
self.log_level = WARNING
self.log = Logger('BTgymDataServer_{}'.format(self.task),
level=self.log_level)
self.process = multiprocessing.current_process()
self.log.info('PID: {}'.format(self.process.pid))
# Set up a comm. channel for server as ZMQ socket:
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind(self.network_address)
# Actually load data to BTgymDataset instance, will reset it later on:
try:
assert not self.dataset.data.empty
except (AssertionError, AttributeError) as e:
self.dataset.read_csv()
# Describe dataset:
self.dataset_stat = self.dataset.describe()
# Main loop:
while True:
# Stick here until receive any request:
service_input = socket.recv_pyobj()
self.log.debug('Received <{}>'.format(service_input))
if 'ctrl' in service_input:
# It's time to exit:
if service_input['ctrl'] == '_stop':
# Server shutdown logic:
# send last run statistic, release comm channel and exit:
message = {'ctrl': 'Exiting.'}
self.log.info(str(message))
socket.send_pyobj(message)
socket.close()
context.destroy()
return None
# Reset datafeed:
elif service_input['ctrl'] == '_reset_data':
try:
kwargs = service_input['kwargs']
except KeyError:
kwargs = {}
self.dataset.reset(**kwargs)
# self.global_timestamp = self.dataset.global_timestamp
self.log.notice(
'Initial global_time set to: {} / stamp: {}'.format(
datetime.datetime.fromtimestamp(
self.dataset.global_timestamp),
self.dataset.global_timestamp))
message = {'ctrl': 'Reset with kwargs: {}'.format(kwargs)}
self.log.debug('Data_is_ready: {}'.format(
self.dataset.is_ready))
socket.send_pyobj(message)
self.local_step = 0
# Send dataset sample:
elif service_input['ctrl'] == '_get_data':
if self.dataset.is_ready:
sample = self.get_data(
sample_config=service_input['kwargs'])
message = 'Sending sample_#{}.'.format(self.local_step)
self.log.debug(message)
socket.send_pyobj({
'sample':
sample,
'stat':
self.dataset_stat,
'origin':
'data_server',
'timestamp':
self.dataset.global_timestamp,
})
else:
message = {
'ctrl':
'Dataset not ready, waiting for control key <_reset_data>'
}
self.log.debug('Sent: ' + str(message))
socket.send_pyobj(message) # pairs any other input
# Send dataset statisitc:
elif service_input['ctrl'] == '_get_info':
message = 'Sending info for #{}.'.format(self.local_step)
self.log.debug(message)
# Compose response:
info_dict = dict(dataset_stat=self.dataset_stat,
dataset_columns=list(self.dataset.names),
pid=self.process.pid,
dataset_is_ready=self.dataset.is_ready)
socket.send_pyobj(info_dict)
# Set global time:
elif service_input['ctrl'] == '_set_global_time':
if self.dataset.global_timestamp != 0 and self.dataset.global_timestamp > service_input[
'timestamp']:
message = 'Moving back in time not supported! ' +\
'Current global_time: {}, '.\
format(datetime.datetime.fromtimestamp(self.dataset.global_timestamp)) +\
'attempt to set: {}; nothing done. '.\
format(datetime.datetime.fromtimestamp(service_input['timestamp'])) +\
'Hint: check sampling logic consistency.'
self.log.warning(message)
else:
self.dataset.global_timestamp = service_input[
'timestamp']
message = 'global_time set to: {} / stamp: {}'.\
format(
datetime.datetime.fromtimestamp(self.dataset.global_timestamp),
self.dataset.global_timestamp
)
socket.send_pyobj(message)
self.log.debug(message)
elif service_input['ctrl'] == '_get_global_time':
# Tell time:
message = {'timestamp': self.dataset.global_timestamp}
socket.send_pyobj(message)
else: # ignore any other input
# NOTE: response dictionary must include 'ctrl' key
message = {
'ctrl':
'waiting for control keys: <_reset_data>, <_get_data>, <_get_info>, <_stop>.'
}
self.log.debug('Sent: ' + str(message))
socket.send_pyobj(message) # pairs any other input
else:
message = {
'ctrl':
'No <ctrl> key received, got:\n{}'.format(service_input)
}
self.log.debug(str(message))
socket.send_pyobj(message) # pairs input
class AMLDG():
"""
Asynchronous implementation of MLDG algorithm (by Da Li et al.)
for one-shot adaptation in dynamically changing environments.
Papers:
Da Li et al.,
"Learning to Generalize: Meta-Learning for Domain Generalization"
https://arxiv.org/abs/1710.03463
Maruan Al-Shedivat et al.,
"Continuous Adaptation via Meta-Learning in Nonstationary and Competitive Environments"
https://arxiv.org/abs/1710.03641
"""
def __init__(
self,
env,
task,
log_level,
aac_class_ref=SubAAC,
runner_config=None,
aac_lambda=1.0,
guided_lambda=1.0,
rollout_length=20,
trial_source_target_cycle=(1, 0),
num_episodes_per_trial=1, # one-shot adaptation
_aux_render_modes=('action_prob', 'value_fn', 'lstm_1_h',
'lstm_2_h'),
name='AMLDG',
**kwargs):
try:
self.aac_class_ref = aac_class_ref
self.task = task
self.name = name
self.summary_writer = None
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(name, task), level=log_level)
self.rollout_length = rollout_length
if runner_config is None:
self.runner_config = {
'class_ref': BaseSynchroRunner,
'kwargs': {},
}
else:
self.runner_config = runner_config
self.env_list = env
assert isinstance(self.env_list, list) and len(self.env_list) == 2, \
'Expected pair of environments, got: {}'.format(self.env_list)
# Instantiate two sub-trainers: one for meta-test and one for meta-train environments:
self.runner_config['kwargs']['data_sample_config'] = {
'mode': 1
} # master
self.runner_config['kwargs']['name'] = 'master'
self.train_aac = aac_class_ref(
env=self.env_list[0], # train data will be master environment
task=self.task,
log_level=log_level,
runner_config=self.runner_config,
aac_lambda=aac_lambda,
guided_lambda=guided_lambda,
rollout_length=self.rollout_length,
trial_source_target_cycle=trial_source_target_cycle,
num_episodes_per_trial=num_episodes_per_trial,
_use_target_policy=False,
_use_global_network=True,
_aux_render_modes=_aux_render_modes,
name=self.name + '/metaTrain',
**kwargs)
self.runner_config['kwargs']['data_sample_config'] = {
'mode': 0
} # slave
self.runner_config['kwargs']['name'] = 'slave'
self.test_aac = aac_class_ref(
env=self.env_list[-1], # test data -> slave env.
task=self.task,
log_level=log_level,
runner_config=self.runner_config,
aac_lambda=aac_lambda,
guided_lambda=guided_lambda,
rollout_length=self.rollout_length,
trial_source_target_cycle=trial_source_target_cycle,
num_episodes_per_trial=num_episodes_per_trial,
_use_target_policy=False,
_use_global_network=False,
global_step_op=self.train_aac.global_step,
global_episode_op=self.train_aac.global_episode,
inc_episode_op=self.train_aac.inc_episode,
_aux_render_modes=_aux_render_modes,
name=self.name + '/metaTest',
**kwargs)
self.local_steps = self.train_aac.local_steps
self.model_summary_freq = self.train_aac.model_summary_freq
self._make_train_op()
self.test_aac.model_summary_op = tf.summary.merge(
[
self.test_aac.model_summary_op,
self._combine_meta_summaries()
],
name='meta_model_summary')
except:
msg = 'AMLDG.__init()__ exception occurred' + \
'\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n'
self.log.exception(msg)
raise RuntimeError(msg)
def _make_train_op(self):
"""
Defines tensors holding training op graph for meta-train, meta-test and meta-optimisation.
"""
# Handy aliases:
pi = self.train_aac.local_network # local meta-train policy
pi_prime = self.test_aac.local_network # local meta-test policy
pi_global = self.train_aac.network # global shared policy
self.test_aac.sync = self.test_aac.sync_pi = tf.group(
*[v1.assign(v2) for v1, v2 in zip(pi_prime.var_list, pi.var_list)])
# Shared counters:
self.global_step = self.train_aac.global_step
self.global_episode = self.train_aac.global_episode
self.test_aac.global_step = self.train_aac.global_step
self.test_aac.global_episode = self.train_aac.global_episode
self.test_aac.inc_episode = self.train_aac.inc_episode
self.train_aac.inc_episode = None
self.inc_step = self.train_aac.inc_step
# Meta-opt. loss:
self.loss = self.train_aac.loss + self.test_aac.loss
# Clipped gradients:
self.train_aac.grads, _ = tf.clip_by_global_norm(
tf.gradients(self.train_aac.loss, pi.var_list), 40.0)
self.test_aac.grads, _ = tf.clip_by_global_norm(
tf.gradients(self.test_aac.loss, pi_prime.var_list), 40.0)
# Aliases:
pi.grads = self.train_aac.grads
pi_prime.grads = self.test_aac.grads
# # Learned meta opt. scaling (equivalent to learned meta-update step-size),
# # conditioned on test input:
# self.meta_grads_scale = tf.reduce_mean(pi_prime.meta_grads_scale)
# meta_grads_scale_var_list = [var for var in pi_prime.var_list if 'meta_grads_scale' in var.name]
# meta_grads_scale_var_list_global = [
# var for var in pi_global.var_list if 'meta_grads_scale' in var.name
# ]
# self.log.warning('meta_grads_scale_var_list: {}'.format(meta_grads_scale_var_list))
# Meta_optimisation gradients as sum of meta-train and meta-test gradients:
self.grads = []
for g1, g2 in zip(pi.grads, pi_prime.grads):
if g1 is not None and g2 is not None:
meta_g = g1 + g2
# meta_g = (1 - self.meta_grads_scale) * g1 + self.meta_grads_scale * g2
else:
meta_g = None # need to map correctly to vars
self.grads.append(meta_g)
# # Second order grads for learned grad. scaling param:
# meta_grads_scale_grads, _ = tf.clip_by_global_norm(
# tf.gradients([g for g in self.grads if g is not None], meta_grads_scale_var_list),
# 40.0
# )
# # Second order grads wrt global variables:
# meta_grads_scale_grads_and_vars = list(zip(meta_grads_scale_grads, meta_grads_scale_var_list_global))
# self.log.warning('meta_grads_scale_grads:\n{}'.format(meta_grads_scale_grads))
# self.log.warning('meta_grads_scale_grads_and_vars:\n{}'.format(meta_grads_scale_grads_and_vars))
#self.log.warning('self.grads_len: {}'.format(len(list(self.grads))))
# Gradients to update local meta-test policy (from train data):
train_grads_and_vars = list(zip(pi.grads, pi_prime.var_list))
# self.log.warning('train_grads_and_vars_len: {}'.format(len(train_grads_and_vars)))
# Meta-gradients to be sent to parameter server:
meta_grads_and_vars = list(
zip(self.grads,
pi_global.var_list)) #+ meta_grads_scale_grads_and_vars
# Remove empty entries:
meta_grads_and_vars = [(g, v) for (g, v) in meta_grads_and_vars
if g is not None]
# for item in meta_grads_and_vars:
# self.log.warning('\nmeta_g_v: {}'.format(item))
# Set global_step increment equal to observation space batch size:
obs_space_keys = list(self.train_aac.local_network.on_state_in.keys())
assert 'external' in obs_space_keys, \
'Expected observation space to contain `external` mode, got: {}'.format(obs_space_keys)
self.train_aac.inc_step = self.train_aac.global_step.assign_add(
tf.shape(self.train_aac.local_network.on_state_in['external'])[0])
# Pi to pi_prime local adaptation op:
self.train_op = self.train_aac.optimizer.apply_gradients(
train_grads_and_vars)
# Optimizer for meta-update, sharing same learn rate (change?):
self.optimizer = tf.train.AdamOptimizer(
self.train_aac.train_learn_rate, epsilon=1e-5)
# Global meta-optimisation op:
self.meta_train_op = self.optimizer.apply_gradients(
meta_grads_and_vars)
self.log.debug('meta_train_op defined')
def _combine_meta_summaries(self):
"""
Additional summaries here.
"""
meta_model_summaries = [
tf.summary.scalar('meta_grad_global_norm',
tf.global_norm(self.grads)),
tf.summary.scalar('total_meta_loss', self.loss),
# tf.summary.scalar('meta_grad_scale', self.meta_grads_scale)
]
return meta_model_summaries
def start(self, sess, summary_writer, **kwargs):
"""
Executes all initializing operations,
starts environment runner[s].
Supposed to be called by parent worker just before training loop starts.
Args:
sess: tf session object.
kwargs: not used by default.
"""
try:
# Copy weights from global to local:
sess.run(self.train_aac.sync_pi)
sess.run(self.test_aac.sync_pi)
# Start thread_runners:
self.train_aac._start_runners( # master first
sess,
summary_writer,
init_context=None,
data_sample_config=self.train_aac.get_sample_config(mode=1))
self.test_aac._start_runners(
sess,
summary_writer,
init_context=None,
data_sample_config=self.test_aac.get_sample_config(mode=0))
self.summary_writer = summary_writer
self.log.notice('Runners started.')
except:
msg = 'start() exception occurred' + \
'\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n'
self.log.exception(msg)
raise RuntimeError(msg)
def process(self, sess):
"""
Meta-train/test procedure for one-shot learning.
Single call runs single meta-test episode.
Args:
sess (tensorflow.Session): tf session obj.
"""
try:
# Copy from parameter server:
sess.run(self.train_aac.sync_pi)
sess.run(self.test_aac.sync_pi)
# self.log.warning('Init Sync ok.')
# Get data configuration,
# (want both data streams come from same trial,
# and trial type we got can be either from source or target domain);
# note: data_config counters get updated once per .process() call
train_data_config = self.train_aac.get_sample_config(
mode=1) # master env., samples trial
test_data_config = self.train_aac.get_sample_config(
mode=0) # slave env, catches up with same trial
# self.log.warning('train_data_config: {}'.format(train_data_config))
# self.log.warning('test_data_config: {}'.format(test_data_config))
# If this step data comes from source or target domain
# (i.e. is it either meta-optimised or true test episode):
is_target = train_data_config['trial_config']['sample_type']
done = False
# Collect initial meta-train trajectory rollout:
train_data = self.train_aac.get_data(
data_sample_config=train_data_config, force_new_episode=True)
feed_dict = self.train_aac.process_data(sess,
train_data,
is_train=True)
# self.log.warning('Init Train data ok.')
# Disable possibility of master data runner acquiring new trials,
# in case meta-train episode termintaes earlier than meta-test -
# we than need to get additional meta-train trajectories from exactly same distribution (trial):
train_data_config['trial_config']['get_new'] = 0
roll_num = 0
# Collect entire meta-test episode rollout by rollout:
while not done:
# self.log.warning('Roll #{}'.format(roll_num))
wirte_model_summary = \
self.local_steps % self.model_summary_freq == 0
# self.log.warning(
# 'Train data trial_num: {}'.format(
# np.asarray(train_data['on_policy'][0]['state']['metadata']['trial_num'])
# )
# )
# Paranoid checks against data sampling logic faults to prevent possible cheating:
train_trial_chksum = np.average(
train_data['on_policy'][0]['state']['metadata']
['trial_num'])
# Update pi_prime parameters wrt collected train data:
if wirte_model_summary:
fetches = [self.train_op, self.train_aac.model_summary_op]
else:
fetches = [self.train_op]
fetched = sess.run(fetches, feed_dict=feed_dict)
# self.log.warning('Train gradients ok.')
# Collect test rollout using updated pi_prime policy:
test_data = self.test_aac.get_data(
data_sample_config=test_data_config)
# If meta-test episode has just ended?
done = np.asarray(test_data['terminal']).any()
# self.log.warning(
# 'Test data trial_num: {}'.format(
# np.asarray(test_data['on_policy'][0]['state']['metadata']['trial_num'])
# )
# )
test_trial_chksum = np.average(
test_data['on_policy'][0]['state']['metadata']
['trial_num'])
# Ensure slave runner data consistency, can correct if episode just started:
if roll_num == 0 and train_trial_chksum != test_trial_chksum:
test_data = self.test_aac.get_data(
data_sample_config=test_data_config,
force_new_episode=True)
done = np.asarray(test_data['terminal']).any()
faulty_chksum = test_trial_chksum
test_trial_chksum = np.average(
test_data['on_policy'][0]['state']['metadata']
['trial_num'])
self.log.warning('Test trial corrected: {} -> {}'.format(
faulty_chksum, test_trial_chksum))
# self.log.warning(
# 'roll # {}: train_trial_chksum: {}, test_trial_chksum: {}'.
# format(roll_num, train_trial_chksum, test_trial_chksum)
# )
if train_trial_chksum != test_trial_chksum:
# Still got error? - highly probable algorithm logic fault. Issue warning.
msg = 'Train/test trials mismatch found!\nGot train trials: {},\nTest trials: {}'. \
format(
train_data['on_policy'][0]['state']['metadata']['trial_num'][0],
test_data['on_policy'][0]['state']['metadata']['trial_num'][0]
)
msg2 = 'Train data config: {}\n Test data config: {}'.format(
train_data_config, test_data_config)
self.log.warning(msg)
self.log.warning(msg2)
# Check episode type for consistency; if failed - another data sampling logic fault, warn:
try:
assert (np.asarray(test_data['on_policy'][0]['state']
['metadata']['type']) == 1).any()
assert (np.asarray(train_data['on_policy'][0]['state']
['metadata']['type']) == 0).any()
except AssertionError:
msg = 'Train/test episodes types mismatch found!\nGot train ep. type: {},\nTest ep.type: {}'. \
format(
train_data['on_policy'][0]['state']['metadata']['type'],
test_data['on_policy'][0]['state']['metadata']['type']
)
self.log.warning(msg)
# self.log.warning('Test data ok.')
if not is_target:
# Process test data and perform meta-optimisation step:
feed_dict.update(
self.test_aac.process_data(sess,
test_data,
is_train=True))
if wirte_model_summary:
meta_fetches = [
self.meta_train_op, self.test_aac.model_summary_op,
self.inc_step
]
else:
meta_fetches = [self.meta_train_op, self.inc_step]
meta_fetched = sess.run(meta_fetches, feed_dict=feed_dict)
# self.log.warning('Meta-gradients ok.')
else:
# True test, no updates sent to parameter server:
meta_fetched = [None, None]
# self.log.warning('Meta-opt. rollout ok.')
if wirte_model_summary:
meta_model_summary = meta_fetched[-2]
model_summary = fetched[-1]
else:
meta_model_summary = None
model_summary = None
# Next step housekeeping:
# copy from parameter server:
sess.run(self.train_aac.sync_pi)
sess.run(self.test_aac.sync_pi)
# self.log.warning('Sync ok.')
# Collect next train trajectory rollout:
train_data = self.train_aac.get_data(
data_sample_config=train_data_config)
feed_dict = self.train_aac.process_data(sess,
train_data,
is_train=True)
# self.log.warning('Train data ok.')
# Write down summaries:
self.test_aac.process_summary(sess, test_data,
meta_model_summary)
self.train_aac.process_summary(sess, train_data, model_summary)
self.local_steps += 1
roll_num += 1
except:
msg = 'process() exception occurred' + \
'\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n'
self.log.exception(msg)
raise RuntimeError(msg)
class NodesDiscovery:
def __init__(self):
handler = TimedRotatingFileHandler('../logs/nodes_discovery.log')
handler.push_application()
self.logger = Logger(name='nodes discovery', level='info')
self.node_hosts, self.nodes_port, self.file_sd_filename, self.nodes_file_backup_name, self.exclude_file, self.metric_filename, self.metric_store_path = self.get_conf()
self.nodes = {}
self.ips = {}
self.nodes_list = []
self.ips_list = []
@staticmethod
def get_conf():
##从配置文件获取配置
cp = ConfigParser.ConfigParser()
with codecs.open(os.path.join(BASE_DIR, './config/config.ini'), 'r', encoding='utf-8') as f:
cp.readfp(f)
node_hosts = eval(cp.get('nodes', 'node_hosts').strip())
nodes_port = int(cp.get('nodes', 'node_port').strip())
file_sd_filename = cp.get('file_ds', 'file_sd_filename').strip()
nodes_file_backup_name = cp.get('file_ds', 'nodes_file_backup_name').strip()
exclude_file = cp.get('file_ds', 'agent_exclude_file').strip()
metric_filename = cp.get('prome', 'metric_filename').strip()
metric_store_path = cp.get('prome', 'metric_store_path').strip()
return node_hosts, nodes_port, os.path.join(BASE_DIR, file_sd_filename), os.path.join(BASE_DIR, nodes_file_backup_name), os.path.join(BASE_DIR, exclude_file), os.path.join(BASE_DIR, metric_filename), metric_store_path
def node_scaner(self, ip_range):
port_scaner = nmap.PortScanner()
try:
# 调用扫描方法，参数指定扫描主机hosts，nmap扫描命令行参数arguments
port_scaner.scan(hosts=ip_range, arguments=' -v -sS -O -p {0} --excludefile {1}'.format(
str(self.nodes_port), self.exclude_file), sudo=True)
except Exception as e:
self.logger.error("Scan erro:" + str(e))
self.logger.info('nmap port scanner finished!')
for host in port_scaner.all_hosts(): # 遍历扫描主机
self.logger.debug(port_scaner[host])
if port_scaner[host]['status']['state'] == 'up':
host = port_scaner[host]['addresses']['ipv4']
try:
nodes_state = port_scaner[host]['tcp'][9100]['state']
except Exception as e:
self.logger.error('Error while get state of host %s: %s' % (host, e))
continue
else:
self.logger.debug("Host %s %s is %s" % (str(host), str(self.nodes_port), str(nodes_state)))
if nodes_state == 'open':
os_classes = port_scaner[host]['osmatch']
for os_class_info in os_classes:
for os_family_info in os_class_info['osclass']:
os_family = os_family_info['osfamily']
self.logger.debug("Host %s system is %s" % (str(host), os_family))
if os_family == 'Linux':
self.nodes_list.append(host + ':' + str(self.nodes_port))
self.ips_list.append(host)
self.logger.debug('debug info of nodes: [ip: %s]' % host)
break
break
self.logger.info('Finished for ips %s' % ip_range)
def host_to_file_sd(self, hosts_list):
hosts_conf = json.load(open(self.file_sd_filename, 'r'))
self.logger.debug('Latest nodes hosts: %s' % hosts_list)
for hosts in hosts_conf:
if hosts['labels']['job'] == 'nodes':
original_nodes_list = hosts['targets']
self.logger.debug('Nodes before update: %s' % original_nodes_list)
self.logger.debug('Nodes updated: %s' % hosts_list)
# ensure_nodes = list(set(original_nodes_list) - set(hosts_list))
# uninstall_nodes = []
# for nodes_host in ensure_nodes:
# port_scaner = nmap.PortScanner()
# try:
# # 调用扫描方法，参数指定扫描主机hosts，nmap扫描命令行参数arguments
# port_scaner.scan(hosts=nodes_host, arguments=' -v -sS -p {0},22'.format(
# str(self.nodes_port)), sudo=True)
# except Exception as e:
# self.logger.error("Scan erro:" + str(e))
# for host in port_scaner.all_hosts(): # 遍历扫描主机
# self.logger.debug(port_scaner[host])
# if port_scaner[host]['status']['state'] == 'up':
# host = port_scaner[host]['addresses']['ipv4']
# try:
# nodes_state = port_scaner[host]['tcp'][9100]['state']
# except Exception as e:
# self.logger.error('Error while get state of host %s: %s' % (host, e))
# continue
# else:
# self.logger.debug(
# "Host %s %s is %s" % (str(host), str(self.nodes_port), str(nodes_state)))
# if nodes_state != 'open':
# uninstall_nodes.append(host)
nodes_all = list(set(hosts_list).union(set(original_nodes_list)))
hosts['targets'] = nodes_all
self.logger.info('nodes hosts file updated!')
hosts_file = json.dumps(hosts_conf, indent=4, ensure_ascii=False, sort_keys=False, encoding='utf-8')
try:
with open(self.file_sd_filename, 'w') as f:
f.write(hosts_file)
except Exception as e:
self.logger.error('Write node_exporter info failed: %s' % e)
else:
shutil.copy(self.file_sd_filename, '../../file_ds/nodes.json')
def node_scan(self):
self.logger.debug('hosts groups from configure file: %s' % self.node_hosts)
for ip_range in self.node_hosts:
self.logger.debug('Scan ip range %s' % ip_range)
if ip_range:
self.node_scaner(ip_range)
if self.nodes_list:
self.host_to_file_sd(self.nodes_list)
self.logger.info('We finished here!')
def nodes_file_backup(self):
self.logger.info('Backup %s before start scanning' % self.file_sd_filename)
shutil.copy(self.file_sd_filename, self.nodes_file_backup_name)
def send_metrics(self):
nodes_metrics = NodesMetrics()
nodes_by_discovery, nodes_hosts = nodes_metrics.get_nodes_discovery()
#nodes_type, changed_hosts = nodes_metrics.get_lost_nodes_discovery()
current_nodes_by_discovery, lost_nodes_by_discovery, lost_nodes, new_nodes_by_discovery, new_nodes = nodes_metrics.get_lost_nodes_discovery()
#metrics_str = 'nodes_discovery_hosts{nodes_type="' + nodes_type + '",changed_hosts="' + changed_hosts + '"} ' + str(nodes_by_discovery)
lost_nodes_str = lost_nodes
new_nodes_str = new_nodes
all_nodes_metrics_str = 'nodes_discovery_hosts ' + str(current_nodes_by_discovery)
lost_nodes_metrics_str = 'nodes_discovery_lost_hosts ' + str(lost_nodes_by_discovery)
add_nodes_metrics_str = 'nodes_discovery_added_hosts ' + str(new_nodes_by_discovery)
self.logger.warning('Lost nodes by discovery: %s' % lost_nodes)
self.logger.info('New nodes by discovery: %s' % new_nodes)
try:
with open(self.metric_filename, 'w') as f:
#f.write(metrics_str)
f.write(all_nodes_metrics_str)
f.write('\n')
f.write(lost_nodes_metrics_str)
f.write('\n')
f.write(add_nodes_metrics_str)
f.write('\n')
except Exception as e:
self.logger.error('Write nodes discovery metrics failed: %s' % e)
else:
self.logger.info('Send nodes discovery metrics : %s %s %s' % (all_nodes_metrics_str, lost_nodes_metrics_str, add_nodes_metrics_str))
shutil.copy(self.metric_filename, os.path.join(self.metric_store_path, 'all_nodes.prom'))
#return lost_nodes_str
if lost_nodes_str:
lost_nodes = lost_nodes_str.split(' ')
nodes_discover.logger.warning('Nodes losts: %s' % lost_nodes)
nodes_discover.host_to_file_sd(lost_nodes)
from service.port import IPortUser, Port
from service.price import Price
from service.settings import HTMLExportSettings, SettingsProvider
from service.update import Update
import gui.fitCommands as cmd
pyfalog = Logger(__name__)
disableOverrideEditor = False
try:
from gui.propertyEditor import AttributeEditor
except ImportError as e:
AttributeEditor = None
pyfalog.warning("Error loading Attribute Editor: %s.\nAccess to Attribute Editor is disabled." % e.message)
disableOverrideEditor = True
pyfalog.debug("Done loading mainframe imports")
# dummy panel(no paint no erasebk)
class PFPanel(wx.Panel):
def __init__(self, parent):
wx.Panel.__init__(self, parent)
self.Bind(wx.EVT_PAINT, self.OnPaint)
self.Bind(wx.EVT_ERASE_BACKGROUND, self.OnBkErase)
def OnPaint(self, event):
event.Skip()
import os
import sys
from logbook import Logger
pyfalog = Logger(__name__)
# Load variable overrides specific to distribution type
try:
import configforced
except ImportError:
pyfalog.warning("Failed to import: configforced")
configforced = None
# Turns on debug mode
debug = False
# Defines if our saveddata will be in pyfa root or not
saveInRoot = False
# Version data
version = "1.34.0"
tag = "Stable"
expansionName = " Arms Race"
expansionVersion = "1.3"
evemonMinVersion = "4081"
pyfaPath = None
savePath = None
saveDB = None
gameDB = None
logPath = None
class BaseClient(RpcObject):
"""RPC客户端"""
@property
def name(self):
return self.__class__.__name__
def __init__(self,
reqAddress,
subAddress,
vaild=True,
logdir="../log",
logzmqhost=None,
tickerAddress=None):
"""Constructor"""
self.__initLog(logdir, logzmqhost)
super(BaseClient, self).__init__()
assert isinstance(vaild, bool)
self.vaild = vaild
self.logdir = logdir
# 使用 JSON 解包
self.useMsgpack()
# 循环为激活状态
self.__active = False # 客户端的工作状态
# 与服务器端的交互线程
self.__reqAddress = reqAddress
self.__subAddress = subAddress
self.__context = zmq.Context()
self.__socketREQ = self.__context.socket(zmq.REQ) # 请求发出socket
self.__socketSUB = self.__context.socket(zmq.SUB) # 广播订阅socket
# 工作线程相关，用于处理服务器推送的数据
self.__thread = threading.Thread(target=self.run) # 客户端的工作线程
# Ticker 数据订阅
self.__tickerAddress = tickerAddress
self.__socketSUBTicker = self.__context.socket(zmq.SUB) # 响应数据索引插入
self.__threadSUBTicker = threading.Thread(
name="{}TickSUB".format(self.name),
target=self.subscribeTicker) # 客户端的工作线程
# ----------------------------------------------------------------------
def __initLog(self, logdir, logzmqhost):
"""
在子进程中初始化客户端的句柄
:param logdir:
:param logzmqhost:
:return:
"""
logfile = os.path.join(logdir, '{}.log'.format(self.name))
log.initLog(logfile, logzmqhost)
# 使用服务名作为日志的来源名
self.log = Logger(self.name)
# ----------------------------------------------------------------------
def __getattr__(self, name):
"""实现远程调用功能"""
# 执行远程调用任务
def dorpc(*args, **kwargs):
# 生成请求
req = [name, args, kwargs]
# 序列化打包请求
reqb = self.pack(req)
# 发送请求并等待回应
self.__socketREQ.send(reqb)
repb = self.__socketREQ.recv()
# 序列化解包回应
rep = self.unpack(repb)
# 若正常则返回结果，调用失败则触发异常
if rep[0]:
return rep[1]
else:
raise RemoteException(rep[1])
return dorpc
# ----------------------------------------------------------------------
def start(self):
"""启动客户端"""
# 连接端口
self.__socketREQ.connect(self.__reqAddress)
self.__socketSUB.connect(self.__subAddress)
# 订阅 Ticker 数据
self.__socketSUBTicker.connect(self.__tickerAddress)
self.__socketSUBTicker.setsockopt(zmq.SUBSCRIBE, b'')
# 将服务器设为启动
self.__active = True
# 启动工作线程
if not self.__thread.isAlive():
self.__thread.start()
self.__activeInsert = True
if not self.__threadSUBTicker.isAlive():
self.__threadSUBTicker.start()
# ----------------------------------------------------------------------
def stop(self):
"""停止客户端"""
# 将客户端设为停止
self.__active = False
# 等待工作线程退出
if self.__thread.isAlive():
self.__thread.join()
if self.__threadInsert.isAlive():
self.__threadSUBTicker.join()
@log.stdout
@log.file
def run(self):
while self.__active:
# 接受广播
self.subRev()
# 发送数据
try:
self.reqSend()
except:
traceback.print_exc()
# ----------------------------------------------------------------------
def subRev(self):
"""客户端运行函数"""
# 使用poll来等待事件到达，等待1秒（1000毫秒）
if not self.__socketSUB.poll(1000):
return
# 从订阅socket收取广播数据
topic, datab = self.__socketSUB.recv_multipart()
# 序列化解包
data = self.unpack(datab)
self.log.debug("获得订阅消息{}".format(str(data)))
# 调用回调函数处理
self.callback(topic, data)
@log.stdout
def reqSend(self):
"""
发送数据
:return:
"""
# req = ['foo', ('argg'), {"kwargs": 1}]
#
# # 序列化
# reqb = self.pack(req)
#
# self.log.debug("往服务器端发送数据")
#
# self.__socketREQ.send(reqb)
#
# self.log.debug("等待数据返回")
# datab = self.__socketREQ.recv_json()
# # 序列化解包
# rep = self.unpack(datab)
#
# if rep[0]:
# return rep[1]
# else:
# raise RemoteException(rep[1])
def callback(self, topic, data):
"""回调函数，必须由用户实现"""
raise NotImplementedError
# ----------------------------------------------------------------------
def subscribeTopic(self, topic):
"""
订阅特定主题的广播数据
可以使用topic=''来订阅所有的主题
"""
self.__socketSUB.setsockopt(zmq.SUBSCRIBE, topic)
@classmethod
def process(cls, queue=None, *args, **kwargs):
"""客户端主程序入口"""
# 创建客户端
client = cls(*args, **kwargs)
client.subscribeTopic(b'')
client.start()
if queue is not None:
queue.get()
else:
while True:
if input("输入exit退出:") != "exit":
continue
if input("是否退出(yes/no):") == "yes":
break
client.stopServer()
@log.stdout
def subscribeTicker(self):
"""
插入数据
:return:
"""
self.log.warning("开始订阅数据")
while self.__active:
self._subscribeTicker()
@log.stdout
@log.file
def _subscribeTicker(self):
if not self.__socketSUBTicker.poll(1000):
# 等待广播
return
# 接受广播
ticker = self.__socketSUBTicker.recv_json()
self.log.debug("收到订阅的ticker: {}".format(str(ticker)))
class Worker(object):
redis_worker_namespace_prefix = 'rq:worker:'
redis_workers_keys = 'rq:workers'
@classmethod
def all(cls, connection=None):
"""Returns an iterable of all Workers.
"""
if connection is None:
connection = get_current_connection()
reported_working = connection.smembers(cls.redis_workers_keys)
workers = [cls.find_by_key(key, connection) for key in reported_working]
return compact(workers)
@classmethod
def find_by_key(cls, worker_key, connection=None):
"""Returns a Worker instance, based on the naming conventions for
naming the internal Redis keys. Can be used to reverse-lookup Workers
by their Redis keys.
"""
prefix = cls.redis_worker_namespace_prefix
name = worker_key[len(prefix):]
if not worker_key.startswith(prefix):
raise ValueError('Not a valid RQ worker key: %s' % (worker_key,))
if connection is None:
connection = get_current_connection()
if not connection.exists(worker_key):
return None
name = worker_key[len(prefix):]
worker = cls([], name)
queues = connection.hget(worker.key, 'queues')
worker._state = connection.hget(worker.key, 'state') or '?'
if queues:
worker.queues = map(Queue, queues.split(','))
return worker
def __init__(self, queues, name=None, rv_ttl=500, connection=None): # noqa
if connection is None:
connection = get_current_connection()
self.connection = connection
if isinstance(queues, Queue):
queues = [queues]
self._name = name
self.queues = queues
self.validate_queues()
self.rv_ttl = rv_ttl
self._state = 'starting'
self._is_horse = False
self._horse_pid = 0
self._stopped = False
self.log = Logger('worker')
self.failed_queue = get_failed_queue(connection=self.connection)
def validate_queues(self): # noqa
"""Sanity check for the given queues."""
if not iterable(self.queues):
raise ValueError('Argument queues not iterable.')
for queue in self.queues:
if not isinstance(queue, Queue):
raise NoQueueError('Give each worker at least one Queue.')
def queue_names(self):
"""Returns the queue names of this worker's queues."""
return map(lambda q: q.name, self.queues)
def queue_keys(self):
"""Returns the Redis keys representing this worker's queues."""
return map(lambda q: q.key, self.queues)
@property # noqa
def name(self):
"""Returns the name of the worker, under which it is registered to the
monitoring system.
By default, the name of the worker is constructed from the current
(short) host name and the current PID.
"""
if self._name is None:
hostname = socket.gethostname()
shortname, _, _ = hostname.partition('.')
self._name = '%s.%s' % (shortname, self.pid)
return self._name
@property
def key(self):
"""Returns the worker's Redis hash key."""
return self.redis_worker_namespace_prefix + self.name
@property
def pid(self):
"""The current process ID."""
return os.getpid()
@property
def horse_pid(self):
"""The horse's process ID. Only available in the worker. Will return
0 in the horse part of the fork.
"""
return self._horse_pid
@property
def is_horse(self):
"""Returns whether or not this is the worker or the work horse."""
return self._is_horse
def procline(self, message):
"""Changes the current procname for the process.
This can be used to make `ps -ef` output more readable.
"""
setprocname('rq: %s' % (message,))
def register_birth(self): # noqa
"""Registers its own birth."""
self.log.debug('Registering birth of worker %s' % (self.name,))
if self.connection.exists(self.key) and \
not self.connection.hexists(self.key, 'death'):
raise ValueError(
'There exists an active worker named \'%s\' '
'already.' % (self.name,))
key = self.key
now = time.time()
queues = ','.join(self.queue_names())
with self.connection.pipeline() as p:
p.delete(key)
p.hset(key, 'birth', now)
p.hset(key, 'queues', queues)
p.sadd(self.redis_workers_keys, key)
p.execute()
def register_death(self):
"""Registers its own death."""
self.log.debug('Registering death')
with self.connection.pipeline() as p:
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
p.srem(self.redis_workers_keys, self.key)
p.hset(self.key, 'death', time.time())
p.expire(self.key, 60)
p.execute()
def set_state(self, new_state):
self._state = new_state
self.connection.hset(self.key, 'state', new_state)
def get_state(self):
return self._state
state = property(get_state, set_state)
@property
def stopped(self):
return self._stopped
def _install_signal_handlers(self):
"""Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
def request_force_stop(signum, frame):
"""Terminates the application (cold shutdown).
"""
self.log.warning('Cold shut down.')
# Take down the horse with the worker
if self.horse_pid:
msg = 'Taking down horse %d with me.' % self.horse_pid
self.log.debug(msg)
try:
os.kill(self.horse_pid, signal.SIGKILL)
except OSError as e:
# ESRCH ("No such process") is fine with us
if e.errno != errno.ESRCH:
self.log.debug('Horse already down.')
raise
raise SystemExit()
def request_stop(signum, frame):
"""Stops the current worker loop but waits for child processes to
end gracefully (warm shutdown).
"""
self.log.debug('Got %s signal.' % signal_name(signum))
signal.signal(signal.SIGINT, request_force_stop)
signal.signal(signal.SIGTERM, request_force_stop)
if self.is_horse:
self.log.debug('Ignoring signal %s.' % signal_name(signum))
return
msg = 'Warm shut down. Press Ctrl+C again for a cold shutdown.'
self.log.warning(msg)
self._stopped = True
self.log.debug('Stopping after current horse is finished.')
signal.signal(signal.SIGINT, request_stop)
signal.signal(signal.SIGTERM, request_stop)
def work(self, burst=False): # noqa
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
self._install_signal_handlers()
did_perform_work = False
self.register_birth()
self.state = 'starting'
try:
while True:
if self.stopped:
self.log.info('Stopping on request.')
break
self.state = 'idle'
qnames = self.queue_names()
self.procline('Listening on %s' % ','.join(qnames))
self.log.info('')
self.log.info('*** Listening on %s...' % \
green(', '.join(qnames)))
wait_for_job = not burst
try:
result = Queue.dequeue_any(self.queues, wait_for_job, \
connection=self.connection)
if result is None:
break
except UnpickleError as e:
msg = '*** Ignoring unpickleable data on %s.' % \
green(e.queue.name)
self.log.warning(msg)
self.log.debug('Data follows:')
self.log.debug(e.raw_data)
self.log.debug('End of unreadable data.')
self.failed_queue.push_job_id(e.job_id)
continue
job, queue = result
self.log.info('%s: %s (%s)' % (green(queue.name),
blue(job.description), job.id))
self.state = 'busy'
self.fork_and_perform_job(job)
did_perform_work = True
finally:
if not self.is_horse:
self.register_death()
return did_perform_work
def fork_and_perform_job(self, job):
"""Spawns a work horse to perform the actual work and passes it a job.
The worker will wait for the work horse and make sure it executes
within the given timeout bounds, or will end the work horse with
SIGALRM.
"""
child_pid = Process()#
if child_pid == 0:
self.main_work_horse(job)
else:
self._horse_pid = child_pid
self.procline('Forked %d at %d' % (child_pid, time.time()))
while True:
try:
os.waitpid(child_pid, 0)
break
except OSError as e:
# In case we encountered an OSError due to EINTR (which is
# caused by a SIGINT or SIGTERM signal during
# os.waitpid()), we simply ignore it and enter the next
# iteration of the loop, waiting for the child to end. In
# any other case, this is some other unexpected OS error,
# which we don't want to catch, so we re-raise those ones.
if e.errno != errno.EINTR:
raise
def main_work_horse(self, job):
"""This is the entry point of the newly spawned work horse."""
# After fork()'ing, always assure we are generating random sequences
# that are different from the worker.
random.seed()
self._is_horse = True
self.log = Logger('horse')
success = self.perform_job(job)
# os._exit() is the way to exit from childs after a fork(), in
# constrast to the regular sys.exit()
os._exit(int(not success))
def perform_job(self, job):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.procline('Processing %s from %s since %s' % (
job.func_name,
job.origin, time.time()))
try:
with death_penalty_after(job.timeout or 180):
rv = job.perform()
except Exception as e:
fq = self.failed_queue
self.log.exception(red(str(e)))
self.log.warning('Moving job to %s queue.' % fq.name)
fq.quarantine(job, exc_info=traceback.format_exc())
return False
if rv is None:
self.log.info('Job OK')
else:
self.log.info('Job OK, result = %s' % (yellow(unicode(rv)),))
if rv is not None:
p = self.connection.pipeline()
p.hset(job.key, 'result', dumps(rv))
p.expire(job.key, self.rv_ttl)
p.execute()
else:
# Cleanup immediately
job.delete()
return True
class MonitordSocketClient:
lastAlert_time = 0.0
lastAlert_zvei = "0"
logger = 0
def __init__(self):
self.loadConfig()
def loadConfig(self):
with open("config.json") as config_file:
config = json.load(config_file)
if len(config["config"]) != 7:
raise Exception(
"Please check config file. Wrong count of config options")
if len(config["trigger"]) == 0:
raise Exception("Please check config file. No trigger defined")
self.config = config["config"]
self.triggers = config["trigger"]
def startNewDataThread(self, data):
x = threading.Thread(target=self.processData, args=(data, ))
x.start()
def processData(self, raw_data):
data = raw_data.split(":")
if data[0] == "100":
self.logger.info("Received welcome. MonitorD version: " + data[1])
elif data[0] == "300":
self.newAlert(data[3])
else:
self.logger.error("Received unknown command. Command: " +
repr(raw_data))
return
def newAlert(self, zvei):
if self.checkIfDoubleAlert(zvei):
return
self.logger.info("{}Received alarm.".format("[" + str(zvei) + "]: "))
subprocess.check_output(
'/usr/bin/zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k "receivedZVEI" -o "'
+ str(zvei) + '"',
shell=True)
trigger = self.checkIfAlertinFilter(zvei)
if not trigger:
self.logger.info(
"{}Received alarm not in filter. Stopping...".format(
"[" + str(zvei) + "]: "))
return
self.logger.info(
"{}!!!Received alarm in filter {} (Time: {}) Starting...".format(
"[" + str(zvei) + "]: ", trigger["name"],
str(datetime.now().time())))
if self.isTestAlert(trigger):
self.logger.info(
"{}Testalart time. Stopping...".format("[" + str(zvei) +
"]: "))
return
self.logger.info("{}Start alarm tasks...".format("[" + str(zvei) +
"]: "))
self.doAlertThings(zvei, trigger)
return
def checkIfDoubleAlert(self, zvei):
double_alert = False
if zvei == self.lastAlert_zvei:
if time.time() - self.lastAlert_time < 10:
double_alert = True
self.lastAlert_time = time.time()
self.lastAlert_zvei = zvei
return double_alert
def checkIfAlertinFilter(self, zvei):
for key, config in self.triggers:
if key == zvei:
return config
return False
def isTestAlert(self, trigger):
begin_time = dtime(trigger["hour_start"], trigger["minute_start"])
end_time = dtime(trigger["hour_end"], trigger["minute_end"])
check_time = datetime.now().time()
return datetime.today().weekday() == trigger[
"weekday"] and check_time >= begin_time and check_time <= end_time
def doAlertThings(self, zvei, trigger):
payload = trigger["request"]
for request_try in range(self.config["retries"]):
r = requests.get(self.config["url"], params=payload)
self.logger.debug(pprint.saferepr(r.url))
self.logger.debug(pprint.saferepr(r.status_code))
self.logger.debug(pprint.saferepr(r.content))
self.logger.debug(pprint.saferepr(r.headers))
if not r.status_code == requests.codes.ok:
self.logger.error("{}Failed to send alert. Try: {}/{}".format(
"[" + str(zvei) + "]: ", str(request_try + 1),
str(self.config["retries"])))
time.sleep(self.config["retry_delay"])
continue
else:
self.logger.info(
"{}Successfully send alert".format("[" + str(zvei) +
"]: "))
break
if trigger["local"]:
try:
self.logger.info("Starting light control")
debugOutput = subprocess.check_output(
'/usr/bin/python3 /opt/light/FFWLightControlTrigger.py',
shell=True)
self.logger.debug(pprint.saferepr(debugOutput))
except:
self.logger.error("Light control exception")
try:
self.logger.info("Starting TV control")
subprocess.check_output(
'/usr/bin/php /usr/local/bin/automate_tv 900 &',
shell=True)
except:
self.logger.info("TV control exception")
return
def reportConnectionError(self, count):
self.logger.error("Connection Error Count:" + count)
return
def main(self):
logbook.set_datetime_format("local")
StreamHandler(sys.stdout,
level=self.config["loglevel"]).push_application()
MonitoringFileHandler(
self.config["logpath"],
mode='a',
encoding='utf-8',
bubble=True,
level=self.config["loglevel"]).push_application()
self.logger = Logger('monitord_socket.py')
self.logger.info("starting...")
connectionErrorCount = 0
while True:
self.logger.info('Connection to {}:{}'.format(
self.config["host"], self.config["port"]))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((self.config["host"], self.config["port"]))
except:
connectionErrorCount += 1
self.reportConnectionError(connectionErrorCount)
time.sleep(10)
continue
connectionErrorCount = 0
try:
while True:
data = s.recv(1024)
if data:
self.logger.debug('Received => ' + repr(data))
self.startNewDataThread(data.decode('ascii'))
else:
self.logger.warning('Connection lost! Restarting...')
break
except KeyboardInterrupt:
print("\nShutting down...")
self.logger.warning('Shutting down...')
exit()
s.close()
time.sleep(5)
logger.error('lock HTTPError: %s, key %r, retries left %s' % (e, key, num_retries))
if e.code in (409, 417): # key already exists or paxos failed, retry
num_retries -= 1
time.sleep(random.random())
else:
break
except urllib2.URLError, e:
logger.error('lock URLError: %s, key %r, retries left %s' % (e, key, num_retries))
num_retries -= 1
time.sleep(random.random())
else:
# check if we lock right
with self._lock:
if key in self._keys:
if self._keys[key] == self.id:
logger.warning('################## key %s already locked by me' % (key, ))
else:
logger.warning('------------------ key %s already locked by worker %s' % (key, self._keys[key]))
else:
self._keys[key] = self.id
logger.debug('locked %r' % (key,))
return True
return False
def unlock(self, key):
num_retries = 10
while num_retries > 0:
try:
class MetaAAC_1_0():
"""
Meta-trainer class.
INITIAL: Implementation of MLDG algorithm tuned
for adaptation in dynamically changing environments
Papers:
Da Li et al.,
"Learning to Generalize: Meta-Learning for Domain Generalization"
https://arxiv.org/abs/1710.03463
Maruan Al-Shedivat et al.,
"Continuous Adaptation via Meta-Learning in Nonstationary and Competitive Environments"
https://arxiv.org/abs/1710.03641
"""
def __init__(self,
env,
task,
log_level,
aac_class_ref=SubAAC,
runner_config=None,
aac_lambda=1.0,
guided_lambda=1.0,
trial_source_target_cycle=(1, 0),
num_episodes_per_trial=1,
_aux_render_modes=('action_prob', 'value_fn', 'lstm_1_h',
'lstm_2_h'),
name='MetaAAC',
**kwargs):
try:
self.aac_class_ref = aac_class_ref
self.task = task
self.name = name
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(name, task), level=log_level)
# with tf.variable_scope(self.name):
if runner_config is None:
self.runner_config = {
'class_ref': BaseSynchroRunner,
'kwargs': {},
}
else:
self.runner_config = runner_config
self.env_list = env
assert isinstance(self.env_list, list) and len(self.env_list) == 2, \
'Expected pair of environments, got: {}'.format(self.env_list)
# Instantiate to sub-trainers: one for test and one for train environments:
self.runner_config['kwargs']['data_sample_config'] = {
'mode': 0
} # salve
self.runner_config['kwargs']['name'] = 'slave'
self.train_aac = aac_class_ref(
env=self.env_list[-1], # train data will be salve environment
task=self.task,
log_level=log_level,
runner_config=self.runner_config,
aac_lambda=aac_lambda,
guided_lambda=guided_lambda,
trial_source_target_cycle=trial_source_target_cycle,
num_episodes_per_trial=num_episodes_per_trial,
_use_target_policy=False,
_use_global_network=True,
_aux_render_modes=_aux_render_modes,
name=self.name + '_sub_Train',
**kwargs)
self.runner_config['kwargs']['data_sample_config'] = {
'mode': 1
} # master
self.runner_config['kwargs']['name'] = 'master'
self.test_aac = aac_class_ref(
env=self.env_list[0], # test data - master env.
task=self.task,
log_level=log_level,
runner_config=self.runner_config,
aac_lambda=aac_lambda,
guided_lambda=guided_lambda,
trial_source_target_cycle=trial_source_target_cycle,
num_episodes_per_trial=num_episodes_per_trial,
_use_target_policy=False,
_use_global_network=False,
global_step_op=self.train_aac.global_step,
global_episode_op=self.train_aac.global_episode,
inc_episode_op=self.train_aac.inc_episode,
_aux_render_modes=_aux_render_modes,
name=self.name + '_sub_Test',
**kwargs)
self.local_steps = self.train_aac.local_steps
self.model_summary_freq = self.train_aac.model_summary_freq
#self.model_summary_op = self.train_aac.model_summary_op
self._make_train_op()
self.test_aac.model_summary_op = tf.summary.merge(
[
self.test_aac.model_summary_op,
self._combine_meta_summaries()
],
name='meta_model_summary')
except:
msg = 'MetaAAC_0_1.__init()__ exception occurred' + \
'\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n'
self.log.exception(msg)
raise RuntimeError(msg)
def _make_train_op(self):
"""
Defines:
tensors holding training op graph for sub trainers and self;
"""
pi = self.train_aac.local_network
pi_prime = self.test_aac.local_network
self.test_aac.sync = self.test_aac.sync_pi = tf.group(
*[v1.assign(v2) for v1, v2 in zip(pi_prime.var_list, pi.var_list)])
self.global_step = self.train_aac.global_step
self.global_episode = self.train_aac.global_episode
self.test_aac.global_step = self.train_aac.global_step
self.test_aac.global_episode = self.train_aac.global_episode
self.test_aac.inc_episode = self.train_aac.inc_episode
self.train_aac.inc_episode = None
self.inc_step = self.train_aac.inc_step
# Meta-loss:
self.loss = 0.5 * self.train_aac.loss + 0.5 * self.test_aac.loss
# Clipped gradients:
self.train_aac.grads, _ = tf.clip_by_global_norm(
tf.gradients(self.train_aac.loss, pi.var_list), 40.0)
self.log.warning('self.train_aac.grads: {}'.format(
len(list(self.train_aac.grads))))
# self.test_aac.grads, _ = tf.clip_by_global_norm(
# tf.gradients(self.test_aac.loss, pi_prime.var_list),
# 40.0
# )
# Meta-gradient:
grads_i, _ = tf.clip_by_global_norm(
tf.gradients(self.train_aac.loss, pi.var_list), 40.0)
grads_i_next, _ = tf.clip_by_global_norm(
tf.gradients(self.test_aac.loss, pi_prime.var_list), 40.0)
self.grads = []
for g1, g2 in zip(grads_i, grads_i_next):
if g1 is not None and g2 is not None:
meta_g = 0.5 * g1 + 0.5 * g2
else:
meta_g = None
self.grads.append(meta_g)
#self.log.warning('self.grads_len: {}'.format(len(list(self.grads))))
# Gradients to update local copy of pi_prime (from train data):
train_grads_and_vars = list(
zip(self.train_aac.grads, pi_prime.var_list))
self.log.warning('train_grads_and_vars_len: {}'.format(
len(train_grads_and_vars)))
# Meta-gradients to be sent to parameter server:
meta_grads_and_vars = list(
zip(self.grads, self.train_aac.network.var_list))
self.log.warning('meta_grads_and_vars_len: {}'.format(
len(meta_grads_and_vars)))
# Set global_step increment equal to observation space batch size:
obs_space_keys = list(self.train_aac.local_network.on_state_in.keys())
assert 'external' in obs_space_keys, \
'Expected observation space to contain `external` mode, got: {}'.format(obs_space_keys)
self.train_aac.inc_step = self.train_aac.global_step.assign_add(
tf.shape(self.train_aac.local_network.on_state_in['external'])[0])
self.train_op = self.train_aac.optimizer.apply_gradients(
train_grads_and_vars)
# Optimizer for meta-update:
self.optimizer = tf.train.AdamOptimizer(
self.train_aac.train_learn_rate, epsilon=1e-5)
# TODO: own alpha-leran rate
self.meta_train_op = self.optimizer.apply_gradients(
meta_grads_and_vars)
self.log.debug('meta_train_op defined')
def _combine_meta_summaries(self):
meta_model_summaries = [
tf.summary.scalar("meta_grad_global_norm",
tf.global_norm(self.grads)),
tf.summary.scalar("total_meta_loss", self.loss),
]
return meta_model_summaries
def start(self, sess, summary_writer, **kwargs):
"""
Executes all initializing operations,
starts environment runner[s].
Supposed to be called by parent worker just before training loop starts.
Args:
sess: tf session object.
kwargs: not used by default.
"""
try:
# Copy weights from global to local:
sess.run(self.train_aac.sync_pi)
sess.run(self.test_aac.sync_pi)
# Start thread_runners:
self.test_aac._start_runners(sess, summary_writer) # master first
self.train_aac._start_runners(sess, summary_writer)
self.summary_writer = summary_writer
self.log.notice('Runners started.')
except:
msg = 'start() exception occurred' + \
'\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n'
self.log.exception(msg)
raise RuntimeError(msg)
def process(self, sess):
"""
Meta-train step.
Args:
sess (tensorflow.Session): tf session obj.
"""
try:
# Say `No` to redundant summaries:
wirte_model_summary = \
self.local_steps % self.model_summary_freq == 0
# Copy from parameter server:
sess.run(self.train_aac.sync_pi)
sess.run(self.test_aac.sync_pi)
#self.log.warning('Sync ok.')
# Collect train trajectory:
train_data = self.train_aac.get_data()
feed_dict = self.train_aac.process_data(sess,
train_data,
is_train=True)
#self.log.warning('Train data ok.')
# Update pi_prime parameters wrt collected data:
if wirte_model_summary:
fetches = [self.train_op, self.train_aac.model_summary_op]
else:
fetches = [self.train_op]
fetched = sess.run(fetches, feed_dict=feed_dict)
#self.log.warning('Train gradients ok.')
# Collect test trajectory wrt updated pi_prime parameters:
test_data = self.test_aac.get_data()
feed_dict.update(
self.test_aac.process_data(sess, test_data, is_train=True))
#self.log.warning('Test data ok.')
# Perform meta-update:
if wirte_model_summary:
meta_fetches = [
self.meta_train_op, self.test_aac.model_summary_op,
self.inc_step
]
else:
meta_fetches = [self.meta_train_op, self.inc_step]
meta_fetched = sess.run(meta_fetches, feed_dict=feed_dict)
#self.log.warning('Meta-gradients ok.')
if wirte_model_summary:
meta_model_summary = meta_fetched[-2]
model_summary = fetched[-1]
else:
meta_model_summary = None
model_summary = None
# Write down summaries:
self.test_aac.process_summary(sess, test_data, meta_model_summary)
self.train_aac.process_summary(sess, train_data, model_summary)
self.local_steps += 1
# TODO: ...what about sampling control?
except:
msg = 'process() exception occurred' + \
'\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n'
self.log.exception(msg)
raise RuntimeError(msg)
class Worker(object):
redis_worker_namespace_prefix = 'rq:worker:'
redis_workers_keys = 'rq:workers'
@classmethod
def all(cls, connection=None):
"""Returns an iterable of all Workers.
"""
if connection is None:
connection = get_current_connection()
reported_working = connection.smembers(cls.redis_workers_keys)
workers = [cls.find_by_key(key, connection) for key in
reported_working]
return compact(workers)
@classmethod
def find_by_key(cls, worker_key, connection=None):
"""Returns a Worker instance, based on the naming conventions for
naming the internal Redis keys. Can be used to reverse-lookup Workers
by their Redis keys.
"""
prefix = cls.redis_worker_namespace_prefix
name = worker_key[len(prefix):]
if not worker_key.startswith(prefix):
raise ValueError('Not a valid RQ worker key: %s' % (worker_key,))
if connection is None:
connection = get_current_connection()
if not connection.exists(worker_key):
return None
name = worker_key[len(prefix):]
worker = cls([], name, connection=connection)
queues = connection.hget(worker.key, 'queues')
worker._state = connection.hget(worker.key, 'state') or '?'
if queues:
worker.queues = [Queue(queue, connection=connection)
for queue in queues.split(',')]
return worker
def __init__(self, queues, name=None, default_result_ttl=DEFAULT_RESULT_TTL,
connection=None, exc_handler=None): # noqa
if connection is None:
connection = get_current_connection()
self.connection = connection
if isinstance(queues, Queue):
queues = [queues]
self._name = name
self.queues = queues
self.validate_queues()
self._exc_handlers = []
self.default_result_ttl = default_result_ttl
self._state = 'starting'
self._is_horse = False
self._horse_pid = 0
self._stopped = False
self.log = Logger('worker')
self.failed_queue = get_failed_queue(connection=self.connection)
# By default, push the "move-to-failed-queue" exception handler onto
# the stack
self.push_exc_handler(self.move_to_failed_queue)
if exc_handler is not None:
self.push_exc_handler(exc_handler)
def validate_queues(self): # noqa
"""Sanity check for the given queues."""
if not iterable(self.queues):
raise ValueError('Argument queues not iterable.')
for queue in self.queues:
if not isinstance(queue, Queue):
raise NoQueueError('Give each worker at least one Queue.')
def queue_names(self):
"""Returns the queue names of this worker's queues."""
return map(lambda q: q.name, self.queues)
def queue_keys(self):
"""Returns the Redis keys representing this worker's queues."""
return map(lambda q: q.key, self.queues)
@property # noqa
def name(self):
"""Returns the name of the worker, under which it is registered to the
monitoring system.
By default, the name of the worker is constructed from the current
(short) host name and the current PID.
"""
if self._name is None:
hostname = socket.gethostname()
shortname, _, _ = hostname.partition('.')
self._name = '%s.%s' % (shortname, self.pid)
return self._name
@property
def key(self):
"""Returns the worker's Redis hash key."""
return self.redis_worker_namespace_prefix + self.name
@property
def pid(self):
"""The current process ID."""
return os.getpid()
@property
def horse_pid(self):
"""The horse's process ID. Only available in the worker. Will return
0 in the horse part of the fork.
"""
return self._horse_pid
@property
def is_horse(self):
"""Returns whether or not this is the worker or the work horse."""
return self._is_horse
def procline(self, message):
"""Changes the current procname for the process.
This can be used to make `ps -ef` output more readable.
"""
setprocname('rq: %s' % (message,))
def register_birth(self): # noqa
"""Registers its own birth."""
self.log.debug('Registering birth of worker %s' % (self.name,))
if self.connection.exists(self.key) and \
not self.connection.hexists(self.key, 'death'):
raise ValueError(
'There exists an active worker named \'%s\' '
'already.' % (self.name,))
key = self.key
now = time.time()
queues = ','.join(self.queue_names())
with self.connection.pipeline() as p:
p.delete(key)
p.hset(key, 'birth', now)
p.hset(key, 'queues', queues)
p.sadd(self.redis_workers_keys, key)
p.execute()
def register_death(self):
"""Registers its own death."""
self.log.debug('Registering death')
with self.connection.pipeline() as p:
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
p.srem(self.redis_workers_keys, self.key)
p.hset(self.key, 'death', time.time())
p.expire(self.key, 60)
p.execute()
def set_state(self, new_state):
self._state = new_state
self.connection.hset(self.key, 'state', new_state)
def get_state(self):
return self._state
state = property(get_state, set_state)
@property
def stopped(self):
return self._stopped
def _install_signal_handlers(self):
"""Installs signal handlers for handling SIGINT and SIGTERM
gracefully.
"""
def request_force_stop(signum, frame):
"""Terminates the application (cold shutdown).
"""
self.log.warning('Cold shut down.')
# Take down the horse with the worker
if self.horse_pid:
msg = 'Taking down horse %d with me.' % self.horse_pid
self.log.debug(msg)
try:
os.kill(self.horse_pid, signal.SIGKILL)
except OSError as e:
# ESRCH ("No such process") is fine with us
if e.errno != errno.ESRCH:
self.log.debug('Horse already down.')
raise
raise SystemExit()
def request_stop(signum, frame):
"""Stops the current worker loop but waits for child processes to
end gracefully (warm shutdown).
"""
self.log.debug('Got signal %s.' % signal_name(signum))
signal.signal(signal.SIGINT, request_force_stop)
signal.signal(signal.SIGTERM, request_force_stop)
msg = 'Warm shut down requested.'
self.log.warning(msg)
# If shutdown is requested in the middle of a job, wait until
# finish before shutting down
if self.state == 'busy':
self._stopped = True
self.log.debug('Stopping after current horse is finished. '
'Press Ctrl+C again for a cold shutdown.')
else:
raise StopRequested()
signal.signal(signal.SIGINT, request_stop)
signal.signal(signal.SIGTERM, request_stop)
def work(self, burst=False): # noqa
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
self._install_signal_handlers()
did_perform_work = False
self.register_birth()
self.log.info('RQ worker started, version %s' % VERSION)
self.state = 'starting'
try:
while True:
if self.stopped:
self.log.info('Stopping on request.')
break
self.state = 'idle'
qnames = self.queue_names()
self.procline('Listening on %s' % ','.join(qnames))
self.log.info('')
self.log.info('*** Listening on %s...' % \
green(', '.join(qnames)))
wait_for_job = not burst
try:
result = Queue.dequeue_any(self.queues, wait_for_job, \
connection=self.connection)
if result is None:
break
except StopRequested:
break
except UnpickleError as e:
msg = '*** Ignoring unpickleable data on %s.' % \
green(e.queue.name)
self.log.warning(msg)
self.log.debug('Data follows:')
self.log.debug(e.raw_data)
self.log.debug('End of unreadable data.')
self.failed_queue.push_job_id(e.job_id)
continue
self.state = 'busy'
job, queue = result
# Use the public setter here, to immediately update Redis
job.status = Status.STARTED
self.log.info('%s: %s (%s)' % (green(queue.name),
blue(job.description), job.id))
self.fork_and_perform_job(job)
did_perform_work = True
finally:
if not self.is_horse:
self.register_death()
return did_perform_work
def fork_and_perform_job(self, job):
"""Spawns a work horse to perform the actual work and passes it a job.
The worker will wait for the work horse and make sure it executes
within the given timeout bounds, or will end the work horse with
SIGALRM.
"""
child_pid = os.fork()
if child_pid == 0:
self.main_work_horse(job)
else:
self._horse_pid = child_pid
self.procline('Forked %d at %d' % (child_pid, time.time()))
while True:
try:
os.waitpid(child_pid, 0)
break
except OSError as e:
# In case we encountered an OSError due to EINTR (which is
# caused by a SIGINT or SIGTERM signal during
# os.waitpid()), we simply ignore it and enter the next
# iteration of the loop, waiting for the child to end. In
# any other case, this is some other unexpected OS error,
# which we don't want to catch, so we re-raise those ones.
if e.errno != errno.EINTR:
raise
def main_work_horse(self, job):
"""This is the entry point of the newly spawned work horse."""
# After fork()'ing, always assure we are generating random sequences
# that are different from the worker.
random.seed()
# Always ignore Ctrl+C in the work horse, as it might abort the
# currently running job.
# The main worker catches the Ctrl+C and requests graceful shutdown
# after the current work is done. When cold shutdown is requested, it
# kills the current job anyway.
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
self._is_horse = True
self.log = Logger('horse')
success = self.perform_job(job)
# os._exit() is the way to exit from childs after a fork(), in
# constrast to the regular sys.exit()
os._exit(int(not success))
def perform_job(self, job):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.procline('Processing %s from %s since %s' % (
job.func_name,
job.origin, time.time()))
try:
with death_penalty_after(job.timeout or 180):
rv = job.perform()
# Pickle the result in the same try-except block since we need to
# use the same exc handling when pickling fails
pickled_rv = dumps(rv)
job._status = Status.FINISHED
except:
# Use the public setter here, to immediately update Redis
job.status = Status.FAILED
self.handle_exception(job, *sys.exc_info())
return False
if rv is None:
self.log.info('Job OK')
else:
self.log.info('Job OK, result = %s' % (yellow(unicode(rv)),))
# How long we persist the job result depends on the value of
# result_ttl:
# - If result_ttl is 0, cleanup the job immediately.
# - If it's a positive number, set the job to expire in X seconds.
# - If result_ttl is negative, don't set an expiry to it (persist
# forever)
result_ttl = self.default_result_ttl if job.result_ttl is None else job.result_ttl # noqa
if result_ttl == 0:
job.delete()
self.log.info('Result discarded immediately.')
else:
p = self.connection.pipeline()
p.hset(job.key, 'result', pickled_rv)
p.hset(job.key, 'status', job._status)
if result_ttl > 0:
p.expire(job.key, result_ttl)
self.log.info('Result is kept for %d seconds.' % result_ttl)
else:
self.log.warning('Result will never expire, clean up result key manually.')
p.execute()
return True
def handle_exception(self, job, *exc_info):
"""Walks the exception handler stack to delegate exception handling."""
exc_string = ''.join(
traceback.format_exception_only(*exc_info[:2]) +
traceback.format_exception(*exc_info))
self.log.error(exc_string)
for handler in reversed(self._exc_handlers):
self.log.debug('Invoking exception handler %s' % (handler,))
fallthrough = handler(job, *exc_info)
# Only handlers with explicit return values should disable further
# exc handling, so interpret a None return value as True.
if fallthrough is None:
fallthrough = True
if not fallthrough:
break
def move_to_failed_queue(self, job, *exc_info):
"""Default exception handler: move the job to the failed queue."""
exc_string = ''.join(traceback.format_exception(*exc_info))
self.log.warning('Moving job to %s queue.' % self.failed_queue.name)
self.failed_queue.quarantine(job, exc_info=exc_string)
def push_exc_handler(self, handler_func):
"""Pushes an exception handler onto the exc handler stack."""
self._exc_handlers.append(handler_func)
def pop_exc_handler(self):
"""Pops the latest exception handler off of the exc handler stack."""
return self._exc_handlers.pop()
class Memory(object):
"""
Replay memory with rebalanced replay based on reward value.
Note:
must be filled up before calling sampling methods.
"""
def __init__(self,
history_size,
max_sample_size,
priority_sample_size,
log_level=WARNING,
rollout_provider=None,
task=-1,
reward_threshold=0.1,
use_priority_sampling=False):
"""
Args:
history_size: number of experiences stored;
max_sample_size: maximum allowed sample size (e.g. off-policy rollout length);
priority_sample_size: sample size of priority_sample() method
log_level: int, logbook.level;
rollout_provider: callable returning list of Rollouts NOT USED
task: parent worker id;
reward_threshold: if |experience.reward| > reward_threshold: experience is saved as 'prioritized';
"""
self._history_size = history_size
self._frames = deque(maxlen=history_size)
self.reward_threshold = reward_threshold
self.max_sample_size = int(max_sample_size)
self.priority_sample_size = int(priority_sample_size)
self.rollout_provider = rollout_provider
self.task = task
self.log_level = log_level
StreamHandler(sys.stdout).push_application()
self.log = Logger('ReplayMemory_{}'.format(self.task),
level=self.log_level)
self.use_priority_sampling = use_priority_sampling
# Indices for non-priority frames:
self._zero_reward_indices = deque()
# Indices for priority frames:
self._non_zero_reward_indices = deque()
self._top_frame_index = 0
if use_priority_sampling:
self.sample_priority = self._sample_priority
else:
self.sample_priority = self._sample_dummy
def add(self, frame):
"""
Appends single experience frame to memory.
Args:
frame: dictionary of values.
"""
if frame['terminal'] and len(
self._frames) > 0 and self._frames[-1]['terminal']:
# Discard if terminal frame continues
self.log.warning(
"Memory_{}: Sequential terminal frame encountered. Discarded.".
format(self.task))
self.log.warning('{} -- {}'.format(self._frames[-1]['position'],
frame['position']))
return
frame_index = self._top_frame_index + len(self._frames)
was_full = self.is_full()
# Append frame:
self._frames.append(frame)
# Decide and append index:
if frame_index >= self.max_sample_size - 1:
if abs(frame['reward']) <= self.reward_threshold:
self._zero_reward_indices.append(frame_index)
else:
self._non_zero_reward_indices.append(frame_index)
if was_full:
# Decide from which index to discard:
self._top_frame_index += 1
cut_frame_index = self._top_frame_index + self.max_sample_size - 1
# Cut frame if its index is lower than cut_frame_index:
if len(self._zero_reward_indices) > 0 and \
self._zero_reward_indices[0] < cut_frame_index:
self._zero_reward_indices.popleft()
if len(self._non_zero_reward_indices) > 0 and \
self._non_zero_reward_indices[0] < cut_frame_index:
self._non_zero_reward_indices.popleft()
def add_rollout(self, rollout):
"""
Adds frames from given rollout to memory with respect to episode continuation.
Args:
rollout: `Rollout` instance.
"""
# Check if current rollout is direct extension of last stored frame sequence:
if len(self._frames) > 0 and not self._frames[-1]['terminal']:
# E.g. check if it is same local episode and successive frame order:
if self._frames[-1]['position']['episode'] == rollout['position']['episode'][0] and \
self._frames[-1]['position']['step'] + 1 == rollout['position']['step'][0]:
# Means it is ok to just extend previously stored episode
pass
else:
# Means part or tail of previously recorded episode is somehow lost,
# so we need to mark stored episode as 'ended':
self._frames[-1]['terminal'] = True
self.log.warning('{} changed to terminal'.format(
self._frames[-1]['position']))
# If we get a lot of such messages it is an indication something is going wrong.
# Add experiences one by one:
# TODO: pain-slow.
for i in range(len(rollout['terminal'])):
frame = rollout.get_frame(i)
self.add(frame)
def is_full(self):
return len(self._frames) >= self._history_size
def fill(self):
"""
Fills replay memory with initial experiences. NOT USED.
Supposed to be called by parent worker() just before training begins.
Args:
rollout_getter: callable, returning list of Rollouts.
"""
if self.rollout_provider is not None:
while not self.is_full():
for rollout in self.rollout_provider():
self.add_rollout(rollout)
self.log.info('Memory_{}: filled.'.format(self.task))
else:
raise AttributeError(
'Rollout_provider is None, can not fill memory.')
def sample_uniform(self, sequence_size):
"""
Uniformly samples sequence of successive frames of size `sequence_size` or less (~off-policy rollout).
Args:
sequence_size: maximum sample size.
Returns:
instance of Rollout of size <= sequence_size.
"""
start_pos = np.random.randint(0,
self._history_size - sequence_size - 1)
# Shift by one if hit terminal frame:
if self._frames[start_pos]['terminal']:
start_pos += 1 # assuming that there are no successive terminal frames.
sampled_rollout = Rollout()
for i in range(sequence_size):
frame = self._frames[start_pos + i]
sampled_rollout.add(frame)
if frame['terminal']:
break # it's ok to return less than `sequence_size` frames if `terminal` frame encountered.
return sampled_rollout
def _sample_priority(self,
size=None,
exact_size=False,
skewness=2,
sample_attempts=100):
"""
Implements rebalanced replay.
Samples sequence of successive frames from distribution skewed by means of reward of last sample frame.
Args:
size: sample size, must be <= self.max_sample_size;
exact_size: whether accept sample with size less than 'size'
or re-sample to get sample of exact size (used for reward prediction task);
skewness: int>=1, sampling probability denominator, such as probability of sampling sequence with
last frame having non-zero reward is: p[non_zero]=1/skewness;
sample_attempts: if exact_size=True, sets number of re-sampling attempts
to get sample of continuous experiences (no `Terminal` frames inside except last one);
if number is reached - sample returned 'as is'.
Returns:
instance of Rollout().
"""
if size is None:
size = self.priority_sample_size
if size > self.max_sample_size:
size = self.max_sample_size
# Toss skewed coin:
if np.random.randint(int(skewness)) == 0:
from_zero = False
else:
from_zero = True
if len(self._zero_reward_indices) == 0:
# zero rewards container was empty
from_zero = False
elif len(self._non_zero_reward_indices) == 0:
# non zero rewards container was empty
from_zero = True
# Try to sample sequence of given length from one episode.
# Take maximum of 'sample_attempts', if no luck
# (e.g too short episodes and/or too big sampling size) ->
# return inconsistent sample and issue warning.
check_sequence = True
for attempt in range(sample_attempts):
if from_zero:
index = np.random.randint(len(self._zero_reward_indices))
end_frame_index = self._zero_reward_indices[index]
else:
index = np.random.randint(len(self._non_zero_reward_indices))
end_frame_index = self._non_zero_reward_indices[index]
start_frame_index = end_frame_index - size + 1
raw_start_frame_index = start_frame_index - self._top_frame_index
sampled_rollout = Rollout()
is_full = True
if attempt == sample_attempts - 1:
check_sequence = False
self.log.warning(
'Memory_{}: failed to sample {} successive frames, sampled as is.'
.format(self.task, size))
for i in range(size - 1):
frame = self._frames[raw_start_frame_index + i]
sampled_rollout.add(frame)
if check_sequence:
if frame['terminal']:
if exact_size:
is_full = False
#print('attempt:', attempt)
#print('frame.terminal:', frame['terminal'])
break
# Last frame can be terminal anyway:
frame = self._frames[raw_start_frame_index + size - 1]
sampled_rollout.add(frame)
if is_full:
break
return sampled_rollout
@staticmethod
def _sample_dummy(**kwargs):
return None
return mongos_instance.client[database].list_collections()
else:
return mongos_instance.client[database].list_collection_names()
if __name__ == '__main__':
Cluster = Mongo(host=ConnString.ip,
port=ConnString.port,
username=ConnString.user,
password=ConnString.password)
if ConnString.auth_enabled:
Cluster.auth()
p.warning("###### Need to be enableSharding ######")
for db in get_unpartion_db(Cluster):
p.warning(db)
p.warning("###### Need to be shardCollection ######")
for i in get_database_list(Cluster):
if i in ["config", "admin", "test"]:
continue
for table in get_collection_list(Cluster, database=i):
full_table_name = i + '.' + table
if full_table_name not in \
[sharded_table['_id'] for sharded_table in get_sharded_collection(Cluster)]:
p.warning(full_table_name)
Cluster.close()
if hasattr(sys, 'frozen'):
pyfalog.info("Running in a frozen state.")
else:
pyfalog.info("Running in a thawed state.")
if not hasattr(sys, 'frozen') and wxversion:
try:
if options.force28 is True:
pyfalog.info("Selecting wx version: 2.8. (Forced)")
wxversion.select('2.8')
else:
pyfalog.info("Selecting wx versions: 3.0, 2.8")
wxversion.select(['3.0', '2.8'])
except:
pyfalog.warning("Unable to select wx version. Attempting to import wx without specifying the version.")
else:
if not wxversion:
pyfalog.warning("wxVersion not found. Attempting to import wx without specifying the version.")
try:
# noinspection PyPackageRequirements
import wx
except:
exit_message = "Cannot import wxPython. You can download wxPython (2.8+) from http://www.wxpython.org/"
raise PreCheckException(exit_message)
pyfalog.info("wxPython version: {0}.", str(wx.VERSION_STRING))
if sqlalchemy is None:
exit_message = "\nCannot find sqlalchemy.\nYou can download sqlalchemy (0.6+) from http://www.sqlalchemy.org/"
filename = os.path.join(file_path, 'ltcli-rotate.log')
rotating_file_handler = RotatingFileHandler(filename=filename,
level=file_level,
bubble=True,
max_size=each_size,
backup_count=backup_count)
rotating_file_handler.format_string = formatter['file']
rotating_file_handler.push_application()
logger.debug('start logging on file: {}'.format(filename))
else:
try:
os.mkdir(file_path)
except Exception:
logger.error("CreateDirError: {}".format(file_path))
msg = message.get('error_logging_in_file')
logger.warning(msg)
def set_level(level):
"""Change log level.
:param level: debug / info / warning / error
"""
level_list = ['debug', 'info', 'warning', 'error', 'warn']
if level not in level_list:
level_list.remove('warn')
logger.error("LogLevelError: '{}'. Select in {}".format(
level, level_list))
return
code = get_log_code(level)
stream_handler.level = code
class BaseDataGenerator():
"""
Base synthetic data provider class.
"""
def __init__(self,
episode_duration=None,
timeframe=1,
generator_fn=null_generator,
generator_params=None,
name='BaseSyntheticDataGenerator',
data_names=('default_asset', ),
global_time=None,
task=0,
log_level=WARNING,
_nested_class_ref=None,
_nested_params=None,
**kwargs):
"""
Args:
episode_duration: dict, duration of episode in days/hours/mins
generator_fn callabale, should return generated data as 1D np.array
generator_params dict,
timeframe: int, data periodicity in minutes
name: str
data_names: iterable of str
global_time: dict {y, m, d} to set custom global time (only for plotting)
task: int
log_level: logbook.Logger level
**kwargs:
"""
# Logging:
self.log_level = log_level
self.task = task
self.name = name
self.filename = self.name + '_sample'
self.data_names = data_names
self.data_name = self.data_names[0]
self.sample_instance = None
self.metadata = {
'sample_num': 0,
'type': None,
'parent_sample_type': None
}
self.data = None
self.data_stat = None
self.sample_num = 0
self.is_ready = False
if _nested_class_ref is None:
self.nested_class_ref = BaseDataGenerator
else:
self.nested_class_ref = _nested_class_ref
if _nested_params is None:
self.nested_params = dict(
episode_duration=episode_duration,
timeframe=timeframe,
generator_fn=generator_fn,
generator_params=generator_params,
name=name,
data_names=data_names,
task=task,
log_level=log_level,
_nested_class_ref=_nested_class_ref,
_nested_params=_nested_params,
)
else:
self.nested_params = _nested_params
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(self.name, self.task),
level=self.log_level)
# Default sample time duration:
if episode_duration is None:
self.episode_duration = dict(
days=0,
hours=23,
minutes=55,
)
else:
self.episode_duration = episode_duration
# Btfeed parsing setup:
self.timeframe = timeframe
self.names = ['open']
self.datetime = 0
self.open = 1
self.high = -1
self.low = -1
self.close = -1
self.volume = -1
self.openinterest = -1
# base data feed related:
self.params = {}
if global_time is None:
self.global_time = datetime.datetime(year=2018, month=1, day=1)
else:
self.global_time = datetime.datetime(**global_time)
self.global_timestamp = self.global_time.timestamp()
# Infer time indexes and sample number of records:
self.train_index = pd.timedelta_range(
start=datetime.timedelta(days=0, hours=0, minutes=0),
end=datetime.timedelta(**self.episode_duration),
freq='{}min'.format(self.timeframe))
self.test_index = pd.timedelta_range(
start=self.train_index[-1] +
datetime.timedelta(minutes=self.timeframe),
periods=len(self.train_index),
freq='{}min'.format(self.timeframe))
self.train_index += self.global_time
self.test_index += self.global_time
self.episode_num_records = len(self.train_index)
self.generator_fn = generator_fn
if generator_params is None:
self.generator_params = {}
else:
self.generator_params = generator_params
def set_logger(self, level=None, task=None):
"""
Sets logbook logger.
Args:
level: logbook.level, int
task: task id, int
"""
if task is not None:
self.task = task
if level is not None:
self.log = Logger('{}_{}'.format(self.name, self.task),
level=level)
def reset(self, **kwargs):
self.sample_num = 0
self.is_ready = True
def read_csv(self, **kwargs):
self.data = self.generate_data(self.generator_params)
def generate_data(self, generator_params, type=0):
"""
Generates data trajectory (episode)
Args:
generator_params: dict, data generating parmeters
type: 0 - generate train data | 1 - generate test data
Returns:
data as pandas dataframe
"""
assert type in [
0, 1
], 'Expected sample type be either 0 (train), or 1 (test) got: {}'.format(
type)
# Generate datapoints:
data_array = self.generator_fn(num_points=self.episode_num_records,
**generator_params)
assert len(data_array.shape) == 1 and data_array.shape[0] == self.episode_num_records,\
'Expected generated data to be 1D array of length {}, got data shape: {}'.format(
self.episode_num_records,
data_array.shape
)
negs = data_array[data_array < 0]
if negs.any():
self.log.warning(
' Set to zero {} negative generated values'.format(
negs.shape[0]))
data_array[data_array < 0] = 0.0
# Make dataframe:
if type:
index = self.test_index
else:
index = self.train_index
# data_dict = {name: data_array for name in self.names}
# data_dict['hh:mm:ss'] = index
df = pd.DataFrame(data={name: data_array
for name in self.names},
index=index)
# df = df.set_index('hh:mm:ss')
return df
def sample(self, get_new=True, sample_type=0, **kwargs):
"""
Samples continuous subset of data.
Args:
get_new (bool): not used;
sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets
respectively.
Returns:
Dataset instance with number of records ~ max_episode_len,
"""
try:
assert sample_type in [0, 1]
except AssertionError:
self.log.exception(
'Sampling attempt: expected sample type be in {}, got: {}'.\
format([0, 1], sample_type)
)
raise AssertionError
if self.metadata['type'] is not None:
if self.metadata['type'] != sample_type:
self.log.warning(
'Attempted to sample type {} given current sample type {}, overriden.'
.format(self.metadata['type'], sample_type))
sample_type = self.metadata['type']
# Generate data:
sampled_data = self.generate_data(self.generator_params,
type=sample_type)
self.sample_instance = self.nested_class_ref(**self.nested_params)
self.sample_instance.filename += '_{}'.format(self.sample_num)
self.log.info('New sample id: <{}>.'.format(
self.sample_instance.filename))
self.sample_instance.data = sampled_data
# Add_metadata
self.sample_instance.metadata['type'] = 'synthetic_data_sample'
self.sample_instance.metadata['first_row'] = 0
self.sample_instance.metadata['last_row'] = self.episode_num_records
self.sample_instance.metadata['type'] = sample_type
self.sample_instance.metadata['sample_num'] = self.sample_num
self.sample_instance.metadata['parent_sample_num'] = self.metadata[
'sample_num']
self.sample_instance.metadata['parent_sample_type'] = self.metadata[
'type']
self.sample_num += 1
return self.sample_instance
def describe(self):
"""
Returns summary dataset statistic as pandas dataframe:
- records count,
- data mean,
- data std dev,
- min value,
- 25% percentile,
- 50% percentile,
- 75% percentile,
- max value
for every data column.
"""
# Pretty straightforward, using standard pandas utility.
# The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again,
# thus avoiding passing big files to BT server:
flush_data = False
try:
assert not self.data.empty
pass
except (AssertionError, AttributeError) as e:
self.read_csv()
flush_data = True
self.data_stat = self.data.describe()
self.log.info('Data summary:\n{}'.format(self.data_stat.to_string()))
if flush_data:
self.data = None
self.log.info('Flushed data.')
return self.data_stat
def to_btfeed(self):
"""
Performs BTgymData-->bt.feed conversion.
Returns:
dict of type: {data_line_name: bt.datafeed instance}.
"""
try:
assert not self.data.empty
btfeed = btfeeds.PandasDirectData(dataname=self.data,
timeframe=self.timeframe,
datetime=self.datetime,
open=self.open,
high=self.high,
low=self.low,
close=self.close,
volume=self.volume,
openinterest=self.openinterest)
btfeed.numrecords = self.data.shape[0]
return {self.data_name: btfeed}
except (AssertionError, AttributeError) as e:
msg = 'Instance holds no data. Hint: forgot to call .read_csv()?'
self.log.error(msg)
raise AssertionError(msg)
def set_global_timestamp(self, timestamp):
pass
mpl_version = int(mpl.__version__[0]) or -1
if mpl_version >= 2:
mpl.use('wxagg')
mplImported = True
else:
mplImported = False
from matplotlib.patches import Patch
from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as Canvas
from matplotlib.figure import Figure
graphFrame_enabled = True
mplImported = True
except ImportError as e:
pyfalog.warning("Matplotlib failed to import. Likely missing or incompatible version.")
mpl_version = -1
Patch = mpl = Canvas = Figure = None
graphFrame_enabled = False
mplImported = False
except Exception:
# We can get exceptions deep within matplotlib. Catch those. See GH #1046
tb = traceback.format_exc()
pyfalog.critical("Exception when importing Matplotlib. Continuing without importing.")
pyfalog.critical(tb)
mpl_version = -1
Patch = mpl = Canvas = Figure = None
graphFrame_enabled = False
mplImported = False
class Plotter:
def __init__(self):
StreamHandler(stdout).push_application()
self.logger = Logger(self.__class__.__name__)
set_datetime_format("local")
try:
with open('config.yaml', 'r') as stream:
self.config = yaml.load(stream)
except yaml.YAMLError as e:
logger.critical(e)
exit()
except IOError as e:
logger.critical(e)
exit()
def plotYN(self):
if self.config['out']['plot'] == False:
self.logger.info('Output not analysed and plotted.')
exit()
def read_data(self):
if self.config['out']['redis']:
self.fr_redis()
elif self.config['out']['file']:
self.fr_file()
else:
self.logger.warning(
'Reading data from neither redis nor file. Aborting')
exit()
def fr_redis(self):
self.logger.info('Reading data from redis key.')
import redis
redis = redis.StrictRedis(self.config['sys_settings']['redis_ip'])
key = self.config['out']['prefix'] + ':' + self.config['in']['master']
if redis.exists(key):
self.data = pickle.loads(redis.get(key), encoding='bytes')
else:
self.logger.warning(
f'Redis key {key} does not exist; might have expired.')
self.fr_file()
def fr_file(self):
self.logger.info('Reading data from file.')
fname = self.concat()
with open(fname + '.pickle', 'rb') as handle:
self.data = pickle.load(handle, encoding='bytes')
def concat(self):
fname = self.config['in']['master'].split('/')[-1]
path = self.config['out']['path']
return path + fname
def detrend(self):
self.logger.info('Detrending data')
df = pd.DataFrame(self.data, columns=['raw'])
df['detrended'] = signal.detrend(df['raw']) + df['raw'].mean()
self.df = df
def normalise(self):
self.norm = self.df / self.df.mean()
def make_fig(self):
import cufflinks as cf #To be removed later; use plotly directly
title = self.concat()
self.fig = self.norm.iplot(asFigure=True,
title=title,
colorscale='polar',
xTitle='frame',
yTitle='Normalised intensity')
def plot(self):
import plotly
fname = self.concat() + '.html'
self.logger.info(f'Plot saved as {fname}')
plotly.offline.plot(self.fig, filename=fname)
def describe_data(self):
out = pd.DataFrame()
for key in self.df.columns:
tmp = self.df[key].describe()
tmp['CV%'] = self.df[key].std() / self.df[key].mean() * 100
out = out.append(tmp)
order = [
'count', 'mean', 'std', 'CV%', 'min', '25%', '50%', '75%', 'max'
]
out = out[order].T
fname = self.concat() + '.csv'
out.to_csv(fname)
self.logger.info(f'Descriptive stats: \n {out}')
self.logger.info(f'Data description saved as {fname}')
def run(self):
self.plotYN()
self.read_data()
self.detrend()
self.normalise()
self.describe_data()
self.make_fig()
self.plot()
class BTgymEnv(gym.Env):
"""
OpenAI Gym API shell for Backtrader backtesting/trading library.
"""
# Datafeed Server management:
data_master = True
data_network_address = 'tcp://127.0.0.1:' # using localhost.
data_port = 4999
data_server = None
data_server_pid = None
data_context = None
data_socket = None
data_server_response = None
# Dataset:
dataset = None # BTgymDataset instance.
dataset_stat = None
# Backtrader engine:
engine = None # bt.Cerbro subclass for server to execute.
# Strategy:
strategy = None # strategy to use if no <engine> class been passed.
# Server and network:
server = None # Server process.
context = None # ZMQ context.
socket = None # ZMQ socket, client side.
port = 5500 # network port to use.
network_address = 'tcp://127.0.0.1:' # using localhost.
ctrl_actions = ('_done', '_reset', '_stop', '_getstat', '_render'
) # server control messages.
server_response = None
# Connection timeout:
connect_timeout = 60 # server connection timeout in seconds.
#connect_timeout_step = 0.01 # time between retries in seconds.
# Rendering:
render_enabled = True
render_modes = [
'human',
'episode',
]
# `episode` - plotted episode results.
# `human` - raw_state observation in conventional human-readable format.
# <obs_space_key> - rendering of arbitrary state presented in observation_space with same key.
renderer = None # Rendering support.
rendered_rgb = dict() # Keep last rendered images for each mode.
# Logging and id:
log = None
log_level = None # logbook level: NOTICE, WARNING, INFO, DEBUG etc. or its integer equivalent;
verbose = 0 # verbosity mode, valid only if no `log_level` arg has been provided:
# 0 - WARNING, 1 - INFO, 2 - DEBUG.
task = 0
closed = True
def __init__(self, **kwargs):
"""
Keyword Args:
filename=None (str, list): csv data file.
**datafeed_args (any): any datafeed-related args, passed through to
default btgym.datafeed class.
dataset=None (btgym.datafeed): BTgymDataDomain instance,
overrides `filename` or any other datafeed-related args.
strategy=None (btgym.startegy): strategy to be used by `engine`, any subclass of
btgym.strategy.base.BTgymBaseStrateg
engine=None (bt.Cerebro): environment simulation engine, any bt.Cerebro subclass,
overrides `strategy` arg.
network_address=`tcp://127.0.0.1:` (str): BTGym_server address.
port=5500 (int): network port to use for server - API_shell communication.
data_master=True (bool): let this environment control over data_server;
data_network_address=`tcp://127.0.0.1:` (str): data_server address.
data_port=4999 (int): network port to use for server -- data_server communication.
connect_timeout=60 (int): server connection timeout in seconds.
render_enabled=True (bool): enable rendering for this environment;
render_modes=['human', 'episode'] (list): `episode` - plotted episode results;
`human` - raw_state observation.
**render_args (any): any render-related args, passed through to renderer class.
verbose=0 (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG}
log_level=None (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13},
overrides `verbose` arg;
log=None (logbook.Logger): external logbook logger,
overrides `log_level` and `verbose` args.
task=0 (int): environment id
Environment kwargs applying logic::
if <engine> kwarg is given:
do not use default engine and strategy parameters;
ignore <strategy> kwarg and all strategy and engine-related kwargs.
else (no <engine>):
use default engine parameters;
if any engine-related kwarg is given:
override corresponding default parameter;
if <strategy> is given:
do not use default strategy parameters;
if any strategy related kwarg is given:
override corresponding strategy parameter;
else (no <strategy>):
use default strategy parameters;
if any strategy related kwarg is given:
override corresponding strategy parameter;
if <dataset> kwarg is given:
do not use default dataset parameters;
ignore dataset related kwargs;
else (no <dataset>):
use default dataset parameters;
if any dataset related kwarg is given:
override corresponding dataset parameter;
If any <other> kwarg is given:
override corresponding default parameter.
"""
# Parameters and default values:
self.params = dict(
# Backtrader engine mandatory parameters:
engine=dict(
start_cash=10.0, # initial trading capital.
broker_commission=
0.001, # trade execution commission, default is 0.1% of operation value.
fixed_stake=10, # single trade stake is fixed type by def.
),
# Dataset mandatory parameters:
dataset=dict(filename=None, ),
strategy=dict(state_shape=dict(), ),
render=dict(),
)
p2 = dict( # IS HERE FOR REFERENCE ONLY
# Strategy related parameters:
# Observation state shape is dictionary of Gym spaces,
# at least should contain `raw_state` field.
# By convention first dimension of every Gym Box space is time embedding one;
# one can define any shape; should match env.observation_space.shape.
# observation space state min/max values,
# For `raw_state' - absolute min/max values from BTgymDataset will be used.
state_shape=dict(raw_state=spaces.Box(
shape=(10, 4), low=-100, high=100, dtype=np.float32)),
drawdown_call=
None, # episode maximum drawdown threshold, default is 90% of initial value.
portfolio_actions=None,
# agent actions,
# should consist with BTgymStrategy order execution logic;
# defaults are: 0 - 'do nothing', 1 - 'buy', 2 - 'sell', 3 - 'close position'.
skip_frame=None,
# Number of environment steps to skip before returning next response,
# e.g. if set to 10 -- agent will interact with environment every 10th episode step;
# Every other step agent's action is assumed to be 'hold'.
# Note: INFO part of environment response is a list of all skipped frame's info's,
# i.e. [info[-9], info[-8], ..., info[0].
)
# Update self attributes, remove used kwargs:
for key in dir(self):
if key in kwargs.keys():
setattr(self, key, kwargs.pop(key))
self.metadata = {'render.modes': self.render_modes}
# Logging and verbosity control:
if self.log is None:
StreamHandler(sys.stdout).push_application()
if self.log_level is None:
log_levels = [(0, NOTICE), (1, INFO), (2, DEBUG)]
self.log_level = WARNING
for key, value in log_levels:
if key == self.verbose:
self.log_level = value
self.log = Logger('BTgymAPIshell_{}'.format(self.task),
level=self.log_level)
# Network parameters:
self.network_address += str(self.port)
self.data_network_address += str(self.data_port)
# Set server rendering:
if self.render_enabled:
self.renderer = BTgymRendering(self.metadata['render.modes'],
log_level=self.log_level,
**kwargs)
else:
self.renderer = BTgymNullRendering()
self.log.info(
'Rendering disabled. Call to render() will return null-plug image.'
)
# Append logging:
self.renderer.log = self.log
# Update params -1: pull from renderer, remove used kwargs:
self.params['render'].update(self.renderer.params)
for key in self.params['render'].keys():
if key in kwargs.keys():
_ = kwargs.pop(key)
if self.data_master:
# DATASET preparation, only data_master executes this:
#
if self.dataset is not None:
# If BTgymDataset instance has been passed:
# do nothing.
msg = 'Custom Dataset class used.'
else:
# If no BTgymDataset has been passed,
# Make default dataset with given CSV file:
try:
os.path.isfile(str(self.params['dataset']['filename']))
except:
raise FileNotFoundError(
'Dataset source data file not specified/not found')
# Use kwargs to instantiate dataset:
self.dataset = BTgymDataset(**kwargs)
msg = 'Base Dataset class used.'
# Append logging:
self.dataset.set_logger(self.log_level, self.task)
# Update params -2: pull from dataset, remove used kwargs:
self.params['dataset'].update(self.dataset.params)
for key in self.params['dataset'].keys():
if key in kwargs.keys():
_ = kwargs.pop(key)
self.log.info(msg)
# Connect/Start data server (and get dataset statistic):
self.log.info('Connecting data_server...')
self._start_data_server()
self.log.info('...done.')
# ENGINE preparation:
# Update params -3: pull engine-related kwargs, remove used:
for key in self.params['engine'].keys():
if key in kwargs.keys():
self.params['engine'][key] = kwargs.pop(key)
if self.engine is not None:
# If full-blown bt.Cerebro() subclass has been passed:
# Update info:
msg = 'Custom Cerebro class used.'
self.strategy = msg
for key in self.params['engine'].keys():
self.params['engine'][key] = msg
# Note: either way, bt.observers.DrawDown observer [and logger] will be added to any BTgymStrategy instance
# by BTgymServer process at runtime.
else:
# Default configuration for Backtrader computational engine (Cerebro),
# if no bt.Cerebro() custom subclass has been passed,
# get base class Cerebro(), using kwargs on top of defaults:
self.engine = bt.Cerebro()
msg = 'Base Cerebro class used.'
# First, set STRATEGY configuration:
if self.strategy is not None:
# If custom strategy has been passed:
msg2 = 'Custom Strategy class used.'
else:
# Base class strategy :
self.strategy = BTgymBaseStrategy
msg2 = 'Base Strategy class used.'
# Add, using kwargs on top of defaults:
#self.log.debug('kwargs for strategy: {}'.format(kwargs))
strat_idx = self.engine.addstrategy(self.strategy, **kwargs)
msg += ' ' + msg2
# Second, set Cerebro-level configuration:
self.engine.broker.setcash(self.params['engine']['start_cash'])
self.engine.broker.setcommission(
self.params['engine']['broker_commission'])
self.engine.addsizer(bt.sizers.SizerFix,
stake=self.params['engine']['fixed_stake'])
self.log.info(msg)
# Define observation space shape, minimum / maximum values and agent action space.
# Retrieve values from configured engine or...
# ...Update params -4:
# Pull strategy defaults to environment params dict :
for t_key, t_value in self.engine.strats[0][0][0].params._gettuple():
self.params['strategy'][t_key] = t_value
# Update it with values from strategy 'passed-to params':
for key, value in self.engine.strats[0][0][2].items():
self.params['strategy'][key] = value
# ... Push it all back (don't ask):
for key, value in self.params['strategy'].items():
self.engine.strats[0][0][2][key] = value
# For 'raw_state' min/max values,
# the only way is to infer from raw Dataset price values (we already got those from data_server):
if 'raw_state' in self.params['strategy']['state_shape'].keys():
# Exclude 'volume' from columns we count:
self.dataset_columns.remove('volume')
#print(self.params['strategy'])
#print('self.engine.strats[0][0][2]:', self.engine.strats[0][0][2])
#print('self.engine.strats[0][0][0].params:', self.engine.strats[0][0][0].params._gettuple())
# Override with absolute price min and max values:
self.params['strategy']['state_shape']['raw_state'].low =\
self.engine.strats[0][0][2]['state_shape']['raw_state'].low =\
np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) +\
self.dataset_stat.loc['min', self.dataset_columns].min()
self.params['strategy']['state_shape']['raw_state'].high = \
self.engine.strats[0][0][2]['state_shape']['raw_state'].high = \
np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) + \
self.dataset_stat.loc['max', self.dataset_columns].max()
self.log.info(
'Inferring `state_raw` high/low values form dataset: {:.6f} / {:.6f}.'
.format(
self.dataset_stat.loc['min', self.dataset_columns].min(),
self.dataset_stat.loc['max', self.dataset_columns].max()))
# Set observation space shape from engine/strategy parameters:
self.observation_space = DictSpace(
self.params['strategy']['state_shape'])
self.log.debug('Obs. shape: {}'.format(self.observation_space.spaces))
#self.log.debug('Obs. min:\n{}\nmax:\n{}'.format(self.observation_space.low, self.observation_space.high))
# Set action space and corresponding server messages:
self.action_space = spaces.Discrete(
len(self.params['strategy']['portfolio_actions']))
self.server_actions = self.params['strategy']['portfolio_actions']
# Finally:
self.server_response = None
self.env_response = None
#if not self.data_master:
self._start_server()
self.closed = False
self.log.info('Environment is ready.')
def _seed(self, seed=None):
"""
Sets env. random seed.
Args:
seed: int or None
"""
np.random.seed(seed)
@staticmethod
def _comm_with_timeout(
socket,
message,
):
"""
Exchanges messages via socket, timeout sensitive.
Args:
socket: zmq connected socket to communicate via;
message: message to send;
Note:
socket zmq.RCVTIMEO and zmq.SNDTIMEO should be set to some finite number of milliseconds.
Returns:
dictionary:
`status`: communication result;
`message`: received message if status == `ok` or None;
`time`: remote side response time.
"""
response = dict(
status='ok',
message=None,
)
try:
socket.send_pyobj(message)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
response['status'] = 'send_failed_due_to_connect_timeout'
else:
response['status'] = 'send_failed_for_unknown_reason'
return response
start = time.time()
try:
response['message'] = socket.recv_pyobj()
response['time'] = time.time() - start
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
response['status'] = 'receive_failed_due_to_connect_timeout'
else:
response['status'] = 'receive_failed_for_unknown_reason'
return response
return response
def _start_server(self):
"""
Configures backtrader REQ/REP server instance and starts server process.
"""
# Ensure network resources:
# 1. Release client-side, if any:
if self.context:
self.context.destroy()
self.socket = None
# 2. Kill any process using server port:
cmd = "kill $( lsof -i:{} -t ) > /dev/null 2>&1".format(self.port)
os.system(cmd)
# Set up client channel:
self.context = zmq.Context()
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.RCVTIMEO, self.connect_timeout * 1000)
self.socket.setsockopt(zmq.SNDTIMEO, self.connect_timeout * 1000)
self.socket.connect(self.network_address)
# Configure and start server:
self.server = BTgymServer(
cerebro=self.engine,
render=self.renderer,
network_address=self.network_address,
data_network_address=self.data_network_address,
connect_timeout=self.connect_timeout,
log_level=self.log_level,
task=self.task,
)
self.server.daemon = False
self.server.start()
# Wait for server to startup:
time.sleep(1)
# Check connection:
self.log.info('Server started, pinging {} ...'.format(
self.network_address))
self.server_response = self._comm_with_timeout(
socket=self.socket, message={'ctrl': 'ping!'})
if self.server_response['status'] in 'ok':
self.log.info('Server seems ready with response: <{}>'.format(
self.server_response['message']))
else:
msg = 'Server unreachable with status: <{}>.'.format(
self.server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
self._closed = False
def _stop_server(self):
"""
Stops BT server process, releases network resources.
"""
if self.server:
if self._force_control_mode():
# In case server is running and client side is ok:
self.socket.send_pyobj({'ctrl': '_stop'})
self.server_response = self.socket.recv_pyobj()
else:
self.server.terminate()
self.server.join()
self.server_response = 'Server process terminated.'
self.log.info('{} Exit code: {}'.format(self.server_response,
self.server.exitcode))
# Release client-side, if any:
if self.context:
self.context.destroy()
self.socket = None
def _force_control_mode(self):
"""Puts BT server to control mode.
"""
# Check is there any faults with server process and connection?
network_error = [
(not self.server or not self.server.is_alive(),
'No running server found. Hint: forgot to call reset()?'),
(not self.context
or self.context.closed, 'No network connection found.'),
]
for (err, msg) in network_error:
if err:
self.log.info(msg)
self.server_response = msg
return False
# If everything works, insist to go 'control':
self.server_response = {}
attempt = 0
while 'ctrl' not in self.server_response:
self.socket.send_pyobj({'ctrl': '_done'})
self.server_response = self.socket.recv_pyobj()
attempt += 1
self.log.debug(
'FORCE CONTROL MODE attempt: {}.\nResponse: {}'.format(
attempt, self.server_response))
return True
def _assert_response(self, response):
"""
Simple watcher:
roughly checks if we really talking to environment (== episode is running).
Rises exception if response given is not as expected.
"""
try:
assert type(response) == tuple and len(response) == 4
except AssertionError:
msg = 'Unexpected environment response: {}\nHint: Forgot to call reset() or reset_data()?'.format(
response)
self.log.exception(msg)
raise AssertionError(msg)
self.log.debug('Response checker received:\n{}\nas type: {}'.format(
response, type(response)))
def _print_space(self, space, _tab=''):
"""
Parses observation space shape or response.
Args:
space: gym observation space or state.
Returns:
description as string.
"""
response = ''
if type(space) in [dict, OrderedDict]:
for key, value in space.items():
response += '\n{}{}:{}\n'.format(
_tab, key, self._print_space(value, ' '))
elif type(space) in [spaces.Dict, DictSpace]:
for s in space.spaces:
response += self._print_space(s, ' ')
elif type(space) in [tuple, list]:
for i in space:
response += self._print_space(i, ' ')
elif type(space) == np.ndarray:
response += '\n{}array of shape: {}, low: {}, high: {}'.format(
_tab, space.shape, space.min(), space.max())
else:
response += '\n{}{}, '.format(_tab, space)
try:
response += 'low: {}, high: {}'.format(space.low.min(),
space.high.max())
except (KeyError, AttributeError, ArithmeticError,
ValueError) as e:
pass
#response += '\n{}'.format(e)
return response
def reset(self, **kwargs):
"""
Implementation of OpenAI Gym env.reset method. Starts new episode. Episode data are sampled
according to data provider class logic, controlled via kwargs. Refer `BTgym_Server` and data provider
classes for details.
Args:
kwargs: any kwargs; this dictionary is passed through to BTgym_server side without any checks and
modifications; currently used for data sampling control;
Returns:
observation space state
Notes:
Current kwargs accepted is::
episode_config=dict(
get_new=True,
sample_type=0,
b_alpha=1,
b_beta=1
),
trial_config=dict(
get_new=True,
sample_type=0,
b_alpha=1,
b_beta=1
)
"""
# Data Server check:
if self.data_master:
if not self.data_server or not self.data_server.is_alive():
self.log.info('No running data_server found, starting...')
self._start_data_server()
# Domain dataset status check:
self.data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': '_get_info'})
if not self.data_server_response['message']['dataset_is_ready']:
self.log.info(
'Data domain `reset()` called prior to `reset_data()` with [possibly inconsistent] defaults.'
)
self.reset_data()
# Server process check:
if not self.server or not self.server.is_alive():
self.log.info('No running server found, starting...')
self._start_server()
if self._force_control_mode():
self.server_response = self._comm_with_timeout(socket=self.socket,
message={
'ctrl':
'_reset',
'kwargs': kwargs
})
# Get initial environment response:
self.env_response = self.step(0)
# Check (once) if it is really (o,r,d,i) tuple:
self._assert_response(self.env_response)
# Check (once) if state_space is as expected:
try:
#assert self.observation_space.contains(self.env_response[0])
pass
except (AssertionError, AttributeError) as e:
msg1 = self._print_space(self.observation_space.spaces)
msg2 = self._print_space(self.env_response[0])
msg3 = ''
for step_info in self.env_response[-1]:
msg3 += '{}\n'.format(step_info)
msg = ('\nState observation shape/range mismatch!\n' +
'Space set by env: \n{}\n' +
'Space returned by server: \n{}\n' +
'Full response:\n{}\n' + 'Reward: {}\n' + 'Done: {}\n' +
'Info:\n{}\n' +
'Hint: Wrong Strategy.get_state() parameters?').format(
msg1,
msg2,
self.env_response[0],
self.env_response[1],
self.env_response[2],
msg3,
)
self.log.exception(msg)
self._stop_server()
raise AssertionError(msg)
return self.env_response[0] #["raw_state"][np.newaxis]
else:
msg = 'Something went wrong. env.reset() can not get response from server.'
self.log.exception(msg)
raise ChildProcessError(msg)
def step(self, action):
"""
Implementation of OpenAI Gym env.step() method.
Makes a step in the environment.
Args:
action: int, number representing action from env.action_space
Returns:
tuple (Observation, Reward, Info, Done)
"""
# Are you in the list, ready to go and all that?
if self.action_space.contains(action)\
and not self._closed\
and (self.socket is not None)\
and not self.socket.closed:
pass
else:
msg = ('\nAt least one of these is true:\n' +
'Action error: (space is {}, action sent is {}): {}\n' +
'Environment closed: {}\n' +
'Network error [socket doesnt exists or closed]: {}\n' +
'Hint: forgot to call reset()?').format(
self.action_space,
action,
not self.action_space.contains(action),
self._closed,
not self.socket or self.socket.closed,
)
self.log.exception(msg)
raise AssertionError(msg)
# Send action to backtrader engine, receive environment response
env_response = self._comm_with_timeout(
socket=self.socket,
message={'action': self.server_actions[action]})
if not env_response['status'] in 'ok':
msg = '.step(): server unreachable with status: <{}>.'.format(
env_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# self.env_response = env_response ['message']
tempNew_state, tempReward, tempDone, tempInfo = env_response['message']
tempNew_state = tempNew_state["raw_state"][np.newaxis]
self.env_response = tempNew_state, tempReward, tempDone, tempInfo
return self.env_response
def close(self):
"""
Implementation of OpenAI Gym env.close method.
Puts BTgym server in Control Mode.
"""
self.log.debug('close.call()')
self._stop_server()
self._stop_data_server()
self.log.info('Environment closed.')
def get_stat(self):
"""
Returns last run episode statistics.
Note:
when invoked, forces running episode to terminate.
"""
if self._force_control_mode():
self.socket.send_pyobj({'ctrl': '_getstat'})
return self.socket.recv_pyobj()
else:
return self.server_response
def render(self, mode='other_mode', close=False):
"""
Implementation of OpenAI Gym env.render method.
Visualises current environment state.
Args:
`mode`: str, any of these::
`human` - current state observation as price lines;
`episode` - plotted results of last completed episode.
[other_key] - corresponding to any custom observation space key
"""
if close:
return None
if not self._closed\
and self.socket\
and not self.socket.closed:
pass
else:
msg = ('\nCan'
't get renderings.'
'\nAt least one of these is true:\n' +
'Environment closed: {}\n' +
'Network error [socket doesnt exists or closed]: {}\n' +
'Hint: forgot to call reset()?').format(
self._closed,
not self.socket or self.socket.closed,
)
self.log.warning(msg)
return None
if mode not in self.render_modes:
raise ValueError('Unexpected render mode {}'.format(mode))
self.socket.send_pyobj({'ctrl': '_render', 'mode': mode})
rgb_array_dict = self.socket.recv_pyobj()
self.rendered_rgb.update(rgb_array_dict)
return self.rendered_rgb[mode]
def _stop(self):
"""
Finishes current episode if any, does nothing otherwise. Leaves server running.
"""
if self._force_control_mode():
self.log.info('Episode stop forced.')
def _restart_server(self):
"""Restarts server.
"""
self._stop_server()
self._start_server()
self.log.info('Server restarted.')
def _start_data_server(self):
"""
For data_master environment:
- configures backtrader REQ/REP server instance and starts server process.
For others:
- establishes network connection to existing data_server.
"""
self.data_server = None
# Ensure network resources:
# 1. Release client-side, if any:
if self.data_context:
self.data_context.destroy()
self.data_socket = None
# Only data_master launches/stops data_server process:
if self.data_master:
# 2. Kill any process using server port:
cmd = "kill $( lsof -i:{} -t ) > /dev/null 2>&1".format(
self.data_port)
os.system(cmd)
# Configure and start server:
self.data_server = BTgymDataFeedServer(
dataset=self.dataset,
network_address=self.data_network_address,
log_level=self.log_level,
task=self.task)
self.data_server.daemon = False
self.data_server.start()
# Wait for server to startup
time.sleep(1)
# Set up client channel:
self.data_context = zmq.Context()
self.data_socket = self.data_context.socket(zmq.REQ)
self.data_socket.setsockopt(zmq.RCVTIMEO, self.connect_timeout * 1000)
self.data_socket.setsockopt(zmq.SNDTIMEO, self.connect_timeout * 1000)
self.data_socket.connect(self.data_network_address)
# Check connection:
self.log.debug('Pinging data_server at: {} ...'.format(
self.data_network_address))
self.data_server_response = self._comm_with_timeout(
socket=self.data_socket, message={'ctrl': 'ping!'})
if self.data_server_response['status'] in 'ok':
self.log.debug(
'Data_server seems ready with response: <{}>'.format(
self.data_server_response['message']))
else:
msg = 'Data_server unreachable with status: <{}>.'.\
format(self.data_server_response['status'])
self.log.error(msg)
raise ConnectionError(msg)
# Get info and statistic:
self.dataset_stat, self.dataset_columns, self.data_server_pid = self._get_dataset_info(
)
def _stop_data_server(self):
"""
For data_master:
- stops BT server process, releases network resources.
"""
if self.data_master:
if self.data_server is not None and self.data_server.is_alive():
# In case server is running and is ok:
self.data_socket.send_pyobj({'ctrl': '_stop'})
self.data_server_response = self.data_socket.recv_pyobj()
else:
self.data_server.terminate()
self.data_server.join()
self.data_server_response = 'Data_server process terminated.'
self.log.info('{} Exit code: {}'.format(self.data_server_response,
self.data_server.exitcode))
if self.data_context:
self.data_context.destroy()
self.data_socket = None
def _restart_data_server(self):
"""
Restarts data_server.
"""
if self.data_master:
self._stop_data_server()
self._start_data_server()
def _get_dataset_info(self):
"""
Retrieves dataset descriptive statistic.
"""
self.data_socket.send_pyobj({'ctrl': '_get_info'})
self.data_server_response = self.data_socket.recv_pyobj()
return self.data_server_response['dataset_stat'],\
self.data_server_response['dataset_columns'],\
self.data_server_response['pid']
def reset_data(self, **kwargs):
"""
Resets data provider class used, whatever it means for that class. Gets data_server ready to provide data.
Supposed to be called before first env.reset().
Note:
when invoked, forces running episode to terminate.
Args:
**kwargs: data provider class .reset() method specific.
"""
if self.closed:
self._start_server()
if self.data_master:
self._start_data_server()
self.closed = False
else:
_ = self._force_control_mode()
if self.data_master:
if self.data_server is None or not self.data_server.is_alive():
self._restart_data_server()
self.data_server_response = self._comm_with_timeout(
socket=self.data_socket,
message={
'ctrl': '_reset_data',
'kwargs': kwargs
})
if self.data_server_response['status'] in 'ok':
self.log.debug(
'Dataset seems ready with response: <{}>'.format(
self.data_server_response['message']))
else:
msg = 'Data_server unreachable with status: <{}>.'. \
format(self.data_server_response['status'])
self.log.error(msg)
raise SystemExit(msg)
else:
pass
class Schema(object):
url = None
version = None
homepage = None
deps = []
dirs_to_symlink = ['bin', 'sbin', 'etc', 'lib', 'include', 'var']
def __init__(self, env):
self.env = env.copy()
self.log = Logger()
if self.version is None:
self.version = _figureout_version(self.url)
if self.url.startswith('git://'):
self.deps.append('git')
self.retriver = self.git
else:
self.retriver = self.wget
self.env['prefix'] = os.path.join(
env['osbench_root'],
'workbench',
env['schema'],
self.version
)
self.env['version'] = self.version
# Methods to override
def install(self):
pass
def is_installed(self):
"""Right now we'll simple check if program
was built and installed into workbench."""
return os.path.exists(self.env['prefix'])
# Internals
def _get_source(self):
if not self.url:
return
self.log.info('Getting source code')
with higher_log_indent():
self.retriver(self.url)
patch_names = sorted(name for name in dir(self) if name.startswith('patch_'))
for name in patch_names:
filename = name.replace('_', '-') + '.diff'
with open(filename, 'w') as f:
f.write(self._get_patch(name))
self.call('patch -p1 < ' + filename)
def _get_patch(self, name):
data = getattr(self, name)
if len(data[:200].split('\n')) == 1:
# then probably this is a file or URL
patch_filename = os.path.join(
os.path.dirname(self.env['schema_filename']),
data
)
if os.path.exists(patch_filename):
data = open(patch_filename).read()
data = self._substitute_vars(data)
return data
def _get_environment_vars(self):
""" Returns dict with variables to set in shell and to replace in the templates.
"""
return dict(
OSBENCH_ROOT=self.env['osbench_root'],
OSBENCH_PREFIX=self.env['prefix'],
)
def _substitute_vars(self, text):
for name, value in self._get_environment_vars().items():
text = text.replace(name, value)
return text
def wget(self, url):
self.call('wget "{0}"'.format(self.url))
files = os.listdir('.')
assert len(files) == 1
filename = files[0]
# remove url params if they was added to the filename
stripped_filename = filename.split('?', 1)[0]
if stripped_filename.endswith('.gz') or \
stripped_filename.endswith('.tgz'):
tar_options = '-zxvf'
elif stripped_filename.endswith('.bz2'):
tar_options = '-jxvf'
else:
raise RuntimeError('Unknown archive format.')
self.call('tar {0} "{1}"'.format(tar_options, filename))
os.unlink(filename)
dirs = os.listdir('.')
assert len(dirs) == 1
os.chdir(dirs[0])
def git(self, url):
self.call("git clone '{0}'".format(url))
dirs = os.listdir('.')
assert len(dirs) == 1
os.chdir(dirs[0])
def _install_deps(self):
if self.deps:
self.log.info('Installing dependencies')
with higher_log_indent():
for dep in self.deps:
if dep.startswith('system.'):
self.call('sudo apt-get --yes install %s' % dep[7:])
def _install(self, interactive=False):
self.log.info('Installing "{schema}"'.format(**self.env))
with higher_log_indent():
self._install_deps()
root = tempfile.mkdtemp(prefix='diy-')
if not os.path.exists(self.env['prefix']):
os.makedirs(self.env['prefix'])
try:
os.chdir(root)
self._get_source()
if interactive:
self.log.info('Entering into the interactive mode')
with higher_log_indent():
shell = os.environ['SHELL']
self.call('git init')
self.call('git add -A')
for name, value in self._get_environment_vars().items():
os.environ[name] = value
self.call(shell, pass_output=True)
else:
self.log.info('Running schema\'s install method')
with higher_log_indent():
self.install()
self._link()
finally:
self.call('rm -fr "{0}"'.format(root))
def _uninstall(self):
"""Uninstalls the schema from the prefix.
It removes symlinks and deletes installed files from workbenches.
"""
self.log.info('Uninstalling "{schema}"'.format(**self.env))
with higher_log_indent():
self._unlink()
self._delete()
def _delete(self):
shutil.rmtree(self.env['prefix'])
def _join_path(self, *args):
return os.path.normpath(os.path.join(*args))
def _link(self):
self.log.info('Making symlinks')
with higher_log_indent():
for dir_name in self.dirs_to_symlink:
s_dir = self._join_path(self.env['prefix'], dir_name)
t_dir = self._join_path(self.env['osbench_root'], dir_name)
if not os.path.exists(t_dir):
self.log.debug('Creating directory "{0}"', t_dir)
os.makedirs(t_dir)
if os.path.exists(s_dir):
for root, dirs, files in os.walk(s_dir):
# making root, relative
root = os.path.relpath(root, s_dir)
for dir_name in dirs:
full_dir_name = self._join_path(
t_dir, root, dir_name
)
if not os.path.exists(full_dir_name):
self.log.debug('Creating directory "{0}"', full_dir_name)
os.makedirs(full_dir_name)
for filename in files:
source = self._join_path(s_dir, root, filename)
target = self._join_path(t_dir, root, filename)
if os.path.exists(target):
if os.path.islink(target):
if os.readlink(target) == source:
self.log.debug('Symlink {target} already exists', target=target)
continue
else:
self.log.warning('Unlinking file {target}, pointing to {source}',
target=target, source=source)
os.unlink(target)
else:
self.log.warning('File {target} already exists and it is not a link',
target=target)
if not os.path.exists(target):
self.log.debug('Creating symlink from "{source}" to {target}',
source=source, target=target)
os.symlink(source, target)
def _unlink(self):
self.log.info('Removing symlinks')
for ftype, name in self.get_files_to_unlink():
if ftype == 'file':
os.unlink(name)
else:
try:
os.rmdir(name)
except OSError:
pass
def get_files_to_unlink(self):
with higher_log_indent():
for dir_name in self.dirs_to_symlink:
s_dir = self._join_path(self.env['prefix'], dir_name)
t_dir = self._join_path(self.env['osbench_root'], dir_name)
if os.path.exists(s_dir):
for root, dirs, files in os.walk(s_dir, topdown=False):
# making root, relative
root = os.path.relpath(root, s_dir)
for filename in files:
source = self._join_path(s_dir, root, filename)
target = self._join_path(t_dir, root, filename)
if os.path.islink(target) and \
os.path.realpath(target) == source:
yield ('file', target)
for dir_name in dirs:
full_dir_name = self._join_path(
t_dir, root, dir_name
)
yield ('dir', full_dir_name)
# Utilities
def call(self, command, pass_output=False):
command = command.format(**self.env)
self.log.info('Running "{0}"'.format(command))
with higher_log_indent():
options = dict(
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True,
)
if pass_output:
del options['stdout']
del options['stderr']
proc = subprocess.Popen(
command,
**options
)
full_output = []
if not pass_output:
for line in proc.stdout:
line = line.decode('utf-8').strip(u'\n')
full_output.append(line)
self.log.debug(line)
return_code = proc.wait()
if return_code != 0:
for line in full_output:
self.log.error(line)
raise BadExitCode('subprogram exit with non zero exit code')
def makedirs(self, *dirs):
"""Makes dirs inside the prefix.
Use this command inside your `install` method.
"""
for d in dirs:
fullname = os.path.join(self.env['prefix'], d)
if not os.path.exists(fullname):
self.log.info('Creating directory "{0}".'.format(fullname))
os.makedirs(fullname)
def create_file_with_content(self, filename, content, mode=None):
"""Creates a file inside 'prefix'.
Use this command inside your `install` method.
Note: Source and target directory should exists.
Warning: if there is some file already, it will be overwritten.
"""
filename = os.path.join(self.env['prefix'], filename)
self.log.info('Creating file "{0}"'.format(filename))
with open(filename, 'w') as f:
f.write(self._substitute_vars(content))
if mode is not None:
self.call('chmod "{0}" "{1}"'.format(mode, filename))
def copy_file(self, from_filename, to_filename, mode=None):
"""Copies file, to a directory inside 'prefix'.
from_filename could be relative to the current directory, or
use variables to be expanded to self.env.
Use this command inside your `install` method.
Note: Source and target directory should exists.
Warning: if there is some file already, it will be overwritten.
"""
with open(from_filename.format(**self.env), 'r') as f:
self.create_file_with_content(to_filename, f.read(), mode=mode)
mpl_version = int(mpl.__version__[0]) or -1
if mpl_version >= 2:
mpl.use('wxagg')
mplImported = True
else:
mplImported = False
from matplotlib.patches import Patch
from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as Canvas
from matplotlib.figure import Figure
graphFrame_enabled = True
mplImported = True
except ImportError as e:
pyfalog.warning(
"Matplotlib failed to import. Likely missing or incompatible version."
)
mpl_version = -1
Patch = mpl = Canvas = Figure = None
graphFrame_enabled = False
mplImported = False
except Exception:
# We can get exceptions deep within matplotlib. Catch those. See GH #1046
tb = traceback.format_exc()
pyfalog.critical(
"Exception when importing Matplotlib. Continuing without importing.")
pyfalog.critical(tb)
mpl_version = -1
Patch = mpl = Canvas = Figure = None
graphFrame_enabled = False
mplImported = False
import wx
from logbook import CRITICAL, DEBUG, ERROR, FingersCrossedHandler, INFO, Logger, NestedSetup, NullHandler, \
StreamHandler, TimedRotatingFileHandler, WARNING
import hashlib
from eos.const import FittingSlot
from cryptography.fernet import Fernet
pyfalog = Logger(__name__)
# Load variable overrides specific to distribution type
try:
import configforced
except ImportError:
pyfalog.warning("Failed to import: configforced")
configforced = None
# Turns on debug mode
debug = False
# Defines if our saveddata will be in pyfa root or not
saveInRoot = False
evemonMinVersion = "4081"
minItemSearchLength = 3
pyfaPath = None
savePath = None
saveDB = None
expire_on_commit=False)()
pyfalog.debug('Getting gamedata version')
# This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new
# game db because we haven't reached gamedata_meta.create_all()
try:
config.gamedata_version = gamedata_session.execute(
"SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'"
).fetchone()[0]
config.gamedata_date = gamedata_session.execute(
"SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'dump_time'"
).fetchone()[0]
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
pyfalog.warning("Missing gamedata version.")
pyfalog.critical(e)
config.gamedata_version = None
config.gamedata_date = None
pyfalog.debug('Initializing saveddata')
saveddata_connectionstring = config.saveddata_connectionstring
if saveddata_connectionstring is not None:
if callable(saveddata_connectionstring):
saveddata_engine = create_engine(creator=saveddata_connectionstring,
echo=config.debug)
else:
saveddata_engine = create_engine(saveddata_connectionstring,
echo=config.debug)
saveddata_meta = MetaData()
num_retries -= 1
time.sleep(random.random())
else:
break
except urllib2.URLError, e:
logger.error('lock URLError: %s, key %r, retries left %s' %
(e, key, num_retries))
num_retries -= 1
time.sleep(random.random())
else:
# check if we lock right
with self._lock:
if key in self._keys:
if self._keys[key] == self.id:
logger.warning(
'################## key %s already locked by me'
% (key, ))
else:
logger.warning(
'------------------ key %s already locked by worker %s'
% (key, self._keys[key]))
else:
self._keys[key] = self.id
logger.debug('locked %r' % (key, ))
return True
return False
def unlock(self, key):
num_retries = 10
class BaseDataGenerator:
"""
Base synthetic data provider class.
"""
def __init__(self,
episode_duration=None,
timeframe=1,
generator_fn=base_random_generator_fn,
generator_parameters_fn=base_generator_parameters_fn,
generator_parameters_config=None,
spread_generator_fn=None,
spread_generator_parameters=None,
name='BaseSyntheticDataGenerator',
data_names=('default_asset', ),
parsing_params=None,
target_period=-1,
global_time=None,
task=0,
log_level=WARNING,
_nested_class_ref=None,
_nested_params=None,
**kwargs):
"""
Args:
episode_duration: dict, duration of episode in days/hours/mins
generator_fn callabale, should return generated data as 1D np.array
generator_parameters_fn: callable, should return dictionary of generator_fn kwargs
generator_parameters_config: dict, generator_parameters_fn args
spread_generator_fn: callable, should return values of spread to form {High, Low}
spread_generator_parameters: dict, spread_generator_fn args
timeframe: int, data periodicity in minutes
name: str
data_names: iterable of str
target_period: int or dict, if set to -1 - disables `test` sampling
global_time: dict {y, m, d} to set custom global time (only for plotting)
task: int
log_level: logbook.Logger level
**kwargs:
"""
# Logging:
self.log_level = log_level
self.task = task
self.name = name
self.filename = self.name + '_sample'
self.target_period = target_period
self.data_names = data_names
self.data_name = self.data_names[0]
self.sample_instance = None
self.metadata = {
'sample_num': 0,
'type': None,
'parent_sample_type': None
}
self.data = None
self.data_stat = None
self.sample_num = 0
self.is_ready = False
if _nested_class_ref is None:
self.nested_class_ref = BaseDataGenerator
else:
self.nested_class_ref = _nested_class_ref
if _nested_params is None:
self.nested_params = dict(
episode_duration=episode_duration,
timeframe=timeframe,
generator_fn=generator_fn,
generator_parameters_fn=generator_parameters_fn,
generator_parameters_config=generator_parameters_config,
name=name,
data_names=data_names,
task=task,
log_level=log_level,
_nested_class_ref=_nested_class_ref,
_nested_params=_nested_params,
)
else:
self.nested_params = _nested_params
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_{}'.format(self.name, self.task),
level=self.log_level)
# Default sample time duration:
if episode_duration is None:
self.episode_duration = dict(
days=0,
hours=23,
minutes=55,
)
else:
self.episode_duration = episode_duration
# Btfeed parsing setup:
if parsing_params is None:
self.parsing_params = dict(names=['ask', 'bid', 'mid'],
datetime=0,
timeframe=1,
open='mid',
high='ask',
low='bid',
close='mid',
volume=-1,
openinterest=-1)
else:
self.parsing_params = parsing_params
self.columns_map = {
'open': 'mean',
'high': 'maximum',
'low': 'minimum',
'close': 'mean',
'bid': 'minimum',
'ask': 'maximum',
'mid': 'mean',
'volume': 'nothing',
}
self.nested_params['parsing_params'] = self.parsing_params
for key, value in self.parsing_params.items():
setattr(self, key, value)
# base data feed related:
self.params = {}
if global_time is None:
self.global_time = datetime.datetime(year=2018, month=1, day=1)
else:
self.global_time = datetime.datetime(**global_time)
self.global_timestamp = self.global_time.timestamp()
# Infer time indexes and sample number of records:
self.train_index = pd.timedelta_range(
start=datetime.timedelta(days=0, hours=0, minutes=0),
end=datetime.timedelta(**self.episode_duration),
freq='{}min'.format(self.timeframe))
self.test_index = pd.timedelta_range(
start=self.train_index[-1] +
datetime.timedelta(minutes=self.timeframe),
periods=len(self.train_index),
freq='{}min'.format(self.timeframe))
self.train_index += self.global_time
self.test_index += self.global_time
self.episode_num_records = len(self.train_index)
self.generator_fn = generator_fn
self.generator_parameters_fn = generator_parameters_fn
if generator_parameters_config is not None:
self.generator_parameters_config = generator_parameters_config
else:
self.generator_parameters_config = {}
self.spread_generator_fn = spread_generator_fn
if spread_generator_parameters is not None:
self.spread_generator_parameters = spread_generator_parameters
else:
self.spread_generator_parameters = {}
def set_logger(self, level=None, task=None):
"""
Sets logbook logger.
Args:
level: logbook.level, int
task: task id, int
"""
if task is not None:
self.task = task
if level is not None:
self.log = Logger('{}_{}'.format(self.name, self.task),
level=level)
def reset(self, **kwargs):
self.read_csv()
self.sample_num = 0
self.is_ready = True
def read_csv(self, **kwargs):
self.data = self.generate_data(
self.generator_parameters_fn(**self.generator_parameters_config))
def generate_data(self, generator_params, sample_type=0):
"""
Generates data trajectory, performs base consistency checks.
Args:
generator_params: dict, data_generating_function parameters
sample_type: 0 - generate train data | 1 - generate test data
Returns:
data as pandas dataframe
"""
assert sample_type in [0, 1],\
'Expected sample type be either 0 (train), or 1 (test) got: {}'.format(sample_type)
# Generate data points:
data_array = self.generator_fn(num_points=self.episode_num_records,
**generator_params)
assert len(data_array.shape) == 1 and data_array.shape[0] == self.episode_num_records,\
'Expected generated data to be 1D array of length {}, got data shape: {}'.format(
self.episode_num_records,
data_array.shape
)
if self.spread_generator_fn is not None:
spread_array = self.spread_generator_fn(
num_points=self.episode_num_records,
**self.spread_generator_parameters)
assert len(spread_array.shape) == 1 and spread_array.shape[0] == self.episode_num_records, \
'Expected generated spread to be 1D array of length {}, got data shape: {}'.format(
self.episode_num_records,
spread_array.shape
)
else:
spread_array = np.zeros(self.episode_num_records)
data_dict = {
'mean': data_array,
'maximum': data_array + .5 * spread_array,
'minimum': data_array - .5 * spread_array,
'nothing': data_array * 0.0,
}
# negs = data_dict['minimum'] < 0
# if negs.any():
# self.log.warning('{} negative generated values detected'.format(negs.shape[0]))
# Make dataframe:
if sample_type:
index = self.test_index
else:
index = self.train_index
# Map dictionary of data to dataframe columns:
df = pd.DataFrame(data={
name: data_dict[self.columns_map[name]]
for name in self.names
},
index=index)
# df = df.set_index('hh:mm:ss')
return df
def sample(self, get_new=True, sample_type=0, **kwargs):
"""
Samples continuous subset of data.
Args:
get_new (bool): not used;
sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets
respectively.
Returns:
Dataset instance with number of records ~ max_episode_len.
"""
try:
assert sample_type in [0, 1]
except AssertionError:
msg = 'Sampling attempt: expected sample type be in {}, got: {}'.format(
[0, 1], sample_type)
self.log.error(msg)
raise ValueError(msg)
if self.target_period == -1 and sample_type:
msg = 'Attempt to sample type {} given disabled target_period'.format(
sample_type)
self.log.error(msg)
raise ValueError(msg)
if self.metadata['type'] is not None:
if self.metadata['type'] != sample_type:
self.log.warning(
'Attempt to sample type {} given current sample type {}, overriden.'
.format(sample_type, self.metadata['type']))
sample_type = self.metadata['type']
# Get sample:
self.sample_instance = self.sample_synthetic(sample_type)
self.sample_instance.metadata['type'] = sample_type
self.sample_instance.metadata['sample_num'] = self.sample_num
self.sample_instance.metadata['parent_sample_num'] = self.metadata[
'sample_num']
self.sample_instance.metadata['parent_sample_type'] = self.metadata[
'type']
self.sample_num += 1
return self.sample_instance
def sample_synthetic(self, sample_type=0):
"""
Get data_generator instance containing synthetic data.
Args:
sample_type (int or bool): 0 (train) or 1 (test) - get sample with train or test time periods
respectively.
Returns:
nested_class_ref instance
"""
# Generate data:
generator_params = self.generator_parameters_fn(
**self.generator_parameters_config)
data = self.generate_data(generator_params, sample_type=sample_type)
# Make data_class instance:
sample_instance = self.nested_class_ref(**self.nested_params)
sample_instance.filename += '_{}'.format(self.sample_num)
self.log.info('New sample id: <{}>.'.format(sample_instance.filename))
# Add data and metadata:
sample_instance.data = data
sample_instance.metadata['generator'] = generator_params
sample_instance.metadata['first_row'] = 0
sample_instance.metadata['last_row'] = self.episode_num_records
return sample_instance
def describe(self):
"""
Returns summary dataset statistic as pandas dataframe:
- records count,
- data mean,
- data std dev,
- min value,
- 25% percentile,
- 50% percentile,
- 75% percentile,
- max value
for every data column.
"""
# Pretty straightforward, using standard pandas utility.
# The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again,
# thus avoiding passing big files to BT server:
flush_data = False
try:
assert not self.data.empty
pass
except (AssertionError, AttributeError) as e:
self.read_csv()
flush_data = True
self.data_stat = self.data.describe()
self.log.info('Data summary:\n{}'.format(self.data_stat.to_string()))
if flush_data:
self.data = None
self.log.info('Flushed data.')
return self.data_stat
def to_btfeed(self):
"""
Performs BTgymData-->bt.feed conversion.
Returns:
dict of type: {data_line_name: bt.datafeed instance}.
"""
try:
assert not self.data.empty
btfeed = btfeeds.PandasDirectData(dataname=self.data,
timeframe=self.timeframe,
datetime=self.datetime,
open=self.open,
high=self.high,
low=self.low,
close=self.close,
volume=self.volume,
openinterest=self.openinterest)
btfeed.numrecords = self.data.shape[0]
return {self.data_name: btfeed}
except (AssertionError, AttributeError) as e:
msg = 'Instance holds no data. Hint: forgot to call .read_csv()?'
self.log.error(msg)
raise AssertionError(msg)
def set_global_timestamp(self, timestamp):
pass
gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=config.debug)
else:
gamedata_engine = create_engine(gamedata_connectionstring, echo=config.debug)
gamedata_meta = MetaData()
gamedata_meta.bind = gamedata_engine
gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)()
# This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new
# game db because we haven't reached gamedata_meta.create_all()
try:
config.gamedata_version = gamedata_session.execute(
"SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'"
).fetchone()[0]
except Exception as e:
pyfalog.warning("Missing gamedata version.")
pyfalog.critical(e)
config.gamedata_version = None
saveddata_connectionstring = config.saveddata_connectionstring
if saveddata_connectionstring is not None:
if callable(saveddata_connectionstring):
saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=config.debug)
else:
saveddata_engine = create_engine(saveddata_connectionstring, echo=config.debug)
saveddata_meta = MetaData()
saveddata_meta.bind = saveddata_engine
saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)()
else:
saveddata_meta = None
