class CustomLogger(object): def __init__( self, log_level=LogLevel.INFO, format_str='[{record.time:%Y-%m-%d %H:%M:%S}] - {record.channel} - {record.level_name} ' '- {record.message}'): self.logger = Logger('WindAdapter') set_datetime_format('local') StreamHandler(sys.stdout, format_string=format_str).push_application() FileHandler('WindAdapter.log', bubble=True, format_string=format_str).push_application() self.set_level(log_level) def set_level(self, log_level): if log_level.lower() == LogLevel.INFO: self.logger.level = logbook.INFO elif log_level.lower() == LogLevel.WARNING: self.logger.level = logbook.WARNING elif log_level.lower() == LogLevel.CRITICAL: self.logger.level = logbook.CRITICAL elif log_level.lower() == LogLevel.NOTSET: self.logger.level = logbook.NOTSET def info(self, msg): self.logger.info(msg) def warning(self, msg): self.logger.warning(msg) def critical(self, msg): self.logger.critical(msg)
def rpc_server(socket, protocol, dispatcher): log = Logger('rpc_server') log.debug('starting up...') while True: try: message = socket.recv_multipart() except Exception as e: log.warning('Failed to receive message from client, ignoring...') log.exception(e) continue log.debug('Received message %s from %r' % (message[-1], message[0])) # assuming protocol is threadsafe and dispatcher is theadsafe, as long # as its immutable def handle_client(message): try: request = protocol.parse_request(message[-1]) except RPCError as e: log.exception(e) response = e.error_respond() else: response = dispatcher.dispatch(request) log.debug('Response okay: %r' % response) # send reply message[-1] = response.serialize() log.debug('Replying %s to %r' % (message[-1], message[0])) socket.send_multipart(message) gevent.spawn(handle_client, message)
def rpc_server(socket, protocol, dispatcher): log = Logger('rpc_server') log.debug('starting up...') while True: try: message = socket.recv_multipart() except Exception as e: log.warning('Failed to receive message from client, ignoring...') log.exception(e) continue log.debug('Received message %s from %r', message[-1], message[0]) # assuming protocol is threadsafe and dispatcher is theadsafe, as long # as its immutable def handle_client(message): try: request = protocol.parse_request(message[-1]) except RPCError as e: log.exception(e) response = e.error_respond() else: response = dispatcher.dispatch(request) log.debug('Response okay: %r', response) # send reply message[-1] = response.serialize() log.debug('Replying %s to %r', message[-1], message[0]) socket.send_multipart(message) gevent.spawn(handle_client, message)
class ProjectManager(object): """ Class for controlling project's settings. A project is a collection of setting for some data transformation. Each project is yaml file with settings. Project PID is a yaml file name. """ config_path = None def __init__(self, settings_class, config_path=None): """ Init projects_folder. Manager loads settings from default location in PySatDNA root or from given config_path. Then create work and projects folders if they were not found. And set config values for settings class. """ self.manager_logger = Logger('Manager logger') if config_path: self.load_config(config_path) else: self.load_config(self.config_path) self.projects_folder = self.config["projects_folder"] self.work_folder = self.config["path_work_folder"] self.settings_class = settings_class self.settings_class.config = self.config self.force_folder_creation = False if self.projects_folder and not os.path.isdir(self.projects_folder): os.makedirs(self.projects_folder) if self.work_folder and not os.path.isdir(self.work_folder): os.makedirs(self.work_folder) def load_config(self, config_path): ''' Load OS-specific configs.''' if config_path: file_path = config_path if not os.path.isfile(file_path): message = "ERROR with open config file: %s" % file_path self.manager_logger.error(message) raise ProjectManagerException(message) else: self.os = platform.system() if self.os == "Windows": file_path = "../config.win.yaml" elif self.os == "Darwin": file_path = "../config.mac.yaml" else: file_path = os.path.expanduser("~/Dropbox/workspace/PySatDNA/config.yaml") if not os.path.isfile(file_path): file_path = os.path.expanduser("~/Dropbox/PySatDNA/config.dobi.yaml") try: with open(file_path) as fh: self.config = yaml.load(fh) except Exception, e: self.manager_logger.error("ERROR with open config file: %s" % file_path) self.manager_logger.warning("Loading default settings") self.config = { 'path_work_folder': 'data', 'path_workspace_folder': '../..', 'projects_folder': 'projects', }
class Plugin(object): def __init__(self, site, name=None): self.name = name or remove_suffix('plugin', self.__class__.__name__) self.log = Logger(self.__class__.__name__.lower()) self.log.debug('{} initialized'.format(self.name)) self.base_dir = Path(inspect.getfile(self.__class__)).parent # initialize templates template_path = self.base_dir / 'templates' if template_path.exists(): self.jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader(str(template_path)), extensions=['jinja2.ext.with_'] ) # load possible default configuration self.register(site) @property def DEFAULTS_FILE(self): return self.base_dir / 'defaults.cfg' def register(self, site): pass def enable_app(self, app): pass def render_template(self, template_name, **kwargs): if not hasattr(self, 'jinja_env'): return RuntimeError('Plugin {} has no template path'.format( self.__class__.__name__ )) tpl = self.jinja_env.get_template(template_name) return tpl.render(**kwargs) def output_template(self, template_name, dest, _mode=0o644, **kwargs): if not dest.parent.exists(): self.log.warning('Path {} did not exist and was created'.format( dest.parent, )) dest.parent.mkdir(parents=True) with new_file(dest, _mode) as out: self.log.info('Writing {}'.format(dest.resolve())) out.write(self.render_template(template_name, **kwargs))
class Root(resource.Resource): isLeaf = True def __init__(self, lock): self._lock = lock self.log = Logger('web') resource.Resource.__init__(self) @delayed def render_GET(self, request): try: key = _get_key_from_path(request.path) if key == 'info/keys': request.write('%r\n' % (self._lock._keys,)) elif key == 'info/status': for line in self._lock.get_status(): request.write('%s %s\n' % line) elif key == 'info/log': for line in self._lock._log: request.write('%s\n' % line) else: value = yield self._lock.get_key(key) request.write(value) except KeyNotFound: request.setResponseCode(NOT_FOUND) returnValue('') @proxy_to_master @delayed def render_POST(self, request): try: key = _get_key_from_path(request.path) data = request.args.get('data', [''])[0] self.log.info('Set key %s=%r' % (key, data)) yield self._lock.set_key(key, data) except KeyAlreadyExists, e: self.log.warning(e) request.setResponseCode(CONFLICT) except PaxosError, e: self.log.warning(e) request.setResponseCode(EXPECTATION_FAILED)
class NodesMetrics: def __init__(self): handler = TimedRotatingFileHandler('../logs/nodes_metrics.log') handler.push_application() self.logger = Logger(name='nodes metrics', level='info') self.nodes_filename, self.nodes_file_backup_name, self.exclude_file, self.url, self.query_all_ip, self.query_current_ip = self.get_conf() self.nodes = {} self.ips = {} self.nodes_list = [] self.ips_list = [] @staticmethod def get_conf(): ##从配置文件获取配置 cp = ConfigParser.ConfigParser() with codecs.open(os.path.join(BASE_DIR, './config/config.ini'), 'r', encoding='utf-8') as f: cp.readfp(f) nodes_filename = cp.get('file_ds', 'file_sd_filename').strip() nodes_file_backup_name = cp.get('file_ds', 'nodes_file_backup_name').strip() exclude_file = cp.get('file_ds', 'agent_exclude_file').strip() url = cp.get('prome', 'url').strip() query_all_ip = cp.get('prome', 'query_all_ip').strip() query_current_ip = cp.get('prome', 'query_current_ip').strip() return os.path.join(BASE_DIR, nodes_filename), os.path.join(BASE_DIR, nodes_file_backup_name), os.path.join(BASE_DIR, exclude_file), url, query_all_ip, query_current_ip #@staticmethod def get_ips(self, res): ips = [] if res.status_code == 200: data = res.json()['data']['result'] for metric in data: # print(metric) ip = metric['metric']['instance_ip'] ips.append(ip) # print(ip) #print('All ips in prome: %s' % ips) #self.logger.info('All ips in prome: %s' % ips) return ips # res = requests.get(url + query_current_ip) # # current_ips = get_ips(res) #@staticmethod def get_hosts(self, file_name): all_ips = [] with open(file_name, 'r') as f: ips = f.read() ip_list = ips.split('\n') if file_name != self.exclude_file: for ip in ip_list: if ip.strip().endswith(':9100",'): nodes_str = ip.split('"')[1] ip_str = nodes_str.split(':')[0] #print(nodes_str) all_ips.append(ip_str) else: all_ips = ip_list print('All nodes from file %s is: %s' % (file_name, all_ips)) self.logger.info('All nodes from file %s is: %s' % (file_name, all_ips)) return all_ips def get_nodes_discovery(self): new_ips = self.get_hosts(self.nodes_filename) nodes_hosts = list(set(new_ips)) current_nodes = len(nodes_hosts) print(current_nodes) if current_nodes: self.logger.info('************************************') self.logger.info( 'There are %s nodes after nodes discovery finished: %s' % \ (current_nodes, nodes_hosts)) self.logger.info('************************************') else: self.logger.info('************************************') self.logger.error( 'There are 0 nodes founded by nodes discovery: %s') self.logger.info('************************************') all_nodes = ' '.join(nodes_hosts) return current_nodes, all_nodes def get_stop_nodes(self): new_ips = self.get_hosts(self.nodes_filename) print(len(new_ips)) res = requests.get(url + query_all_ip) all_ips = self.get_ips(res) self.logger.info(all_ips) print('All nodes are:') print(all_ips) old_ips = self.get_hosts(self.exclude_file) failed_hosts = list(set(new_ips) - set(all_ips) - set(old_ips)) stop_nodes = len(failed_hosts) #failed_hosts = list(set(new_ips) - set(all_ips)) self.logger.info('************************************') self.logger.warning( 'There are %s nodes stopped now' % stop_nodes) self.logger.info('************************************') return stop_nodes, failed_hosts
if hasattr(sys, 'frozen'): pyfalog.info("Running in a frozen state.") else: pyfalog.info("Running in a thawed state.") if not hasattr(sys, 'frozen') and wxversion: try: if options.force28 is True: pyfalog.info("Selecting wx version: 2.8. (Forced)") wxversion.select('2.8') else: pyfalog.info("Selecting wx versions: 3.0, 2.8") wxversion.select(['3.0', '2.8']) except: pyfalog.warning( "Unable to select wx version. Attempting to import wx without specifying the version." ) else: if not wxversion: pyfalog.warning( "wxVersion not found. Attempting to import wx without specifying the version." ) try: # noinspection PyPackageRequirements import wx except: exit_message = "Cannot import wxPython. You can download wxPython (2.8+) from http://www.wxpython.org/" raise PreCheckException(exit_message) pyfalog.info("wxPython version: {0}.", str(wx.VERSION_STRING))
class Worker(object): redis_worker_namespace_prefix = 'rq:worker:' redis_workers_keys = 'rq:workers' @classmethod def all(cls): """Returns an iterable of all Workers. """ conn = get_current_connection() reported_working = conn.smembers(cls.redis_workers_keys) return compact(map(cls.find_by_key, reported_working)) @classmethod def find_by_key(cls, worker_key): """Returns a Worker instance, based on the naming conventions for naming the internal Redis keys. Can be used to reverse-lookup Workers by their Redis keys. """ prefix = cls.redis_worker_namespace_prefix name = worker_key[len(prefix):] if not worker_key.startswith(prefix): raise ValueError('Not a valid RQ worker key: %s' % (worker_key, )) conn = get_current_connection() if not conn.exists(worker_key): return None name = worker_key[len(prefix):] worker = cls([], name) queues = conn.hget(worker.key, 'queues') worker._state = conn.hget(worker.key, 'state') or '?' if queues: worker.queues = map(Queue, queues.split(',')) return worker def __init__(self, queues, name=None, rv_ttl=500, connection=None): # noqa if connection is None: connection = get_current_connection() self.connection = connection if isinstance(queues, Queue): queues = [queues] self._name = name self.queues = queues self.validate_queues() self.rv_ttl = rv_ttl self._state = 'starting' self._is_horse = False self._horse_pid = 0 self._stopped = False self.log = Logger('worker') self.failed_queue = get_failed_queue(connection=self.connection) def validate_queues(self): # noqa """Sanity check for the given queues.""" if not iterable(self.queues): raise ValueError('Argument queues not iterable.') for queue in self.queues: if not isinstance(queue, Queue): raise NoQueueError('Give each worker at least one Queue.') def queue_names(self): """Returns the queue names of this worker's queues.""" return map(lambda q: q.name, self.queues) def queue_keys(self): """Returns the Redis keys representing this worker's queues.""" return map(lambda q: q.key, self.queues) @property # noqa def name(self): """Returns the name of the worker, under which it is registered to the monitoring system. By default, the name of the worker is constructed from the current (short) host name and the current PID. """ if self._name is None: hostname = socket.gethostname() shortname, _, _ = hostname.partition('.') self._name = '%s.%s' % (shortname, self.pid) return self._name @property def key(self): """Returns the worker's Redis hash key.""" return self.redis_worker_namespace_prefix + self.name @property def pid(self): """The current process ID.""" return os.getpid() @property def horse_pid(self): """The horse's process ID. Only available in the worker. Will return 0 in the horse part of the fork. """ return self._horse_pid @property def is_horse(self): """Returns whether or not this is the worker or the work horse.""" return self._is_horse def procline(self, message): """Changes the current procname for the process. This can be used to make `ps -ef` output more readable. """ procname.setprocname('rq: %s' % (message, )) def register_birth(self): # noqa """Registers its own birth.""" self.log.debug('Registering birth of worker %s' % (self.name, )) if self.connection.exists(self.key) and \ not self.connection.hexists(self.key, 'death'): raise ValueError('There exists an active worker named \'%s\' ' 'already.' % (self.name, )) key = self.key now = time.time() queues = ','.join(self.queue_names()) with self.connection.pipeline() as p: p.delete(key) p.hset(key, 'birth', now) p.hset(key, 'queues', queues) p.sadd(self.redis_workers_keys, key) p.execute() def register_death(self): """Registers its own death.""" self.log.debug('Registering death') with self.connection.pipeline() as p: # We cannot use self.state = 'dead' here, because that would # rollback the pipeline p.srem(self.redis_workers_keys, self.key) p.hset(self.key, 'death', time.time()) p.expire(self.key, 60) p.execute() def set_state(self, new_state): self._state = new_state self.connection.hset(self.key, 'state', new_state) def get_state(self): return self._state state = property(get_state, set_state) @property def stopped(self): return self._stopped def _install_signal_handlers(self): """Installs signal handlers for handling SIGINT and SIGTERM gracefully. """ def request_force_stop(signum, frame): """Terminates the application (cold shutdown). """ self.log.warning('Cold shut down.') # Take down the horse with the worker if self.horse_pid: msg = 'Taking down horse %d with me.' % self.horse_pid self.log.debug(msg) try: os.kill(self.horse_pid, signal.SIGKILL) except OSError as e: # ESRCH ("No such process") is fine with us if e.errno != errno.ESRCH: self.log.debug('Horse already down.') raise raise SystemExit() def request_stop(signum, frame): """Stops the current worker loop but waits for child processes to end gracefully (warm shutdown). """ self.log.debug('Got %s signal.' % signal_name(signum)) signal.signal(signal.SIGINT, request_force_stop) signal.signal(signal.SIGTERM, request_force_stop) if self.is_horse: self.log.debug('Ignoring signal %s.' % signal_name(signum)) return msg = 'Warm shut down. Press Ctrl+C again for a cold shutdown.' self.log.warning(msg) self._stopped = True self.log.debug('Stopping after current horse is finished.') signal.signal(signal.SIGINT, request_stop) signal.signal(signal.SIGTERM, request_stop) def work(self, burst=False): # noqa """Starts the work loop. Pops and performs all jobs on the current list of queues. When all queues are empty, block and wait for new jobs to arrive on any of the queues, unless `burst` mode is enabled. The return value indicates whether any jobs were processed. """ self._install_signal_handlers() did_perform_work = False self.register_birth() self.state = 'starting' try: while True: if self.stopped: self.log.info('Stopping on request.') break self.state = 'idle' qnames = self.queue_names() self.procline('Listening on %s' % ','.join(qnames)) self.log.info('') self.log.info('*** Listening on %s...' % \ green(', '.join(qnames))) wait_for_job = not burst try: result = Queue.dequeue_any(self.queues, wait_for_job, \ connection=self.connection) if result is None: break except UnpickleError as e: msg = '*** Ignoring unpickleable data on %s.' % \ green(e.queue.name) self.log.warning(msg) self.log.debug('Data follows:') self.log.debug(e.raw_data) self.log.debug('End of unreadable data.') self.failed_queue.push_job_id(e.job_id) continue job, queue = result self.log.info( '%s: %s (%s)' % (green(queue.name), blue(job.description), job.id)) self.state = 'busy' self.fork_and_perform_job(job) did_perform_work = True finally: if not self.is_horse: self.register_death() return did_perform_work def fork_and_perform_job(self, job): """Spawns a work horse to perform the actual work and passes it a job. The worker will wait for the work horse and make sure it executes within the given timeout bounds, or will end the work horse with SIGALRM. """ child_pid = os.fork() if child_pid == 0: self.main_work_horse(job) else: self._horse_pid = child_pid self.procline('Forked %d at %d' % (child_pid, time.time())) while True: try: os.waitpid(child_pid, 0) break except OSError as e: # In case we encountered an OSError due to EINTR (which is # caused by a SIGINT or SIGTERM signal during # os.waitpid()), we simply ignore it and enter the next # iteration of the loop, waiting for the child to end. In # any other case, this is some other unexpected OS error, # which we don't want to catch, so we re-raise those ones. if e.errno != errno.EINTR: raise def main_work_horse(self, job): """This is the entry point of the newly spawned work horse.""" # After fork()'ing, always assure we are generating random sequences # that are different from the worker. random.seed() self._is_horse = True self.log = Logger('horse') success = self.perform_job(job) # os._exit() is the way to exit from childs after a fork(), in # constrast to the regular sys.exit() os._exit(int(not success)) def perform_job(self, job): """Performs the actual work of a job. Will/should only be called inside the work horse's process. """ self.procline('Processing %s from %s since %s' % (job.func_name, job.origin, time.time())) try: with death_pentalty_after(job.timeout or 180): rv = job.perform() except Exception as e: fq = self.failed_queue self.log.exception(red(str(e))) self.log.warning('Moving job to %s queue.' % fq.name) fq.quarantine(job, exc_info=traceback.format_exc()) return False if rv is None: self.log.info('Job OK') else: self.log.info('Job OK, result = %s' % (yellow(unicode(rv)), )) if rv is not None: p = self.connection.pipeline() p.hset(job.key, 'result', dumps(rv)) p.expire(job.key, self.rv_ttl) p.execute() else: # Cleanup immediately job.delete() return True
class BTgymBaseData: """ Base BTgym data provider class. Provides core data loading, sampling, splitting and converting functionality. Do not use directly. Enables Pipe:: CSV[source data]-->pandas[for efficient sampling]-->bt.feeds """ def __init__( self, filename=None, parsing_params=None, sampling_params=None, name='base_data', data_names=('default_asset',), task=0, frozen_time_split=None, log_level=WARNING, _config_stack=None, **kwargs ): """ Args: filename: Str or list of str, should be given either here or when calling read_csv(), see `Notes`. specific_params CSV to Pandas parsing sep: ';' header: 0 index_col: 0 parse_dates: True names: ['open', 'high', 'low', 'close', 'volume'] specific_params Pandas to BT.feeds conversion timeframe=1: 1 minute. datetime: 0 open: 1 high: 2 low: 3 close: 4 volume: -1 openinterest: -1 specific_params Sampling sample_class_ref: None - if not None, than sample() method will return instance of specified class, which itself must be subclass of BaseBTgymDataset, else returns instance of the base data class. start_weekdays: [0, 1, 2, 3, ] - Only weekdays from the list will be used for sample start. start_00: True - sample start time will be set to first record of the day (usually 00:00). sample_duration: {'days': 1, 'hours': 23, 'minutes': 55} - Maximum sample time duration in days, hours, minutes time_gap: {''days': 0, hours': 5, 'minutes': 0} - Data omittance threshold: maximum no-data time gap allowed within sample in days, hours. Thereby, if set to be < 1 day, samples containing weekends and holidays gaps will be rejected. test_period: {'days': 0, 'hours': 0, 'minutes': 0} - setting this param to non-zero duration forces instance.data split to train / test subsets with test subset duration equal to `test_period` with `time_gap` tolerance. Train data always precedes test one: [0_record<-train_data->split_point_record<-test_data->last_record]. sample_expanding: None, reserved for child classes. Note: - CSV file can contain duplicate records, checks will be performed and all duplicates will be removed; - CSV file should be properly sorted by date_time in ascending order, no sorting checks performed. - When supplying list of file_names, all files should be also listed ascending by their time period, no correct sampling will be possible otherwise. - Default parameters are source-specific and made to correctly parse 1 minute Forex generic ASCII data files from www.HistData.com. Tune according to your data source. """ self.filename = filename if parsing_params is None: self.parsing_params = dict( # Default parameters for source-specific CSV datafeed class, # correctly parses 1 minute Forex generic ASCII # data files from www.HistData.com: # CSV to Pandas params. sep=';', header=0, index_col=0, parse_dates=True, names=['open', 'high', 'low', 'close', 'volume'], # Pandas to BT.feeds params: timeframe=1, # 1 minute. datetime=0, open=1, high=2, low=3, close=4, volume=-1, openinterest=-1, ) else: self.parsing_params = parsing_params if sampling_params is None: self.sampling_params = dict( # Sampling params: start_weekdays=[], # Only weekdays from the list will be used for episode start. start_00=False, # Sample start time will be set to first record of the day (usually 00:00). sample_duration=dict( # Maximum sample time duration in days, hours, minutes: days=0, hours=0, minutes=0 ), time_gap=dict( # Maximum data time gap allowed within sample in days, hours. Thereby, days=0, # if set to be < 1 day, samples containing weekends and holidays gaps will be rejected. hours=0, ), test_period=dict( # Time period to take test samples from, in days, hours, minutes: days=0, hours=0, minutes=0 ), expanding=False, ) else: self.sampling_params = sampling_params self.name = name # String will be used as key name for bt_feed data-line: self.task = task self.log_level = log_level self.data_names = data_names self.data_name = self.data_names[0] self.data = None # Will hold actual data as pandas dataframe self.is_ready = False self.global_timestamp = 0 self.start_timestamp = 0 self.final_timestamp = 0 self.data_stat = None # Dataset descriptive statistic as pandas dataframe self.data_range_delta = None # Dataset total duration timedelta self.max_time_gap = None self.time_gap = None self.max_sample_len_delta = None self.sample_duration = None self.sample_num_records = 0 self.start_weekdays = {0, 1, 2, 3, 4, 5, 6} self.start_00 = False self.expanding = False self.sample_instance = None self.test_range_delta = None self.train_range_delta = None self.test_num_records = 0 self.train_num_records = 0 self.total_num_records = 0 self.train_interval = [0, 0] self.test_interval = [0, 0] self.test_period = {'days': 0, 'hours': 0, 'minutes': 0} self.train_period = {'days': 0, 'hours': 0, 'minutes': 0} self._test_period_backshift_delta = datetime.timedelta(**{'days': 0, 'hours': 0, 'minutes': 0}) self.sample_num = 0 self.task = 0 self.metadata = {'sample_num': 0, 'type': None} self.set_params(self.parsing_params) self.set_params(self.sampling_params) self._config_stack = copy.deepcopy(_config_stack) try: nested_config = self._config_stack.pop() except (IndexError, AttributeError) as e: # IF stack is empty, sample of this instance itself is not supposed to be sampled. nested_config = dict( class_ref=None, kwargs=dict( parsing_params=self.parsing_params, sample_params=None, name='data_stream', task=self.task, log_level=self.log_level, _config_stack=None, ) ) # Configure sample instance parameters: self.nested_class_ref = nested_config['class_ref'] self.nested_params = nested_config['kwargs'] self.sample_name = '{}_w_{}_'.format(self.nested_params['name'], self.task) self.nested_params['_config_stack'] = self._config_stack # Logging: StreamHandler(sys.stdout).push_application() self.log = Logger('{}_{}'.format(self.name, self.task), level=self.log_level) # Legacy parameter dictionary, left here for BTgym API_shell: self.params = {} self.params.update(self.parsing_params) self.params.update(self.sampling_params) if frozen_time_split is not None: self.frozen_time_split = datetime.datetime(**frozen_time_split) else: self.frozen_time_split = None self.frozen_split_timestamp = None def set_params(self, params_dict): """ Batch attribute setter. Args: params_dict: dictionary of parameters to be set as instance attributes. """ for key, value in params_dict.items(): setattr(self, key, value) def set_logger(self, level=None, task=None): """ Sets logbook logger. Args: level: logbook.level, int task: task id, int """ if task is not None: self.task = task if level is not None: self.log = Logger('{}_{}'.format(self.name, self.task), level=level) def set_global_timestamp(self, timestamp): if self.data is not None: self.global_timestamp = self.data.index[0].timestamp() def reset(self, data_filename=None, **kwargs): """ Gets instance ready. Args: data_filename: [opt] string or list of strings. kwargs: not used. """ self._reset(data_filename=data_filename, **kwargs) def _reset(self, data_filename=None, timestamp=None, **kwargs): self.read_csv(data_filename) # Add global timepoints: self.start_timestamp = self.data.index[0].timestamp() self.final_timestamp = self.data.index[-1].timestamp() if self.frozen_time_split is not None: frozen_index = self.data.index.get_loc(self.frozen_time_split, method='ffill') self.frozen_split_timestamp = self.data.index[frozen_index].timestamp() self.set_global_timestamp(self.frozen_split_timestamp) else: self.frozen_split_timestamp = None self.set_global_timestamp(timestamp) self.log.debug( 'time stamps start: {}, current: {} final: {}'.format( self.start_timestamp, self.global_timestamp, self.final_timestamp ) ) # Maximum data time gap allowed within sample as pydatetimedelta obj: self.max_time_gap = datetime.timedelta(**self.time_gap) # Max. gap number of records: self.max_gap_num_records = int(self.max_time_gap.total_seconds() / (60 * self.timeframe)) # ... maximum episode time duration: self.max_sample_len_delta = datetime.timedelta(**self.sample_duration) # Maximum possible number of data records (rows) within episode: self.sample_num_records = int(self.max_sample_len_delta.total_seconds() / (60 * self.timeframe)) self.backshift_num_records = round(self._test_period_backshift_delta.total_seconds() / (60 * self.timeframe)) # Train/test timedeltas: if self.train_period is None or self.test_period == -1: # No train data assumed, test only: self.train_num_records = 0 self.test_num_records = self.data.shape[0] - self.backshift_num_records break_point = self.backshift_num_records self.train_interval = [0, 0] self.test_interval = [self.backshift_num_records, self.data.shape[0]] else: # Train and maybe test data assumed: if self.test_period is not None: self.test_range_delta = datetime.timedelta(**self.test_period) self.test_num_records = round(self.test_range_delta.total_seconds() / (60 * self.timeframe)) self.train_num_records = self.data.shape[0] - self.test_num_records break_point = self.train_num_records self.train_interval = [0, break_point] self.test_interval = [break_point - self.backshift_num_records, self.data.shape[0]] else: self.test_num_records = 0 self.train_num_records = self.data.shape[0] break_point = self.train_num_records self.train_interval = [0, break_point] self.test_interval = [0, 0] if self.train_num_records > 0: try: assert self.train_num_records + self.max_gap_num_records >= self.sample_num_records except AssertionError: self.log.exception( 'Train subset should contain at least one sample, ' + 'got: train_set size: {} rows, sample_size: {} rows, tolerance: {} rows'. format(self.train_num_records, self.sample_num_records, self.max_gap_num_records) ) raise AssertionError if self.test_num_records > 0: try: assert self.test_num_records + self.max_gap_num_records >= self.sample_num_records except AssertionError: self.log.exception( 'Test subset should contain at least one sample, ' + 'got: test_set size: {} rows, sample_size: {} rows, tolerance: {} rows'. format(self.test_num_records, self.sample_num_records, self.max_gap_num_records) ) raise AssertionError self.sample_num = 0 self.is_ready = True def read_csv(self, data_filename=None, force_reload=False): """ Populates instance by loading data: CSV file --> pandas dataframe. Args: data_filename: [opt] csv data filename as string or list of such strings. force_reload: ignore loaded data. """ if self.data is not None and not force_reload: data_range = pd.to_datetime(self.data.index) self.total_num_records = self.data.shape[0] self.data_range_delta = (data_range[-1] - data_range[0]).to_pytimedelta() self.log.debug('data has been already loaded. Use `force_reload=True` to reload') return if data_filename: self.filename = data_filename # override data source if one is given if type(self.filename) == str: self.filename = [self.filename] dataframes = [] for filename in self.filename: try: assert filename and os.path.isfile(filename) current_dataframe = pd.read_csv( filename, sep=self.sep, header=self.header, index_col=self.index_col, parse_dates=self.parse_dates, names=self.names, ) # Check and remove duplicate datetime indexes: duplicates = current_dataframe.index.duplicated(keep='first') how_bad = duplicates.sum() if how_bad > 0: current_dataframe = current_dataframe[~duplicates] self.log.warning('Found {} duplicated date_time records in <{}>.\ Removed all but first occurrences.'.format(how_bad, filename)) dataframes += [current_dataframe] self.log.info('Loaded {} records from <{}>.'.format(dataframes[-1].shape[0], filename)) except: msg = 'Data file <{}> not specified / not found / parser error.'.format(str(filename)) self.log.error(msg) raise FileNotFoundError(msg) self.data = pd.concat(dataframes) data_range = pd.to_datetime(self.data.index) self.total_num_records = self.data.shape[0] self.data_range_delta = (data_range[-1] - data_range[0]).to_pytimedelta() def describe(self): """ Returns summary dataset statistic as pandas dataframe: - records count, - data mean, - data std dev, - min value, - 25% percentile, - 50% percentile, - 75% percentile, - max value for every data column. """ # Pretty straightforward, using standard pandas utility. # The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again, # thus avoiding passing big files to BT server: flush_data = False try: assert not self.data.empty pass except (AssertionError, AttributeError) as e: self.read_csv() flush_data = True self.data_stat = self.data.describe() self.log.info('Data summary:\n{}'.format(self.data_stat.to_string())) if flush_data: self.data = None self.log.info('Flushed data.') return self.data_stat def to_btfeed(self): """ Performs BTgymData-->bt.feed conversion. Returns: dict of type: {data_line_name: bt.datafeed instance}. """ def bt_timeframe(minutes): timeframe = TimeFrame.Minutes if minutes / 1440 == 1: timeframe = TimeFrame.Days return timeframe try: assert not self.data.empty btfeed = btfeeds.PandasDirectData( dataname=self.data, timeframe=bt_timeframe(self.timeframe), datetime=self.datetime, open=self.open, high=self.high, low=self.low, close=self.close, volume=self.volume, openinterest=self.openinterest ) btfeed.numrecords = self.data.shape[0] return {self.data_name: btfeed} except (AssertionError, AttributeError) as e: msg = 'Instance holds no data. Hint: forgot to call .read_csv()?' self.log.error(msg) raise AssertionError(msg) def sample(self, **kwargs): return self._sample(**kwargs) def _sample( self, get_new=True, sample_type=0, b_alpha=1.0, b_beta=1.0, force_interval=False, interval=None, **kwargs ): """ Samples continuous subset of data. Args: get_new (bool): sample new (True) or reuse (False) last made sample; sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets respectively. b_alpha (float): beta-distribution sampling alpha > 0, valid for train episodes. b_beta (float): beta-distribution sampling beta > 0, valid for train episodes. force_interval(bool): use exact sampling interval (should be given) interval(iterable of int, len2): exact interval to sample from when force_interval=True Returns: if no sample_class_ref param been set: BTgymDataset instance with number of records ~ max_episode_len, where `~` tolerance is set by `time_gap` param; else: `sample_class_ref` instance with same as above number of records. Note: Train sample start position within interval is drawn from beta-distribution with default parameters b_alpha=1, b_beta=1, i.e. uniform one. Beta-distribution makes skewed sampling possible , e.g. to give recent episodes higher probability of being sampled, e.g.: b_alpha=10, b_beta=0.8. Test samples are always uniform one. """ try: assert self.is_ready except AssertionError: msg = 'sampling attempt: data not ready. Hint: forgot to call data.reset()?' self.log.error(msg) raise RuntimeError(msg) try: assert sample_type in [0, 1] except AssertionError: msg = 'sampling attempt: expected sample type be in {}, got: {}'.format([0, 1], sample_type) self.log.error(msg) raise ValueError(msg) if force_interval: try: assert interval is not None and len(list(interval)) == 2 except AssertionError: msg = 'sampling attempt: got force_interval=True, expected interval=[a,b], got: <{}>'.format(interval) self.log.error(msg) raise ValueError(msg) if self.sample_instance is None or get_new: if sample_type == 0: # Get beta_distributed sample in train interval: if force_interval: sample_interval = interval else: sample_interval = self.train_interval self.sample_instance = self._sample_interval( sample_interval, force_interval=force_interval, b_alpha=b_alpha, b_beta=b_beta, name='train_' + self.sample_name, **kwargs ) else: # Get uniform sample in test interval: if force_interval: sample_interval = interval else: sample_interval = self.test_interval self.sample_instance = self._sample_interval( sample_interval, force_interval=force_interval, b_alpha=1, b_beta=1, name='test_' + self.sample_name, **kwargs ) self.sample_instance.metadata['type'] = sample_type # TODO: can move inside sample() self.sample_instance.metadata['sample_num'] = self.sample_num self.sample_instance.metadata['parent_sample_num'] = copy.deepcopy(self.metadata['sample_num']) self.sample_instance.metadata['parent_sample_type'] = copy.deepcopy(self.metadata['type']) self.sample_num += 1 else: # Do nothing: self.log.debug('Reusing sample, id: {}'.format(self.sample_instance.filename)) return self.sample_instance def _sample_random( self, sample_type=0, timestamp=None, name='random_sample_', interval=None, force_interval=False, **kwargs ): """ Randomly samples continuous subset of data. Args: name: str, sample filename id Returns: BTgymDataset instance with number of records ~ max_episode_len, where `~` tolerance is set by `time_gap` param. """ try: assert not self.data.empty except (AssertionError, AttributeError) as e: self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?') raise AssertionError if force_interval: raise NotImplementedError('Force_interval for random sampling not implemented.') self.log.debug('Maximum sample time duration set to: {}.'.format(self.max_sample_len_delta)) self.log.debug('Respective number of steps: {}.'.format(self.sample_num_records)) self.log.debug('Maximum allowed data time gap set to: {}.\n'.format(self.max_time_gap)) sampled_data = None sample_len = 0 # Sanity check param: max_attempts = 100 attempts = 0 # # Keep sampling random enter points until all conditions are met: while attempts <= max_attempts: # Randomly sample record (row) from entire datafeed: first_row = int((self.data.shape[0] - self.sample_num_records - 1) * random.random()) sample_first_day = self.data[first_row:first_row + 1].index[0] self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday())) # Keep sampling until good day: while not sample_first_day.weekday() in self.start_weekdays and attempts <= max_attempts: self.log.debug('Not a good day to start, resampling...') first_row = int((self.data.shape[0] - self.sample_num_records - 1) * random.random()) sample_first_day = self.data[first_row:first_row + 1].index[0] self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday())) attempts +=1 # Check if managed to get proper weekday: assert attempts <= max_attempts, \ 'Quitting after {} sampling attempts. Hint: check sampling params / dataset consistency.'. \ format(attempts) # If 00 option set, get index of first record of that day: if self.start_00: adj_timedate = sample_first_day.date() self.log.debug('Start time adjusted to <00:00>') else: adj_timedate = sample_first_day first_row = self.data.index.get_loc(adj_timedate, method='nearest') # Easy part: last_row = first_row + self.sample_num_records # + 1 sampled_data = self.data[first_row: last_row] sample_len = (sampled_data.index[-1] - sampled_data.index[0]).to_pytimedelta() self.log.debug('Actual sample duration: {}.'.format(sample_len, )) self.log.debug('Total sample time gap: {}.'.format(self.max_sample_len_delta - sample_len)) # Perform data gap check: if self.max_sample_len_delta - sample_len < self.max_time_gap: self.log.debug('Sample accepted.') # If sample OK - compose and return sample: new_instance = self.nested_class_ref(**self.nested_params) new_instance.filename = name + 'n{}_at_{}'.format(self.sample_num, adj_timedate) self.log.info('Sample id: <{}>.'.format(new_instance.filename)) new_instance.data = sampled_data new_instance.metadata['type'] = 'random_sample' new_instance.metadata['first_row'] = first_row new_instance.metadata['last_row'] = last_row return new_instance else: self.log.debug('Duration too big, resampling...\n') attempts += 1 # Got here -> sanity check failed: msg = ( '\nQuitting after {} sampling attempts.\n' + 'Full sample duration: {}\n' + 'Total sample time gap: {}\n' + 'Sample start time: {}\n' + 'Sample finish time: {}\n' + 'Hint: check sampling params / dataset consistency.' ).format( attempts, sample_len, sample_len - self.max_sample_len_delta, sampled_data.index[0], sampled_data.index[-1] ) self.log.error(msg) raise RuntimeError(msg) def _sample_interval( self, interval, b_alpha=1.0, b_beta=1.0, name='interval_sample_', force_interval=False, **kwargs ): """ Samples continuous subset of data, such as entire episode records lie within positions specified by interval. Episode start position within interval is drawn from beta-distribution parametrised by `b_alpha, b_beta`. By default distribution is uniform one. Args: interval: tuple, list or 1d-array of integers of length 2: [lower_row_number, upper_row_number]; b_alpha: float > 0, sampling B-distribution alpha param, def=1; b_beta: float > 0, sampling B-distribution beta param, def=1; name: str, sample filename id force_interval: bool, if true: force exact interval sampling Returns: - BTgymDataset instance such as: 1. number of records ~ max_episode_len, subj. to `time_gap` param; 2. actual episode start position is sampled from `interval`; - `False` if it is not possible to sample instance with set args. """ try: assert not self.data.empty except (AssertionError, AttributeError) as e: self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?') raise AssertionError try: assert len(interval) == 2 except AssertionError: self.log.exception( 'Invalid interval arg: expected list or tuple of size 2, got: {}'.format(interval) ) raise AssertionError if force_interval: return self._sample_exact_interval(interval, name) try: assert b_alpha > 0 and b_beta > 0 except AssertionError: self.log.exception( 'Expected positive B-distribution [alpha, beta] params, got: {}'.format([b_alpha, b_beta]) ) raise AssertionError if interval[-1] - interval[0] + self.max_gap_num_records > self.sample_num_records: sample_num_records = self.sample_num_records else: sample_num_records = interval[-1] - interval[0] self.log.debug('Sample interval: {}'.format(interval)) self.log.debug('Maximum sample time duration set to: {}.'.format(self.max_sample_len_delta)) self.log.debug('Sample number of steps (adjusted to interval): {}.'.format(sample_num_records)) self.log.debug('Maximum allowed data time gap set to: {}.\n'.format(self.max_time_gap)) sampled_data = None sample_len = 0 # Sanity check param: max_attempts = 100 attempts = 0 # # Keep sampling random enter points until all conditions are met: while attempts <= max_attempts: first_row = interval[0] + int( (interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta) ) #print('_sample_interval_sample_num_records: ', sample_num_records) #print('_sample_interval_first_row: ', first_row) sample_first_day = self.data[first_row:first_row + 1].index[0] self.log.debug( 'Sample start row: {}, day: {}, weekday: {}.'. format(first_row, sample_first_day, sample_first_day.weekday()) ) # Keep sampling until good day: while not sample_first_day.weekday() in self.start_weekdays and attempts <= max_attempts: self.log.debug('Not a good day to start, resampling...') first_row = interval[0] + round( (interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta) ) #print('r_sample_interval_sample_num_records: ', sample_num_records) #print('r_sample_interval_first_row: ', first_row) sample_first_day = self.data[first_row:first_row + 1].index[0] self.log.debug( 'Sample start row: {}, day: {}, weekday: {}.'. format(first_row, sample_first_day, sample_first_day.weekday()) ) attempts += 1 # Check if managed to get proper weekday: try: assert attempts <= max_attempts except AssertionError: self.log.exception( 'Quitting after {} sampling attempts. Hint: check sampling params / dataset consistency.'. format(attempts) ) raise RuntimeError # If 00 option set, get index of first record of that day: if self.start_00: adj_timedate = sample_first_day.date() self.log.debug('Start time adjusted to <00:00>') first_row = self.data.index.get_loc(adj_timedate, method='nearest') else: adj_timedate = sample_first_day # first_row = self.data.index.get_loc(adj_timedate, method='nearest') # Easy part: last_row = first_row + sample_num_records # + 1 sampled_data = self.data[first_row: last_row] self.log.debug( 'first_row: {}, last_row: {}, data_shape: {}'.format( first_row, last_row, sampled_data.shape ) ) sample_len = (sampled_data.index[-1] - sampled_data.index[0]).to_pytimedelta() self.log.debug('Actual sample duration: {}.'.format(sample_len)) self.log.debug('Total sample time gap: {}.'.format(self.max_sample_len_delta - sample_len)) # Perform data gap check: if self.max_sample_len_delta - sample_len < self.max_time_gap: self.log.debug('Sample accepted.') # If sample OK - return new dataset: new_instance = self.nested_class_ref(**self.nested_params) new_instance.filename = name + 'num_{}_at_{}'.format(self.sample_num, adj_timedate) self.log.info('New sample id: <{}>.'.format(new_instance.filename)) new_instance.data = sampled_data new_instance.metadata['type'] = 'interval_sample' new_instance.metadata['first_row'] = first_row new_instance.metadata['last_row'] = last_row return new_instance else: self.log.debug('Attempt {}: gap is too big, resampling, ...\n'.format(attempts)) attempts += 1 # Got here -> sanity check failed: msg = ( '\nQuitting after {} sampling attempts.\n' + 'Full sample duration: {}\n' + 'Total sample time gap: {}\n' + 'Sample start time: {}\n' + 'Sample finish time: {}\n' + 'Hint: check sampling params / dataset consistency.' ).format( attempts, sample_len, sample_len - self.max_sample_len_delta, sampled_data.index[0], sampled_data.index[-1] ) self.log.error(msg) raise RuntimeError(msg) def _sample_aligned_interval( self, interval, align_left=False, b_alpha=1.0, b_beta=1.0, name='interval_sample_', force_interval=False, **kwargs ): """ Samples continuous subset of data, such as entire episode records lie within positions specified by interval Episode start position within interval is drawn from beta-distribution parametrised by `b_alpha, b_beta`. By default distribution is uniform one. Args: interval: tuple, list or 1d-array of integers of length 2: [lower_row_number, upper_row_number]; align: if True - try to align sample to beginning of interval; b_alpha: float > 0, sampling B-distribution alpha param, def=1; b_beta: float > 0, sampling B-distribution beta param, def=1; name: str, sample filename id force_interval: bool, if true: force exact interval sampling Returns: - BTgymDataset instance such as: 1. number of records ~ max_episode_len, subj. to `time_gap` param; 2. actual episode start position is sampled from `interval`; - `False` if it is not possible to sample instance with set args. """ try: assert not self.data.empty except (AssertionError, AttributeError) as e: self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?') raise AssertionError try: assert len(interval) == 2 except AssertionError: self.log.exception( 'Invalid interval arg: expected list or tuple of size 2, got: {}'.format(interval) ) raise AssertionError if force_interval: return self._sample_exact_interval(interval, name) try: assert b_alpha > 0 and b_beta > 0 except AssertionError: self.log.exception( 'Expected positive B-distribution [alpha, beta] params, got: {}'.format([b_alpha, b_beta]) ) raise AssertionError sample_num_records = self.sample_num_records self.log.debug('Maximum sample time duration set to: {}.'.format(self.max_sample_len_delta)) self.log.debug('Respective number of steps: {}.'.format(sample_num_records)) self.log.debug('Maximum allowed data time gap set to: {}.\n'.format(self.max_time_gap)) # Sanity check param: if align_left: max_attempts = interval[-1] - interval[0] else: # Sanity check: max_attempts = 100 attempts = 0 align_shift = 0 # Sample enter point as close to beginning until all conditions are met: while attempts <= max_attempts: if align_left: first_row = interval[0] + align_shift else: first_row = interval[0] + int( (interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta) ) #print('_sample_interval_sample_num_records: ', sample_num_records) self.log.debug('_sample_interval_first_row: {}'.format(first_row)) sample_first_day = self.data[first_row:first_row + 1].index[0] self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday())) # Keep sampling until good day: while not sample_first_day.weekday() in self.start_weekdays and attempts <= max_attempts: align_shift += 1 self.log.debug('Not a good day to start, resampling...') if align_left: first_row = interval[0] + align_shift else: first_row = interval[0] + int( (interval[-1] - interval[0] - sample_num_records) * random_beta(a=b_alpha, b=b_beta) ) #print('r_sample_interval_sample_num_records: ', sample_num_records) self.log.debug('_sample_interval_first_row: {}'.format(first_row)) sample_first_day = self.data[first_row:first_row + 1].index[0] self.log.debug('Sample start: {}, weekday: {}.'.format(sample_first_day, sample_first_day.weekday())) attempts += 1 # Check if managed to get proper weekday: try: assert attempts <= max_attempts except AssertionError: self.log.exception( 'Quitting after {} sampling attempts. Hint: check sampling params / dataset consistency.'. format(attempts) ) raise RuntimeError # If 00 option set, get index of first record of that day: if self.start_00: adj_timedate = sample_first_day.date() self.log.debug('Start time adjusted to <00:00>') first_row = self.data.index.get_loc(adj_timedate, method='nearest') else: adj_timedate = sample_first_day # first_row = self.data.index.get_loc(adj_timedate, method='nearest') # Easy part: last_row = first_row + sample_num_records # + 1 sampled_data = self.data[first_row: last_row] sample_len = (sampled_data.index[-1] - sampled_data.index[0]).to_pytimedelta() self.log.debug('Actual sample duration: {}.'.format(sample_len)) self.log.debug('Total sample time gap: {}.'.format(sample_len - self.max_sample_len_delta)) # Perform data gap check: if sample_len - self.max_sample_len_delta < self.max_time_gap: self.log.debug('Sample accepted.') # If sample OK - return new dataset: new_instance = self.nested_class_ref(**self.nested_params) new_instance.filename = name + 'num_{}_at_{}'.format(self.sample_num, adj_timedate) self.log.info('New sample id: <{}>.'.format(new_instance.filename)) new_instance.data = sampled_data new_instance.metadata['type'] = 'interval_sample' new_instance.metadata['first_row'] = first_row new_instance.metadata['last_row'] = last_row return new_instance else: self.log.debug('Attempt {}: duration too big, resampling, ...\n'.format(attempts)) attempts += 1 align_shift += 1 # Got here -> sanity check failed: msg = ('Quitting after {} sampling attempts.' + 'Hint: check sampling params / dataset consistency.').format(attempts) self.log.error(msg) raise RuntimeError(msg) def _sample_exact_interval(self, interval, name='interval_sample_', **kwargs): """ Samples exactly defined interval. Args: interval: tuple, list or 1d-array of integers of length 2: [lower_row_number, upper_row_number]; name: str, sample filename id Returns: BTgymDataset instance. """ try: assert not self.data.empty except (AssertionError, AttributeError) as e: self.log.exception('Instance holds no data. Hint: forgot to call .read_csv()?') raise AssertionError try: assert len(interval) == 2 except AssertionError: self.log.exception( 'Invalid interval arg: expected list or tuple of size 2, got: {}'.format(interval) ) raise AssertionError first_row = interval[0] last_row = interval[-1] sampled_data = self.data[first_row: last_row] sample_first_day = self.data[first_row:first_row + 1].index[0] new_instance = self.nested_class_ref(**self.nested_params) new_instance.filename = name + 'num_{}_at_{}'.format(self.sample_num, sample_first_day) self.log.info('New sample id: <{}>.'.format(new_instance.filename)) new_instance.data = sampled_data new_instance.metadata['type'] = 'interval_sample' new_instance.metadata['first_row'] = first_row new_instance.metadata['last_row'] = last_row return new_instance
class Launcher(): """ Configures and starts distributed TF training session with workers running sets of separate instances of BTgym/Atari environment. """ def __init__(self, env_config=None, cluster_config=None, policy_config=None, trainer_config=None, max_env_steps=None, root_random_seed=None, test_mode=False, purge_previous=0, log_level=None, verbose=0): """ Args: env_config (dict): environment class_config_dict, see 'Note' below. cluster_config (dict): tf cluster configuration, see 'Note' below. policy_config (dict): policy class_config_dict holding corr. policy class args. trainer_config (dict): trainer class_config_dict holding corr. trainer class args. max_env_steps (int): total number of environment steps to run training on. root_random_seed (int): int or None test_mode (bool): if True - use Atari gym env., BTGym otherwise. purge_previous (int): keep or remove previous log files and saved checkpoints from log_dir: {0 - keep, 1 - ask, 2 - remove}. verbose (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG}. log_level (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13}, overrides `verbose` arg. Note: class_config_dict: dictionary containing at least two keys: - `class_ref`: reference to class constructor or function; - `kwargs`: dictionary of keyword arguments, see corr. environment class args. cluster_config: dictionary containing at least these keys: - 'host': cluster host, def: '127.0.0.1' - 'port': cluster port, def: 12222 - 'num_workers': number of workers to run, def: 1 - 'num_ps': number of parameter servers, def: 1 - 'num_envs': number of environments to run in parallel for each worker, def: 1 - 'log_dir': directory to save model and summaries, def: './tmp/btgym_aac_log' """ self.env_config = dict(class_ref=None, kwargs=dict( port=5000, data_port=4999, gym_id=None, )) self.cluster_config = dict( host='127.0.0.1', port=12222, num_workers=1, num_ps=1, log_dir='./tmp/btgym_aac_log', num_envs=1, ) self.policy_config = dict(class_ref=BaseAacPolicy, kwargs=dict(lstm_layers=(256, ))) self.trainer_config = dict(class_ref=A3C, kwargs={}) self.max_env_steps = 100 * 10**6 self.ports_to_use = [] self.root_random_seed = root_random_seed self.purge_previous = purge_previous self.test_mode = test_mode self.log_level = log_level self.verbose = verbose if max_env_steps is not None: self.max_env_steps = max_env_steps self.env_config = self._update_config_dict(self.env_config, env_config) self.cluster_config = self._update_config_dict(self.cluster_config, cluster_config) self.policy_config = self._update_config_dict(self.policy_config, policy_config) self.trainer_config = self._update_config_dict(self.trainer_config, trainer_config) self.trainer_config['kwargs']['test_mode'] = self.test_mode # Logging config: # TODO: Warnings --> to Notices StreamHandler(sys.stdout).push_application() if self.log_level is None: log_levels = [(0, WARNING), (1, INFO), (2, DEBUG)] self.log_level = WARNING for key, value in log_levels: if key == self.verbose: self.log_level = value self.log = Logger('Launcher_shell', level=self.log_level) # Seeding: if self.root_random_seed is not None: np.random.seed(self.root_random_seed) self.log.info('Random seed: {}'.format(self.root_random_seed)) # Seeding for workers: workers_rnd_seeds = list( np.random.randint( 0, 2**30, self.cluster_config['num_workers'] + self.cluster_config['num_ps'])) # Log_dir: if os.path.exists(self.cluster_config['log_dir']): # Remove previous log files and saved model if opted: if self.purge_previous > 0: confirm = 'y' if self.purge_previous < 2: confirm = input( '<{}> already exists. Override[y/n]? '.format( self.cluster_config['log_dir'])) if confirm in 'y': files = glob.glob(self.cluster_config['log_dir'] + '/*') p = psutil.Popen([ 'rm', '-R', ] + files, stdout=PIPE, stderr=PIPE) self.log.warning('Files in <{}> purged.'.format( self.cluster_config['log_dir'])) else: self.log.warning('Appending to <{}>.'.format( self.cluster_config['log_dir'])) else: os.makedirs(self.cluster_config['log_dir']) self.log.warning('<{}> created.'.format( self.cluster_config['log_dir'])) for kwarg in ['port', 'data_port']: assert kwarg in self.env_config['kwargs'].keys() assert self.env_config['class_ref'] is not None # Make cluster specification dict: self.cluster_spec = self.make_cluster_spec(self.cluster_config) # Configure workers: self.workers_config_list = [] env_ports = np.arange(self.cluster_config['num_envs']) worker_port = self.env_config['kwargs'][ 'port'] # start value for BTGym comm. port #for k, v in self.env_config['kwargs'].items(): # try: # c = copy.deepcopy(v) # print('key: {} -- deepcopy ok.'.format(k)) # except: # print('key: {} -- deepcopy failed!.'.format(k)) # TODO: Hacky, cause dataset is threadlocked; do: pass dataset as class_ref + kwargs_dict: if self.test_mode: dataset_instance = None else: dataset_instance = self.env_config['kwargs'].pop('dataset') for key, spec_list in self.cluster_spec.items(): task_index = 0 # referenced farther as worker id for _id in spec_list: env_config = copy.deepcopy(self.env_config) worker_config = {} if key in 'worker': # Configure worker BTgym environment: if task_index == 0: env_config['kwargs'][ 'data_master'] = True # set worker_0 as chief and data_master env_config['kwargs']['dataset'] = dataset_instance env_config['kwargs']['render_enabled'] = True else: env_config['kwargs']['data_master'] = False env_config['kwargs'][ 'render_enabled'] = False # disable rendering for all but chief # Add list of connection ports for every parallel env for each worker: env_config['kwargs']['port'] = list(worker_port + env_ports) worker_port += self.cluster_config['num_envs'] worker_config.update({ 'env_config': env_config, 'policy_config': self.policy_config, 'trainer_config': self.trainer_config, 'cluster_spec': self.cluster_spec, 'job_name': key, 'task': task_index, 'test_mode': self.test_mode, 'log_dir': self.cluster_config['log_dir'], 'max_env_steps': self.max_env_steps, #'log': self.log, 'log_level': self.log_level, 'random_seed': workers_rnd_seeds.pop() }) self.clear_port(env_config['kwargs']['port']) self.workers_config_list.append(worker_config) task_index += 1 self.clear_port(self.env_config['kwargs']['data_port']) self.log.debug('Launcher ready.') def make_cluster_spec(self, config): """ Composes cluster specification dictionary. """ cluster = {} all_ps = [] port = config['port'] for _ in range(config['num_ps']): self.clear_port(port) self.ports_to_use.append(port) all_ps.append('{}:{}'.format(config['host'], port)) port += 1 cluster['ps'] = all_ps all_workers = [] for _ in range(config['num_workers']): self.clear_port(port) self.ports_to_use.append(port) all_workers.append('{}:{}'.format(config['host'], port)) port += 1 cluster['worker'] = all_workers return cluster def clear_port(self, port_list): """ Kills process on specified ports list, if any. """ if not isinstance(port_list, list): port_list = [port_list] for port in port_list: p = psutil.Popen(['lsof', '-i:{}'.format(port), '-t'], stdout=PIPE, stderr=PIPE) pid = p.communicate()[0].decode()[:-1] # retrieving PID if pid is not '': p = psutil.Popen(['kill', pid]) self.log.info('port {} cleared'.format(port)) def _update_config_dict(self, old_dict, new_dict=None): """ Service, updates nested dictionary with values from other one of same structure. Args: old_dict: dict to update to new_dict: dict to update from Returns: new updated dict """ if type(new_dict) is not dict: new_dict = old_dict # ~identity op for key, value in new_dict.items(): if type(value) == dict: old_dict[key] = self._update_config_dict(old_dict[key], value) else: old_dict[key] = value return old_dict def run(self): """ Launches processes: distributed workers; parameter_server. """ workers_list = [] p_servers_list = [] chief_worker = None def signal_handler(signal, frame): nonlocal workers_list nonlocal chief_worker nonlocal p_servers_list def stop_worker(worker_list): for worker in worker_list: worker.terminate() stop_worker(workers_list) stop_worker([chief_worker]) stop_worker(p_servers_list) # Start workers: for worker_config in self.workers_config_list: # Make: worker = Worker(**worker_config) # Launch: worker.daemon = False worker.start() if worker.job_name in 'worker': # Allow data-master to launch datafeed_server: if worker_config['env_config']['kwargs']['data_master']: time.sleep(5) chief_worker = worker else: workers_list.append(worker) else: p_servers_list.append(worker) # TODO: auto-launch tensorboard? signal.signal(signal.SIGINT, signal_handler) # Halt here: msg = 'Press `Ctrl-C` or [Kernel]->[Interrupt] to stop training and close launcher.' print(msg) self.log.info(msg) signal.pause() # Wait every worker to finish: for worker in workers_list: worker.join() self.log.info('worker_{} has joined.'.format(worker.task)) chief_worker.join() self.log.info('chief_worker_{} has joined.'.format(chief_worker.task)) for ps in p_servers_list: ps.join() self.log.info('parameter_server_{} has joined.'.format(ps.task)) # TODO: close tensorboard self.log.info('Launcher closed.')
class Paxos(object): def __init__(self, transport, on_learn, on_prepare=None, on_stale=None, quorum_timeout=3, logger_group=None, ): self._logger = Logger('paxos') if logger_group is not None: logger_group.add_logger(self._logger) self.transport = transport self.on_learn = on_learn self.on_prepare = on_prepare self.on_stale = on_stale self.quorum_timeout = quorum_timeout self.id = 0 self.max_seen_id = 0 self.last_accepted_id = 0 self._logger.debug('2 last_accepted_id=%(last_accepted_id)s' % self.__dict__) self.proposed_value = None self.deferred = None self.queue = deque() # queue of (value, deferred) to propose self._learn_queue = [] # sorted list with learn requests which come out of order # delayed calls for timeouts self._accepted_timeout = None self._acks_timeout = None self._waiting_to_learn_id = deque() def recv(self, message, client): message = shlex.split(message) command = getattr(self, message[0]) command(client=client, *message[1:]) def propose(self, value): deferred = Deferred() if self.proposed_value is None: self._start_paxos(value, deferred) else: self.queue.append((value, deferred)) self._logger.debug('Request for %s was queued, queue size is %s (because we are proposing %s now)' % ( value, len(self.queue), self.proposed_value, ) ) return deferred def _start_paxos(self, value, deferred): """Starts paxos iteration proposing given value.""" self.id = self.max_seen_id + 1 self.proposed_value = value self.deferred = deferred self._num_acks_to_wait = self.transport.quorum_size def _timeout_callback(): self._logger.info('+++ prepare timeout') # TODO sometimes self.deferred is None when this callbach is called self.deferred.errback(PrepareTimeout()) self.deferred = None self.proposed_value = None self._acks_timeout = reactor.callLater(self.quorum_timeout, _timeout_callback) self.transport.broadcast('paxos_prepare %s %s' % (self.id, self.last_accepted_id)) def paxos_prepare(self, num, last_accepted_id, client): num = int(num) last_accepted_id = int(last_accepted_id) if last_accepted_id > self.last_accepted_id: # Move to the "stale" state self._logger.debug('stale last_accepted_id(%s) < self.last_accepted_id(%s)' % ( last_accepted_id, self.last_accepted_id )) self.on_stale(last_accepted_id) else: if num > self.max_seen_id: if self.on_prepare is not None: self.on_prepare(num, client) self.max_seen_id = num self._send_to(client, 'paxos_ack %s' % num) def paxos_ack(self, num, client): num = int(num) if self.proposed_value is not None and num == self.id: self._num_acks_to_wait -= 1 if self._num_acks_to_wait == 0: _stop_waiting(self._acks_timeout) self._num_accepts_to_wait = self.transport.quorum_size def _timeout_callback(): self._logger.info('+++ accept timeout') self.deferred.errback(AcceptTimeout()) self.deferred = None self.proposed_value = None self._accepted_timeout = reactor.callLater( self.quorum_timeout, _timeout_callback ) self.transport.broadcast('paxos_accept %s "%s"' % (self.id, escape(self.proposed_value))) def paxos_accept(self, num, value, client): num = int(num) if num == self.max_seen_id: if self.id == num: # we have a deferred to return result in this round self._waiting_to_learn_id.append((num, self.deferred)) else: # may be we have deferred but it is for another Paxos round self._waiting_to_learn_id.append((num, None)) self._send_to(client, 'paxos_accepted %s' % num) def paxos_accepted(self, num, client): num = int(num) if self.proposed_value is not None and num == self.id: self._num_accepts_to_wait -= 1 if self._num_accepts_to_wait == 0: _stop_waiting(self._accepted_timeout) self.transport.broadcast('paxos_learn %s "%s"' % (self.id, escape(self.proposed_value))) def paxos_learn(self, num, value, client): self._logger.info('paxos.learn %s' % value) num = int(num) if self._waiting_to_learn_id and num == self._waiting_to_learn_id[0][0]: num, deferred = self._waiting_to_learn_id.popleft() try: result = self.on_learn(num, value, client) except Exception, e: self._logger.exception('paxos.learn %s' % value) result = e self.last_accepted_id = num self._logger.debug('1 last_accepted_id=%(last_accepted_id)s' % self.__dict__) if deferred is not None and value == self.proposed_value: # this works for current round coordinator only # because it must return result to the client # and to start a new round for next request if isinstance(result, Exception): self._logger.warning('returning error from paxos.learn %s, %s' % (value, result)) deferred.errback(result) else: self._logger.warning('returning success from paxos.learn %s' % value) deferred.callback(result) self._logger.debug('queue size: %s' % len(self.queue)) if self.queue: # start new Paxos instance # for next value from the queue next_value, deferred = self.queue.pop() self._logger.debug('next value from the queue: %s' % next_value) self._start_paxos(next_value, deferred) else: self.proposed_value = None self.deferred = None if self._learn_queue: self._logger.debug('relearning remembered values') # clear queue because it will be filled again if needed queue, self._learn_queue = self._learn_queue, [] for args in queue: self.paxos_learn(*args) else:
class Plug(object): """The Plug is the communication pipe between a Driver and Onitu. The driver should instantiate a new Plug as soon as possible, define his handlers (see :func:`Plug.handler`) and call :func:`Plug.start` when it's ready to receive notifications. """ def __init__(self): super(Plug, self).__init__() self.redis = connect_to_redis() self.name = None self.logger = None self.router = None self.worker = None self.options = {} self._handlers = {} def start(self, name): """This method should be called when the driver is ready to communicate with Onitu. Takes a parameter `name` that corresponds to the first parameter given to the Driver at start. :func:`start` launches two threads : - The :class:`worker.Worker`, listening to notifications from the Referee and handling them by calling the handlers defined by the Driver - The :class:`router.Router`, listening to requests by other Drivers that need a chunk of a file and getting this chunk by calling the `get_chunk` handler. """ self.name = name self.logger = Logger(self.name) self.options = self.redis.hgetall('drivers:{}:options'.format(name)) self.logger.info("Started") self.router = Router(name, self.redis, self._handlers.get('get_chunk')) self.router.start() self.worker = Worker(self) self.worker.start() self.worker.resume_transfers() def wait(self): """Waits until the :class:`Plug` is killed by another process. """ self.router.join() def handler(self, task=None): """Decorator used to bind to a function assigned to a specific task. Example:: @plug.handler('get_chunk') def read(filename, offset, size): with open(filename, 'rb') as f: f.seek(offset) return f.read(size) Currently, the supported tasks are : - `get_chunk`, which takes the name of the file, the offset and the size of the chunk in parameter, and should return a string. - `start_upload`, which takes a `Metadata` in parameter, returns nothing, and is called before each transfer of a complete file. - `upload_chunk`, which takes the name of the file, the offset and the chunk to be uploaded and return nothing. - `end_upload`, which takes a `Metadata` in parameter, returns nothing, and is called at the end of each transfer of a complete file. A Driver can implement any, all or none of the tasks above. """ def decorator(handler): self._handlers[task if task else handler.__name__] = handler return handler return decorator def update_file(self, metadata): """This method should be called by the Driver after each update of a file or after the creation of a file. It takes a `Metadata` object in parameter that should have been updated with the new value of the properties. """ fid = self.redis.hget('files', metadata.filename) if not fid: fid = self.redis.incr('last_id') self.redis.hset('files', metadata.filename, fid) self.redis.hset('drivers:{}:files'.format(self.name), fid, "") metadata.owners = [self.name] metadata._fid = fid elif self.redis.sismember('drivers:{}:transfers'.format(self.name), fid): # The event has been triggered during a transfer, we # have to cancel it. self.logger.warning( "About to send an event for '{}' when downloading it, " "aborting the event", metadata.filename) return metadata.uptodate = [self.name] metadata.write() self.redis.rpush('events', "{}:{}".format(self.name, fid)) def get_metadata(self, filename): """Returns a `Metadata` object corresponding to the given filename. """ metadata = Metadata.get_by_filename(self, filename) if not metadata: metadata = Metadata(plug=self, filename=filename) metadata.entry = self.name return metadata
class BaseDataSet(): """ Basic data preparation: csv -> pd.dataframe -> sparse matrix """ def __init__(self, filename, path='data/', rates_filename='rub_exchange_rates.csv'): StreamHandler(sys.stdout).push_application() self.path = path self.filename = filename self.rates_filename = rates_filename self.log = Logger('BaseData/' + self.filename) self.mcc_codes_table = pd.read_html(requests.get( 'https://mcc-codes.ru/code', headers={ 'User-agent': 'Mozilla/5.0' }).text, converters={'MCC': str})[0] self.mcc_map = self.mcc_codes_table[[ u'MCC', u'Группа' ]].set_index('MCC').to_dict()[u'Группа'] self.rates = pd.read_csv(self.path + self.rates_filename) self.data = None self.aggregated_data = None self.sparse_array_data = None self.mcc_groups = None self.grouped_by_id = None @staticmethod def extract_month(s): day, month, year = s.split('/') month = int(month) month += (int(year) - 2016) * 12 return month def _to_rur(self, obj): if obj['currency'] != 810: # Get currency: curr = iso4217parse.parse(int(obj['currency']))[0] if curr is not None: curr_alpha3 = curr[0] else: raise ValueError('Unknown currency code: {}'.format( obj['currency'])) curr_amount = obj['amount'] # Convert: try: obj['amount'] = curr_amount / self.rates.loc[curr_alpha3][ 'rate'] # print('Converted {} {} to {} RUR'.format(curr_amount, curr_alpha3, obj['amount'])) except KeyError as e: # Ex. rate not found, fake 1000 rub on tis transaction: self.log.warning( 'Rate missing for {} {}, substituted by 1000 RUR'.format( obj['amount'], curr_alpha3)) obj['amount'] = 1000 obj['currency'] = 643 return obj @staticmethod def _day_to_int(obj): obj['rel_day'] = obj['rel_day'].days return obj def load_csv(self, truncate=None, **kwargs): self.data = pd.read_csv(self.path + self.filename) self.log.info('Loaded data shape: {}'.format(self.data.shape)) if truncate is not None: assert truncate < self.data.shape[0],\ 'Truncation index {} is bigger than data size {}'.format(truncate, self.data.shape[0]) self.data = self.data[0:truncate] self.log.info( 'Data truncated down to first {} rows'.format(truncate)) self.data['PERIOD'] = pd.to_datetime(self.data['PERIOD'], format='%m/%d/%Y') self.data['TRDATETIME'] = pd.to_datetime(self.data['TRDATETIME'], format='%d%b%y:%X') self.data['rel_day'] = self.data.apply( lambda zero: datetime.timedelta(), axis=1) self.data['channel_type'] = self.data['channel_type'].fillna( '0').apply(lambda s: int(s[-1])) self.data['mcc_group'] = self.data['MCC'].astype(str).map(self.mcc_map) self.mcc_groups = list(set(self.data.mcc_group.unique())) self.grouped_by_id = None def to_relative_days(self, **kwargs): self.grouped_by_id = self.data.groupby('cl_id') for cl_id, group in self.grouped_by_id: start_tstamp = copy.deepcopy(group['TRDATETIME'].min()) idx = copy.deepcopy(group.index) self.data.loc[idx, 'rel_day'] = (self.data.loc[idx, 'TRDATETIME'] - start_tstamp) self.data = self.data.transform(self._day_to_int, axis=1) def to_rur(self, **kwargs): self.data = self.data.transform(self._to_rur, axis=1) def aggregate_by_daily_sums(self, **kwargs): self.grouped_by_id = self.data.groupby('cl_id') current_idx = 0 col_names = ['cl_id', 'rel_day', 'sum_POS'] col_names += ['sum_{}'.format(group) for group in self.mcc_groups] col_names += ['target_flag', 'target_sum'] aggr_data = pd.DataFrame(index=None, columns=col_names).fillna(0) for cl_id, cl_group in self.grouped_by_id: id_by_day = cl_group.groupby('rel_day') for day, ts_group in id_by_day: day_by_mcc = ts_group.groupby('MCC') day_sum_pos = 0 s = pd.Series(name=current_idx, index=col_names).fillna(0) s['cl_id'] = cl_id s['rel_day'] = day try: s['target_flag'] = ts_group.target_flag.all() s['target_sum'] = ts_group.target_sum.mean() except AttributeError: s['target_flag'] = float('NaN') s['target_sum'] = float('NaN') for mcc_id, ts in day_by_mcc: day_sum_pos += ts[ts.trx_category == 'POS']['amount'].sum() s['sum_{}'.format( ts.mcc_group.values[0])] += ts['amount'].sum() s['sum_POS'] = day_sum_pos aggr_data = aggr_data.append(s) current_idx += 1 return aggr_data def save_csv(self, data, file_prefix='_', **kwargs): data.to_csv(self.path + file_prefix + self.filename, index=True, index_label=False) def save_sparse(self, data, file_prefix='sparse_array_', **kwargs): assert isinstance( data, sparse.COO), 'Expected sparse.COO data type, got: {}'.format( type(data)) sparse.save_npz(self.path + file_prefix + self.filename[:-4] + '.npz', data) def to_sparse_array(self, min_days=90, max_days=None, rebase=True, **kwargs): if self.aggregated_data is not None: clients = [] if max_days is None: max_days = int(self.aggregated_data['rel_day'].max() ) + 1 # infer from data if max_days < 91: max_days = 91 self.log.info( 'max_days inferred from data: {}'.format(max_days)) if rebase: assert min_days < max_days, 'It is not possible to rebase with min_days={}, max_days={}'.format( min_days, max_days) self.log.info( 'Rebasing observations with min_days={}'.format(min_days)) num_col = int(self.aggregated_data.shape[-1]) - 1 client_shape = [1, max_days, num_col] agg_by_id = self.aggregated_data.groupby('cl_id') for cl_id, cl_group in agg_by_id: client_array = np.zeros(client_shape) client_values = cl_group.values client_index = client_values[:, 1].astype( int) # second column - rel_day values --> to 0_dim values if rebase: # Rebase but allow no less than 90 days observation period: client_max_day = client_index.max() if client_max_day < min_days: client_max_day = min_days rebase_index = max_days - client_max_day - 1 client_index += rebase_index client_array[:, client_index, :] = client_values[:, 1:] # remove cl_id (gone to new dim) and rel_day # Fill all records for single client: client_array[..., 0] = int(cl_id) # id if np.isnan(cl_group.target_sum).any(): client_array[..., -1] = float('NaN') client_array[..., -2] = float('NaN') else: client_array[..., -1] = cl_group.target_sum.mean() client_array[..., -2] = cl_group.target_flag.all() # Save as sparse 3d array: clients.append(sparse.COO(client_array.astype('float32'))) full_array_sparse = sparse.concatenate(clients, axis=0) else: self.log.warning( 'No aggregated data found, call .aggregate_by_daily_sums() method first.' ) full_array_sparse = None return full_array_sparse def process(self, level=2, **kwargs): if self.data is None: self.log.info('Loading data...') else: self.log.info('Reloading data...') self.load_csv(**kwargs) self.log.info('Converting to RUB...') self.to_rur(**kwargs) self.log.info('Calculating relative days...') self.to_relative_days(**kwargs) if level > 0: self.log.info('Aggregating by daily sums...') self.aggregated_data = self.aggregate_by_daily_sums(**kwargs) self.log.info('Saving aggregated csv file...') self.save_csv(self.aggregated_data, file_prefix='aggr_') if level > 1: self.log.info('Making sparse data array...') self.sparse_array_data = self.to_sparse_array(**kwargs) self.log.info('Saving sparse array...') self.save_sparse(self.sparse_array_data) self.log.info('Done.')
class BTgymDataFeedServer(multiprocessing.Process): """ Data provider server class. Enables efficient data sampling for asynchronous multiply BTgym environments execution. Manages global back-testing time. """ process = None dataset_stat = None def __init__(self, dataset=None, network_address=None, log_level=None, task=0): """ Configures data server instance. Args: dataset: data domain instance; network_address: ...to bind to. log_level: int, logbook.level task: id """ super(BTgymDataFeedServer, self).__init__() self.log_level = log_level self.task = task self.log = None self.local_step = 0 self.dataset = dataset self.network_address = network_address self.default_sample_config = copy.deepcopy(DataSampleConfig) self.debug_pre_sample_fails = 0 self.debug_pre_sample_attempts = 0 # self.global_timestamp = 0 def get_data(self, sample_config=None): """ Get Trial sample according to parameters received. If no parameters being passed - makes sample with default parameters. Args: sample_config: sampling parameters configuration dictionary Returns: sample: if `sample_params` arg has been passed and dataset is ready None: otherwise """ if self.dataset.is_ready: if sample_config is not None: # We do not allow configuration timestamps which point earlier than current global_timestamp; # if config timestamp points later - it is ok because global time will be shifted accordingly after # [traget test] sample will get into work. if sample_config['timestamp'] is None: sample_config['timestamp'] = 0 # If config timestamp is outdated - refresh with latest: if sample_config['timestamp'] < self.dataset.global_timestamp: sample_config['timestamp'] = copy.deepcopy( self.dataset.global_timestamp) self.log.debug( 'Sampling with params: {}'.format(sample_config)) sample = self.dataset.sample(**sample_config) else: self.default_sample_config['timestamp'] = copy.deepcopy( self.dataset.global_timestamp) self.log.debug('Sampling with default params: {}'.format( self.default_sample_config)) sample = self.dataset.sample(**self.default_sample_config) self.local_step += 1 else: # Dataset not ready, make dummy: sample = None return sample def run(self): """ Server process runtime body. """ # Logging: from logbook import Logger, StreamHandler, WARNING import sys StreamHandler(sys.stdout).push_application() if self.log_level is None: self.log_level = WARNING self.log = Logger('BTgymDataServer_{}'.format(self.task), level=self.log_level) self.process = multiprocessing.current_process() self.log.info('PID: {}'.format(self.process.pid)) # Set up a comm. channel for server as ZMQ socket: context = zmq.Context() socket = context.socket(zmq.REP) socket.bind(self.network_address) # Actually load data to BTgymDataset instance, will reset it later on: try: assert not self.dataset.data.empty except (AssertionError, AttributeError) as e: self.dataset.read_csv() # Describe dataset: self.dataset_stat = self.dataset.describe() # Main loop: while True: # Stick here until receive any request: service_input = socket.recv_pyobj() self.log.debug('Received <{}>'.format(service_input)) if 'ctrl' in service_input: # It's time to exit: if service_input['ctrl'] == '_stop': # Server shutdown logic: # send last run statistic, release comm channel and exit: message = {'ctrl': 'Exiting.'} self.log.info(str(message)) socket.send_pyobj(message) socket.close() context.destroy() return None # Reset datafeed: elif service_input['ctrl'] == '_reset_data': try: kwargs = service_input['kwargs'] except KeyError: kwargs = {} self.dataset.reset(**kwargs) # self.global_timestamp = self.dataset.global_timestamp self.log.notice( 'Initial global_time set to: {} / stamp: {}'.format( datetime.datetime.fromtimestamp( self.dataset.global_timestamp), self.dataset.global_timestamp)) message = {'ctrl': 'Reset with kwargs: {}'.format(kwargs)} self.log.debug('Data_is_ready: {}'.format( self.dataset.is_ready)) socket.send_pyobj(message) self.local_step = 0 # Send dataset sample: elif service_input['ctrl'] == '_get_data': if self.dataset.is_ready: sample = self.get_data( sample_config=service_input['kwargs']) message = 'Sending sample_#{}.'.format(self.local_step) self.log.debug(message) socket.send_pyobj({ 'sample': sample, 'stat': self.dataset_stat, 'origin': 'data_server', 'timestamp': self.dataset.global_timestamp, }) else: message = { 'ctrl': 'Dataset not ready, waiting for control key <_reset_data>' } self.log.debug('Sent: ' + str(message)) socket.send_pyobj(message) # pairs any other input # Send dataset statisitc: elif service_input['ctrl'] == '_get_info': message = 'Sending info for #{}.'.format(self.local_step) self.log.debug(message) # Compose response: info_dict = dict(dataset_stat=self.dataset_stat, dataset_columns=list(self.dataset.names), pid=self.process.pid, dataset_is_ready=self.dataset.is_ready) socket.send_pyobj(info_dict) # Set global time: elif service_input['ctrl'] == '_set_global_time': if self.dataset.global_timestamp != 0 and self.dataset.global_timestamp > service_input[ 'timestamp']: message = 'Moving back in time not supported! ' +\ 'Current global_time: {}, '.\ format(datetime.datetime.fromtimestamp(self.dataset.global_timestamp)) +\ 'attempt to set: {}; nothing done. '.\ format(datetime.datetime.fromtimestamp(service_input['timestamp'])) +\ 'Hint: check sampling logic consistency.' self.log.warning(message) else: self.dataset.global_timestamp = service_input[ 'timestamp'] message = 'global_time set to: {} / stamp: {}'.\ format( datetime.datetime.fromtimestamp(self.dataset.global_timestamp), self.dataset.global_timestamp ) socket.send_pyobj(message) self.log.debug(message) elif service_input['ctrl'] == '_get_global_time': # Tell time: message = {'timestamp': self.dataset.global_timestamp} socket.send_pyobj(message) else: # ignore any other input # NOTE: response dictionary must include 'ctrl' key message = { 'ctrl': 'waiting for control keys: <_reset_data>, <_get_data>, <_get_info>, <_stop>.' } self.log.debug('Sent: ' + str(message)) socket.send_pyobj(message) # pairs any other input else: message = { 'ctrl': 'No <ctrl> key received, got:\n{}'.format(service_input) } self.log.debug(str(message)) socket.send_pyobj(message) # pairs input
class AMLDG(): """ Asynchronous implementation of MLDG algorithm (by Da Li et al.) for one-shot adaptation in dynamically changing environments. Papers: Da Li et al., "Learning to Generalize: Meta-Learning for Domain Generalization" https://arxiv.org/abs/1710.03463 Maruan Al-Shedivat et al., "Continuous Adaptation via Meta-Learning in Nonstationary and Competitive Environments" https://arxiv.org/abs/1710.03641 """ def __init__( self, env, task, log_level, aac_class_ref=SubAAC, runner_config=None, aac_lambda=1.0, guided_lambda=1.0, rollout_length=20, trial_source_target_cycle=(1, 0), num_episodes_per_trial=1, # one-shot adaptation _aux_render_modes=('action_prob', 'value_fn', 'lstm_1_h', 'lstm_2_h'), name='AMLDG', **kwargs): try: self.aac_class_ref = aac_class_ref self.task = task self.name = name self.summary_writer = None StreamHandler(sys.stdout).push_application() self.log = Logger('{}_{}'.format(name, task), level=log_level) self.rollout_length = rollout_length if runner_config is None: self.runner_config = { 'class_ref': BaseSynchroRunner, 'kwargs': {}, } else: self.runner_config = runner_config self.env_list = env assert isinstance(self.env_list, list) and len(self.env_list) == 2, \ 'Expected pair of environments, got: {}'.format(self.env_list) # Instantiate two sub-trainers: one for meta-test and one for meta-train environments: self.runner_config['kwargs']['data_sample_config'] = { 'mode': 1 } # master self.runner_config['kwargs']['name'] = 'master' self.train_aac = aac_class_ref( env=self.env_list[0], # train data will be master environment task=self.task, log_level=log_level, runner_config=self.runner_config, aac_lambda=aac_lambda, guided_lambda=guided_lambda, rollout_length=self.rollout_length, trial_source_target_cycle=trial_source_target_cycle, num_episodes_per_trial=num_episodes_per_trial, _use_target_policy=False, _use_global_network=True, _aux_render_modes=_aux_render_modes, name=self.name + '/metaTrain', **kwargs) self.runner_config['kwargs']['data_sample_config'] = { 'mode': 0 } # slave self.runner_config['kwargs']['name'] = 'slave' self.test_aac = aac_class_ref( env=self.env_list[-1], # test data -> slave env. task=self.task, log_level=log_level, runner_config=self.runner_config, aac_lambda=aac_lambda, guided_lambda=guided_lambda, rollout_length=self.rollout_length, trial_source_target_cycle=trial_source_target_cycle, num_episodes_per_trial=num_episodes_per_trial, _use_target_policy=False, _use_global_network=False, global_step_op=self.train_aac.global_step, global_episode_op=self.train_aac.global_episode, inc_episode_op=self.train_aac.inc_episode, _aux_render_modes=_aux_render_modes, name=self.name + '/metaTest', **kwargs) self.local_steps = self.train_aac.local_steps self.model_summary_freq = self.train_aac.model_summary_freq self._make_train_op() self.test_aac.model_summary_op = tf.summary.merge( [ self.test_aac.model_summary_op, self._combine_meta_summaries() ], name='meta_model_summary') except: msg = 'AMLDG.__init()__ exception occurred' + \ '\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n' self.log.exception(msg) raise RuntimeError(msg) def _make_train_op(self): """ Defines tensors holding training op graph for meta-train, meta-test and meta-optimisation. """ # Handy aliases: pi = self.train_aac.local_network # local meta-train policy pi_prime = self.test_aac.local_network # local meta-test policy pi_global = self.train_aac.network # global shared policy self.test_aac.sync = self.test_aac.sync_pi = tf.group( *[v1.assign(v2) for v1, v2 in zip(pi_prime.var_list, pi.var_list)]) # Shared counters: self.global_step = self.train_aac.global_step self.global_episode = self.train_aac.global_episode self.test_aac.global_step = self.train_aac.global_step self.test_aac.global_episode = self.train_aac.global_episode self.test_aac.inc_episode = self.train_aac.inc_episode self.train_aac.inc_episode = None self.inc_step = self.train_aac.inc_step # Meta-opt. loss: self.loss = self.train_aac.loss + self.test_aac.loss # Clipped gradients: self.train_aac.grads, _ = tf.clip_by_global_norm( tf.gradients(self.train_aac.loss, pi.var_list), 40.0) self.test_aac.grads, _ = tf.clip_by_global_norm( tf.gradients(self.test_aac.loss, pi_prime.var_list), 40.0) # Aliases: pi.grads = self.train_aac.grads pi_prime.grads = self.test_aac.grads # # Learned meta opt. scaling (equivalent to learned meta-update step-size), # # conditioned on test input: # self.meta_grads_scale = tf.reduce_mean(pi_prime.meta_grads_scale) # meta_grads_scale_var_list = [var for var in pi_prime.var_list if 'meta_grads_scale' in var.name] # meta_grads_scale_var_list_global = [ # var for var in pi_global.var_list if 'meta_grads_scale' in var.name # ] # self.log.warning('meta_grads_scale_var_list: {}'.format(meta_grads_scale_var_list)) # Meta_optimisation gradients as sum of meta-train and meta-test gradients: self.grads = [] for g1, g2 in zip(pi.grads, pi_prime.grads): if g1 is not None and g2 is not None: meta_g = g1 + g2 # meta_g = (1 - self.meta_grads_scale) * g1 + self.meta_grads_scale * g2 else: meta_g = None # need to map correctly to vars self.grads.append(meta_g) # # Second order grads for learned grad. scaling param: # meta_grads_scale_grads, _ = tf.clip_by_global_norm( # tf.gradients([g for g in self.grads if g is not None], meta_grads_scale_var_list), # 40.0 # ) # # Second order grads wrt global variables: # meta_grads_scale_grads_and_vars = list(zip(meta_grads_scale_grads, meta_grads_scale_var_list_global)) # self.log.warning('meta_grads_scale_grads:\n{}'.format(meta_grads_scale_grads)) # self.log.warning('meta_grads_scale_grads_and_vars:\n{}'.format(meta_grads_scale_grads_and_vars)) #self.log.warning('self.grads_len: {}'.format(len(list(self.grads)))) # Gradients to update local meta-test policy (from train data): train_grads_and_vars = list(zip(pi.grads, pi_prime.var_list)) # self.log.warning('train_grads_and_vars_len: {}'.format(len(train_grads_and_vars))) # Meta-gradients to be sent to parameter server: meta_grads_and_vars = list( zip(self.grads, pi_global.var_list)) #+ meta_grads_scale_grads_and_vars # Remove empty entries: meta_grads_and_vars = [(g, v) for (g, v) in meta_grads_and_vars if g is not None] # for item in meta_grads_and_vars: # self.log.warning('\nmeta_g_v: {}'.format(item)) # Set global_step increment equal to observation space batch size: obs_space_keys = list(self.train_aac.local_network.on_state_in.keys()) assert 'external' in obs_space_keys, \ 'Expected observation space to contain `external` mode, got: {}'.format(obs_space_keys) self.train_aac.inc_step = self.train_aac.global_step.assign_add( tf.shape(self.train_aac.local_network.on_state_in['external'])[0]) # Pi to pi_prime local adaptation op: self.train_op = self.train_aac.optimizer.apply_gradients( train_grads_and_vars) # Optimizer for meta-update, sharing same learn rate (change?): self.optimizer = tf.train.AdamOptimizer( self.train_aac.train_learn_rate, epsilon=1e-5) # Global meta-optimisation op: self.meta_train_op = self.optimizer.apply_gradients( meta_grads_and_vars) self.log.debug('meta_train_op defined') def _combine_meta_summaries(self): """ Additional summaries here. """ meta_model_summaries = [ tf.summary.scalar('meta_grad_global_norm', tf.global_norm(self.grads)), tf.summary.scalar('total_meta_loss', self.loss), # tf.summary.scalar('meta_grad_scale', self.meta_grads_scale) ] return meta_model_summaries def start(self, sess, summary_writer, **kwargs): """ Executes all initializing operations, starts environment runner[s]. Supposed to be called by parent worker just before training loop starts. Args: sess: tf session object. kwargs: not used by default. """ try: # Copy weights from global to local: sess.run(self.train_aac.sync_pi) sess.run(self.test_aac.sync_pi) # Start thread_runners: self.train_aac._start_runners( # master first sess, summary_writer, init_context=None, data_sample_config=self.train_aac.get_sample_config(mode=1)) self.test_aac._start_runners( sess, summary_writer, init_context=None, data_sample_config=self.test_aac.get_sample_config(mode=0)) self.summary_writer = summary_writer self.log.notice('Runners started.') except: msg = 'start() exception occurred' + \ '\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n' self.log.exception(msg) raise RuntimeError(msg) def process(self, sess): """ Meta-train/test procedure for one-shot learning. Single call runs single meta-test episode. Args: sess (tensorflow.Session): tf session obj. """ try: # Copy from parameter server: sess.run(self.train_aac.sync_pi) sess.run(self.test_aac.sync_pi) # self.log.warning('Init Sync ok.') # Get data configuration, # (want both data streams come from same trial, # and trial type we got can be either from source or target domain); # note: data_config counters get updated once per .process() call train_data_config = self.train_aac.get_sample_config( mode=1) # master env., samples trial test_data_config = self.train_aac.get_sample_config( mode=0) # slave env, catches up with same trial # self.log.warning('train_data_config: {}'.format(train_data_config)) # self.log.warning('test_data_config: {}'.format(test_data_config)) # If this step data comes from source or target domain # (i.e. is it either meta-optimised or true test episode): is_target = train_data_config['trial_config']['sample_type'] done = False # Collect initial meta-train trajectory rollout: train_data = self.train_aac.get_data( data_sample_config=train_data_config, force_new_episode=True) feed_dict = self.train_aac.process_data(sess, train_data, is_train=True) # self.log.warning('Init Train data ok.') # Disable possibility of master data runner acquiring new trials, # in case meta-train episode termintaes earlier than meta-test - # we than need to get additional meta-train trajectories from exactly same distribution (trial): train_data_config['trial_config']['get_new'] = 0 roll_num = 0 # Collect entire meta-test episode rollout by rollout: while not done: # self.log.warning('Roll #{}'.format(roll_num)) wirte_model_summary = \ self.local_steps % self.model_summary_freq == 0 # self.log.warning( # 'Train data trial_num: {}'.format( # np.asarray(train_data['on_policy'][0]['state']['metadata']['trial_num']) # ) # ) # Paranoid checks against data sampling logic faults to prevent possible cheating: train_trial_chksum = np.average( train_data['on_policy'][0]['state']['metadata'] ['trial_num']) # Update pi_prime parameters wrt collected train data: if wirte_model_summary: fetches = [self.train_op, self.train_aac.model_summary_op] else: fetches = [self.train_op] fetched = sess.run(fetches, feed_dict=feed_dict) # self.log.warning('Train gradients ok.') # Collect test rollout using updated pi_prime policy: test_data = self.test_aac.get_data( data_sample_config=test_data_config) # If meta-test episode has just ended? done = np.asarray(test_data['terminal']).any() # self.log.warning( # 'Test data trial_num: {}'.format( # np.asarray(test_data['on_policy'][0]['state']['metadata']['trial_num']) # ) # ) test_trial_chksum = np.average( test_data['on_policy'][0]['state']['metadata'] ['trial_num']) # Ensure slave runner data consistency, can correct if episode just started: if roll_num == 0 and train_trial_chksum != test_trial_chksum: test_data = self.test_aac.get_data( data_sample_config=test_data_config, force_new_episode=True) done = np.asarray(test_data['terminal']).any() faulty_chksum = test_trial_chksum test_trial_chksum = np.average( test_data['on_policy'][0]['state']['metadata'] ['trial_num']) self.log.warning('Test trial corrected: {} -> {}'.format( faulty_chksum, test_trial_chksum)) # self.log.warning( # 'roll # {}: train_trial_chksum: {}, test_trial_chksum: {}'. # format(roll_num, train_trial_chksum, test_trial_chksum) # ) if train_trial_chksum != test_trial_chksum: # Still got error? - highly probable algorithm logic fault. Issue warning. msg = 'Train/test trials mismatch found!\nGot train trials: {},\nTest trials: {}'. \ format( train_data['on_policy'][0]['state']['metadata']['trial_num'][0], test_data['on_policy'][0]['state']['metadata']['trial_num'][0] ) msg2 = 'Train data config: {}\n Test data config: {}'.format( train_data_config, test_data_config) self.log.warning(msg) self.log.warning(msg2) # Check episode type for consistency; if failed - another data sampling logic fault, warn: try: assert (np.asarray(test_data['on_policy'][0]['state'] ['metadata']['type']) == 1).any() assert (np.asarray(train_data['on_policy'][0]['state'] ['metadata']['type']) == 0).any() except AssertionError: msg = 'Train/test episodes types mismatch found!\nGot train ep. type: {},\nTest ep.type: {}'. \ format( train_data['on_policy'][0]['state']['metadata']['type'], test_data['on_policy'][0]['state']['metadata']['type'] ) self.log.warning(msg) # self.log.warning('Test data ok.') if not is_target: # Process test data and perform meta-optimisation step: feed_dict.update( self.test_aac.process_data(sess, test_data, is_train=True)) if wirte_model_summary: meta_fetches = [ self.meta_train_op, self.test_aac.model_summary_op, self.inc_step ] else: meta_fetches = [self.meta_train_op, self.inc_step] meta_fetched = sess.run(meta_fetches, feed_dict=feed_dict) # self.log.warning('Meta-gradients ok.') else: # True test, no updates sent to parameter server: meta_fetched = [None, None] # self.log.warning('Meta-opt. rollout ok.') if wirte_model_summary: meta_model_summary = meta_fetched[-2] model_summary = fetched[-1] else: meta_model_summary = None model_summary = None # Next step housekeeping: # copy from parameter server: sess.run(self.train_aac.sync_pi) sess.run(self.test_aac.sync_pi) # self.log.warning('Sync ok.') # Collect next train trajectory rollout: train_data = self.train_aac.get_data( data_sample_config=train_data_config) feed_dict = self.train_aac.process_data(sess, train_data, is_train=True) # self.log.warning('Train data ok.') # Write down summaries: self.test_aac.process_summary(sess, test_data, meta_model_summary) self.train_aac.process_summary(sess, train_data, model_summary) self.local_steps += 1 roll_num += 1 except: msg = 'process() exception occurred' + \ '\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n' self.log.exception(msg) raise RuntimeError(msg)
class NodesDiscovery: def __init__(self): handler = TimedRotatingFileHandler('../logs/nodes_discovery.log') handler.push_application() self.logger = Logger(name='nodes discovery', level='info') self.node_hosts, self.nodes_port, self.file_sd_filename, self.nodes_file_backup_name, self.exclude_file, self.metric_filename, self.metric_store_path = self.get_conf() self.nodes = {} self.ips = {} self.nodes_list = [] self.ips_list = [] @staticmethod def get_conf(): ##从配置文件获取配置 cp = ConfigParser.ConfigParser() with codecs.open(os.path.join(BASE_DIR, './config/config.ini'), 'r', encoding='utf-8') as f: cp.readfp(f) node_hosts = eval(cp.get('nodes', 'node_hosts').strip()) nodes_port = int(cp.get('nodes', 'node_port').strip()) file_sd_filename = cp.get('file_ds', 'file_sd_filename').strip() nodes_file_backup_name = cp.get('file_ds', 'nodes_file_backup_name').strip() exclude_file = cp.get('file_ds', 'agent_exclude_file').strip() metric_filename = cp.get('prome', 'metric_filename').strip() metric_store_path = cp.get('prome', 'metric_store_path').strip() return node_hosts, nodes_port, os.path.join(BASE_DIR, file_sd_filename), os.path.join(BASE_DIR, nodes_file_backup_name), os.path.join(BASE_DIR, exclude_file), os.path.join(BASE_DIR, metric_filename), metric_store_path def node_scaner(self, ip_range): port_scaner = nmap.PortScanner() try: # 调用扫描方法,参数指定扫描主机hosts,nmap扫描命令行参数arguments port_scaner.scan(hosts=ip_range, arguments=' -v -sS -O -p {0} --excludefile {1}'.format( str(self.nodes_port), self.exclude_file), sudo=True) except Exception as e: self.logger.error("Scan erro:" + str(e)) self.logger.info('nmap port scanner finished!') for host in port_scaner.all_hosts(): # 遍历扫描主机 self.logger.debug(port_scaner[host]) if port_scaner[host]['status']['state'] == 'up': host = port_scaner[host]['addresses']['ipv4'] try: nodes_state = port_scaner[host]['tcp'][9100]['state'] except Exception as e: self.logger.error('Error while get state of host %s: %s' % (host, e)) continue else: self.logger.debug("Host %s %s is %s" % (str(host), str(self.nodes_port), str(nodes_state))) if nodes_state == 'open': os_classes = port_scaner[host]['osmatch'] for os_class_info in os_classes: for os_family_info in os_class_info['osclass']: os_family = os_family_info['osfamily'] self.logger.debug("Host %s system is %s" % (str(host), os_family)) if os_family == 'Linux': self.nodes_list.append(host + ':' + str(self.nodes_port)) self.ips_list.append(host) self.logger.debug('debug info of nodes: [ip: %s]' % host) break break self.logger.info('Finished for ips %s' % ip_range) def host_to_file_sd(self, hosts_list): hosts_conf = json.load(open(self.file_sd_filename, 'r')) self.logger.debug('Latest nodes hosts: %s' % hosts_list) for hosts in hosts_conf: if hosts['labels']['job'] == 'nodes': original_nodes_list = hosts['targets'] self.logger.debug('Nodes before update: %s' % original_nodes_list) self.logger.debug('Nodes updated: %s' % hosts_list) # ensure_nodes = list(set(original_nodes_list) - set(hosts_list)) # uninstall_nodes = [] # for nodes_host in ensure_nodes: # port_scaner = nmap.PortScanner() # try: # # 调用扫描方法,参数指定扫描主机hosts,nmap扫描命令行参数arguments # port_scaner.scan(hosts=nodes_host, arguments=' -v -sS -p {0},22'.format( # str(self.nodes_port)), sudo=True) # except Exception as e: # self.logger.error("Scan erro:" + str(e)) # for host in port_scaner.all_hosts(): # 遍历扫描主机 # self.logger.debug(port_scaner[host]) # if port_scaner[host]['status']['state'] == 'up': # host = port_scaner[host]['addresses']['ipv4'] # try: # nodes_state = port_scaner[host]['tcp'][9100]['state'] # except Exception as e: # self.logger.error('Error while get state of host %s: %s' % (host, e)) # continue # else: # self.logger.debug( # "Host %s %s is %s" % (str(host), str(self.nodes_port), str(nodes_state))) # if nodes_state != 'open': # uninstall_nodes.append(host) nodes_all = list(set(hosts_list).union(set(original_nodes_list))) hosts['targets'] = nodes_all self.logger.info('nodes hosts file updated!') hosts_file = json.dumps(hosts_conf, indent=4, ensure_ascii=False, sort_keys=False, encoding='utf-8') try: with open(self.file_sd_filename, 'w') as f: f.write(hosts_file) except Exception as e: self.logger.error('Write node_exporter info failed: %s' % e) else: shutil.copy(self.file_sd_filename, '../../file_ds/nodes.json') def node_scan(self): self.logger.debug('hosts groups from configure file: %s' % self.node_hosts) for ip_range in self.node_hosts: self.logger.debug('Scan ip range %s' % ip_range) if ip_range: self.node_scaner(ip_range) if self.nodes_list: self.host_to_file_sd(self.nodes_list) self.logger.info('We finished here!') def nodes_file_backup(self): self.logger.info('Backup %s before start scanning' % self.file_sd_filename) shutil.copy(self.file_sd_filename, self.nodes_file_backup_name) def send_metrics(self): nodes_metrics = NodesMetrics() nodes_by_discovery, nodes_hosts = nodes_metrics.get_nodes_discovery() #nodes_type, changed_hosts = nodes_metrics.get_lost_nodes_discovery() current_nodes_by_discovery, lost_nodes_by_discovery, lost_nodes, new_nodes_by_discovery, new_nodes = nodes_metrics.get_lost_nodes_discovery() #metrics_str = 'nodes_discovery_hosts{nodes_type="' + nodes_type + '",changed_hosts="' + changed_hosts + '"} ' + str(nodes_by_discovery) lost_nodes_str = lost_nodes new_nodes_str = new_nodes all_nodes_metrics_str = 'nodes_discovery_hosts ' + str(current_nodes_by_discovery) lost_nodes_metrics_str = 'nodes_discovery_lost_hosts ' + str(lost_nodes_by_discovery) add_nodes_metrics_str = 'nodes_discovery_added_hosts ' + str(new_nodes_by_discovery) self.logger.warning('Lost nodes by discovery: %s' % lost_nodes) self.logger.info('New nodes by discovery: %s' % new_nodes) try: with open(self.metric_filename, 'w') as f: #f.write(metrics_str) f.write(all_nodes_metrics_str) f.write('\n') f.write(lost_nodes_metrics_str) f.write('\n') f.write(add_nodes_metrics_str) f.write('\n') except Exception as e: self.logger.error('Write nodes discovery metrics failed: %s' % e) else: self.logger.info('Send nodes discovery metrics : %s %s %s' % (all_nodes_metrics_str, lost_nodes_metrics_str, add_nodes_metrics_str)) shutil.copy(self.metric_filename, os.path.join(self.metric_store_path, 'all_nodes.prom')) #return lost_nodes_str if lost_nodes_str: lost_nodes = lost_nodes_str.split(' ') nodes_discover.logger.warning('Nodes losts: %s' % lost_nodes) nodes_discover.host_to_file_sd(lost_nodes)
from service.port import IPortUser, Port from service.price import Price from service.settings import HTMLExportSettings, SettingsProvider from service.update import Update import gui.fitCommands as cmd pyfalog = Logger(__name__) disableOverrideEditor = False try: from gui.propertyEditor import AttributeEditor except ImportError as e: AttributeEditor = None pyfalog.warning("Error loading Attribute Editor: %s.\nAccess to Attribute Editor is disabled." % e.message) disableOverrideEditor = True pyfalog.debug("Done loading mainframe imports") # dummy panel(no paint no erasebk) class PFPanel(wx.Panel): def __init__(self, parent): wx.Panel.__init__(self, parent) self.Bind(wx.EVT_PAINT, self.OnPaint) self.Bind(wx.EVT_ERASE_BACKGROUND, self.OnBkErase) def OnPaint(self, event): event.Skip()
import os import sys from logbook import Logger pyfalog = Logger(__name__) # Load variable overrides specific to distribution type try: import configforced except ImportError: pyfalog.warning("Failed to import: configforced") configforced = None # Turns on debug mode debug = False # Defines if our saveddata will be in pyfa root or not saveInRoot = False # Version data version = "1.34.0" tag = "Stable" expansionName = " Arms Race" expansionVersion = "1.3" evemonMinVersion = "4081" pyfaPath = None savePath = None saveDB = None gameDB = None logPath = None
class BaseClient(RpcObject): """RPC客户端""" @property def name(self): return self.__class__.__name__ def __init__(self, reqAddress, subAddress, vaild=True, logdir="../log", logzmqhost=None, tickerAddress=None): """Constructor""" self.__initLog(logdir, logzmqhost) super(BaseClient, self).__init__() assert isinstance(vaild, bool) self.vaild = vaild self.logdir = logdir # 使用 JSON 解包 self.useMsgpack() # 循环为激活状态 self.__active = False # 客户端的工作状态 # 与服务器端的交互线程 self.__reqAddress = reqAddress self.__subAddress = subAddress self.__context = zmq.Context() self.__socketREQ = self.__context.socket(zmq.REQ) # 请求发出socket self.__socketSUB = self.__context.socket(zmq.SUB) # 广播订阅socket # 工作线程相关,用于处理服务器推送的数据 self.__thread = threading.Thread(target=self.run) # 客户端的工作线程 # Ticker 数据订阅 self.__tickerAddress = tickerAddress self.__socketSUBTicker = self.__context.socket(zmq.SUB) # 响应数据索引插入 self.__threadSUBTicker = threading.Thread( name="{}TickSUB".format(self.name), target=self.subscribeTicker) # 客户端的工作线程 # ---------------------------------------------------------------------- def __initLog(self, logdir, logzmqhost): """ 在子进程中初始化客户端的句柄 :param logdir: :param logzmqhost: :return: """ logfile = os.path.join(logdir, '{}.log'.format(self.name)) log.initLog(logfile, logzmqhost) # 使用服务名作为日志的来源名 self.log = Logger(self.name) # ---------------------------------------------------------------------- def __getattr__(self, name): """实现远程调用功能""" # 执行远程调用任务 def dorpc(*args, **kwargs): # 生成请求 req = [name, args, kwargs] # 序列化打包请求 reqb = self.pack(req) # 发送请求并等待回应 self.__socketREQ.send(reqb) repb = self.__socketREQ.recv() # 序列化解包回应 rep = self.unpack(repb) # 若正常则返回结果,调用失败则触发异常 if rep[0]: return rep[1] else: raise RemoteException(rep[1]) return dorpc # ---------------------------------------------------------------------- def start(self): """启动客户端""" # 连接端口 self.__socketREQ.connect(self.__reqAddress) self.__socketSUB.connect(self.__subAddress) # 订阅 Ticker 数据 self.__socketSUBTicker.connect(self.__tickerAddress) self.__socketSUBTicker.setsockopt(zmq.SUBSCRIBE, b'') # 将服务器设为启动 self.__active = True # 启动工作线程 if not self.__thread.isAlive(): self.__thread.start() self.__activeInsert = True if not self.__threadSUBTicker.isAlive(): self.__threadSUBTicker.start() # ---------------------------------------------------------------------- def stop(self): """停止客户端""" # 将客户端设为停止 self.__active = False # 等待工作线程退出 if self.__thread.isAlive(): self.__thread.join() if self.__threadInsert.isAlive(): self.__threadSUBTicker.join() @log.stdout @log.file def run(self): while self.__active: # 接受广播 self.subRev() # 发送数据 try: self.reqSend() except: traceback.print_exc() # ---------------------------------------------------------------------- def subRev(self): """客户端运行函数""" # 使用poll来等待事件到达,等待1秒(1000毫秒) if not self.__socketSUB.poll(1000): return # 从订阅socket收取广播数据 topic, datab = self.__socketSUB.recv_multipart() # 序列化解包 data = self.unpack(datab) self.log.debug("获得订阅消息{}".format(str(data))) # 调用回调函数处理 self.callback(topic, data) @log.stdout def reqSend(self): """ 发送数据 :return: """ # req = ['foo', ('argg'), {"kwargs": 1}] # # # 序列化 # reqb = self.pack(req) # # self.log.debug("往服务器端发送数据") # # self.__socketREQ.send(reqb) # # self.log.debug("等待数据返回") # datab = self.__socketREQ.recv_json() # # 序列化解包 # rep = self.unpack(datab) # # if rep[0]: # return rep[1] # else: # raise RemoteException(rep[1]) def callback(self, topic, data): """回调函数,必须由用户实现""" raise NotImplementedError # ---------------------------------------------------------------------- def subscribeTopic(self, topic): """ 订阅特定主题的广播数据 可以使用topic=''来订阅所有的主题 """ self.__socketSUB.setsockopt(zmq.SUBSCRIBE, topic) @classmethod def process(cls, queue=None, *args, **kwargs): """客户端主程序入口""" # 创建客户端 client = cls(*args, **kwargs) client.subscribeTopic(b'') client.start() if queue is not None: queue.get() else: while True: if input("输入exit退出:") != "exit": continue if input("是否退出(yes/no):") == "yes": break client.stopServer() @log.stdout def subscribeTicker(self): """ 插入数据 :return: """ self.log.warning("开始订阅数据") while self.__active: self._subscribeTicker() @log.stdout @log.file def _subscribeTicker(self): if not self.__socketSUBTicker.poll(1000): # 等待广播 return # 接受广播 ticker = self.__socketSUBTicker.recv_json() self.log.debug("收到订阅的ticker: {}".format(str(ticker)))
class Worker(object): redis_worker_namespace_prefix = 'rq:worker:' redis_workers_keys = 'rq:workers' @classmethod def all(cls, connection=None): """Returns an iterable of all Workers. """ if connection is None: connection = get_current_connection() reported_working = connection.smembers(cls.redis_workers_keys) workers = [cls.find_by_key(key, connection) for key in reported_working] return compact(workers) @classmethod def find_by_key(cls, worker_key, connection=None): """Returns a Worker instance, based on the naming conventions for naming the internal Redis keys. Can be used to reverse-lookup Workers by their Redis keys. """ prefix = cls.redis_worker_namespace_prefix name = worker_key[len(prefix):] if not worker_key.startswith(prefix): raise ValueError('Not a valid RQ worker key: %s' % (worker_key,)) if connection is None: connection = get_current_connection() if not connection.exists(worker_key): return None name = worker_key[len(prefix):] worker = cls([], name) queues = connection.hget(worker.key, 'queues') worker._state = connection.hget(worker.key, 'state') or '?' if queues: worker.queues = map(Queue, queues.split(',')) return worker def __init__(self, queues, name=None, rv_ttl=500, connection=None): # noqa if connection is None: connection = get_current_connection() self.connection = connection if isinstance(queues, Queue): queues = [queues] self._name = name self.queues = queues self.validate_queues() self.rv_ttl = rv_ttl self._state = 'starting' self._is_horse = False self._horse_pid = 0 self._stopped = False self.log = Logger('worker') self.failed_queue = get_failed_queue(connection=self.connection) def validate_queues(self): # noqa """Sanity check for the given queues.""" if not iterable(self.queues): raise ValueError('Argument queues not iterable.') for queue in self.queues: if not isinstance(queue, Queue): raise NoQueueError('Give each worker at least one Queue.') def queue_names(self): """Returns the queue names of this worker's queues.""" return map(lambda q: q.name, self.queues) def queue_keys(self): """Returns the Redis keys representing this worker's queues.""" return map(lambda q: q.key, self.queues) @property # noqa def name(self): """Returns the name of the worker, under which it is registered to the monitoring system. By default, the name of the worker is constructed from the current (short) host name and the current PID. """ if self._name is None: hostname = socket.gethostname() shortname, _, _ = hostname.partition('.') self._name = '%s.%s' % (shortname, self.pid) return self._name @property def key(self): """Returns the worker's Redis hash key.""" return self.redis_worker_namespace_prefix + self.name @property def pid(self): """The current process ID.""" return os.getpid() @property def horse_pid(self): """The horse's process ID. Only available in the worker. Will return 0 in the horse part of the fork. """ return self._horse_pid @property def is_horse(self): """Returns whether or not this is the worker or the work horse.""" return self._is_horse def procline(self, message): """Changes the current procname for the process. This can be used to make `ps -ef` output more readable. """ setprocname('rq: %s' % (message,)) def register_birth(self): # noqa """Registers its own birth.""" self.log.debug('Registering birth of worker %s' % (self.name,)) if self.connection.exists(self.key) and \ not self.connection.hexists(self.key, 'death'): raise ValueError( 'There exists an active worker named \'%s\' ' 'already.' % (self.name,)) key = self.key now = time.time() queues = ','.join(self.queue_names()) with self.connection.pipeline() as p: p.delete(key) p.hset(key, 'birth', now) p.hset(key, 'queues', queues) p.sadd(self.redis_workers_keys, key) p.execute() def register_death(self): """Registers its own death.""" self.log.debug('Registering death') with self.connection.pipeline() as p: # We cannot use self.state = 'dead' here, because that would # rollback the pipeline p.srem(self.redis_workers_keys, self.key) p.hset(self.key, 'death', time.time()) p.expire(self.key, 60) p.execute() def set_state(self, new_state): self._state = new_state self.connection.hset(self.key, 'state', new_state) def get_state(self): return self._state state = property(get_state, set_state) @property def stopped(self): return self._stopped def _install_signal_handlers(self): """Installs signal handlers for handling SIGINT and SIGTERM gracefully. """ def request_force_stop(signum, frame): """Terminates the application (cold shutdown). """ self.log.warning('Cold shut down.') # Take down the horse with the worker if self.horse_pid: msg = 'Taking down horse %d with me.' % self.horse_pid self.log.debug(msg) try: os.kill(self.horse_pid, signal.SIGKILL) except OSError as e: # ESRCH ("No such process") is fine with us if e.errno != errno.ESRCH: self.log.debug('Horse already down.') raise raise SystemExit() def request_stop(signum, frame): """Stops the current worker loop but waits for child processes to end gracefully (warm shutdown). """ self.log.debug('Got %s signal.' % signal_name(signum)) signal.signal(signal.SIGINT, request_force_stop) signal.signal(signal.SIGTERM, request_force_stop) if self.is_horse: self.log.debug('Ignoring signal %s.' % signal_name(signum)) return msg = 'Warm shut down. Press Ctrl+C again for a cold shutdown.' self.log.warning(msg) self._stopped = True self.log.debug('Stopping after current horse is finished.') signal.signal(signal.SIGINT, request_stop) signal.signal(signal.SIGTERM, request_stop) def work(self, burst=False): # noqa """Starts the work loop. Pops and performs all jobs on the current list of queues. When all queues are empty, block and wait for new jobs to arrive on any of the queues, unless `burst` mode is enabled. The return value indicates whether any jobs were processed. """ self._install_signal_handlers() did_perform_work = False self.register_birth() self.state = 'starting' try: while True: if self.stopped: self.log.info('Stopping on request.') break self.state = 'idle' qnames = self.queue_names() self.procline('Listening on %s' % ','.join(qnames)) self.log.info('') self.log.info('*** Listening on %s...' % \ green(', '.join(qnames))) wait_for_job = not burst try: result = Queue.dequeue_any(self.queues, wait_for_job, \ connection=self.connection) if result is None: break except UnpickleError as e: msg = '*** Ignoring unpickleable data on %s.' % \ green(e.queue.name) self.log.warning(msg) self.log.debug('Data follows:') self.log.debug(e.raw_data) self.log.debug('End of unreadable data.') self.failed_queue.push_job_id(e.job_id) continue job, queue = result self.log.info('%s: %s (%s)' % (green(queue.name), blue(job.description), job.id)) self.state = 'busy' self.fork_and_perform_job(job) did_perform_work = True finally: if not self.is_horse: self.register_death() return did_perform_work def fork_and_perform_job(self, job): """Spawns a work horse to perform the actual work and passes it a job. The worker will wait for the work horse and make sure it executes within the given timeout bounds, or will end the work horse with SIGALRM. """ child_pid = Process()# if child_pid == 0: self.main_work_horse(job) else: self._horse_pid = child_pid self.procline('Forked %d at %d' % (child_pid, time.time())) while True: try: os.waitpid(child_pid, 0) break except OSError as e: # In case we encountered an OSError due to EINTR (which is # caused by a SIGINT or SIGTERM signal during # os.waitpid()), we simply ignore it and enter the next # iteration of the loop, waiting for the child to end. In # any other case, this is some other unexpected OS error, # which we don't want to catch, so we re-raise those ones. if e.errno != errno.EINTR: raise def main_work_horse(self, job): """This is the entry point of the newly spawned work horse.""" # After fork()'ing, always assure we are generating random sequences # that are different from the worker. random.seed() self._is_horse = True self.log = Logger('horse') success = self.perform_job(job) # os._exit() is the way to exit from childs after a fork(), in # constrast to the regular sys.exit() os._exit(int(not success)) def perform_job(self, job): """Performs the actual work of a job. Will/should only be called inside the work horse's process. """ self.procline('Processing %s from %s since %s' % ( job.func_name, job.origin, time.time())) try: with death_penalty_after(job.timeout or 180): rv = job.perform() except Exception as e: fq = self.failed_queue self.log.exception(red(str(e))) self.log.warning('Moving job to %s queue.' % fq.name) fq.quarantine(job, exc_info=traceback.format_exc()) return False if rv is None: self.log.info('Job OK') else: self.log.info('Job OK, result = %s' % (yellow(unicode(rv)),)) if rv is not None: p = self.connection.pipeline() p.hset(job.key, 'result', dumps(rv)) p.expire(job.key, self.rv_ttl) p.execute() else: # Cleanup immediately job.delete() return True
class MonitordSocketClient: lastAlert_time = 0.0 lastAlert_zvei = "0" logger = 0 def __init__(self): self.loadConfig() def loadConfig(self): with open("config.json") as config_file: config = json.load(config_file) if len(config["config"]) != 7: raise Exception( "Please check config file. Wrong count of config options") if len(config["trigger"]) == 0: raise Exception("Please check config file. No trigger defined") self.config = config["config"] self.triggers = config["trigger"] def startNewDataThread(self, data): x = threading.Thread(target=self.processData, args=(data, )) x.start() def processData(self, raw_data): data = raw_data.split(":") if data[0] == "100": self.logger.info("Received welcome. MonitorD version: " + data[1]) elif data[0] == "300": self.newAlert(data[3]) else: self.logger.error("Received unknown command. Command: " + repr(raw_data)) return def newAlert(self, zvei): if self.checkIfDoubleAlert(zvei): return self.logger.info("{}Received alarm.".format("[" + str(zvei) + "]: ")) subprocess.check_output( '/usr/bin/zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k "receivedZVEI" -o "' + str(zvei) + '"', shell=True) trigger = self.checkIfAlertinFilter(zvei) if not trigger: self.logger.info( "{}Received alarm not in filter. Stopping...".format( "[" + str(zvei) + "]: ")) return self.logger.info( "{}!!!Received alarm in filter {} (Time: {}) Starting...".format( "[" + str(zvei) + "]: ", trigger["name"], str(datetime.now().time()))) if self.isTestAlert(trigger): self.logger.info( "{}Testalart time. Stopping...".format("[" + str(zvei) + "]: ")) return self.logger.info("{}Start alarm tasks...".format("[" + str(zvei) + "]: ")) self.doAlertThings(zvei, trigger) return def checkIfDoubleAlert(self, zvei): double_alert = False if zvei == self.lastAlert_zvei: if time.time() - self.lastAlert_time < 10: double_alert = True self.lastAlert_time = time.time() self.lastAlert_zvei = zvei return double_alert def checkIfAlertinFilter(self, zvei): for key, config in self.triggers: if key == zvei: return config return False def isTestAlert(self, trigger): begin_time = dtime(trigger["hour_start"], trigger["minute_start"]) end_time = dtime(trigger["hour_end"], trigger["minute_end"]) check_time = datetime.now().time() return datetime.today().weekday() == trigger[ "weekday"] and check_time >= begin_time and check_time <= end_time def doAlertThings(self, zvei, trigger): payload = trigger["request"] for request_try in range(self.config["retries"]): r = requests.get(self.config["url"], params=payload) self.logger.debug(pprint.saferepr(r.url)) self.logger.debug(pprint.saferepr(r.status_code)) self.logger.debug(pprint.saferepr(r.content)) self.logger.debug(pprint.saferepr(r.headers)) if not r.status_code == requests.codes.ok: self.logger.error("{}Failed to send alert. Try: {}/{}".format( "[" + str(zvei) + "]: ", str(request_try + 1), str(self.config["retries"]))) time.sleep(self.config["retry_delay"]) continue else: self.logger.info( "{}Successfully send alert".format("[" + str(zvei) + "]: ")) break if trigger["local"]: try: self.logger.info("Starting light control") debugOutput = subprocess.check_output( '/usr/bin/python3 /opt/light/FFWLightControlTrigger.py', shell=True) self.logger.debug(pprint.saferepr(debugOutput)) except: self.logger.error("Light control exception") try: self.logger.info("Starting TV control") subprocess.check_output( '/usr/bin/php /usr/local/bin/automate_tv 900 &', shell=True) except: self.logger.info("TV control exception") return def reportConnectionError(self, count): self.logger.error("Connection Error Count:" + count) return def main(self): logbook.set_datetime_format("local") StreamHandler(sys.stdout, level=self.config["loglevel"]).push_application() MonitoringFileHandler( self.config["logpath"], mode='a', encoding='utf-8', bubble=True, level=self.config["loglevel"]).push_application() self.logger = Logger('monitord_socket.py') self.logger.info("starting...") connectionErrorCount = 0 while True: self.logger.info('Connection to {}:{}'.format( self.config["host"], self.config["port"])) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((self.config["host"], self.config["port"])) except: connectionErrorCount += 1 self.reportConnectionError(connectionErrorCount) time.sleep(10) continue connectionErrorCount = 0 try: while True: data = s.recv(1024) if data: self.logger.debug('Received => ' + repr(data)) self.startNewDataThread(data.decode('ascii')) else: self.logger.warning('Connection lost! Restarting...') break except KeyboardInterrupt: print("\nShutting down...") self.logger.warning('Shutting down...') exit() s.close() time.sleep(5)
logger.error('lock HTTPError: %s, key %r, retries left %s' % (e, key, num_retries)) if e.code in (409, 417): # key already exists or paxos failed, retry num_retries -= 1 time.sleep(random.random()) else: break except urllib2.URLError, e: logger.error('lock URLError: %s, key %r, retries left %s' % (e, key, num_retries)) num_retries -= 1 time.sleep(random.random()) else: # check if we lock right with self._lock: if key in self._keys: if self._keys[key] == self.id: logger.warning('################## key %s already locked by me' % (key, )) else: logger.warning('------------------ key %s already locked by worker %s' % (key, self._keys[key])) else: self._keys[key] = self.id logger.debug('locked %r' % (key,)) return True return False def unlock(self, key): num_retries = 10 while num_retries > 0: try:
class MetaAAC_1_0(): """ Meta-trainer class. INITIAL: Implementation of MLDG algorithm tuned for adaptation in dynamically changing environments Papers: Da Li et al., "Learning to Generalize: Meta-Learning for Domain Generalization" https://arxiv.org/abs/1710.03463 Maruan Al-Shedivat et al., "Continuous Adaptation via Meta-Learning in Nonstationary and Competitive Environments" https://arxiv.org/abs/1710.03641 """ def __init__(self, env, task, log_level, aac_class_ref=SubAAC, runner_config=None, aac_lambda=1.0, guided_lambda=1.0, trial_source_target_cycle=(1, 0), num_episodes_per_trial=1, _aux_render_modes=('action_prob', 'value_fn', 'lstm_1_h', 'lstm_2_h'), name='MetaAAC', **kwargs): try: self.aac_class_ref = aac_class_ref self.task = task self.name = name StreamHandler(sys.stdout).push_application() self.log = Logger('{}_{}'.format(name, task), level=log_level) # with tf.variable_scope(self.name): if runner_config is None: self.runner_config = { 'class_ref': BaseSynchroRunner, 'kwargs': {}, } else: self.runner_config = runner_config self.env_list = env assert isinstance(self.env_list, list) and len(self.env_list) == 2, \ 'Expected pair of environments, got: {}'.format(self.env_list) # Instantiate to sub-trainers: one for test and one for train environments: self.runner_config['kwargs']['data_sample_config'] = { 'mode': 0 } # salve self.runner_config['kwargs']['name'] = 'slave' self.train_aac = aac_class_ref( env=self.env_list[-1], # train data will be salve environment task=self.task, log_level=log_level, runner_config=self.runner_config, aac_lambda=aac_lambda, guided_lambda=guided_lambda, trial_source_target_cycle=trial_source_target_cycle, num_episodes_per_trial=num_episodes_per_trial, _use_target_policy=False, _use_global_network=True, _aux_render_modes=_aux_render_modes, name=self.name + '_sub_Train', **kwargs) self.runner_config['kwargs']['data_sample_config'] = { 'mode': 1 } # master self.runner_config['kwargs']['name'] = 'master' self.test_aac = aac_class_ref( env=self.env_list[0], # test data - master env. task=self.task, log_level=log_level, runner_config=self.runner_config, aac_lambda=aac_lambda, guided_lambda=guided_lambda, trial_source_target_cycle=trial_source_target_cycle, num_episodes_per_trial=num_episodes_per_trial, _use_target_policy=False, _use_global_network=False, global_step_op=self.train_aac.global_step, global_episode_op=self.train_aac.global_episode, inc_episode_op=self.train_aac.inc_episode, _aux_render_modes=_aux_render_modes, name=self.name + '_sub_Test', **kwargs) self.local_steps = self.train_aac.local_steps self.model_summary_freq = self.train_aac.model_summary_freq #self.model_summary_op = self.train_aac.model_summary_op self._make_train_op() self.test_aac.model_summary_op = tf.summary.merge( [ self.test_aac.model_summary_op, self._combine_meta_summaries() ], name='meta_model_summary') except: msg = 'MetaAAC_0_1.__init()__ exception occurred' + \ '\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n' self.log.exception(msg) raise RuntimeError(msg) def _make_train_op(self): """ Defines: tensors holding training op graph for sub trainers and self; """ pi = self.train_aac.local_network pi_prime = self.test_aac.local_network self.test_aac.sync = self.test_aac.sync_pi = tf.group( *[v1.assign(v2) for v1, v2 in zip(pi_prime.var_list, pi.var_list)]) self.global_step = self.train_aac.global_step self.global_episode = self.train_aac.global_episode self.test_aac.global_step = self.train_aac.global_step self.test_aac.global_episode = self.train_aac.global_episode self.test_aac.inc_episode = self.train_aac.inc_episode self.train_aac.inc_episode = None self.inc_step = self.train_aac.inc_step # Meta-loss: self.loss = 0.5 * self.train_aac.loss + 0.5 * self.test_aac.loss # Clipped gradients: self.train_aac.grads, _ = tf.clip_by_global_norm( tf.gradients(self.train_aac.loss, pi.var_list), 40.0) self.log.warning('self.train_aac.grads: {}'.format( len(list(self.train_aac.grads)))) # self.test_aac.grads, _ = tf.clip_by_global_norm( # tf.gradients(self.test_aac.loss, pi_prime.var_list), # 40.0 # ) # Meta-gradient: grads_i, _ = tf.clip_by_global_norm( tf.gradients(self.train_aac.loss, pi.var_list), 40.0) grads_i_next, _ = tf.clip_by_global_norm( tf.gradients(self.test_aac.loss, pi_prime.var_list), 40.0) self.grads = [] for g1, g2 in zip(grads_i, grads_i_next): if g1 is not None and g2 is not None: meta_g = 0.5 * g1 + 0.5 * g2 else: meta_g = None self.grads.append(meta_g) #self.log.warning('self.grads_len: {}'.format(len(list(self.grads)))) # Gradients to update local copy of pi_prime (from train data): train_grads_and_vars = list( zip(self.train_aac.grads, pi_prime.var_list)) self.log.warning('train_grads_and_vars_len: {}'.format( len(train_grads_and_vars))) # Meta-gradients to be sent to parameter server: meta_grads_and_vars = list( zip(self.grads, self.train_aac.network.var_list)) self.log.warning('meta_grads_and_vars_len: {}'.format( len(meta_grads_and_vars))) # Set global_step increment equal to observation space batch size: obs_space_keys = list(self.train_aac.local_network.on_state_in.keys()) assert 'external' in obs_space_keys, \ 'Expected observation space to contain `external` mode, got: {}'.format(obs_space_keys) self.train_aac.inc_step = self.train_aac.global_step.assign_add( tf.shape(self.train_aac.local_network.on_state_in['external'])[0]) self.train_op = self.train_aac.optimizer.apply_gradients( train_grads_and_vars) # Optimizer for meta-update: self.optimizer = tf.train.AdamOptimizer( self.train_aac.train_learn_rate, epsilon=1e-5) # TODO: own alpha-leran rate self.meta_train_op = self.optimizer.apply_gradients( meta_grads_and_vars) self.log.debug('meta_train_op defined') def _combine_meta_summaries(self): meta_model_summaries = [ tf.summary.scalar("meta_grad_global_norm", tf.global_norm(self.grads)), tf.summary.scalar("total_meta_loss", self.loss), ] return meta_model_summaries def start(self, sess, summary_writer, **kwargs): """ Executes all initializing operations, starts environment runner[s]. Supposed to be called by parent worker just before training loop starts. Args: sess: tf session object. kwargs: not used by default. """ try: # Copy weights from global to local: sess.run(self.train_aac.sync_pi) sess.run(self.test_aac.sync_pi) # Start thread_runners: self.test_aac._start_runners(sess, summary_writer) # master first self.train_aac._start_runners(sess, summary_writer) self.summary_writer = summary_writer self.log.notice('Runners started.') except: msg = 'start() exception occurred' + \ '\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n' self.log.exception(msg) raise RuntimeError(msg) def process(self, sess): """ Meta-train step. Args: sess (tensorflow.Session): tf session obj. """ try: # Say `No` to redundant summaries: wirte_model_summary = \ self.local_steps % self.model_summary_freq == 0 # Copy from parameter server: sess.run(self.train_aac.sync_pi) sess.run(self.test_aac.sync_pi) #self.log.warning('Sync ok.') # Collect train trajectory: train_data = self.train_aac.get_data() feed_dict = self.train_aac.process_data(sess, train_data, is_train=True) #self.log.warning('Train data ok.') # Update pi_prime parameters wrt collected data: if wirte_model_summary: fetches = [self.train_op, self.train_aac.model_summary_op] else: fetches = [self.train_op] fetched = sess.run(fetches, feed_dict=feed_dict) #self.log.warning('Train gradients ok.') # Collect test trajectory wrt updated pi_prime parameters: test_data = self.test_aac.get_data() feed_dict.update( self.test_aac.process_data(sess, test_data, is_train=True)) #self.log.warning('Test data ok.') # Perform meta-update: if wirte_model_summary: meta_fetches = [ self.meta_train_op, self.test_aac.model_summary_op, self.inc_step ] else: meta_fetches = [self.meta_train_op, self.inc_step] meta_fetched = sess.run(meta_fetches, feed_dict=feed_dict) #self.log.warning('Meta-gradients ok.') if wirte_model_summary: meta_model_summary = meta_fetched[-2] model_summary = fetched[-1] else: meta_model_summary = None model_summary = None # Write down summaries: self.test_aac.process_summary(sess, test_data, meta_model_summary) self.train_aac.process_summary(sess, train_data, model_summary) self.local_steps += 1 # TODO: ...what about sampling control? except: msg = 'process() exception occurred' + \ '\n\nPress `Ctrl-C` or jupyter:[Kernel]->[Interrupt] for clean exit.\n' self.log.exception(msg) raise RuntimeError(msg)
class Worker(object): redis_worker_namespace_prefix = 'rq:worker:' redis_workers_keys = 'rq:workers' @classmethod def all(cls, connection=None): """Returns an iterable of all Workers. """ if connection is None: connection = get_current_connection() reported_working = connection.smembers(cls.redis_workers_keys) workers = [cls.find_by_key(key, connection) for key in reported_working] return compact(workers) @classmethod def find_by_key(cls, worker_key, connection=None): """Returns a Worker instance, based on the naming conventions for naming the internal Redis keys. Can be used to reverse-lookup Workers by their Redis keys. """ prefix = cls.redis_worker_namespace_prefix name = worker_key[len(prefix):] if not worker_key.startswith(prefix): raise ValueError('Not a valid RQ worker key: %s' % (worker_key,)) if connection is None: connection = get_current_connection() if not connection.exists(worker_key): return None name = worker_key[len(prefix):] worker = cls([], name, connection=connection) queues = connection.hget(worker.key, 'queues') worker._state = connection.hget(worker.key, 'state') or '?' if queues: worker.queues = [Queue(queue, connection=connection) for queue in queues.split(',')] return worker def __init__(self, queues, name=None, default_result_ttl=DEFAULT_RESULT_TTL, connection=None, exc_handler=None): # noqa if connection is None: connection = get_current_connection() self.connection = connection if isinstance(queues, Queue): queues = [queues] self._name = name self.queues = queues self.validate_queues() self._exc_handlers = [] self.default_result_ttl = default_result_ttl self._state = 'starting' self._is_horse = False self._horse_pid = 0 self._stopped = False self.log = Logger('worker') self.failed_queue = get_failed_queue(connection=self.connection) # By default, push the "move-to-failed-queue" exception handler onto # the stack self.push_exc_handler(self.move_to_failed_queue) if exc_handler is not None: self.push_exc_handler(exc_handler) def validate_queues(self): # noqa """Sanity check for the given queues.""" if not iterable(self.queues): raise ValueError('Argument queues not iterable.') for queue in self.queues: if not isinstance(queue, Queue): raise NoQueueError('Give each worker at least one Queue.') def queue_names(self): """Returns the queue names of this worker's queues.""" return map(lambda q: q.name, self.queues) def queue_keys(self): """Returns the Redis keys representing this worker's queues.""" return map(lambda q: q.key, self.queues) @property # noqa def name(self): """Returns the name of the worker, under which it is registered to the monitoring system. By default, the name of the worker is constructed from the current (short) host name and the current PID. """ if self._name is None: hostname = socket.gethostname() shortname, _, _ = hostname.partition('.') self._name = '%s.%s' % (shortname, self.pid) return self._name @property def key(self): """Returns the worker's Redis hash key.""" return self.redis_worker_namespace_prefix + self.name @property def pid(self): """The current process ID.""" return os.getpid() @property def horse_pid(self): """The horse's process ID. Only available in the worker. Will return 0 in the horse part of the fork. """ return self._horse_pid @property def is_horse(self): """Returns whether or not this is the worker or the work horse.""" return self._is_horse def procline(self, message): """Changes the current procname for the process. This can be used to make `ps -ef` output more readable. """ setprocname('rq: %s' % (message,)) def register_birth(self): # noqa """Registers its own birth.""" self.log.debug('Registering birth of worker %s' % (self.name,)) if self.connection.exists(self.key) and \ not self.connection.hexists(self.key, 'death'): raise ValueError( 'There exists an active worker named \'%s\' ' 'already.' % (self.name,)) key = self.key now = time.time() queues = ','.join(self.queue_names()) with self.connection.pipeline() as p: p.delete(key) p.hset(key, 'birth', now) p.hset(key, 'queues', queues) p.sadd(self.redis_workers_keys, key) p.execute() def register_death(self): """Registers its own death.""" self.log.debug('Registering death') with self.connection.pipeline() as p: # We cannot use self.state = 'dead' here, because that would # rollback the pipeline p.srem(self.redis_workers_keys, self.key) p.hset(self.key, 'death', time.time()) p.expire(self.key, 60) p.execute() def set_state(self, new_state): self._state = new_state self.connection.hset(self.key, 'state', new_state) def get_state(self): return self._state state = property(get_state, set_state) @property def stopped(self): return self._stopped def _install_signal_handlers(self): """Installs signal handlers for handling SIGINT and SIGTERM gracefully. """ def request_force_stop(signum, frame): """Terminates the application (cold shutdown). """ self.log.warning('Cold shut down.') # Take down the horse with the worker if self.horse_pid: msg = 'Taking down horse %d with me.' % self.horse_pid self.log.debug(msg) try: os.kill(self.horse_pid, signal.SIGKILL) except OSError as e: # ESRCH ("No such process") is fine with us if e.errno != errno.ESRCH: self.log.debug('Horse already down.') raise raise SystemExit() def request_stop(signum, frame): """Stops the current worker loop but waits for child processes to end gracefully (warm shutdown). """ self.log.debug('Got signal %s.' % signal_name(signum)) signal.signal(signal.SIGINT, request_force_stop) signal.signal(signal.SIGTERM, request_force_stop) msg = 'Warm shut down requested.' self.log.warning(msg) # If shutdown is requested in the middle of a job, wait until # finish before shutting down if self.state == 'busy': self._stopped = True self.log.debug('Stopping after current horse is finished. ' 'Press Ctrl+C again for a cold shutdown.') else: raise StopRequested() signal.signal(signal.SIGINT, request_stop) signal.signal(signal.SIGTERM, request_stop) def work(self, burst=False): # noqa """Starts the work loop. Pops and performs all jobs on the current list of queues. When all queues are empty, block and wait for new jobs to arrive on any of the queues, unless `burst` mode is enabled. The return value indicates whether any jobs were processed. """ self._install_signal_handlers() did_perform_work = False self.register_birth() self.log.info('RQ worker started, version %s' % VERSION) self.state = 'starting' try: while True: if self.stopped: self.log.info('Stopping on request.') break self.state = 'idle' qnames = self.queue_names() self.procline('Listening on %s' % ','.join(qnames)) self.log.info('') self.log.info('*** Listening on %s...' % \ green(', '.join(qnames))) wait_for_job = not burst try: result = Queue.dequeue_any(self.queues, wait_for_job, \ connection=self.connection) if result is None: break except StopRequested: break except UnpickleError as e: msg = '*** Ignoring unpickleable data on %s.' % \ green(e.queue.name) self.log.warning(msg) self.log.debug('Data follows:') self.log.debug(e.raw_data) self.log.debug('End of unreadable data.') self.failed_queue.push_job_id(e.job_id) continue self.state = 'busy' job, queue = result # Use the public setter here, to immediately update Redis job.status = Status.STARTED self.log.info('%s: %s (%s)' % (green(queue.name), blue(job.description), job.id)) self.fork_and_perform_job(job) did_perform_work = True finally: if not self.is_horse: self.register_death() return did_perform_work def fork_and_perform_job(self, job): """Spawns a work horse to perform the actual work and passes it a job. The worker will wait for the work horse and make sure it executes within the given timeout bounds, or will end the work horse with SIGALRM. """ child_pid = os.fork() if child_pid == 0: self.main_work_horse(job) else: self._horse_pid = child_pid self.procline('Forked %d at %d' % (child_pid, time.time())) while True: try: os.waitpid(child_pid, 0) break except OSError as e: # In case we encountered an OSError due to EINTR (which is # caused by a SIGINT or SIGTERM signal during # os.waitpid()), we simply ignore it and enter the next # iteration of the loop, waiting for the child to end. In # any other case, this is some other unexpected OS error, # which we don't want to catch, so we re-raise those ones. if e.errno != errno.EINTR: raise def main_work_horse(self, job): """This is the entry point of the newly spawned work horse.""" # After fork()'ing, always assure we are generating random sequences # that are different from the worker. random.seed() # Always ignore Ctrl+C in the work horse, as it might abort the # currently running job. # The main worker catches the Ctrl+C and requests graceful shutdown # after the current work is done. When cold shutdown is requested, it # kills the current job anyway. signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_DFL) self._is_horse = True self.log = Logger('horse') success = self.perform_job(job) # os._exit() is the way to exit from childs after a fork(), in # constrast to the regular sys.exit() os._exit(int(not success)) def perform_job(self, job): """Performs the actual work of a job. Will/should only be called inside the work horse's process. """ self.procline('Processing %s from %s since %s' % ( job.func_name, job.origin, time.time())) try: with death_penalty_after(job.timeout or 180): rv = job.perform() # Pickle the result in the same try-except block since we need to # use the same exc handling when pickling fails pickled_rv = dumps(rv) job._status = Status.FINISHED except: # Use the public setter here, to immediately update Redis job.status = Status.FAILED self.handle_exception(job, *sys.exc_info()) return False if rv is None: self.log.info('Job OK') else: self.log.info('Job OK, result = %s' % (yellow(unicode(rv)),)) # How long we persist the job result depends on the value of # result_ttl: # - If result_ttl is 0, cleanup the job immediately. # - If it's a positive number, set the job to expire in X seconds. # - If result_ttl is negative, don't set an expiry to it (persist # forever) result_ttl = self.default_result_ttl if job.result_ttl is None else job.result_ttl # noqa if result_ttl == 0: job.delete() self.log.info('Result discarded immediately.') else: p = self.connection.pipeline() p.hset(job.key, 'result', pickled_rv) p.hset(job.key, 'status', job._status) if result_ttl > 0: p.expire(job.key, result_ttl) self.log.info('Result is kept for %d seconds.' % result_ttl) else: self.log.warning('Result will never expire, clean up result key manually.') p.execute() return True def handle_exception(self, job, *exc_info): """Walks the exception handler stack to delegate exception handling.""" exc_string = ''.join( traceback.format_exception_only(*exc_info[:2]) + traceback.format_exception(*exc_info)) self.log.error(exc_string) for handler in reversed(self._exc_handlers): self.log.debug('Invoking exception handler %s' % (handler,)) fallthrough = handler(job, *exc_info) # Only handlers with explicit return values should disable further # exc handling, so interpret a None return value as True. if fallthrough is None: fallthrough = True if not fallthrough: break def move_to_failed_queue(self, job, *exc_info): """Default exception handler: move the job to the failed queue.""" exc_string = ''.join(traceback.format_exception(*exc_info)) self.log.warning('Moving job to %s queue.' % self.failed_queue.name) self.failed_queue.quarantine(job, exc_info=exc_string) def push_exc_handler(self, handler_func): """Pushes an exception handler onto the exc handler stack.""" self._exc_handlers.append(handler_func) def pop_exc_handler(self): """Pops the latest exception handler off of the exc handler stack.""" return self._exc_handlers.pop()
class Memory(object): """ Replay memory with rebalanced replay based on reward value. Note: must be filled up before calling sampling methods. """ def __init__(self, history_size, max_sample_size, priority_sample_size, log_level=WARNING, rollout_provider=None, task=-1, reward_threshold=0.1, use_priority_sampling=False): """ Args: history_size: number of experiences stored; max_sample_size: maximum allowed sample size (e.g. off-policy rollout length); priority_sample_size: sample size of priority_sample() method log_level: int, logbook.level; rollout_provider: callable returning list of Rollouts NOT USED task: parent worker id; reward_threshold: if |experience.reward| > reward_threshold: experience is saved as 'prioritized'; """ self._history_size = history_size self._frames = deque(maxlen=history_size) self.reward_threshold = reward_threshold self.max_sample_size = int(max_sample_size) self.priority_sample_size = int(priority_sample_size) self.rollout_provider = rollout_provider self.task = task self.log_level = log_level StreamHandler(sys.stdout).push_application() self.log = Logger('ReplayMemory_{}'.format(self.task), level=self.log_level) self.use_priority_sampling = use_priority_sampling # Indices for non-priority frames: self._zero_reward_indices = deque() # Indices for priority frames: self._non_zero_reward_indices = deque() self._top_frame_index = 0 if use_priority_sampling: self.sample_priority = self._sample_priority else: self.sample_priority = self._sample_dummy def add(self, frame): """ Appends single experience frame to memory. Args: frame: dictionary of values. """ if frame['terminal'] and len( self._frames) > 0 and self._frames[-1]['terminal']: # Discard if terminal frame continues self.log.warning( "Memory_{}: Sequential terminal frame encountered. Discarded.". format(self.task)) self.log.warning('{} -- {}'.format(self._frames[-1]['position'], frame['position'])) return frame_index = self._top_frame_index + len(self._frames) was_full = self.is_full() # Append frame: self._frames.append(frame) # Decide and append index: if frame_index >= self.max_sample_size - 1: if abs(frame['reward']) <= self.reward_threshold: self._zero_reward_indices.append(frame_index) else: self._non_zero_reward_indices.append(frame_index) if was_full: # Decide from which index to discard: self._top_frame_index += 1 cut_frame_index = self._top_frame_index + self.max_sample_size - 1 # Cut frame if its index is lower than cut_frame_index: if len(self._zero_reward_indices) > 0 and \ self._zero_reward_indices[0] < cut_frame_index: self._zero_reward_indices.popleft() if len(self._non_zero_reward_indices) > 0 and \ self._non_zero_reward_indices[0] < cut_frame_index: self._non_zero_reward_indices.popleft() def add_rollout(self, rollout): """ Adds frames from given rollout to memory with respect to episode continuation. Args: rollout: `Rollout` instance. """ # Check if current rollout is direct extension of last stored frame sequence: if len(self._frames) > 0 and not self._frames[-1]['terminal']: # E.g. check if it is same local episode and successive frame order: if self._frames[-1]['position']['episode'] == rollout['position']['episode'][0] and \ self._frames[-1]['position']['step'] + 1 == rollout['position']['step'][0]: # Means it is ok to just extend previously stored episode pass else: # Means part or tail of previously recorded episode is somehow lost, # so we need to mark stored episode as 'ended': self._frames[-1]['terminal'] = True self.log.warning('{} changed to terminal'.format( self._frames[-1]['position'])) # If we get a lot of such messages it is an indication something is going wrong. # Add experiences one by one: # TODO: pain-slow. for i in range(len(rollout['terminal'])): frame = rollout.get_frame(i) self.add(frame) def is_full(self): return len(self._frames) >= self._history_size def fill(self): """ Fills replay memory with initial experiences. NOT USED. Supposed to be called by parent worker() just before training begins. Args: rollout_getter: callable, returning list of Rollouts. """ if self.rollout_provider is not None: while not self.is_full(): for rollout in self.rollout_provider(): self.add_rollout(rollout) self.log.info('Memory_{}: filled.'.format(self.task)) else: raise AttributeError( 'Rollout_provider is None, can not fill memory.') def sample_uniform(self, sequence_size): """ Uniformly samples sequence of successive frames of size `sequence_size` or less (~off-policy rollout). Args: sequence_size: maximum sample size. Returns: instance of Rollout of size <= sequence_size. """ start_pos = np.random.randint(0, self._history_size - sequence_size - 1) # Shift by one if hit terminal frame: if self._frames[start_pos]['terminal']: start_pos += 1 # assuming that there are no successive terminal frames. sampled_rollout = Rollout() for i in range(sequence_size): frame = self._frames[start_pos + i] sampled_rollout.add(frame) if frame['terminal']: break # it's ok to return less than `sequence_size` frames if `terminal` frame encountered. return sampled_rollout def _sample_priority(self, size=None, exact_size=False, skewness=2, sample_attempts=100): """ Implements rebalanced replay. Samples sequence of successive frames from distribution skewed by means of reward of last sample frame. Args: size: sample size, must be <= self.max_sample_size; exact_size: whether accept sample with size less than 'size' or re-sample to get sample of exact size (used for reward prediction task); skewness: int>=1, sampling probability denominator, such as probability of sampling sequence with last frame having non-zero reward is: p[non_zero]=1/skewness; sample_attempts: if exact_size=True, sets number of re-sampling attempts to get sample of continuous experiences (no `Terminal` frames inside except last one); if number is reached - sample returned 'as is'. Returns: instance of Rollout(). """ if size is None: size = self.priority_sample_size if size > self.max_sample_size: size = self.max_sample_size # Toss skewed coin: if np.random.randint(int(skewness)) == 0: from_zero = False else: from_zero = True if len(self._zero_reward_indices) == 0: # zero rewards container was empty from_zero = False elif len(self._non_zero_reward_indices) == 0: # non zero rewards container was empty from_zero = True # Try to sample sequence of given length from one episode. # Take maximum of 'sample_attempts', if no luck # (e.g too short episodes and/or too big sampling size) -> # return inconsistent sample and issue warning. check_sequence = True for attempt in range(sample_attempts): if from_zero: index = np.random.randint(len(self._zero_reward_indices)) end_frame_index = self._zero_reward_indices[index] else: index = np.random.randint(len(self._non_zero_reward_indices)) end_frame_index = self._non_zero_reward_indices[index] start_frame_index = end_frame_index - size + 1 raw_start_frame_index = start_frame_index - self._top_frame_index sampled_rollout = Rollout() is_full = True if attempt == sample_attempts - 1: check_sequence = False self.log.warning( 'Memory_{}: failed to sample {} successive frames, sampled as is.' .format(self.task, size)) for i in range(size - 1): frame = self._frames[raw_start_frame_index + i] sampled_rollout.add(frame) if check_sequence: if frame['terminal']: if exact_size: is_full = False #print('attempt:', attempt) #print('frame.terminal:', frame['terminal']) break # Last frame can be terminal anyway: frame = self._frames[raw_start_frame_index + size - 1] sampled_rollout.add(frame) if is_full: break return sampled_rollout @staticmethod def _sample_dummy(**kwargs): return None
return mongos_instance.client[database].list_collections() else: return mongos_instance.client[database].list_collection_names() if __name__ == '__main__': Cluster = Mongo(host=ConnString.ip, port=ConnString.port, username=ConnString.user, password=ConnString.password) if ConnString.auth_enabled: Cluster.auth() p.warning("###### Need to be enableSharding ######") for db in get_unpartion_db(Cluster): p.warning(db) p.warning("###### Need to be shardCollection ######") for i in get_database_list(Cluster): if i in ["config", "admin", "test"]: continue for table in get_collection_list(Cluster, database=i): full_table_name = i + '.' + table if full_table_name not in \ [sharded_table['_id'] for sharded_table in get_sharded_collection(Cluster)]: p.warning(full_table_name) Cluster.close()
if hasattr(sys, 'frozen'): pyfalog.info("Running in a frozen state.") else: pyfalog.info("Running in a thawed state.") if not hasattr(sys, 'frozen') and wxversion: try: if options.force28 is True: pyfalog.info("Selecting wx version: 2.8. (Forced)") wxversion.select('2.8') else: pyfalog.info("Selecting wx versions: 3.0, 2.8") wxversion.select(['3.0', '2.8']) except: pyfalog.warning("Unable to select wx version. Attempting to import wx without specifying the version.") else: if not wxversion: pyfalog.warning("wxVersion not found. Attempting to import wx without specifying the version.") try: # noinspection PyPackageRequirements import wx except: exit_message = "Cannot import wxPython. You can download wxPython (2.8+) from http://www.wxpython.org/" raise PreCheckException(exit_message) pyfalog.info("wxPython version: {0}.", str(wx.VERSION_STRING)) if sqlalchemy is None: exit_message = "\nCannot find sqlalchemy.\nYou can download sqlalchemy (0.6+) from http://www.sqlalchemy.org/"
filename = os.path.join(file_path, 'ltcli-rotate.log') rotating_file_handler = RotatingFileHandler(filename=filename, level=file_level, bubble=True, max_size=each_size, backup_count=backup_count) rotating_file_handler.format_string = formatter['file'] rotating_file_handler.push_application() logger.debug('start logging on file: {}'.format(filename)) else: try: os.mkdir(file_path) except Exception: logger.error("CreateDirError: {}".format(file_path)) msg = message.get('error_logging_in_file') logger.warning(msg) def set_level(level): """Change log level. :param level: debug / info / warning / error """ level_list = ['debug', 'info', 'warning', 'error', 'warn'] if level not in level_list: level_list.remove('warn') logger.error("LogLevelError: '{}'. Select in {}".format( level, level_list)) return code = get_log_code(level) stream_handler.level = code
class BaseDataGenerator(): """ Base synthetic data provider class. """ def __init__(self, episode_duration=None, timeframe=1, generator_fn=null_generator, generator_params=None, name='BaseSyntheticDataGenerator', data_names=('default_asset', ), global_time=None, task=0, log_level=WARNING, _nested_class_ref=None, _nested_params=None, **kwargs): """ Args: episode_duration: dict, duration of episode in days/hours/mins generator_fn callabale, should return generated data as 1D np.array generator_params dict, timeframe: int, data periodicity in minutes name: str data_names: iterable of str global_time: dict {y, m, d} to set custom global time (only for plotting) task: int log_level: logbook.Logger level **kwargs: """ # Logging: self.log_level = log_level self.task = task self.name = name self.filename = self.name + '_sample' self.data_names = data_names self.data_name = self.data_names[0] self.sample_instance = None self.metadata = { 'sample_num': 0, 'type': None, 'parent_sample_type': None } self.data = None self.data_stat = None self.sample_num = 0 self.is_ready = False if _nested_class_ref is None: self.nested_class_ref = BaseDataGenerator else: self.nested_class_ref = _nested_class_ref if _nested_params is None: self.nested_params = dict( episode_duration=episode_duration, timeframe=timeframe, generator_fn=generator_fn, generator_params=generator_params, name=name, data_names=data_names, task=task, log_level=log_level, _nested_class_ref=_nested_class_ref, _nested_params=_nested_params, ) else: self.nested_params = _nested_params StreamHandler(sys.stdout).push_application() self.log = Logger('{}_{}'.format(self.name, self.task), level=self.log_level) # Default sample time duration: if episode_duration is None: self.episode_duration = dict( days=0, hours=23, minutes=55, ) else: self.episode_duration = episode_duration # Btfeed parsing setup: self.timeframe = timeframe self.names = ['open'] self.datetime = 0 self.open = 1 self.high = -1 self.low = -1 self.close = -1 self.volume = -1 self.openinterest = -1 # base data feed related: self.params = {} if global_time is None: self.global_time = datetime.datetime(year=2018, month=1, day=1) else: self.global_time = datetime.datetime(**global_time) self.global_timestamp = self.global_time.timestamp() # Infer time indexes and sample number of records: self.train_index = pd.timedelta_range( start=datetime.timedelta(days=0, hours=0, minutes=0), end=datetime.timedelta(**self.episode_duration), freq='{}min'.format(self.timeframe)) self.test_index = pd.timedelta_range( start=self.train_index[-1] + datetime.timedelta(minutes=self.timeframe), periods=len(self.train_index), freq='{}min'.format(self.timeframe)) self.train_index += self.global_time self.test_index += self.global_time self.episode_num_records = len(self.train_index) self.generator_fn = generator_fn if generator_params is None: self.generator_params = {} else: self.generator_params = generator_params def set_logger(self, level=None, task=None): """ Sets logbook logger. Args: level: logbook.level, int task: task id, int """ if task is not None: self.task = task if level is not None: self.log = Logger('{}_{}'.format(self.name, self.task), level=level) def reset(self, **kwargs): self.sample_num = 0 self.is_ready = True def read_csv(self, **kwargs): self.data = self.generate_data(self.generator_params) def generate_data(self, generator_params, type=0): """ Generates data trajectory (episode) Args: generator_params: dict, data generating parmeters type: 0 - generate train data | 1 - generate test data Returns: data as pandas dataframe """ assert type in [ 0, 1 ], 'Expected sample type be either 0 (train), or 1 (test) got: {}'.format( type) # Generate datapoints: data_array = self.generator_fn(num_points=self.episode_num_records, **generator_params) assert len(data_array.shape) == 1 and data_array.shape[0] == self.episode_num_records,\ 'Expected generated data to be 1D array of length {}, got data shape: {}'.format( self.episode_num_records, data_array.shape ) negs = data_array[data_array < 0] if negs.any(): self.log.warning( ' Set to zero {} negative generated values'.format( negs.shape[0])) data_array[data_array < 0] = 0.0 # Make dataframe: if type: index = self.test_index else: index = self.train_index # data_dict = {name: data_array for name in self.names} # data_dict['hh:mm:ss'] = index df = pd.DataFrame(data={name: data_array for name in self.names}, index=index) # df = df.set_index('hh:mm:ss') return df def sample(self, get_new=True, sample_type=0, **kwargs): """ Samples continuous subset of data. Args: get_new (bool): not used; sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets respectively. Returns: Dataset instance with number of records ~ max_episode_len, """ try: assert sample_type in [0, 1] except AssertionError: self.log.exception( 'Sampling attempt: expected sample type be in {}, got: {}'.\ format([0, 1], sample_type) ) raise AssertionError if self.metadata['type'] is not None: if self.metadata['type'] != sample_type: self.log.warning( 'Attempted to sample type {} given current sample type {}, overriden.' .format(self.metadata['type'], sample_type)) sample_type = self.metadata['type'] # Generate data: sampled_data = self.generate_data(self.generator_params, type=sample_type) self.sample_instance = self.nested_class_ref(**self.nested_params) self.sample_instance.filename += '_{}'.format(self.sample_num) self.log.info('New sample id: <{}>.'.format( self.sample_instance.filename)) self.sample_instance.data = sampled_data # Add_metadata self.sample_instance.metadata['type'] = 'synthetic_data_sample' self.sample_instance.metadata['first_row'] = 0 self.sample_instance.metadata['last_row'] = self.episode_num_records self.sample_instance.metadata['type'] = sample_type self.sample_instance.metadata['sample_num'] = self.sample_num self.sample_instance.metadata['parent_sample_num'] = self.metadata[ 'sample_num'] self.sample_instance.metadata['parent_sample_type'] = self.metadata[ 'type'] self.sample_num += 1 return self.sample_instance def describe(self): """ Returns summary dataset statistic as pandas dataframe: - records count, - data mean, - data std dev, - min value, - 25% percentile, - 50% percentile, - 75% percentile, - max value for every data column. """ # Pretty straightforward, using standard pandas utility. # The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again, # thus avoiding passing big files to BT server: flush_data = False try: assert not self.data.empty pass except (AssertionError, AttributeError) as e: self.read_csv() flush_data = True self.data_stat = self.data.describe() self.log.info('Data summary:\n{}'.format(self.data_stat.to_string())) if flush_data: self.data = None self.log.info('Flushed data.') return self.data_stat def to_btfeed(self): """ Performs BTgymData-->bt.feed conversion. Returns: dict of type: {data_line_name: bt.datafeed instance}. """ try: assert not self.data.empty btfeed = btfeeds.PandasDirectData(dataname=self.data, timeframe=self.timeframe, datetime=self.datetime, open=self.open, high=self.high, low=self.low, close=self.close, volume=self.volume, openinterest=self.openinterest) btfeed.numrecords = self.data.shape[0] return {self.data_name: btfeed} except (AssertionError, AttributeError) as e: msg = 'Instance holds no data. Hint: forgot to call .read_csv()?' self.log.error(msg) raise AssertionError(msg) def set_global_timestamp(self, timestamp): pass
mpl_version = int(mpl.__version__[0]) or -1 if mpl_version >= 2: mpl.use('wxagg') mplImported = True else: mplImported = False from matplotlib.patches import Patch from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as Canvas from matplotlib.figure import Figure graphFrame_enabled = True mplImported = True except ImportError as e: pyfalog.warning("Matplotlib failed to import. Likely missing or incompatible version.") mpl_version = -1 Patch = mpl = Canvas = Figure = None graphFrame_enabled = False mplImported = False except Exception: # We can get exceptions deep within matplotlib. Catch those. See GH #1046 tb = traceback.format_exc() pyfalog.critical("Exception when importing Matplotlib. Continuing without importing.") pyfalog.critical(tb) mpl_version = -1 Patch = mpl = Canvas = Figure = None graphFrame_enabled = False mplImported = False
class Plotter: def __init__(self): StreamHandler(stdout).push_application() self.logger = Logger(self.__class__.__name__) set_datetime_format("local") try: with open('config.yaml', 'r') as stream: self.config = yaml.load(stream) except yaml.YAMLError as e: logger.critical(e) exit() except IOError as e: logger.critical(e) exit() def plotYN(self): if self.config['out']['plot'] == False: self.logger.info('Output not analysed and plotted.') exit() def read_data(self): if self.config['out']['redis']: self.fr_redis() elif self.config['out']['file']: self.fr_file() else: self.logger.warning( 'Reading data from neither redis nor file. Aborting') exit() def fr_redis(self): self.logger.info('Reading data from redis key.') import redis redis = redis.StrictRedis(self.config['sys_settings']['redis_ip']) key = self.config['out']['prefix'] + ':' + self.config['in']['master'] if redis.exists(key): self.data = pickle.loads(redis.get(key), encoding='bytes') else: self.logger.warning( f'Redis key {key} does not exist; might have expired.') self.fr_file() def fr_file(self): self.logger.info('Reading data from file.') fname = self.concat() with open(fname + '.pickle', 'rb') as handle: self.data = pickle.load(handle, encoding='bytes') def concat(self): fname = self.config['in']['master'].split('/')[-1] path = self.config['out']['path'] return path + fname def detrend(self): self.logger.info('Detrending data') df = pd.DataFrame(self.data, columns=['raw']) df['detrended'] = signal.detrend(df['raw']) + df['raw'].mean() self.df = df def normalise(self): self.norm = self.df / self.df.mean() def make_fig(self): import cufflinks as cf #To be removed later; use plotly directly title = self.concat() self.fig = self.norm.iplot(asFigure=True, title=title, colorscale='polar', xTitle='frame', yTitle='Normalised intensity') def plot(self): import plotly fname = self.concat() + '.html' self.logger.info(f'Plot saved as {fname}') plotly.offline.plot(self.fig, filename=fname) def describe_data(self): out = pd.DataFrame() for key in self.df.columns: tmp = self.df[key].describe() tmp['CV%'] = self.df[key].std() / self.df[key].mean() * 100 out = out.append(tmp) order = [ 'count', 'mean', 'std', 'CV%', 'min', '25%', '50%', '75%', 'max' ] out = out[order].T fname = self.concat() + '.csv' out.to_csv(fname) self.logger.info(f'Descriptive stats: \n {out}') self.logger.info(f'Data description saved as {fname}') def run(self): self.plotYN() self.read_data() self.detrend() self.normalise() self.describe_data() self.make_fig() self.plot()
class BTgymEnv(gym.Env): """ OpenAI Gym API shell for Backtrader backtesting/trading library. """ # Datafeed Server management: data_master = True data_network_address = 'tcp://127.0.0.1:' # using localhost. data_port = 4999 data_server = None data_server_pid = None data_context = None data_socket = None data_server_response = None # Dataset: dataset = None # BTgymDataset instance. dataset_stat = None # Backtrader engine: engine = None # bt.Cerbro subclass for server to execute. # Strategy: strategy = None # strategy to use if no <engine> class been passed. # Server and network: server = None # Server process. context = None # ZMQ context. socket = None # ZMQ socket, client side. port = 5500 # network port to use. network_address = 'tcp://127.0.0.1:' # using localhost. ctrl_actions = ('_done', '_reset', '_stop', '_getstat', '_render' ) # server control messages. server_response = None # Connection timeout: connect_timeout = 60 # server connection timeout in seconds. #connect_timeout_step = 0.01 # time between retries in seconds. # Rendering: render_enabled = True render_modes = [ 'human', 'episode', ] # `episode` - plotted episode results. # `human` - raw_state observation in conventional human-readable format. # <obs_space_key> - rendering of arbitrary state presented in observation_space with same key. renderer = None # Rendering support. rendered_rgb = dict() # Keep last rendered images for each mode. # Logging and id: log = None log_level = None # logbook level: NOTICE, WARNING, INFO, DEBUG etc. or its integer equivalent; verbose = 0 # verbosity mode, valid only if no `log_level` arg has been provided: # 0 - WARNING, 1 - INFO, 2 - DEBUG. task = 0 closed = True def __init__(self, **kwargs): """ Keyword Args: filename=None (str, list): csv data file. **datafeed_args (any): any datafeed-related args, passed through to default btgym.datafeed class. dataset=None (btgym.datafeed): BTgymDataDomain instance, overrides `filename` or any other datafeed-related args. strategy=None (btgym.startegy): strategy to be used by `engine`, any subclass of btgym.strategy.base.BTgymBaseStrateg engine=None (bt.Cerebro): environment simulation engine, any bt.Cerebro subclass, overrides `strategy` arg. network_address=`tcp://127.0.0.1:` (str): BTGym_server address. port=5500 (int): network port to use for server - API_shell communication. data_master=True (bool): let this environment control over data_server; data_network_address=`tcp://127.0.0.1:` (str): data_server address. data_port=4999 (int): network port to use for server -- data_server communication. connect_timeout=60 (int): server connection timeout in seconds. render_enabled=True (bool): enable rendering for this environment; render_modes=['human', 'episode'] (list): `episode` - plotted episode results; `human` - raw_state observation. **render_args (any): any render-related args, passed through to renderer class. verbose=0 (int): verbosity mode, {0 - WARNING, 1 - INFO, 2 - DEBUG} log_level=None (int): logbook level {DEBUG=10, INFO=11, NOTICE=12, WARNING=13}, overrides `verbose` arg; log=None (logbook.Logger): external logbook logger, overrides `log_level` and `verbose` args. task=0 (int): environment id Environment kwargs applying logic:: if <engine> kwarg is given: do not use default engine and strategy parameters; ignore <strategy> kwarg and all strategy and engine-related kwargs. else (no <engine>): use default engine parameters; if any engine-related kwarg is given: override corresponding default parameter; if <strategy> is given: do not use default strategy parameters; if any strategy related kwarg is given: override corresponding strategy parameter; else (no <strategy>): use default strategy parameters; if any strategy related kwarg is given: override corresponding strategy parameter; if <dataset> kwarg is given: do not use default dataset parameters; ignore dataset related kwargs; else (no <dataset>): use default dataset parameters; if any dataset related kwarg is given: override corresponding dataset parameter; If any <other> kwarg is given: override corresponding default parameter. """ # Parameters and default values: self.params = dict( # Backtrader engine mandatory parameters: engine=dict( start_cash=10.0, # initial trading capital. broker_commission= 0.001, # trade execution commission, default is 0.1% of operation value. fixed_stake=10, # single trade stake is fixed type by def. ), # Dataset mandatory parameters: dataset=dict(filename=None, ), strategy=dict(state_shape=dict(), ), render=dict(), ) p2 = dict( # IS HERE FOR REFERENCE ONLY # Strategy related parameters: # Observation state shape is dictionary of Gym spaces, # at least should contain `raw_state` field. # By convention first dimension of every Gym Box space is time embedding one; # one can define any shape; should match env.observation_space.shape. # observation space state min/max values, # For `raw_state' - absolute min/max values from BTgymDataset will be used. state_shape=dict(raw_state=spaces.Box( shape=(10, 4), low=-100, high=100, dtype=np.float32)), drawdown_call= None, # episode maximum drawdown threshold, default is 90% of initial value. portfolio_actions=None, # agent actions, # should consist with BTgymStrategy order execution logic; # defaults are: 0 - 'do nothing', 1 - 'buy', 2 - 'sell', 3 - 'close position'. skip_frame=None, # Number of environment steps to skip before returning next response, # e.g. if set to 10 -- agent will interact with environment every 10th episode step; # Every other step agent's action is assumed to be 'hold'. # Note: INFO part of environment response is a list of all skipped frame's info's, # i.e. [info[-9], info[-8], ..., info[0]. ) # Update self attributes, remove used kwargs: for key in dir(self): if key in kwargs.keys(): setattr(self, key, kwargs.pop(key)) self.metadata = {'render.modes': self.render_modes} # Logging and verbosity control: if self.log is None: StreamHandler(sys.stdout).push_application() if self.log_level is None: log_levels = [(0, NOTICE), (1, INFO), (2, DEBUG)] self.log_level = WARNING for key, value in log_levels: if key == self.verbose: self.log_level = value self.log = Logger('BTgymAPIshell_{}'.format(self.task), level=self.log_level) # Network parameters: self.network_address += str(self.port) self.data_network_address += str(self.data_port) # Set server rendering: if self.render_enabled: self.renderer = BTgymRendering(self.metadata['render.modes'], log_level=self.log_level, **kwargs) else: self.renderer = BTgymNullRendering() self.log.info( 'Rendering disabled. Call to render() will return null-plug image.' ) # Append logging: self.renderer.log = self.log # Update params -1: pull from renderer, remove used kwargs: self.params['render'].update(self.renderer.params) for key in self.params['render'].keys(): if key in kwargs.keys(): _ = kwargs.pop(key) if self.data_master: # DATASET preparation, only data_master executes this: # if self.dataset is not None: # If BTgymDataset instance has been passed: # do nothing. msg = 'Custom Dataset class used.' else: # If no BTgymDataset has been passed, # Make default dataset with given CSV file: try: os.path.isfile(str(self.params['dataset']['filename'])) except: raise FileNotFoundError( 'Dataset source data file not specified/not found') # Use kwargs to instantiate dataset: self.dataset = BTgymDataset(**kwargs) msg = 'Base Dataset class used.' # Append logging: self.dataset.set_logger(self.log_level, self.task) # Update params -2: pull from dataset, remove used kwargs: self.params['dataset'].update(self.dataset.params) for key in self.params['dataset'].keys(): if key in kwargs.keys(): _ = kwargs.pop(key) self.log.info(msg) # Connect/Start data server (and get dataset statistic): self.log.info('Connecting data_server...') self._start_data_server() self.log.info('...done.') # ENGINE preparation: # Update params -3: pull engine-related kwargs, remove used: for key in self.params['engine'].keys(): if key in kwargs.keys(): self.params['engine'][key] = kwargs.pop(key) if self.engine is not None: # If full-blown bt.Cerebro() subclass has been passed: # Update info: msg = 'Custom Cerebro class used.' self.strategy = msg for key in self.params['engine'].keys(): self.params['engine'][key] = msg # Note: either way, bt.observers.DrawDown observer [and logger] will be added to any BTgymStrategy instance # by BTgymServer process at runtime. else: # Default configuration for Backtrader computational engine (Cerebro), # if no bt.Cerebro() custom subclass has been passed, # get base class Cerebro(), using kwargs on top of defaults: self.engine = bt.Cerebro() msg = 'Base Cerebro class used.' # First, set STRATEGY configuration: if self.strategy is not None: # If custom strategy has been passed: msg2 = 'Custom Strategy class used.' else: # Base class strategy : self.strategy = BTgymBaseStrategy msg2 = 'Base Strategy class used.' # Add, using kwargs on top of defaults: #self.log.debug('kwargs for strategy: {}'.format(kwargs)) strat_idx = self.engine.addstrategy(self.strategy, **kwargs) msg += ' ' + msg2 # Second, set Cerebro-level configuration: self.engine.broker.setcash(self.params['engine']['start_cash']) self.engine.broker.setcommission( self.params['engine']['broker_commission']) self.engine.addsizer(bt.sizers.SizerFix, stake=self.params['engine']['fixed_stake']) self.log.info(msg) # Define observation space shape, minimum / maximum values and agent action space. # Retrieve values from configured engine or... # ...Update params -4: # Pull strategy defaults to environment params dict : for t_key, t_value in self.engine.strats[0][0][0].params._gettuple(): self.params['strategy'][t_key] = t_value # Update it with values from strategy 'passed-to params': for key, value in self.engine.strats[0][0][2].items(): self.params['strategy'][key] = value # ... Push it all back (don't ask): for key, value in self.params['strategy'].items(): self.engine.strats[0][0][2][key] = value # For 'raw_state' min/max values, # the only way is to infer from raw Dataset price values (we already got those from data_server): if 'raw_state' in self.params['strategy']['state_shape'].keys(): # Exclude 'volume' from columns we count: self.dataset_columns.remove('volume') #print(self.params['strategy']) #print('self.engine.strats[0][0][2]:', self.engine.strats[0][0][2]) #print('self.engine.strats[0][0][0].params:', self.engine.strats[0][0][0].params._gettuple()) # Override with absolute price min and max values: self.params['strategy']['state_shape']['raw_state'].low =\ self.engine.strats[0][0][2]['state_shape']['raw_state'].low =\ np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) +\ self.dataset_stat.loc['min', self.dataset_columns].min() self.params['strategy']['state_shape']['raw_state'].high = \ self.engine.strats[0][0][2]['state_shape']['raw_state'].high = \ np.zeros(self.params['strategy']['state_shape']['raw_state'].shape) + \ self.dataset_stat.loc['max', self.dataset_columns].max() self.log.info( 'Inferring `state_raw` high/low values form dataset: {:.6f} / {:.6f}.' .format( self.dataset_stat.loc['min', self.dataset_columns].min(), self.dataset_stat.loc['max', self.dataset_columns].max())) # Set observation space shape from engine/strategy parameters: self.observation_space = DictSpace( self.params['strategy']['state_shape']) self.log.debug('Obs. shape: {}'.format(self.observation_space.spaces)) #self.log.debug('Obs. min:\n{}\nmax:\n{}'.format(self.observation_space.low, self.observation_space.high)) # Set action space and corresponding server messages: self.action_space = spaces.Discrete( len(self.params['strategy']['portfolio_actions'])) self.server_actions = self.params['strategy']['portfolio_actions'] # Finally: self.server_response = None self.env_response = None #if not self.data_master: self._start_server() self.closed = False self.log.info('Environment is ready.') def _seed(self, seed=None): """ Sets env. random seed. Args: seed: int or None """ np.random.seed(seed) @staticmethod def _comm_with_timeout( socket, message, ): """ Exchanges messages via socket, timeout sensitive. Args: socket: zmq connected socket to communicate via; message: message to send; Note: socket zmq.RCVTIMEO and zmq.SNDTIMEO should be set to some finite number of milliseconds. Returns: dictionary: `status`: communication result; `message`: received message if status == `ok` or None; `time`: remote side response time. """ response = dict( status='ok', message=None, ) try: socket.send_pyobj(message) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: response['status'] = 'send_failed_due_to_connect_timeout' else: response['status'] = 'send_failed_for_unknown_reason' return response start = time.time() try: response['message'] = socket.recv_pyobj() response['time'] = time.time() - start except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: response['status'] = 'receive_failed_due_to_connect_timeout' else: response['status'] = 'receive_failed_for_unknown_reason' return response return response def _start_server(self): """ Configures backtrader REQ/REP server instance and starts server process. """ # Ensure network resources: # 1. Release client-side, if any: if self.context: self.context.destroy() self.socket = None # 2. Kill any process using server port: cmd = "kill $( lsof -i:{} -t ) > /dev/null 2>&1".format(self.port) os.system(cmd) # Set up client channel: self.context = zmq.Context() self.socket = self.context.socket(zmq.REQ) self.socket.setsockopt(zmq.RCVTIMEO, self.connect_timeout * 1000) self.socket.setsockopt(zmq.SNDTIMEO, self.connect_timeout * 1000) self.socket.connect(self.network_address) # Configure and start server: self.server = BTgymServer( cerebro=self.engine, render=self.renderer, network_address=self.network_address, data_network_address=self.data_network_address, connect_timeout=self.connect_timeout, log_level=self.log_level, task=self.task, ) self.server.daemon = False self.server.start() # Wait for server to startup: time.sleep(1) # Check connection: self.log.info('Server started, pinging {} ...'.format( self.network_address)) self.server_response = self._comm_with_timeout( socket=self.socket, message={'ctrl': 'ping!'}) if self.server_response['status'] in 'ok': self.log.info('Server seems ready with response: <{}>'.format( self.server_response['message'])) else: msg = 'Server unreachable with status: <{}>.'.format( self.server_response['status']) self.log.error(msg) raise ConnectionError(msg) self._closed = False def _stop_server(self): """ Stops BT server process, releases network resources. """ if self.server: if self._force_control_mode(): # In case server is running and client side is ok: self.socket.send_pyobj({'ctrl': '_stop'}) self.server_response = self.socket.recv_pyobj() else: self.server.terminate() self.server.join() self.server_response = 'Server process terminated.' self.log.info('{} Exit code: {}'.format(self.server_response, self.server.exitcode)) # Release client-side, if any: if self.context: self.context.destroy() self.socket = None def _force_control_mode(self): """Puts BT server to control mode. """ # Check is there any faults with server process and connection? network_error = [ (not self.server or not self.server.is_alive(), 'No running server found. Hint: forgot to call reset()?'), (not self.context or self.context.closed, 'No network connection found.'), ] for (err, msg) in network_error: if err: self.log.info(msg) self.server_response = msg return False # If everything works, insist to go 'control': self.server_response = {} attempt = 0 while 'ctrl' not in self.server_response: self.socket.send_pyobj({'ctrl': '_done'}) self.server_response = self.socket.recv_pyobj() attempt += 1 self.log.debug( 'FORCE CONTROL MODE attempt: {}.\nResponse: {}'.format( attempt, self.server_response)) return True def _assert_response(self, response): """ Simple watcher: roughly checks if we really talking to environment (== episode is running). Rises exception if response given is not as expected. """ try: assert type(response) == tuple and len(response) == 4 except AssertionError: msg = 'Unexpected environment response: {}\nHint: Forgot to call reset() or reset_data()?'.format( response) self.log.exception(msg) raise AssertionError(msg) self.log.debug('Response checker received:\n{}\nas type: {}'.format( response, type(response))) def _print_space(self, space, _tab=''): """ Parses observation space shape or response. Args: space: gym observation space or state. Returns: description as string. """ response = '' if type(space) in [dict, OrderedDict]: for key, value in space.items(): response += '\n{}{}:{}\n'.format( _tab, key, self._print_space(value, ' ')) elif type(space) in [spaces.Dict, DictSpace]: for s in space.spaces: response += self._print_space(s, ' ') elif type(space) in [tuple, list]: for i in space: response += self._print_space(i, ' ') elif type(space) == np.ndarray: response += '\n{}array of shape: {}, low: {}, high: {}'.format( _tab, space.shape, space.min(), space.max()) else: response += '\n{}{}, '.format(_tab, space) try: response += 'low: {}, high: {}'.format(space.low.min(), space.high.max()) except (KeyError, AttributeError, ArithmeticError, ValueError) as e: pass #response += '\n{}'.format(e) return response def reset(self, **kwargs): """ Implementation of OpenAI Gym env.reset method. Starts new episode. Episode data are sampled according to data provider class logic, controlled via kwargs. Refer `BTgym_Server` and data provider classes for details. Args: kwargs: any kwargs; this dictionary is passed through to BTgym_server side without any checks and modifications; currently used for data sampling control; Returns: observation space state Notes: Current kwargs accepted is:: episode_config=dict( get_new=True, sample_type=0, b_alpha=1, b_beta=1 ), trial_config=dict( get_new=True, sample_type=0, b_alpha=1, b_beta=1 ) """ # Data Server check: if self.data_master: if not self.data_server or not self.data_server.is_alive(): self.log.info('No running data_server found, starting...') self._start_data_server() # Domain dataset status check: self.data_server_response = self._comm_with_timeout( socket=self.data_socket, message={'ctrl': '_get_info'}) if not self.data_server_response['message']['dataset_is_ready']: self.log.info( 'Data domain `reset()` called prior to `reset_data()` with [possibly inconsistent] defaults.' ) self.reset_data() # Server process check: if not self.server or not self.server.is_alive(): self.log.info('No running server found, starting...') self._start_server() if self._force_control_mode(): self.server_response = self._comm_with_timeout(socket=self.socket, message={ 'ctrl': '_reset', 'kwargs': kwargs }) # Get initial environment response: self.env_response = self.step(0) # Check (once) if it is really (o,r,d,i) tuple: self._assert_response(self.env_response) # Check (once) if state_space is as expected: try: #assert self.observation_space.contains(self.env_response[0]) pass except (AssertionError, AttributeError) as e: msg1 = self._print_space(self.observation_space.spaces) msg2 = self._print_space(self.env_response[0]) msg3 = '' for step_info in self.env_response[-1]: msg3 += '{}\n'.format(step_info) msg = ('\nState observation shape/range mismatch!\n' + 'Space set by env: \n{}\n' + 'Space returned by server: \n{}\n' + 'Full response:\n{}\n' + 'Reward: {}\n' + 'Done: {}\n' + 'Info:\n{}\n' + 'Hint: Wrong Strategy.get_state() parameters?').format( msg1, msg2, self.env_response[0], self.env_response[1], self.env_response[2], msg3, ) self.log.exception(msg) self._stop_server() raise AssertionError(msg) return self.env_response[0] #["raw_state"][np.newaxis] else: msg = 'Something went wrong. env.reset() can not get response from server.' self.log.exception(msg) raise ChildProcessError(msg) def step(self, action): """ Implementation of OpenAI Gym env.step() method. Makes a step in the environment. Args: action: int, number representing action from env.action_space Returns: tuple (Observation, Reward, Info, Done) """ # Are you in the list, ready to go and all that? if self.action_space.contains(action)\ and not self._closed\ and (self.socket is not None)\ and not self.socket.closed: pass else: msg = ('\nAt least one of these is true:\n' + 'Action error: (space is {}, action sent is {}): {}\n' + 'Environment closed: {}\n' + 'Network error [socket doesnt exists or closed]: {}\n' + 'Hint: forgot to call reset()?').format( self.action_space, action, not self.action_space.contains(action), self._closed, not self.socket or self.socket.closed, ) self.log.exception(msg) raise AssertionError(msg) # Send action to backtrader engine, receive environment response env_response = self._comm_with_timeout( socket=self.socket, message={'action': self.server_actions[action]}) if not env_response['status'] in 'ok': msg = '.step(): server unreachable with status: <{}>.'.format( env_response['status']) self.log.error(msg) raise ConnectionError(msg) # self.env_response = env_response ['message'] tempNew_state, tempReward, tempDone, tempInfo = env_response['message'] tempNew_state = tempNew_state["raw_state"][np.newaxis] self.env_response = tempNew_state, tempReward, tempDone, tempInfo return self.env_response def close(self): """ Implementation of OpenAI Gym env.close method. Puts BTgym server in Control Mode. """ self.log.debug('close.call()') self._stop_server() self._stop_data_server() self.log.info('Environment closed.') def get_stat(self): """ Returns last run episode statistics. Note: when invoked, forces running episode to terminate. """ if self._force_control_mode(): self.socket.send_pyobj({'ctrl': '_getstat'}) return self.socket.recv_pyobj() else: return self.server_response def render(self, mode='other_mode', close=False): """ Implementation of OpenAI Gym env.render method. Visualises current environment state. Args: `mode`: str, any of these:: `human` - current state observation as price lines; `episode` - plotted results of last completed episode. [other_key] - corresponding to any custom observation space key """ if close: return None if not self._closed\ and self.socket\ and not self.socket.closed: pass else: msg = ('\nCan' 't get renderings.' '\nAt least one of these is true:\n' + 'Environment closed: {}\n' + 'Network error [socket doesnt exists or closed]: {}\n' + 'Hint: forgot to call reset()?').format( self._closed, not self.socket or self.socket.closed, ) self.log.warning(msg) return None if mode not in self.render_modes: raise ValueError('Unexpected render mode {}'.format(mode)) self.socket.send_pyobj({'ctrl': '_render', 'mode': mode}) rgb_array_dict = self.socket.recv_pyobj() self.rendered_rgb.update(rgb_array_dict) return self.rendered_rgb[mode] def _stop(self): """ Finishes current episode if any, does nothing otherwise. Leaves server running. """ if self._force_control_mode(): self.log.info('Episode stop forced.') def _restart_server(self): """Restarts server. """ self._stop_server() self._start_server() self.log.info('Server restarted.') def _start_data_server(self): """ For data_master environment: - configures backtrader REQ/REP server instance and starts server process. For others: - establishes network connection to existing data_server. """ self.data_server = None # Ensure network resources: # 1. Release client-side, if any: if self.data_context: self.data_context.destroy() self.data_socket = None # Only data_master launches/stops data_server process: if self.data_master: # 2. Kill any process using server port: cmd = "kill $( lsof -i:{} -t ) > /dev/null 2>&1".format( self.data_port) os.system(cmd) # Configure and start server: self.data_server = BTgymDataFeedServer( dataset=self.dataset, network_address=self.data_network_address, log_level=self.log_level, task=self.task) self.data_server.daemon = False self.data_server.start() # Wait for server to startup time.sleep(1) # Set up client channel: self.data_context = zmq.Context() self.data_socket = self.data_context.socket(zmq.REQ) self.data_socket.setsockopt(zmq.RCVTIMEO, self.connect_timeout * 1000) self.data_socket.setsockopt(zmq.SNDTIMEO, self.connect_timeout * 1000) self.data_socket.connect(self.data_network_address) # Check connection: self.log.debug('Pinging data_server at: {} ...'.format( self.data_network_address)) self.data_server_response = self._comm_with_timeout( socket=self.data_socket, message={'ctrl': 'ping!'}) if self.data_server_response['status'] in 'ok': self.log.debug( 'Data_server seems ready with response: <{}>'.format( self.data_server_response['message'])) else: msg = 'Data_server unreachable with status: <{}>.'.\ format(self.data_server_response['status']) self.log.error(msg) raise ConnectionError(msg) # Get info and statistic: self.dataset_stat, self.dataset_columns, self.data_server_pid = self._get_dataset_info( ) def _stop_data_server(self): """ For data_master: - stops BT server process, releases network resources. """ if self.data_master: if self.data_server is not None and self.data_server.is_alive(): # In case server is running and is ok: self.data_socket.send_pyobj({'ctrl': '_stop'}) self.data_server_response = self.data_socket.recv_pyobj() else: self.data_server.terminate() self.data_server.join() self.data_server_response = 'Data_server process terminated.' self.log.info('{} Exit code: {}'.format(self.data_server_response, self.data_server.exitcode)) if self.data_context: self.data_context.destroy() self.data_socket = None def _restart_data_server(self): """ Restarts data_server. """ if self.data_master: self._stop_data_server() self._start_data_server() def _get_dataset_info(self): """ Retrieves dataset descriptive statistic. """ self.data_socket.send_pyobj({'ctrl': '_get_info'}) self.data_server_response = self.data_socket.recv_pyobj() return self.data_server_response['dataset_stat'],\ self.data_server_response['dataset_columns'],\ self.data_server_response['pid'] def reset_data(self, **kwargs): """ Resets data provider class used, whatever it means for that class. Gets data_server ready to provide data. Supposed to be called before first env.reset(). Note: when invoked, forces running episode to terminate. Args: **kwargs: data provider class .reset() method specific. """ if self.closed: self._start_server() if self.data_master: self._start_data_server() self.closed = False else: _ = self._force_control_mode() if self.data_master: if self.data_server is None or not self.data_server.is_alive(): self._restart_data_server() self.data_server_response = self._comm_with_timeout( socket=self.data_socket, message={ 'ctrl': '_reset_data', 'kwargs': kwargs }) if self.data_server_response['status'] in 'ok': self.log.debug( 'Dataset seems ready with response: <{}>'.format( self.data_server_response['message'])) else: msg = 'Data_server unreachable with status: <{}>.'. \ format(self.data_server_response['status']) self.log.error(msg) raise SystemExit(msg) else: pass
class Schema(object): url = None version = None homepage = None deps = [] dirs_to_symlink = ['bin', 'sbin', 'etc', 'lib', 'include', 'var'] def __init__(self, env): self.env = env.copy() self.log = Logger() if self.version is None: self.version = _figureout_version(self.url) if self.url.startswith('git://'): self.deps.append('git') self.retriver = self.git else: self.retriver = self.wget self.env['prefix'] = os.path.join( env['osbench_root'], 'workbench', env['schema'], self.version ) self.env['version'] = self.version # Methods to override def install(self): pass def is_installed(self): """Right now we'll simple check if program was built and installed into workbench.""" return os.path.exists(self.env['prefix']) # Internals def _get_source(self): if not self.url: return self.log.info('Getting source code') with higher_log_indent(): self.retriver(self.url) patch_names = sorted(name for name in dir(self) if name.startswith('patch_')) for name in patch_names: filename = name.replace('_', '-') + '.diff' with open(filename, 'w') as f: f.write(self._get_patch(name)) self.call('patch -p1 < ' + filename) def _get_patch(self, name): data = getattr(self, name) if len(data[:200].split('\n')) == 1: # then probably this is a file or URL patch_filename = os.path.join( os.path.dirname(self.env['schema_filename']), data ) if os.path.exists(patch_filename): data = open(patch_filename).read() data = self._substitute_vars(data) return data def _get_environment_vars(self): """ Returns dict with variables to set in shell and to replace in the templates. """ return dict( OSBENCH_ROOT=self.env['osbench_root'], OSBENCH_PREFIX=self.env['prefix'], ) def _substitute_vars(self, text): for name, value in self._get_environment_vars().items(): text = text.replace(name, value) return text def wget(self, url): self.call('wget "{0}"'.format(self.url)) files = os.listdir('.') assert len(files) == 1 filename = files[0] # remove url params if they was added to the filename stripped_filename = filename.split('?', 1)[0] if stripped_filename.endswith('.gz') or \ stripped_filename.endswith('.tgz'): tar_options = '-zxvf' elif stripped_filename.endswith('.bz2'): tar_options = '-jxvf' else: raise RuntimeError('Unknown archive format.') self.call('tar {0} "{1}"'.format(tar_options, filename)) os.unlink(filename) dirs = os.listdir('.') assert len(dirs) == 1 os.chdir(dirs[0]) def git(self, url): self.call("git clone '{0}'".format(url)) dirs = os.listdir('.') assert len(dirs) == 1 os.chdir(dirs[0]) def _install_deps(self): if self.deps: self.log.info('Installing dependencies') with higher_log_indent(): for dep in self.deps: if dep.startswith('system.'): self.call('sudo apt-get --yes install %s' % dep[7:]) def _install(self, interactive=False): self.log.info('Installing "{schema}"'.format(**self.env)) with higher_log_indent(): self._install_deps() root = tempfile.mkdtemp(prefix='diy-') if not os.path.exists(self.env['prefix']): os.makedirs(self.env['prefix']) try: os.chdir(root) self._get_source() if interactive: self.log.info('Entering into the interactive mode') with higher_log_indent(): shell = os.environ['SHELL'] self.call('git init') self.call('git add -A') for name, value in self._get_environment_vars().items(): os.environ[name] = value self.call(shell, pass_output=True) else: self.log.info('Running schema\'s install method') with higher_log_indent(): self.install() self._link() finally: self.call('rm -fr "{0}"'.format(root)) def _uninstall(self): """Uninstalls the schema from the prefix. It removes symlinks and deletes installed files from workbenches. """ self.log.info('Uninstalling "{schema}"'.format(**self.env)) with higher_log_indent(): self._unlink() self._delete() def _delete(self): shutil.rmtree(self.env['prefix']) def _join_path(self, *args): return os.path.normpath(os.path.join(*args)) def _link(self): self.log.info('Making symlinks') with higher_log_indent(): for dir_name in self.dirs_to_symlink: s_dir = self._join_path(self.env['prefix'], dir_name) t_dir = self._join_path(self.env['osbench_root'], dir_name) if not os.path.exists(t_dir): self.log.debug('Creating directory "{0}"', t_dir) os.makedirs(t_dir) if os.path.exists(s_dir): for root, dirs, files in os.walk(s_dir): # making root, relative root = os.path.relpath(root, s_dir) for dir_name in dirs: full_dir_name = self._join_path( t_dir, root, dir_name ) if not os.path.exists(full_dir_name): self.log.debug('Creating directory "{0}"', full_dir_name) os.makedirs(full_dir_name) for filename in files: source = self._join_path(s_dir, root, filename) target = self._join_path(t_dir, root, filename) if os.path.exists(target): if os.path.islink(target): if os.readlink(target) == source: self.log.debug('Symlink {target} already exists', target=target) continue else: self.log.warning('Unlinking file {target}, pointing to {source}', target=target, source=source) os.unlink(target) else: self.log.warning('File {target} already exists and it is not a link', target=target) if not os.path.exists(target): self.log.debug('Creating symlink from "{source}" to {target}', source=source, target=target) os.symlink(source, target) def _unlink(self): self.log.info('Removing symlinks') for ftype, name in self.get_files_to_unlink(): if ftype == 'file': os.unlink(name) else: try: os.rmdir(name) except OSError: pass def get_files_to_unlink(self): with higher_log_indent(): for dir_name in self.dirs_to_symlink: s_dir = self._join_path(self.env['prefix'], dir_name) t_dir = self._join_path(self.env['osbench_root'], dir_name) if os.path.exists(s_dir): for root, dirs, files in os.walk(s_dir, topdown=False): # making root, relative root = os.path.relpath(root, s_dir) for filename in files: source = self._join_path(s_dir, root, filename) target = self._join_path(t_dir, root, filename) if os.path.islink(target) and \ os.path.realpath(target) == source: yield ('file', target) for dir_name in dirs: full_dir_name = self._join_path( t_dir, root, dir_name ) yield ('dir', full_dir_name) # Utilities def call(self, command, pass_output=False): command = command.format(**self.env) self.log.info('Running "{0}"'.format(command)) with higher_log_indent(): options = dict( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, ) if pass_output: del options['stdout'] del options['stderr'] proc = subprocess.Popen( command, **options ) full_output = [] if not pass_output: for line in proc.stdout: line = line.decode('utf-8').strip(u'\n') full_output.append(line) self.log.debug(line) return_code = proc.wait() if return_code != 0: for line in full_output: self.log.error(line) raise BadExitCode('subprogram exit with non zero exit code') def makedirs(self, *dirs): """Makes dirs inside the prefix. Use this command inside your `install` method. """ for d in dirs: fullname = os.path.join(self.env['prefix'], d) if not os.path.exists(fullname): self.log.info('Creating directory "{0}".'.format(fullname)) os.makedirs(fullname) def create_file_with_content(self, filename, content, mode=None): """Creates a file inside 'prefix'. Use this command inside your `install` method. Note: Source and target directory should exists. Warning: if there is some file already, it will be overwritten. """ filename = os.path.join(self.env['prefix'], filename) self.log.info('Creating file "{0}"'.format(filename)) with open(filename, 'w') as f: f.write(self._substitute_vars(content)) if mode is not None: self.call('chmod "{0}" "{1}"'.format(mode, filename)) def copy_file(self, from_filename, to_filename, mode=None): """Copies file, to a directory inside 'prefix'. from_filename could be relative to the current directory, or use variables to be expanded to self.env. Use this command inside your `install` method. Note: Source and target directory should exists. Warning: if there is some file already, it will be overwritten. """ with open(from_filename.format(**self.env), 'r') as f: self.create_file_with_content(to_filename, f.read(), mode=mode)
mpl_version = int(mpl.__version__[0]) or -1 if mpl_version >= 2: mpl.use('wxagg') mplImported = True else: mplImported = False from matplotlib.patches import Patch from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as Canvas from matplotlib.figure import Figure graphFrame_enabled = True mplImported = True except ImportError as e: pyfalog.warning( "Matplotlib failed to import. Likely missing or incompatible version." ) mpl_version = -1 Patch = mpl = Canvas = Figure = None graphFrame_enabled = False mplImported = False except Exception: # We can get exceptions deep within matplotlib. Catch those. See GH #1046 tb = traceback.format_exc() pyfalog.critical( "Exception when importing Matplotlib. Continuing without importing.") pyfalog.critical(tb) mpl_version = -1 Patch = mpl = Canvas = Figure = None graphFrame_enabled = False mplImported = False
import wx from logbook import CRITICAL, DEBUG, ERROR, FingersCrossedHandler, INFO, Logger, NestedSetup, NullHandler, \ StreamHandler, TimedRotatingFileHandler, WARNING import hashlib from eos.const import FittingSlot from cryptography.fernet import Fernet pyfalog = Logger(__name__) # Load variable overrides specific to distribution type try: import configforced except ImportError: pyfalog.warning("Failed to import: configforced") configforced = None # Turns on debug mode debug = False # Defines if our saveddata will be in pyfa root or not saveInRoot = False evemonMinVersion = "4081" minItemSearchLength = 3 pyfaPath = None savePath = None saveDB = None
expire_on_commit=False)() pyfalog.debug('Getting gamedata version') # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new # game db because we haven't reached gamedata_meta.create_all() try: config.gamedata_version = gamedata_session.execute( "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'" ).fetchone()[0] config.gamedata_date = gamedata_session.execute( "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'dump_time'" ).fetchone()[0] except (KeyboardInterrupt, SystemExit): raise except Exception as e: pyfalog.warning("Missing gamedata version.") pyfalog.critical(e) config.gamedata_version = None config.gamedata_date = None pyfalog.debug('Initializing saveddata') saveddata_connectionstring = config.saveddata_connectionstring if saveddata_connectionstring is not None: if callable(saveddata_connectionstring): saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=config.debug) else: saveddata_engine = create_engine(saveddata_connectionstring, echo=config.debug) saveddata_meta = MetaData()
num_retries -= 1 time.sleep(random.random()) else: break except urllib2.URLError, e: logger.error('lock URLError: %s, key %r, retries left %s' % (e, key, num_retries)) num_retries -= 1 time.sleep(random.random()) else: # check if we lock right with self._lock: if key in self._keys: if self._keys[key] == self.id: logger.warning( '################## key %s already locked by me' % (key, )) else: logger.warning( '------------------ key %s already locked by worker %s' % (key, self._keys[key])) else: self._keys[key] = self.id logger.debug('locked %r' % (key, )) return True return False def unlock(self, key): num_retries = 10
class BaseDataGenerator: """ Base synthetic data provider class. """ def __init__(self, episode_duration=None, timeframe=1, generator_fn=base_random_generator_fn, generator_parameters_fn=base_generator_parameters_fn, generator_parameters_config=None, spread_generator_fn=None, spread_generator_parameters=None, name='BaseSyntheticDataGenerator', data_names=('default_asset', ), parsing_params=None, target_period=-1, global_time=None, task=0, log_level=WARNING, _nested_class_ref=None, _nested_params=None, **kwargs): """ Args: episode_duration: dict, duration of episode in days/hours/mins generator_fn callabale, should return generated data as 1D np.array generator_parameters_fn: callable, should return dictionary of generator_fn kwargs generator_parameters_config: dict, generator_parameters_fn args spread_generator_fn: callable, should return values of spread to form {High, Low} spread_generator_parameters: dict, spread_generator_fn args timeframe: int, data periodicity in minutes name: str data_names: iterable of str target_period: int or dict, if set to -1 - disables `test` sampling global_time: dict {y, m, d} to set custom global time (only for plotting) task: int log_level: logbook.Logger level **kwargs: """ # Logging: self.log_level = log_level self.task = task self.name = name self.filename = self.name + '_sample' self.target_period = target_period self.data_names = data_names self.data_name = self.data_names[0] self.sample_instance = None self.metadata = { 'sample_num': 0, 'type': None, 'parent_sample_type': None } self.data = None self.data_stat = None self.sample_num = 0 self.is_ready = False if _nested_class_ref is None: self.nested_class_ref = BaseDataGenerator else: self.nested_class_ref = _nested_class_ref if _nested_params is None: self.nested_params = dict( episode_duration=episode_duration, timeframe=timeframe, generator_fn=generator_fn, generator_parameters_fn=generator_parameters_fn, generator_parameters_config=generator_parameters_config, name=name, data_names=data_names, task=task, log_level=log_level, _nested_class_ref=_nested_class_ref, _nested_params=_nested_params, ) else: self.nested_params = _nested_params StreamHandler(sys.stdout).push_application() self.log = Logger('{}_{}'.format(self.name, self.task), level=self.log_level) # Default sample time duration: if episode_duration is None: self.episode_duration = dict( days=0, hours=23, minutes=55, ) else: self.episode_duration = episode_duration # Btfeed parsing setup: if parsing_params is None: self.parsing_params = dict(names=['ask', 'bid', 'mid'], datetime=0, timeframe=1, open='mid', high='ask', low='bid', close='mid', volume=-1, openinterest=-1) else: self.parsing_params = parsing_params self.columns_map = { 'open': 'mean', 'high': 'maximum', 'low': 'minimum', 'close': 'mean', 'bid': 'minimum', 'ask': 'maximum', 'mid': 'mean', 'volume': 'nothing', } self.nested_params['parsing_params'] = self.parsing_params for key, value in self.parsing_params.items(): setattr(self, key, value) # base data feed related: self.params = {} if global_time is None: self.global_time = datetime.datetime(year=2018, month=1, day=1) else: self.global_time = datetime.datetime(**global_time) self.global_timestamp = self.global_time.timestamp() # Infer time indexes and sample number of records: self.train_index = pd.timedelta_range( start=datetime.timedelta(days=0, hours=0, minutes=0), end=datetime.timedelta(**self.episode_duration), freq='{}min'.format(self.timeframe)) self.test_index = pd.timedelta_range( start=self.train_index[-1] + datetime.timedelta(minutes=self.timeframe), periods=len(self.train_index), freq='{}min'.format(self.timeframe)) self.train_index += self.global_time self.test_index += self.global_time self.episode_num_records = len(self.train_index) self.generator_fn = generator_fn self.generator_parameters_fn = generator_parameters_fn if generator_parameters_config is not None: self.generator_parameters_config = generator_parameters_config else: self.generator_parameters_config = {} self.spread_generator_fn = spread_generator_fn if spread_generator_parameters is not None: self.spread_generator_parameters = spread_generator_parameters else: self.spread_generator_parameters = {} def set_logger(self, level=None, task=None): """ Sets logbook logger. Args: level: logbook.level, int task: task id, int """ if task is not None: self.task = task if level is not None: self.log = Logger('{}_{}'.format(self.name, self.task), level=level) def reset(self, **kwargs): self.read_csv() self.sample_num = 0 self.is_ready = True def read_csv(self, **kwargs): self.data = self.generate_data( self.generator_parameters_fn(**self.generator_parameters_config)) def generate_data(self, generator_params, sample_type=0): """ Generates data trajectory, performs base consistency checks. Args: generator_params: dict, data_generating_function parameters sample_type: 0 - generate train data | 1 - generate test data Returns: data as pandas dataframe """ assert sample_type in [0, 1],\ 'Expected sample type be either 0 (train), or 1 (test) got: {}'.format(sample_type) # Generate data points: data_array = self.generator_fn(num_points=self.episode_num_records, **generator_params) assert len(data_array.shape) == 1 and data_array.shape[0] == self.episode_num_records,\ 'Expected generated data to be 1D array of length {}, got data shape: {}'.format( self.episode_num_records, data_array.shape ) if self.spread_generator_fn is not None: spread_array = self.spread_generator_fn( num_points=self.episode_num_records, **self.spread_generator_parameters) assert len(spread_array.shape) == 1 and spread_array.shape[0] == self.episode_num_records, \ 'Expected generated spread to be 1D array of length {}, got data shape: {}'.format( self.episode_num_records, spread_array.shape ) else: spread_array = np.zeros(self.episode_num_records) data_dict = { 'mean': data_array, 'maximum': data_array + .5 * spread_array, 'minimum': data_array - .5 * spread_array, 'nothing': data_array * 0.0, } # negs = data_dict['minimum'] < 0 # if negs.any(): # self.log.warning('{} negative generated values detected'.format(negs.shape[0])) # Make dataframe: if sample_type: index = self.test_index else: index = self.train_index # Map dictionary of data to dataframe columns: df = pd.DataFrame(data={ name: data_dict[self.columns_map[name]] for name in self.names }, index=index) # df = df.set_index('hh:mm:ss') return df def sample(self, get_new=True, sample_type=0, **kwargs): """ Samples continuous subset of data. Args: get_new (bool): not used; sample_type (int or bool): 0 (train) or 1 (test) - get sample from train or test data subsets respectively. Returns: Dataset instance with number of records ~ max_episode_len. """ try: assert sample_type in [0, 1] except AssertionError: msg = 'Sampling attempt: expected sample type be in {}, got: {}'.format( [0, 1], sample_type) self.log.error(msg) raise ValueError(msg) if self.target_period == -1 and sample_type: msg = 'Attempt to sample type {} given disabled target_period'.format( sample_type) self.log.error(msg) raise ValueError(msg) if self.metadata['type'] is not None: if self.metadata['type'] != sample_type: self.log.warning( 'Attempt to sample type {} given current sample type {}, overriden.' .format(sample_type, self.metadata['type'])) sample_type = self.metadata['type'] # Get sample: self.sample_instance = self.sample_synthetic(sample_type) self.sample_instance.metadata['type'] = sample_type self.sample_instance.metadata['sample_num'] = self.sample_num self.sample_instance.metadata['parent_sample_num'] = self.metadata[ 'sample_num'] self.sample_instance.metadata['parent_sample_type'] = self.metadata[ 'type'] self.sample_num += 1 return self.sample_instance def sample_synthetic(self, sample_type=0): """ Get data_generator instance containing synthetic data. Args: sample_type (int or bool): 0 (train) or 1 (test) - get sample with train or test time periods respectively. Returns: nested_class_ref instance """ # Generate data: generator_params = self.generator_parameters_fn( **self.generator_parameters_config) data = self.generate_data(generator_params, sample_type=sample_type) # Make data_class instance: sample_instance = self.nested_class_ref(**self.nested_params) sample_instance.filename += '_{}'.format(self.sample_num) self.log.info('New sample id: <{}>.'.format(sample_instance.filename)) # Add data and metadata: sample_instance.data = data sample_instance.metadata['generator'] = generator_params sample_instance.metadata['first_row'] = 0 sample_instance.metadata['last_row'] = self.episode_num_records return sample_instance def describe(self): """ Returns summary dataset statistic as pandas dataframe: - records count, - data mean, - data std dev, - min value, - 25% percentile, - 50% percentile, - 75% percentile, - max value for every data column. """ # Pretty straightforward, using standard pandas utility. # The only caveat here is that if actual data has not been loaded yet, need to load, describe and unload again, # thus avoiding passing big files to BT server: flush_data = False try: assert not self.data.empty pass except (AssertionError, AttributeError) as e: self.read_csv() flush_data = True self.data_stat = self.data.describe() self.log.info('Data summary:\n{}'.format(self.data_stat.to_string())) if flush_data: self.data = None self.log.info('Flushed data.') return self.data_stat def to_btfeed(self): """ Performs BTgymData-->bt.feed conversion. Returns: dict of type: {data_line_name: bt.datafeed instance}. """ try: assert not self.data.empty btfeed = btfeeds.PandasDirectData(dataname=self.data, timeframe=self.timeframe, datetime=self.datetime, open=self.open, high=self.high, low=self.low, close=self.close, volume=self.volume, openinterest=self.openinterest) btfeed.numrecords = self.data.shape[0] return {self.data_name: btfeed} except (AssertionError, AttributeError) as e: msg = 'Instance holds no data. Hint: forgot to call .read_csv()?' self.log.error(msg) raise AssertionError(msg) def set_global_timestamp(self, timestamp): pass
gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=config.debug) else: gamedata_engine = create_engine(gamedata_connectionstring, echo=config.debug) gamedata_meta = MetaData() gamedata_meta.bind = gamedata_engine gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)() # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new # game db because we haven't reached gamedata_meta.create_all() try: config.gamedata_version = gamedata_session.execute( "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'" ).fetchone()[0] except Exception as e: pyfalog.warning("Missing gamedata version.") pyfalog.critical(e) config.gamedata_version = None saveddata_connectionstring = config.saveddata_connectionstring if saveddata_connectionstring is not None: if callable(saveddata_connectionstring): saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=config.debug) else: saveddata_engine = create_engine(saveddata_connectionstring, echo=config.debug) saveddata_meta = MetaData() saveddata_meta.bind = saveddata_engine saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)() else: saveddata_meta = None