def _load_sim_file(self, sim_file): '''Load, parse and expand the simulation file.''' sim_fname = os.path.realpath(sim_file) # Load the master file path, _ = os.path.split(sim_fname) with open(sim_fname) as file_: sim = toml.load(file_) # Expand the building building_fname = '{}.toml'.format(sim['building']['model']) with open(os.path.join(path, 'buildings', building_fname)) as file_: sim['building'] = toml.load(file_)['floor'] # Expand the lifts processed_lifts = [] for lift in sim['lifts']: fname = '{}.toml'.format(lift['model']) with open(os.path.join(path, 'lifts', fname)) as file_: exp = toml.load(file_) exp['bottom_floor_number'], exp['top_floor_number'] = lift['range'] exp['location'] = lift['location'] exp['open_doors'] = lift['open_doors'] processed_lifts.append(exp) sim['lifts'] = processed_lifts # If provided, initialise the random seed try: seed(sim['people']['seed']) except KeyError: print('no seed') pass self.description = sim
def read_cargo_toml(key, manifest=None): """Read a value from Cargo.toml manifest. :param key: Key to read from [package], or entire path to a key. It may contain dots. :param manifest: Optional path to the manifest, or a file-like object with it opened :return: The value """ if not isinstance(key, (list, tuple)): key = (key,) if key.startswith('package.') else ('package', key) key = list(chain.from_iterable(k.split('.') for k in key)) if not key: raise ValueError("key must not be empty") # Read the content of Cargo.toml. manifest = manifest or Path.cwd() / 'Cargo.toml' if hasattr(manifest, 'read'): content = toml.load(manifest) else: manifest = Path(manifest) with manifest.open() as f: content = toml.load(f) # Get the value. value = content for k in key: value = value[k] return value
def test_test_mode() -> None: ''' Tests server and client by starting the server in a thread, then requesting meta and sensor data. Asserts that this data is in the correct format and returns None on success. ''' print('Finding default configs...') if os.path.isdir('docs'): server_config_file = 'docs/sensed-config.sample.toml' client_config_file = 'docs/senselog-config.sample.toml' elif os.path.isdir('../docs'): server_config_file = '../docs/sensed-config.sample.toml' client_config_file = '../docs/senselog-config.sample.toml' print('Running sensed test mode test...') with open(server_config_file) as f: server_config = DotMap(toml.load(f)) dt_sensor = datetime.Sensor(server_config) ni_sensor = netifaces.Sensor(server_config) cam_sensor = camera.Sensor(server_config) hat_sensor = hat.Sensor(server_config) server_config.sensors = DotMap({'datetime': dt_sensor, 'netifaces': ni_sensor, 'camera': cam_sensor, 'hat': hat_sensor}) server = socketserver.UDPServer((server_config.sensed.host, server_config.sensed.port), SensedServer) server.sensors = server_config.sensors server.config = server_config server_thread = threading.Thread(target=server.serve_forever) server_thread.daemon = True server_thread.start() with open(client_config_file) as f: client_config = DotMap(toml.load(f)) client = SensedClient(client_config) meta = client.get_all_meta() sensors = client.get_all_sensors() assert isinstance(meta, list) assert isinstance(meta[0], dict) assert isinstance(sensors, list) assert isinstance(sensors[0], dict) print('Packet test passed. Shutting down...') server.shutdown() print('Test complete.')
async def read_user_state(path): global prev global manual book_files = utility.find_files(path, ('brf', 'pef')) main_toml = os.path.join(path, 'sd-card', USER_STATE_FILE) current_book = manual_filename if os.path.exists(main_toml): main_state = toml.load(main_toml) if 'current_book' in main_state: current_book = main_state['current_book'] if not current_book == manual_filename: current_book = os.path.join(path, current_book) if 'current_language' in main_state: current_language = main_state['current_language'] else: current_language = 'en_GB:en' else: current_language = 'en_GB:en' manual_toml = os.path.join(path, to_state_file(manual_filename)) if os.path.exists(manual_toml): t = toml.load(manual_toml) if 'current_page' in t: manual = manual._replace(page_number=t['current_page'] - 1) if 'bookmarks' in t: manual = manual._replace(bookmarks=tuple(sorted(manual.bookmarks + tuple( bm - 1 for bm in t['bookmarks'])))) else: manual = Manual.create(current_language) books = OrderedDict({manual_filename: manual}) for book_file in book_files: toml_file = to_state_file(book_file) book = BookFile(filename=book_file, width=40, height=9) if os.path.exists(toml_file): t = toml.load(toml_file) if 'current_page' in t: book = book._replace(page_number=t['current_page'] - 1) if 'bookmarks' in t: book = book._replace(bookmarks=tuple(sorted(book.bookmarks + tuple( bm - 1 for bm in t['bookmarks'])))) books[book_file] = book books[cleaning_filename] = CleaningAndTesting.create() if current_book not in books: current_book = manual_filename user_state = frozendict(books=FrozenOrderedDict( books), current_book=current_book, current_language=current_language) prev = user_state return user_state.copy(books=user_state['books'])
def main() -> None: data = toml.load(sys.stdin) assert list(data.keys()) == ["source"] # this value is non deterministic data["source"]["vendored-sources"]["directory"] = "@vendor@" lines = [] inner = data["source"] for source, attrs in sorted(inner.items()): lines.append("[source.{}]".format(quote(source))) if source == "vendored-sources": lines.append('"directory" = "@vendor@"\n') else: for key, value in sorted(attrs.items()): attr = "{} = {}".format(quote(key), quote(value)) lines.append(attr) lines.append("") result = "\n".join(lines) real = toml.loads(result) assert real == data, "output = {} while input = {}".format(real, data) print(result)
def load_hosts_from_hosts_toml(self, options=None, hosts=None, hosts_file=None, ): if hosts_file == None and options != None: hosts_file = options.hosts_file if hosts == None and options != None: hosts = options.hosts if not os.path.isfile(hosts_file): org_hosts_file = hosts_file hosts_file = "conf/" + hosts_file if not os.path.isfile(hosts_file): raise Exception("You must have a hosts.toml in order to deploy. File '%s' does not exist." % org_hosts_file) with open(hosts_file) as f: all_hosts_conf = toml.load(f) if not all_hosts_conf.get('hosts'): raise Exception("No hosts defined in hosts.toml") if not len(hosts): if len(all_hosts_conf.get('hosts')): hosts = ['ALL'] if not hosts: raise Exception("You must pass in a comma separated list of hosts as the first argument. Use ALL to setup all hosts") confs = [] for conf in all_hosts_conf.get('hosts'): if not conf.get('host'): raise Exception("No 'host' attribute defined for a host in your hosts.toml") if hosts == ['ALL'] or conf.get('host') in hosts: confs.append(conf) if not confs: raise Exception('No host confs found matching host list: %s' % hosts) for conf in confs: conf['os'] = conf.get('os', 'ubuntu') if not conf['os'] in SUPPORTED_OSES: raise Exception('Sorry, %s is not a supported operating system.') return confs, hosts_file
def __init__(self, parent=None): """Initiate the abstract widget that is displayed in the preferences dialog.""" super(Playback, self).__init__(parent) self.user_config_file = os.path.join(AppDirs('mosaic', 'Mandeep').user_config_dir, 'settings.toml') with open(self.user_config_file) as conffile: config = toml.load(conffile) playback_config = QGroupBox('Playback Configuration') playback_config_layout = QVBoxLayout() playback_config_layout.setAlignment(Qt.AlignTop) self.cover_art_playback = QCheckBox('Cover Art Playback') self.playlist_save_checkbox = QCheckBox('Save Playlist on Close') playback_config_layout.addWidget(self.cover_art_playback) playback_config_layout.addWidget(self.playlist_save_checkbox) playback_config.setLayout(playback_config_layout) main_layout = QVBoxLayout() main_layout.addWidget(playback_config) self.setLayout(main_layout) self.check_playback_setting(config) self.check_playlist_save(config) self.cover_art_playback.clicked.connect(lambda: self.cover_art_playback_setting(config)) self.playlist_save_checkbox.clicked.connect(lambda: self.playlist_save_setting(config))
def run(): homu_cfg = toml.load('/home/servo/homu/cfg.toml') homu_builders = homu_cfg['repo']['servo']['buildbot'] # We need to invoke a new process to read the Buildbot master config # because Buildbot is written in python2. scriptpath = os.path.join(os.path.dirname(__file__), 'get_buildbot_cfg.py') ret = subprocess.run( ['/usr/bin/python2', scriptpath], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) if ret.returncode != 0: return Failure( 'Unable to retrieve buildbot builders:', ret.stderr ) buildbot_builders = json.loads(ret.stdout.decode('utf-8'))['builders'] failure_msg = '' for builder_set in ['builders', 'try_builders']: diff = set(homu_builders[builder_set]) - set(buildbot_builders) if diff: failure_msg += 'Invalid builders for "{}": {}\n' \ .format(builder_set, diff) if failure_msg: return Failure( "Homu config isn't synced with Buildbot config:", failure_msg ) return Success('Buildbot and homu configs are synced')
def _get_poetry_requirements(): try: data = toml.load('poetry.lock') except IOError: raise ShubException('Please make sure the poetry lock file is present') # Adapted from poetry 1.0.0a2 poetry/utils/exporter.py lines = [] for package in data['package']: source = package.get('source') or {} source_type = source.get('type') if source_type == 'git': line = 'git+{}@{}#egg={}'.format( source['url'], source['reference'], package['name'] ) elif source_type in ['directory', 'file']: line = '' line += source['url'] else: line = '{}=={}'.format(package['name'], package['version']) if source_type == 'legacy' and source['url']: line += ' \\\n' line += ' --index-url {}'.format(source['url']) line += '\n' lines.append(line) return ''.join(lines)
def setUp(self): logging.disable(logging.CRITICAL) self.db_file, self.db_filename = tempfile.mkstemp() with open("fixtures/test_sac_config.toml") as f: config = toml.load(f) config['db']['connection_string'] = "sqlite:///{}".format(self.db_filename) self.config = config
def loadconfig(filename): try: config=toml.load(filename) except IOError: with open(filename,"w") as f: pass config={} return config
def layout(self, client, userdata): fixtures = os.path.join(ROOT, 'test/fixtures') profile = webdriver.FirefoxProfile(os.path.join(fixtures, 'profile', client)) for xpi in glob.glob(os.path.join(ROOT, 'xpi/*.xpi')): profile.add_extension(xpi) profile.set_preference('extensions.zotero.translators.better-bibtex.testing', True) with open(os.path.join(os.path.dirname(__file__), 'preferences.toml')) as f: preferences = toml.load(f) for p, v in nested_dict_iter(preferences['general']): profile.set_preference(p, v) if userdata.get('locale', '') == 'fr': for p, v in nested_dict_iter(preferences['fr']): profile.firefox.set_preference(p, v) if userdata.get('first-run', 'false') == 'false': profile.set_preference('extensions.zotero.translators.better-bibtex.citekeyFormat', '[auth][shorttitle][year]') if client == 'jurism': print('\n\n** WORKAROUNDS FOR JURIS-M IN PLACE -- SEE https://github.com/Juris-M/zotero/issues/34 **\n\n') #profile.set_preference('extensions.zotero.dataDir', os.path.join(self.path, 'jurism')) #profile.set_preference('extensions.zotero.useDataDir', True) #profile.set_preference('extensions.zotero.translators.better-bibtex.removeStock', False) profile.update_preferences() shutil.rmtree(self.path, ignore_errors=True) shutil.move(profile.path, self.path)
def __init__(self, overrides=None, valgrind=False): self.pid = None with open(os.path.join(constants.ASSETS_DIR, "test.toml")) as fp: self.cf = toml.load(fp) if overrides: update_nested_dict(self.cf, overrides) self.cf["core"].pop("admin_host", None) # extract some field from cf self.data_store = self.cf["zone_source"]["type"].lower() self.pidfile = self.cf["core"]["pidfile"] self.admin_host = self.cf["core"].get("admin_host", None) self.admin_port = self.cf["core"].get("admin_port", None) self.dns_port = self.cf["core"]["port"] self.dns_host = self.cf["core"]["bind"] # override mongo host and mongo port mongo_conf = self.cf["zone_source"]["mongo"] if self.data_store == "mongo": self.zm = ZoneMongo(constants.MONGO_HOST, constants.MONGO_PORT, mongo_conf["dbname"]) self.valgrind = valgrind self.cf_str = toml.dumps(self.cf) self.fp = tempfile.NamedTemporaryFile() self.fp.write(self.cf_str.encode("utf8")) self.fp.flush() fname = self.fp.name if self.valgrind: # TODO find the bug of possible lost and still reachable memory self.cmd = "valgrind --leak-check=full --show-reachable=no --show-possibly-lost=no %s -c %s" % (DNS_BIN, fname) else: self.cmd = "%s -c %s" % (DNS_BIN, fname) self.vagrant = vagrant.Vagrant(root=os.path.join(constants.REPO_ROOT, "vagrant"))
def load(cls, fp: TextIO, args: Optional[Namespace] = None): """ :param fp: .toml file's file pointer :param args: command line arguments :return: Config instance """ config_dic = toml.load(fp) code_style_config_dic = config_dic.get('codestyle', {}) postprocess_config_dic = config_dic.get('postprocess', {}) etc_config_dic = config_dic.get('etc', {}) if args: code_style_config_dic = _update_config_dict(code_style_config_dic, dict( template_file=args.template, workspace_dir=args.workspace, lang=args.lang)) etc_config_dic = _update_config_dict(etc_config_dic, dict( download_without_login=args.without_login, parallel_download=args.parallel, save_no_session_cache=args.save_no_session_cache)) return Config( code_style_config=CodeStyleConfig(**code_style_config_dic), postprocess_config=PostprocessConfig(**postprocess_config_dic), etc_config=EtcConfig(**etc_config_dic) )
def set_database_path(dbfolder): """Use to write the database path into the config. Parameters ---------- dbfolder : str or pathlib.Path Path to where planetpy will store data it downloads.. """ # First check if there's a config file, so that we don't overwrite # anything: try: config = toml.load(str(configpath)) except IOError: # config file doesn't exist config = {} # create new config dictionary # check if there's an `data_archive` sub-dic try: archive_config = config["data_archive"] except KeyError: config["data_archive"] = {"path": dbfolder} else: archive_config["path"] = dbfolder with open(configpath, "w") as f: ret = toml.dump(config, f) print(f"Saved database path {ret} into {configpath}.")
def read_config(): with open('reversion.toml') as f: conf = toml.load(f) if 'currentversion' not in conf: raise ConfigError("No field named currentversion") if not isinstance(conf['currentversion'], str): raise ConfigError("currentversion should be a string, not {}".format( type(conf['currentversion']))) places = conf.get('place', []) if not places: raise ConfigError("Need at least one replacement site ([[place]] section)") if not isinstance(places, list): raise ConfigError("place must be an array") if not isinstance(places[0], dict): raise ConfigError("place must be an array of tables") for place in places: if 'file' not in place: raise ConfigError("Missing file= field for place") if not isinstance(place['file'], str): raise ConfigError("file must be string") if ('line-regex' in place) and not isinstance(place['line-regex'], str): raise ConfigError("linematch must be string") return conf
def __init__(self, conf_file): self.toml_file = conf_file self.settings: Dict = toml.load(conf_file) self.last_look = arrow.get(self.settings['twitter']['last_look']) self.multiplexing = True self.multi_server = self.settings['pyborg']['multi_server'] self.pyborg = None
def read_config(self): try: config_data = toml.load(self.config_file) except Exception as e: raise ConfigError(self.config_file, e) self._validate_config(config_data) self._config_data = config_data
def main(): host = sys.argv[1] print('Running certbot') with open('hosts.toml') as f: host_conf = toml.load(f) # get certbot domains print('Run certbot for %s' % host_conf) certbot_conf = host_conf.get('certbot', {}) if not certbot_conf: raise ValueError('[certbot] configuration section of the hosts.toml file is empty!') webroot_path = certbot_conf.get("webroot_path", "/var/www/html/") domains = certbot_conf.get('domains', []) email = certbot_conf.get('email') if not domains or not type(domains) == list: raise ValueError('certbot conf has no list of domains entry!') if not os.path.isfile('/usr/bin/certbot'): install('software-properties-common') local['add-apt-repository']['-y', 'ppa:certbot/certbot'] & FG apt_get['update'] & FG install('python-certbot-nginx') cron_path = '/etc/cron.d/certbot-global-auto-renew' if not os.path.isfile(cron_path): with open(cron_path, 'w') as f: f.write(cron_template) os.chmod(cron_path, 0o600) domains = sorted(domains, key=lambda d:len(d)) name = domains[0] conf_path = '/etc/letsencrypt/renewal/' + name + '.conf' matches = True if not os.path.isfile(conf_path): matches = False else: with open(conf_path, 'r') as f: text = f.read() for d in domains: if d + ' = ' + webroot_path.rstrip('/') not in text: matches = False if matches: print('letsencrypt renewal conf exists for domains %s' % domains) return yn = input("Is DNS for the SSL domains pointing to this server? If no, you'll have to confirm domain ownership via editing DNS entries. (y/n) ") if yn.lower() == 'y': args = ['certonly', '--webroot', '-w', webroot_path, '--cert-name', name] for domain in domains: args.extend(['-d', domain]) local['certbot'][args] & FG else: args = ['certonly', '--manual', '--preferred-challenges=dns', '--cert-name', name] for domain in domains: args.extend(['-d', domain]) local['certbot'][args] & FG print("Once DNS is pointing to this server, run certbot again to configure automatic renewal.")
def load(self): """Cache the file onto self.__db, and mark the time it was loaded""" if not os.path.exists(self.path): #TODO (OS): Figure out what to do here. return with open(self.path, 'r') as f: self.__db = toml.load(f) self.__last_load = time()
def read_package_info(cargo_toml=None): """Read the content of [package] section from Cargo.toml. :return: [package] section as dictionary """ cargo_toml = Path(cargo_toml or Path.cwd() / 'Cargo.toml') with cargo_toml.open() as f: manifest = toml.load(f) return manifest['package']
def read_config_file(toml_file): """read config settings from TOML file""" try: from toml import load except ImportError: return {"config_file_error": "'toml' module must be installed to use config file"} else: return load(toml_file)
def __init__(self): for item in CONF_LOCATIONS: conf_path = os.path.expanduser(os.path.join(item, CONF_FILE)) if os.path.exists(conf_path): with open(conf_path) as f: for k,v in toml.load(f).iteritems(): self.__setattr__(k, v) self.hosts = {} self.get_hosts()
def import_from_toml(self, fp): """ Import from a file-like object where TOML markup can be read from. :param fp: A filelike object. :return: Naught. """ data = toml.load(fp) self.import_from_data(data)
def _loadConfig(self,cfgFile): if os.path.exists(cfgFile) is None: return None self.conf = toml.load(cfgFile) if self.conf is None: logging.error('cannot load the config file: ' + str(cfgFile)) else: logging.info('loaded the config file: ' + str(cfgFile))
def parse_config(): with open('config.toml', 'rb') as fin: obj = pytoml.load(fin) if not obj['key']: raise Exception('Key must be defined in config file') if not 'start' in obj: obj['start'] = 0 if not 'end' in obj: obj['end'] = 10000 return obj
def _load_user_config(self): config_file_path = os.sep.join([ self.metadata.distribution_root, 'pyproject.toml', ]) try: config = toml.load(config_file_path) return config[u'tool'][u'dependencychecker'] except (IOError, KeyError, ): return {}
def _add_dependencies(cargo_toml_path, additional_dependencies): with open(cargo_toml_path, 'r+') as f: cargo_toml = toml.load(f) cargo_toml.setdefault('dependencies', {}) for dep in additional_dependencies: name, _, spec = dep.partition(':') cargo_toml['dependencies'][name] = spec or '*' f.seek(0) toml.dump(cargo_toml, f) f.truncate()
def start(self) -> None: self.settings = toml.load(self.conf_file) self.multiplexing = True self.multi_server = self.settings['pyborg']['multi_server'] self.project = aeidon.Project() logger.debug("About to load subs.") self.project.open_main(self.subs_file) logger.debug("Loaded subs!") self.subtitles = self.project.subtitles self.run()
def project_config(): """Deprecated""" project_name = curdir config_path = Path(join(project_name, PROJECT_CONFIG_FILE)) if not exists(config_path): store_settings(DEFAULTS.copy()) return DEFAULTS return toml.load(io.open(config_path)) or {}
import toml var_path = "config.toml" try: config_vars = toml.load(var_path) except FileNotFoundError: print("Le fichier {0} est manquant, veuillez le creer".format(var_path)) raise def password(): """ Retourne le mot de passe du fichier de configuration voir le fichier indiqué par `var_path` pour le modifier """ return config_vars['login_infos']['password'] def screen_infos(dimension): """ Retourne une information concernant l'écran du fichier de configuration voir le fichier indiqué par `var_path` pour le modifier """ return config_vars['screen_infos'][dimension] def tempo_infos(dimension): """ Retourne une information concernant les tempos du fichier de configuration voir le fichier indiqué par `var_path` pour le modifier """
def intent_callback_fuel(hermes, intent_message): for (slot_value, slot) in intent_message.slots.items(): if slot[0].slot_value.value.value == "Diesel": hermes.publish_end_session( intent_message.session_id, tankerkoenig.diesel_price(intent_message)) elif slot[0].slot_value.value.value == "Benzin": hermes.publish_end_session( intent_message.session_id, tankerkoenig.benzin_price(intent_message)) if __name__ == "__main__": config = read_configuration_file() tankerkoenig = Tankerkoenig(config) snips_config = toml.load('/etc/snips.toml') if 'mqtt' in snips_config['snips-common'].keys(): MQTT_BROKER_ADDRESS = snips_config['snips-common']['mqtt'] if 'mqtt_username' in snips_config['snips-common'].keys(): MQTT_USERNAME = snips_config['snips-common']['mqtt_username'] if 'mqtt_password' in snips_config['snips-common'].keys(): MQTT_PASSWORD = snips_config['snips-common']['mqtt_password'] mqtt_opts = MqttOptions(username=MQTT_USERNAME, password=MQTT_PASSWORD, broker_address=MQTT_BROKER_ADDRESS) with Hermes(mqtt_options=mqtt_opts) as h: h.subscribe_intent(add_postfix("fuelInfo"), intent_callback_fuel) h.start()
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import toml g_conf = toml.load('conf/trade.conf')
def test_warnings(): # Expect 1 warning for the non existent toml file with pytest.warns(UserWarning): toml.load(["test.toml", "nonexist.toml"])
def load(self): # Download package try: uri, groupuri = self.repository._decompose_uri() g = gitlab.Gitlab(uri, private_token=self.repository._token) g.auth() except Exception as e: print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{e}") try: group = g.groups.get(groupuri) project = None for p in group.projects.list(all=True): if p.name==self.name: project = g.projects.get(p.id) break if project is None: raise GitlabRepositoryException(f"{self.name}: project not found inside repository {self.repository.name}") except gitlab.exceptions.GitlabGetError as e: print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{self.name}: project not found inside repository {self.repository.name}") except Exception as e: print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{e}") try: # locate release release = None for r in project.releases.list(): if r.tag_name==f"v{self.version}": release = r break except Exception as e: print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{e}") if release is None: raise GitlabRepositoryException(f"{self.name}-{self.version}: no release found from github repository") try: # search zip file asset = None for link in release.assets['links']: if link['name']==f"{self.name}-{self.version}.zip": asset = link except Exception as e: print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{e}") if asset is None: raise GitlabRepositoryException(f"{self.name}-{self.version}.zip: not found from github repository") try: # download unzip package tmpd = tempfile.mkdtemp() tmpf = tempfile.mktemp() # TODO: need this bug to be corrected to make this working # https://gitlab.com/gitlab-org/gitlab/-/issues/28978 headers = {'Private-Token': self.repository._token} r = requests.get(asset['url'], headers=headers, allow_redirects=True) with open(tmpf, mode="wb") as f: f.write(r.content) file = zipfile.ZipFile(tmpf) file.extractall(path=tmpd, members=file.namelist()) self._source_path = tmpd except Exception as e: os.remove(tmpf) print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{e}") # load army.toml try: file = os.path.join(tmpd, "army.toml") log.info(f"Load package '{file}'") self._data = toml.load(file) log.debug(f"content: {self._data}") except toml.decoder.TomlDecodeError as e: os.remove(tmpf) print_stack() log.debug(f"{type(e)} {e}") raise GitlabRepositoryException(f"{e}") os.remove(tmpf)
parser.add_argument( "-n", dest="nsamples", default=1, type=int, help="the number of samples to be generated (default=%(default)s)", ) parser.add_argument("-q", action="store_true", help="do quietly") args = parser.parse_args() VERBOSE = not args.q param = toml.load(args.input) if VERBOSE: print("parameter file is successfully loaded") outdir = param["parameter"].get("outputdir", "work") if not os.path.exists(outdir): os.makedirs(outdir) if VERBOSE: print(f"make output directory ({outdir})") if not os.path.isdir(outdir): raise RuntimeError(f"{outdir} is not a directory.") nsamples = args.nsamples
sys.dont_write_bytecode = True # ??? - Blaise 2020-08-13 import glob import sys import time import logging from argparse import ArgumentParser from slackclient import SlackClient import appdirs import toml import pathlib config = toml.load( pathlib.path(appdirs.user_config_dir("yaqc-cmds", "yaqc-cmds")) / "config.toml") default_channel = config["slack"]["channel"] witch_token = config["slack"]["token"] class RtmBot(object): def __init__(self, token): self.last_ping = 0 self.token = token self.bot_plugins = [] self.slack_client = None def connect(self): """Convenience method that creates Server instance""" self.slack_client = SlackClient(self.token)
def __init__(self, service_config_filepath=SERVICE_CONFIG_PATH): self.config = toml.load(service_config_filepath)
def main(): """ main function """ global START_TIME, MINER_URL, MINER_TOKEN, DAEMON_URL, DAEMON_TOKEN # Start execution time mesurement START_TIME = time.time() # SET API IP PORT AND AUTH if MINER_URL == '': miner_config = toml.load(str(Path.home()) + "/.lotusminer/config.toml") miner_api_ip = "127.0.0.1" miner_api_port = "2345" # try to read configuration file to identify miner url if "API" in miner_config.keys(): if "ListenAddress" in miner_config["API"].keys(): miner_api = miner_config["API"]["ListenAddress"].split("/") miner_api_ip = miner_api[2].replace("0.0.0.0", "127.0.0.1") miner_api_port = miner_api[4] MINER_URL = "http://" + miner_api_ip + ":" + miner_api_port + "/rpc/v0" if DAEMON_URL == '': daemon_config = toml.load(str(Path.home()) + "/.lotus/config.toml") daemon_api_ip = "127.0.0.1" daemon_api_port = "1234" # try to read configuration file to identify daemon url if "API" in daemon_config.keys(): if "ListenAddress" in daemon_config["API"].keys(): daemon_api = daemon_config["API"]["ListenAddress"].split("/") daemon_api_ip = daemon_api[2].replace("0.0.0.0", "127.0.0.1") daemon_api_port = daemon_api[4] DAEMON_URL = "http://" + daemon_api_ip + ":" + daemon_api_port + "/rpc/v0" if MINER_TOKEN == '': with open(str(Path.home()) + "/.lotusminer/token", "r") as text_file: MINER_TOKEN = text_file.read() if DAEMON_TOKEN == '': with open(str(Path.home()) + "/.lotus/token", "r") as text_file: DAEMON_TOKEN = text_file.read() ################################################################################# # MAIN ################################################################################# # SCRAPE METRIC DEFINITION print( "# HELP lotus_scrape_execution_succeed return 1 if lotus-farcaster execution was successfully" ) print("# TYPE lotus_scrape_execution_succeed gauge") # LOCAL TIME METRIC print( "# HELP lotus_local_time time on the node machine when last execution start in epoch" ) print("# TYPE lotus_local_time gauge") print(f'lotus_local_time {{ }} { int(time.time()) }') # RETRIEVE MINER ID actoraddress = miner_get_json("ActorAddress", []) miner_id = actoraddress['result'] # RETRIEVE TIPSET + CHAINHEAD chainhead = daemon_get_json("ChainHead", []) tipsetkey = chainhead["result"]["Cids"] # XXX small hack trying to speedup the script empty_tipsetkey = [] print("# HELP lotus_chain_height return current height") print("# TYPE lotus_chain_height counter") print( f'lotus_chain_height {{ miner_id="{miner_id}" }} {chainhead["result"]["Height"]}' ) checkpoint("ChainHead") # GENERATE CHAIN SYNC STATUS print( "# HELP lotus_chain_sync_diff return daemon sync height diff with chainhead for each daemon worker" ) print("# TYPE lotus_chain_sync_diff gauge") print( "# HELP lotus_chain_sync_status return daemon sync status with chainhead for each daemon worker" ) print("# TYPE lotus_chain_sync_status gauge") sync_status = daemon_get_json("SyncState", []) for worker in sync_status["result"]["ActiveSyncs"]: try: diff_height = worker["Target"]["Height"] - worker["Base"]["Height"] except Exception: diff_height = -1 print( f'lotus_chain_sync_diff {{ miner_id="{ miner_id }", worker_id="{ sync_status["result"]["ActiveSyncs"].index(worker) }" }} { diff_height }' ) print( f'lotus_chain_sync_status {{ miner_id="{ miner_id }", worker_id="{ sync_status["result"]["ActiveSyncs"].index(worker) }" }} { worker["Stage"] }' ) checkpoint("ChainSync") # GENERATE MINER INFO miner_version = miner_get_json("Version", []) checkpoint("Miner") # RETRIEVE MAIN ADDRESSES daemon_stats = daemon_get_json("StateMinerInfo", [miner_id, empty_tipsetkey]) miner_owner = daemon_stats["result"]["Owner"] miner_owner_addr = daemon_get_json( "StateAccountKey", [miner_owner, empty_tipsetkey])["result"] miner_worker = daemon_stats["result"]["Worker"] miner_worker_addr = daemon_get_json( "StateAccountKey", [miner_worker, empty_tipsetkey])["result"] try: miner_control0 = daemon_stats["result"]["ControlAddresses"][0] except: miner_control0 = miner_worker miner_control0_addr = daemon_get_json( "StateAccountKey", [miner_control0, empty_tipsetkey])["result"] print( "# HELP lotus_miner_info lotus miner information like adress version etc" ) print("# TYPE lotus_miner_info gauge") print("# HELP lotus_miner_info_sector_size lotus miner sector size") print("# TYPE lotus_miner_info_sector_size gauge") print( f'lotus_miner_info {{ miner_id = "{miner_id}", version="{ miner_version["result"]["Version"] }", owner="{ miner_owner }", owner_addr="{ miner_owner_addr }", worker="{ miner_worker }", worker_addr="{ miner_worker_addr }", control0="{ miner_control0 }", control0_addr="{ miner_control0_addr }" }} 1' ) print( f'lotus_miner_info_sector_size {{ miner_id = "{miner_id}" }} { daemon_stats["result"]["SectorSize"] }' ) checkpoint("StateMinerInfo") # GENERATE DAEMON INFO daemon_network = daemon_get_json("StateNetworkName", []) daemon_network_version = daemon_get_json("StateNetworkVersion", [empty_tipsetkey]) daemon_version = daemon_get_json("Version", []) print( "# HELP lotus_info lotus daemon information like adress version, value is set to network version number" ) print("# TYPE lotus_info gauge") print( f'lotus_info {{ miner_id="{miner_id}", version="{ daemon_version["result"]["Version"] }", network="{ daemon_network["result"] }"}} { daemon_network_version["result"]}' ) checkpoint("Daemon") # GENERATE WALLET + LOCKED FUNDS BALANCES walletlist = daemon_get_json("WalletList", []) print("# HELP lotus_wallet_balance return wallet balance") print("# TYPE lotus_wallet_balance gauge") for addr in walletlist["result"]: balance = daemon_get_json("WalletBalance", [addr]) short = addr[0:5] + "..." + addr[-5:] print( f'lotus_wallet_balance {{ miner_id="{miner_id}", address="{ addr }", short="{ short }" }} { int(balance["result"])/1000000000000000000 }' ) # Add miner balance : miner_balance_available = daemon_get_json("StateMinerAvailableBalance", [miner_id, empty_tipsetkey]) print( f'lotus_wallet_balance {{ miner_id="{miner_id}", address="{ miner_id }", short="{ miner_id }" }} { int(miner_balance_available["result"])/1000000000000000000 }' ) # Retrieve locked funds balance locked_funds = daemon_get_json("StateReadState", [miner_id, empty_tipsetkey]) print( "# HELP lotus_wallet_locked_balance return miner wallet locked funds") print("# TYPE lotus_wallet_locked_balance gauge") for i in ["PreCommitDeposits", "LockedFunds", "FeeDebt", "InitialPledge"]: print( f'lotus_wallet_locked_balance {{ miner_id="{miner_id}", address="{ miner_id }", locked_type ="{ i }" }} { int(locked_funds["result"]["State"][i])/1000000000000000000 }' ) checkpoint("Balances") # GENERATE POWER powerlist = daemon_get_json("StateMinerPower", [miner_id, empty_tipsetkey]) print("# HELP lotus_power return miner power") print("# TYPE lotus_power gauge") for minerpower in powerlist["result"]["MinerPower"]: print( f'lotus_power {{ miner_id="{miner_id}", scope="miner", power_type="{ minerpower }" }} { powerlist["result"]["MinerPower"][minerpower] }' ) for totalpower in powerlist["result"]["TotalPower"]: print( f'lotus_power {{ miner_id="{miner_id}", scope="network", power_type="{ totalpower }" }} { powerlist["result"]["TotalPower"][totalpower] }' ) # Mining eligibility print( "# HELP lotus_power_mining_eligibility return miner mining eligibility" ) print("# TYPE lotus_power_mining_eligibility gauge") base_info = daemon_get_json( "MinerGetBaseInfo", [miner_id, chainhead["result"]["Height"], tipsetkey]) if base_info["result"] is None: print(f'ERROR MinerGetBaseInfo return no result', file=sys.stderr) print( f'KNOWN_REASON your miner reports wrong info to the chain and thats pretty bad (not just for the dashboard)', file=sys.stderr) print(f'SOLUTION restart your miner and node', file=sys.stderr) print('lotus_scrape_execution_succeed { } 0') sys.exit(0) if base_info["result"]["EligibleForMining"]: eligibility = 1 else: eligibility = 0 print( f'lotus_power_mining_eligibility {{ miner_id="{miner_id}" }} { eligibility }' ) checkpoint("Power") # GENERATE MPOOL mpoolpending = daemon_get_json("MpoolPending", [empty_tipsetkey]) print("# HELP lotus_mpool_total return number of message pending in mpool") print("# TYPE lotus_mpool_total gauge") print( "# HELP lotus_mpool_local_total return total number in mpool comming from local adresses" ) print("# TYPE lotus_power_local_total gauge") print("# HELP lotus_mpool_local_message local message details") print("# TYPE lotus_mpool_local_message gauge") mpool_total = 0 mpool_local_total = 0 for message in mpoolpending["result"]: mpool_total += 1 frm = message["Message"]["From"] if frm in walletlist["result"]: mpool_local_total += 1 if frm == miner_owner_addr: display_addr = "owner" elif frm == miner_worker_addr: display_addr = "worker" elif frm == miner_control0_addr: display_addr = "control0" elif frm != miner_id: display_addr = frm[0:5] + "..." + frm[-5:] print( f'lotus_mpool_local_message {{ miner_id="{miner_id}", from="{ display_addr }", to="{ message["Message"]["To"] }", nonce="{ message["Message"]["Nonce"] }", value="{ message["Message"]["Value"] }", gaslimit="{ message["Message"]["GasLimit"] }", gasfeecap="{ message["Message"]["GasFeeCap"] }", gaspremium="{ message["Message"]["GasPremium"] }", method="{ message["Message"]["Method"] }" }} 1' ) print(f'lotus_mpool_total {{ miner_id="{miner_id}" }} { mpool_total }') print( f'lotus_mpool_local_total {{ miner_id="{miner_id}" }} { mpool_local_total }' ) checkpoint("MPool") # GENERATE NET_PEERS daemon_netpeers = daemon_get_json("NetPeers", []) print("# HELP lotus_netpeers_total return number netpeers") print("# TYPE lotus_netpeers_total gauge") print( f'lotus_netpeers_total {{ miner_id="{miner_id}" }} { len(daemon_netpeers["result"]) }' ) miner_netpeers = miner_get_json("NetPeers", []) print("# HELP lotus_miner_netpeers_total return number netpeers") print("# TYPE lotus_miner_netpeers_total gauge") print( f'lotus_miner_netpeers_total {{ miner_id="{miner_id}" }} { len(miner_netpeers["result"]) }' ) checkpoint("NetPeers") # GENERATE NETSTATS XXX Verfier la qualité des stats ... lotus net, API et Grafana sont tous differents print("# HELP lotus_net_protocol_in return input net per protocol") print("# TYPE lotus_net_protocol_in counter") print("# HELP lotus_net_protocol_out return output per protocol net") print("# TYPE lotus_net_protocol_out counter") protocols_list = daemon_get_json("NetBandwidthStatsByProtocol", []) for protocol in protocols_list["result"]: print( f'lotus_net_protocol_in {{ miner_id="{miner_id}", protocol="{ protocol }" }} { protocols_list["result"][protocol]["TotalIn"] }' ) print( f'lotus_net_protocol_out {{ miner_id="{miner_id}", protocol="{ protocol }" }} { protocols_list["result"][protocol]["TotalOut"] }' ) print("# HELP lotus_miner_net_protocol_in return input net per protocol") print("# TYPE lotus_miner_net_protocol_in counter") print("# HELP lotus_miner_net_protocol_out return output per protocol net") print("# TYPE lotus_miner_net_protocol_out counter") protocols_list = miner_get_json("NetBandwidthStatsByProtocol", []) for protocol in protocols_list["result"]: print( f'lotus_miner_net_protocol_in {{ miner_id="{miner_id}", protocol="{ protocol }" }} { protocols_list["result"][protocol]["TotalIn"] }' ) print( f'lotus_miner_net_protocol_out {{ miner_id="{miner_id}", protocol="{ protocol }" }} { protocols_list["result"][protocol]["TotalOut"] }' ) print("# HELP lotus_net_total_in return input net") print("# TYPE lotus_net_total_in counter") print("# HELP lotus_net_total_out return output net") print("# TYPE lotus_net_total_out counter") net_list = daemon_get_json("NetBandwidthStats", []) print( f'lotus_net_total_in {{ miner_id="{miner_id}" }} { net_list["result"]["TotalIn"] }' ) print( f'lotus_net_total_out {{ miner_id="{miner_id}" }} { net_list["result"]["TotalOut"] }' ) print("# HELP lotus_miner_net_total_in return input net") print("# TYPE lotus_miner_net_total_in counter") print("# HELP lotus_miner_net_total_out return output net") print("# TYPE lotus_miner_net_total_out counter") net_list = miner_get_json("NetBandwidthStats", []) print( f'lotus_miner_net_total_in {{ miner_id="{miner_id}" }} { net_list["result"]["TotalIn"] }' ) print( f'lotus_miner_net_total_out {{ miner_id="{miner_id}" }} { net_list["result"]["TotalOut"] }' ) checkpoint("NetBandwidth") # GENERATE WORKER INFOS workerstats = miner_get_json("WorkerStats", []) # XXX 1.2.1 introduce a new worker_id format. Later we should delete it, its a useless info. #print("# HELP lotus_miner_worker_id All lotus worker information prfer to use workername than workerid which is changing at each restart") #print("# TYPE lotus_miner_worker_id gauge") print( "# HELP lotus_miner_worker_mem_physical_used worker minimal memory used" ) print("# TYPE lotus_miner_worker_mem_physical_used gauge") print("# HELP lotus_miner_worker_mem_vmem_used worker maximum memory used") print("# TYPE lotus_miner_worker_mem_vmem_used gauge") print( "# HELP lotus_miner_worker_mem_reserved worker memory reserved by lotus" ) print("# TYPE lotus_miner_worker_mem_reserved gauge") print("# HELP lotus_miner_worker_gpu_used is the GPU used by lotus") print("# TYPE lotus_miner_worker_gpu_used gauge") print("# HELP lotus_miner_worker_cpu_used number of CPU used by lotus") print("# TYPE lotus_miner_worker_cpu_used gauge") print("# HELP lotus_miner_worker_cpu number of CPU") print("# TYPE lotus_miner_worker_cpu gauge") print("# HELP lotus_miner_worker_gpu number of GPU") print("# TYPE lotus_miner_worker_gpu gauge") print("# HELP lotus_miner_worker_mem_physical server RAM") print("# TYPE lotus_miner_worker_mem_physical gauge") print("# HELP lotus_miner_worker_mem_swap server SWAP") print("# TYPE lotus_miner_worker_mem_swap gauge") for val in workerstats["result"].items(): val = val[1] info = val["Info"] worker_host = info["Hostname"] mem_physical = info["Resources"]["MemPhysical"] mem_swap = info["Resources"]["MemSwap"] mem_reserved = info["Resources"]["MemReserved"] cpus = info["Resources"]["CPUs"] gpus = len(info["Resources"]["GPUs"]) mem_used_min = val["MemUsedMin"] mem_used_max = val["MemUsedMax"] if val["GpuUsed"]: gpu_used = 1 else: gpu_used = 0 cpu_used = val["CpuUse"] print( f'lotus_miner_worker_cpu {{ miner_id="{miner_id}", worker_host="{worker_host}" }} { cpus }' ) print( f'lotus_miner_worker_gpu {{ miner_id="{miner_id}", worker_host="{worker_host}" }} { gpus }' ) print( f'lotus_miner_worker_mem_physical {{ miner_id="{miner_id}", worker_host="{worker_host}" }} { mem_physical }' ) print( f'lotus_miner_worker_mem_swap {{ miner_id="{miner_id}", worker_host="{worker_host}" }} { mem_swap }' ) print( f'lotus_miner_worker_mem_physical_used {{ miner_id="{miner_id}", worker_host="{worker_host}" }} {mem_used_min}' ) print( f'lotus_miner_worker_mem_vmem_used {{ miner_id="{miner_id}", worker_host="{worker_host}" }} {mem_used_max}' ) print( f'lotus_miner_worker_mem_reserved {{ miner_id="{miner_id}", worker_host="{worker_host}" }} {mem_reserved}' ) print( f'lotus_miner_worker_gpu_used {{ miner_id="{miner_id}", worker_host="{worker_host}" }} {gpu_used}' ) print( f'lotus_miner_worker_cpu_used {{ miner_id="{miner_id}", worker_host="{worker_host}" }} {cpu_used}' ) checkpoint("Workers") # GENERATE JOB INFOS workerjobs = miner_get_json("WorkerJobs", []) print( "# HELP lotus_miner_worker_job status of each individual job running on the workers. Value is the duration" ) print("# TYPE lotus_miner_worker_job gauge") for (wrk, job_list) in workerjobs["result"].items(): for job in job_list: job_id = job['ID']['ID'] sector = str(job['Sector']['Number']) try: worker_host = workerstats["result"][wrk]["Info"]["Hostname"] except: # sometime WorkerJobs return invalid worker_id like 0000-000000-0000... in that case return unknown worker_host = "unknown" task = str(job['Task']) job_start_time = str(job['Start']) job_start_time = job_start_time[:-1] run_wait = str(job['RunWait']) job_start_epoch = time.mktime( time.strptime(job_start_time[:19], '%Y-%m-%dT%H:%M:%S')) print( f'lotus_miner_worker_job {{ miner_id="{miner_id}", job_id="{job_id}", worker_host="{ worker_host }", task="{task}", sector_id="{sector}", job_start_time="{job_start_time}", run_wait="{run_wait}" }} { START_TIME - job_start_epoch }' ) checkpoint("Jobs") # GENERATE JOB SCHEDDIAG scheddiag = miner_get_json("SealingSchedDiag", [True]) if scheddiag["result"]["SchedInfo"]["Requests"]: for req in scheddiag["result"]["SchedInfo"]["Requests"]: sector = req["Sector"]["Number"] task = req["TaskType"] print( f'lotus_miner_worker_job {{ miner_id="{miner_id}", job_id="", worker="", task="{task}", sector_id="{sector}", start="", run_wait="99" }} 0' ) checkpoint("SchedDiag") # GENERATE SECTORS print("# HELP lotus_miner_sector_state sector state") print("# TYPE lotus_miner_sector_state gauge") print( "# HELP lotus_miner_sector_event contains important event of the sector life" ) print("# TYPE lotus_miner_sector_event gauge") print( "# HELP lotus_miner_sector_sealing_deals_info contains information related to deals that are not in Proving and Removed state." ) print("# TYPE lotus_miner_sector_sealing_deals_info gauge") sector_list = miner_get_json("SectorsList", []) #sectors_counters = {} # remove duplicate (bug) unique_sector_list = set(sector_list["result"]) for sector in unique_sector_list: detail = miner_get_json("SectorsStatus", [sector, False]) deals = len( detail["result"]["Deals"]) - detail["result"]["Deals"].count(0) creation_date = detail["result"]["Log"][0]["Timestamp"] packed_date = "" finalized_date = "" verified_weight = detail["result"]["VerifiedDealWeight"] for log in range(len(detail["result"]["Log"])): if detail["result"]["Log"][log][ "Kind"] == "event;sealing.SectorPacked": packed_date = detail["result"]["Log"][log]["Timestamp"] if detail["result"]["Log"][log][ "Kind"] == "event;sealing.SectorFinalized": finalized_date = detail["result"]["Log"][log]["Timestamp"] if detail["result"]["Log"][0]["Kind"] == "event;sealing.SectorStartCC": pledged = 1 else: pledged = 0 print( f'lotus_miner_sector_state {{ miner_id="{miner_id}", sector_id="{ sector }", state="{ detail["result"]["State"] }", pledged="{ pledged }", deals="{ deals }", verified_weight="{ verified_weight }" }} 1' ) if packed_date != "": print( f'lotus_miner_sector_event {{ miner_id="{miner_id}", sector_id="{ sector }", event_type="packed" }} { packed_date }' ) if creation_date != "": print( f'lotus_miner_sector_event {{ miner_id="{miner_id}", sector_id="{ sector }", event_type="creation" }} { creation_date }' ) if finalized_date != "": print( f'lotus_miner_sector_event {{ miner_id="{miner_id}", sector_id="{ sector }", event_type="finalized" }} { finalized_date }' ) if detail["result"]["State"] not in ["Proving", "Removed"]: for deal in detail["result"]["Deals"]: if deal != 0: try: deal_info = daemon_get_json("StateMarketStorageDeal", [deal, empty_tipsetkey]) except: deal_is_verified = "unknown" deal_size = "unknown" deal_slash_epoch = "unknown" deal_price_per_epoch = "unknown" deal_provider_collateral = "unknown" deal_client_collateral = "unknown" deal_start_epoch = "unknown" deal_end_epoch = "unknown" else: deal_is_verified = deal_info["result"]["Proposal"][ "VerifiedDeal"] deal_size = deal_info["result"]["Proposal"][ "PieceSize"] deal_slash_epoch = deal_info["result"]["State"][ "SlashEpoch"] deal_price_per_epoch = deal_info["result"]["Proposal"][ "StoragePricePerEpoch"] deal_provider_collateral = deal_info["result"][ "Proposal"]["ProviderCollateral"] deal_client_collateral = deal_info["result"][ "Proposal"]["ClientCollateral"] deal_start_epoch = deal_info["result"]["Proposal"][ "StartEpoch"] deal_end_epoch = deal_info["result"]["Proposal"][ "EndEpoch"] print( f'lotus_miner_sector_sealing_deals_size {{ miner_id="{miner_id}", sector_id="{ sector }", deal_id="{ deal }", deal_is_verified="{ deal_is_verified }", deal_slash_epoch="{ deal_slash_epoch }", deal_price_per_epoch="{ deal_price_per_epoch }",deal_provider_collateral="{ deal_provider_collateral }", deal_client_collateral="{ deal_client_collateral }", deal_size="{ deal_size }", deal_start_epoch="{ deal_start_epoch }", deal_end_epoch="{ deal_end_epoch }" }} 1' ) # GENERATE DEADLINES proven_partitions = daemon_get_json("StateMinerDeadlines", [miner_id, empty_tipsetkey]) deadlines = daemon_get_json("StateMinerProvingDeadline", [miner_id, empty_tipsetkey]) dl_epoch = deadlines["result"]["CurrentEpoch"] dl_index = deadlines["result"]["Index"] dl_open = deadlines["result"]["Open"] dl_numbers = deadlines["result"]["WPoStPeriodDeadlines"] dl_window = deadlines["result"]["WPoStChallengeWindow"] print("# HELP lotus_miner_deadline_info deadlines and WPoSt informations") print("# TYPE lotus_miner_deadline_info gauge") print( f'lotus_miner_deadline_info {{ miner_id="{miner_id}", current_idx="{ dl_index }", current_epoch="{ dl_epoch }",current_open_epoch="{ dl_open }", wpost_period_deadlines="{ dl_numbers }", wpost_challenge_window="{ dl_window }" }} 1' ) print( "# HELP lotus_miner_deadline_active_start remaining time before deadline start" ) print("# TYPE lotus_miner_deadline_active_start gauge") print( "# HELP lotus_miner_deadline_active_sectors_all number of sectors in the deadline" ) print("# TYPE lotus_miner_deadline_active_sectors_all gauge") print( "# HELP lotus_miner_deadline_active_sectors_recovering number of sectors in recovering state" ) print("# TYPE lotus_miner_deadline_active_sectors_recovering gauge") print( "# HELP lotus_miner_deadline_active_sectors_faulty number of faulty sectors" ) print("# TYPE lotus_miner_deadline_active_sectors_faulty gauge") print( "# HELP lotus_miner_deadline_active_sectors_live number of live sectors" ) print("# TYPE lotus_miner_deadline_active_sectors_live gauge") print( "# HELP lotus_miner_deadline_active_sectors_active number of active sectors" ) print("# TYPE lotus_miner_deadline_active_sectors_active gauge") print( "# HELP lotus_miner_deadline_active_partitions number of partitions in the deadline" ) print("# TYPE lotus_miner_deadline_active_partitions gauge") print( "# HELP lotus_miner_deadline_active_partitions_proven number of partitions already proven for the deadline" ) print("# TYPE lotus_miner_deadline_active_partitions_proven gauge") for c_dl in range(dl_numbers): idx = (dl_index + c_dl) % dl_numbers opened = dl_open + dl_window * c_dl partitions = daemon_get_json("StateMinerPartitions", [miner_id, idx, empty_tipsetkey]) if partitions["result"]: faulty = 0 recovering = 0 alls = 0 active = 0 live = 0 count = len(partitions["result"]) proven = bitfield_count( proven_partitions["result"][idx]["PostSubmissions"]) for partition in partitions["result"]: faulty += bitfield_count(partition["FaultySectors"]) recovering += bitfield_count(partition["RecoveringSectors"]) active += bitfield_count(partition["ActiveSectors"]) live += bitfield_count(partition["LiveSectors"]) alls = bitfield_count(partition["AllSectors"]) print( f'lotus_miner_deadline_active_start {{ miner_id="{miner_id}", index="{ idx }" }} { (opened - dl_epoch) * 30 }' ) print( f'lotus_miner_deadline_active_partitions_proven {{ miner_id="{miner_id}", index="{ idx }" }} { proven }' ) print( f'lotus_miner_deadline_active_partitions {{ miner_id="{miner_id}", index="{ idx }" }} { count }' ) print( f'lotus_miner_deadline_active_sectors_all {{ miner_id="{miner_id}", index="{ idx }" }} { alls }' ) print( f'lotus_miner_deadline_active_sectors_recovering {{ miner_id="{miner_id}", index="{ idx }" }} { recovering }' ) print( f'lotus_miner_deadline_active_sectors_faulty {{ miner_id="{miner_id}", index="{ idx }" }} { faulty }' ) print( f'lotus_miner_deadline_active_sectors_active {{ miner_id="{miner_id}", index="{ idx }" }} { active }' ) print( f'lotus_miner_deadline_active_sectors_live {{ miner_id="{miner_id}", index="{ idx }" }} { live }' ) checkpoint("Deadlines") # GENERATE STORAGE INFO print("# HELP lotus_miner_storage_info get storage info state") print("# TYPE lotus_miner_storage_info gauge") print("# HELP lotus_miner_storage_capacity get storage total capacity") print("# TYPE lotus_miner_storage_capacity gauge") print( "# HELP lotus_miner_storage_available get storage available capacity") print("# TYPE lotus_miner_storage_available gauge") print("# HELP lotus_miner_storage_reserved get storage reserved capacity") print("# TYPE lotus_miner_storage_reserved gauge") storage_list = miner_get_json("StorageList", []) storage_local_list = miner_get_json("StorageLocal", []) for storage in storage_list["result"].keys(): storage_info = miner_get_json("StorageInfo", [storage]) if storage in storage_local_list["result"].keys(): storage_path = storage_local_list["result"][storage] else: storage_path = '' storage_id = storage_info["result"]["ID"] storage_url = urlparse(storage_info["result"]["URLs"][0]) storage_host_ip = storage_url.hostname try: storate_host_name = socket.gethostbyaddr(storage_host_ip)[0] except Exception: storate_host_name = storage_host_ip storage_host_port = storage_url.port storage_weight = storage_info["result"]["Weight"] storage_can_seal = storage_info["result"]["CanSeal"] storage_can_store = storage_info["result"]["CanStore"] try: storage_stat = miner_get_json("StorageStat", [storage]) except: storage_capacity = 0 storage_available = 0 storage_reserved = 0 else: storage_capacity = storage_stat["result"]["Capacity"] storage_available = storage_stat["result"]["Available"] storage_reserved = storage_stat["result"]["Reserved"] print( f'lotus_miner_storage_info {{ miner_id="{miner_id}", storage_id="{ storage_id }", storage_url="{ storage_info["result"]["URLs"][0] }", storage_host_name="{ storate_host_name }", storage_host_ip="{ storage_host_ip }", storage_host_port="{ storage_host_port }", weight="{ storage_weight }", can_seal="{ storage_can_seal }", can_store="{ storage_can_store }", path="{ storage_path }" }} 1' ) print( f'lotus_miner_storage_capacity {{ miner_id="{miner_id}", storage_id="{ storage_id }" }} { storage_capacity }' ) print( f'lotus_miner_storage_available {{ miner_id="{miner_id}", storage_id="{ storage_id }" }} { storage_available }' ) print( f'lotus_miner_storage_reserved {{ miner_id="{miner_id}", storage_id="{ storage_id }" }} { storage_reserved }' ) checkpoint("Storage") # GENERATE SCRAPE TIME print( f'lotus_scrape_duration_seconds {{ collector="All" }} {time.time() - START_TIME}' ) print('lotus_scrape_execution_succeed { } 1')
import toml import numpy from utils import retry, accounts, firstProducer, numProducers, intToCurrency config = toml.load('./config.toml') def allocateFunds(b, e): dist = numpy.random.pareto(1.161, e - b).tolist() # 1.161 = 80/20 rule dist.sort() dist.reverse() factor = 2_000_000 / sum(dist) total = 0 for i in range(b, e): funds = round(factor * dist[i - b] * 10000) if i >= firstProducer and i < firstProducer + numProducers: funds = max(funds, round(config['funds']['min_producer_funds'] * 10000)) total += funds accounts[i]['funds'] = funds return total def createStakedAccounts(b, e): ramFunds = round(config['funds']['ram_funds'] * 10000) configuredMinStake = round(config['funds']['min_stake'] * 10000) maxUnstaked = round(config['funds']['max_unstaked'] * 10000) for i in range(b, e): a = accounts[i] funds = a['funds'] print('#' * 80) print('# %d/%d %s %s' % (i, e, a['name'], intToCurrency(funds))) print('#' * 80) if funds < ramFunds: print('skipping %s: not enough funds to cover ram' % a['name'])
import simplejson as json import yaml import toml try: filename="convert" f = open(filename+'.toml', 'r') tomlData = toml.load(f) f.close() except FileNotFoundError:('File not find') ff = open("convert2"+'.yaml', 'w+') yamlData = {'func':'\t', 'arr':'','null':'', 'bool1':'','bool2':'', 'string':'','fred':'', 'emptyArray':'','emptyObject':'', 'emptyString':''} yamlData['func'] = tomlData['func'] yamlData['arr'] = tomlData['arr'] yamlData['null'] = tomlData['null'] yamlData['bool1'] = tomlData['bool1'] yamlData['bool2'] = tomlData['bool2'] yamlData['string'] = tomlData['string'] yamlData['fred'] = tomlData['fred'] yamlData['emptyArray'] = tomlData['emptyArray'] yamlData['emptyObject'] = tomlData['emptyObject'] yamlData['emptyString'] = tomlData['emptyString'] yamlData['code'] = tomlData['code'] yamlData['base64'] = tomlData['base64'] yaml.dump(yamlData, ff)
from pathlib import Path # noqa import toml # noqa ROOT = Path(__file__).parent.absolute() os.chdir(str(ROOT)) with (ROOT.parent.parent / "README.md").open() as readme_file: readme = readme_file.read() with (ROOT.parent.parent / "CHANGELOG.md").open() as history_file: history = history_file.read() with (ROOT.parent / "Cargo.toml").open() as cargo: parsed_toml = toml.load(cargo) version = parsed_toml["package"]["version"] requirements = [ "cffi>=1", "glean_parser==1.28.3", "iso8601>=0.1.10; python_version<='3.6'", ] setup_requirements = ["cffi>=1.0.0"] # The environment variable `GLEAN_BUILD_VARIANT` can be set to `debug` or `release` buildvariant = os.environ.get("GLEAN_BUILD_VARIANT", "debug") if mingw_arch == "i686": shared_object_build_dir = "../../target/i686-pc-windows-gnu"
def plottime(control, future): db = toml.load(control.dbfile) n_exercises = len(db['exercises']) grid_size = math.ceil(n_exercises**0.5) fig, axs = plt.subplots(nrows=grid_size, ncols=grid_size) years = mdates.YearLocator() years_fmt = mdates.DateFormatter('%Y') months = mdates.MonthLocator() months_fmt = mdates.DateFormatter('%Y-%m') days = mdates.WeekdayLocator(byweekday=MO) for i, exercise in enumerate(db['exercises']): this_axs = axs[int(i / grid_size)][i % grid_size] amraps = parse_amraps(db[exercise]['amraps']) if future: x_axis = [ amraps['date'][0] + datetime.timedelta(days=x) for x in range((amraps['date'][-1] - amraps['date'][0]).days) ] else: x_axis = amraps['date'] rm_axis = [] rm_axis_lower = [] rm_axis_upper = [] rms = [1, 5, 10] linestyles = ['-', '--', 'dotted'] for j, date in enumerate(x_axis): if future: orm, slope, sigma_orm, sigma_slope = fit_rmcurve(amraps, date) rm_axis.append([ round(forward_general_epley(orm, x, slope), 1) for x in rms ]) rm_axis_upper.append([ round( forward_general_epley(orm, x, slope + sigma_slope) + sigma_orm, 1) for x in rms ]) rm_axis_lower.append([ round( forward_general_epley(orm, x, slope - sigma_slope) - sigma_orm, 1) for x in rms ]) else: if j == 0: continue old_amraps = {k: v[:j + 1] for k, v in amraps.items()} orm, slope, sigma_orm, sigma_slope = fit_rmcurve(old_amraps) rm_axis.append([ round(forward_general_epley(orm, x, slope), 1) for x in rms ]) rm_axis_upper.append([ round( forward_general_epley(orm, x, slope + sigma_slope) + sigma_orm, 1) for x in rms ]) rm_axis_lower.append([ round( forward_general_epley(orm, x, slope - sigma_slope) - sigma_orm, 1) for x in rms ]) if not future: x_axis = x_axis[1:] for j, rm in enumerate(rms): this_axs.fill_between(x_axis, [x[j] for x in rm_axis_lower], [x[j] for x in rm_axis_upper], color='lightgrey', alpha=0.5) this_axs.plot(x_axis, [x[j] for x in rm_axis], color='k', linestyle=linestyles[j]) this_axs.xaxis.set_major_locator(years) this_axs.xaxis.set_major_formatter(years_fmt) this_axs.xaxis.set_minor_locator(months) this_axs.grid(which='minor') this_axs.set_title(exercise) this_axs.format_xdata = mdates.DateFormatter('%Y-%m-%d') min_date = min([[x.get_xlim()[0] for x in y] for y in axs])[0] max_date = max([[x.get_xlim()[1] for x in y] for y in axs])[0] for y in axs: for x in y: x.set_xlim(min_date, max_date) plt.tight_layout() plt.show()
def get_configuration_from_toml(filename: str) -> Any: """Return the tool.skjold section from the given pyproject.toml location.""" document = toml.load(filename) section = document.get("tool", {}).get("skjold", {}) return section
import toml import os import importlib from setuptools import find_packages, setup pytom = toml.load("pyproject.toml") package_name = pytom["project"]["name"] author_name = " - ".join(pytom["project"]["authors"]) mymodule = importlib.import_module(package_name) data_subdirs = ["templates", "static"] data_files = [] for subdir in data_subdirs: for dir_path, _, file_names in os.walk(os.path.join(package_name, subdir)): data_files += [os.path.join(dir_path, f) for f in file_names] with open("README.md") as fp: long_description = fp.read() with open("requirements.txt") as fp: requirements = [p for p in fp.read().strip().split() if ("git+" not in p)] if __name__ == "__main__": setup( name=package_name, version=mymodule.__version__, description=pytom["project"]["description"], long_description=long_description,
def load_config(path: str) -> MutableMapping: raw_data = toml.load(path) raw_data = convert_legacy_format(raw_data) return ConfigSchema().load(raw_data)
def main(): # Define all valid attributes attrs = { "author": None, "comment": None, "description": None, "email": None, "homepage": None, "license": "NONE", "version": None, } # Initialize parser parser = argparse.ArgumentParser() # fmt: off # Positional arguments parser.add_argument("file", type=str, help="target file to create") # Optional arguments parser.add_argument("-b", "--box", action="store_true", help="apply a box decoration to the header") parser.add_argument("--box-style", type=str, default="round", help="box decoration style") parser.add_argument("-f", "--force", action="store_true", help="force creation of the target file") parser.add_argument("-t", "--template", type=str, default="basic.txt", help="source code header template") parser.add_argument( "--confdir", type=str, default="~/.config/mksrc", help="mksrc config home directory (default: ~/.config/mksrc)") for attr in attrs: # add overrides for all attributes parser.add_argument(f"--{attr}", type=str, help=f"{attr} string attribute") # fmt: on # Parse args args = parser.parse_args() # Resolve paths args.confdir = Path(args.confdir).expanduser() args.config = args.confdir.joinpath("config.toml") args.opath = Path(args.file) args.templates = args.confdir.joinpath("templates") args.template = Path(args.template) if not args.template.is_file(): args.template = args.templates.joinpath(args.template) # Override args args.file = args.opath.name # Read the config file config = toml.load(args.config) # Ensure the output file does not exist if args.opath.exists() and not args.force: raise FileExistsError # Update commentstrings from defaults cms = commentstrings() cms.update(config.get("commentstrings") or {}) config["commentstrings"] = cms # Update attributes from config attrs.update(config["attributes"]) # Autofill any missing attributes for attr in attrs: if args.__dict__.get(attr): attrs[attr] = args.__dict__[attr] # Set any missing mandatory attributes if not attrs.get("comment"): attrs["comment"] = (cms[args.opath.suffix] if cms.get( args.opath.suffix) else (input("Comment: ") or "#")) attrs["comment"] += " %s" if "%s" not in attrs["comment"] else "" if not attrs.get("file"): attrs["file"] = args.opath.name # Create a header from the template with open(args.template, "r") as f: # Read the template file header = f.read() # Fill in attribute tags matches = re.finditer("__([A-Z]+)__", header) # Iterate all matches for match in matches: # Extract the attribute name, value attr = match.group(1) value = attrs.get(attr.lower()) # Prompt the user for missing values if not value: value = input(f"{attr.title()}: ") # Continue if the value is still missing if not value: continue # Replace attribute tags with values header = header.replace(match.group(0), value) # Fill in any dates header = date.today().strftime(header) # Perform any decorations border = { "ascii": { "dl": "+", "dr": "+", "hh": "-", "ul": "+", "ur": "+", "vv": "|", }, "round": { "dl": "╮", "dr": "╭", "ul": "╯", "ur": "╰", }, "sharp": { "dl": "┐", "dr": "┌", "ul": "┘", "ur": "└", }, "star": { "dl": "*", "dr": "*", "hh": "*", "ul": "*", "ur": "*", "vv": "*", }, }[args.box_style] if args.box: header = box(attrs, header, border) # Split the header into lines header = header.splitlines() # Strip trailing whitespace header = map(str.strip, header) # Filter out missing attributes header = filter(lambda line: not re.search("__([A-Z]+)__", line), header) # Insert commentstrings header = map(lambda line: attrs["comment"] % line, header) # Add back newlines header = map(lambda line: "%s\n" % line, header) # Rejoin the header header = "".join(header) # Print the header print(header, end="") # Make intermediate output directories args.opath.parent.mkdir(exist_ok=True) # Write the header to the output file with open(args.opath, "w") as f: f.write(header)
def parse_config(config_file, platform, output_format): """ Parse the briefcase section of the pyproject.toml configuration file. This method only does basic structural parsing of the TOML, looking for, at a minimum, a ``[tool.briefcase.app.<appname>]`` section declaring the existence of a single app. It will also search for: * ``[tool.briefcase]`` - global briefcase settings * ``[tool.briefcase.app.<appname>]`` - settings specific to the app * ``[tool.briefcase.app.<appname>.<platform>]`` - settings specific to the platform * ``[tool.briefcase.app.<appname>.<platform>.<format>]`` - settings specific to the output format A configuration can define multiple apps; the final output is the merged content of the global, app, platform and output format settings for each app, with output format definitions taking precendence over platform, over app-level, over global. The final result is a single (mostly) flat dictionary for each app. :param config_file: A file-like object containing TOML to be parsed. :param platform: The platform being targetted :param output_format: The output format :returns: A dictionary of configuration data. The top level dictionary is keyed by the names of the apps that are declared; each value is itself the configuration data merged from global, app, platform and format definitions. """ try: pyproject = toml.load(config_file) global_config = pyproject['tool']['briefcase'] except toml.TomlDecodeError as e: raise BriefcaseConfigError('Invalid pyproject.toml: {e}'.format(e=e)) except KeyError: raise BriefcaseConfigError('No tool.briefcase section in pyproject.toml') # For consistent results, sort the platforms and formats all_platforms = sorted(get_platforms().keys()) all_formats = sorted(get_output_formats(platform).keys()) try: all_apps = global_config.pop('app') except KeyError: raise BriefcaseConfigError('No Briefcase apps defined in pyproject.toml') # Build the flat configuration for each app, # based on the requested platform and output format app_configs = {} for app_name, app_data in all_apps.items(): # At this point, the base configuration will contain a section # for each configured platform. Iterate over all the known platforms, # and remove these platform configurations. Keep a copy of the platform # configuration if it matches the requested platform, and merge it # into the app's configuration platform_data = None for p in all_platforms: try: platform_block = app_data.pop(p) if p == platform: # If the platform matches the requested format, preserve # it for later use. platform_data = platform_block merge_config(platform_data, platform_data) # The platform configuration will contain a section # for each configured output format. Iterate over all # the output format of the platform, and remove these # output format configurations. Keep a copy of the output # format configuration if it matches the requested output # format, and merge it into the platform's configuration. format_data = None for f in all_formats: try: format_block = platform_data.pop(f) if f == output_format: # If the output format matches the requested # one, preserve it. format_data = format_block except KeyError: pass # If we found a specific configuration for the requested # output format, add it to the platform configuration, # overwriting any platform-level settings with format-level # values. if format_data: merge_config(platform_data, format_data) except KeyError: pass # Now construct the final configuration. # First, convert the requirement definition at the global level merge_config(global_config, global_config) # The app's config starts as a copy of the base briefcase configuation. config = copy.deepcopy(global_config) # The app name is both the key, and a property of the configuration config['app_name'] = app_name # Merge the app-specific requirements merge_config(config, app_data) # If there is platform-specific configuration, merge the requirements, # then overwrite the platform-specific values. # This will already include any format-specific configuration. if platform_data: merge_config(config, platform_data) # Construct a configuration object, and add it to the list # of configurations that are being handled. app_configs[app_name] = config return global_config, app_configs
def main(): # configs args = toml.load(open('config.toml'))['model'] seed = args['seed'] batch_size = args['batch_size'] seq_size = args['seq_size'] init_size = args['init_size'] state_size = args['state_size'] belief_size = args['belief_size'] num_layers = args['num_layers'] obs_std = args['obs_std'] obs_bit = args['obs_bit'] learn_rate = args['learn_rate'] grad_clip = args['grad_clip'] max_iters = args['max_iters'] seg_num = args['seg_num'] seg_len = args['seg_len'] max_beta = args['max_beta'] min_beta = args['min_beta'] beta_anneal = args['beta_anneal'] log_dir = args['log_dir'] test_times = args['test_times'] gpu_ids = args['gpu_ids'] data_path = args['data_path'] check_path = args['check_path'] # fix seed np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True # set logger log_format = '[%(asctime)s] %(message)s' logging.basicConfig(level=logging.INFO, format=log_format, stream=sys.stderr) # set size seq_size = seq_size init_size = init_size # set writer exp_name = set_exp_name(args) writer = SummaryWriter(log_dir + exp_name) LOGGER.info('EXP NAME: ' + exp_name) # load dataset train_loader, test_loader, check_loader = full_dataloader(seq_size, init_size, batch_size, data_path, check_path) LOGGER.info('Dataset loaded') # init models model = EnvModel(belief_size=belief_size, state_size=state_size, num_layers=num_layers, max_seg_len=seg_len, max_seg_num=seg_num) if torch.cuda.is_available(): device = torch.device(f'cuda:{gpu_ids[0]}') model.to(device) model = nn.DataParallel(model, device_ids=gpu_ids) model = model.module else: device = torch.device('cpu') model.to(device) LOGGER.info('Model initialized') # init optimizer optimizer = Adam(params=model.parameters(), lr=learn_rate, amsgrad=True) # test data pre_test_full_list = iter(test_loader).next() pre_test_full_data_list = pre_test_full_list['img'] pre_test_full_point_list = pre_test_full_list['point'] pre_test_full_data_list = preprocess(pre_test_full_data_list.to(device), obs_bit) # for each iter b_idx = 0 while b_idx <= max_iters: # for each batch for train_list in train_loader: b_idx += 1 # mask temp annealing if beta_anneal: model.state_model.mask_beta = (max_beta - min_beta) * 0.999 ** (b_idx / beta_anneal) + min_beta else: model.state_model.mask_beta = max_beta ############## # train time # ############## # get input data train_obs_list = train_list['img'] train_points_list = train_list['point'] train_obs_list = preprocess(train_obs_list.to(device), obs_bit) # run model with train mode model.train() optimizer.zero_grad() results = model(train_obs_list, train_points_list, seq_size, init_size, obs_std) # get train loss and backward update train_total_loss = results['train_loss'] train_total_loss.backward() if grad_clip > 0.0: nn.utils.clip_grad_norm_(model.parameters(), grad_clip) optimizer.step() # log if b_idx % 10 == 0: log_str, log_data = log_train(results, writer, b_idx) LOGGER.info(log_str, *log_data) ############# # test time # ############# if b_idx % test_times == 0: # set data pre_test_init_data_list = pre_test_full_data_list[:, :init_size] post_test_init_data_list = postprocess(pre_test_init_data_list, obs_bit) pre_test_input_data_list = pre_test_full_data_list[:, init_size:(init_size + seq_size)] post_test_input_data_list = postprocess(pre_test_input_data_list, obs_bit) with torch.no_grad(): ################## # test data elbo # ################## model.eval() results = model(pre_test_full_data_list, pre_test_full_point_list, seq_size, init_size, obs_std) post_test_rec_data_list = postprocess(results['rec_data'], obs_bit) output_img, output_mask = plot_rec(post_test_init_data_list, post_test_input_data_list, post_test_rec_data_list, results['mask_data'], results['p_mask'], results['q_mask']) # log log_str, log_data = log_test(results, writer, b_idx) LOGGER.info(log_str, *log_data) writer.add_image('valid/rec_image', output_img.transpose([2, 0, 1]), global_step=b_idx) writer.add_image('valid/mask_image', output_mask.transpose([2, 0, 1]), global_step=b_idx) ################### # full generation # ################### pre_test_gen_data_list, test_mask_data_list = model.full_generation(pre_test_init_data_list, seq_size) post_test_gen_data_list = postprocess(pre_test_gen_data_list, obs_bit) # log output_img = plot_gen(post_test_init_data_list, post_test_gen_data_list, test_mask_data_list) writer.add_image('valid/full_gen_image', output_img.transpose([2, 0, 1]), b_idx) with torch.no_grad(): model.eval() acc = [] precision = [] recall = [] f_value = [] for check in check_loader: check_obs = check['img'] check_point = check['point'] check_obs = preprocess(check_obs.to(device), obs_bit) results = model(check_obs, check_point, seq_size, init_size, obs_std) metrixs = calc_metrixs(results['mask_data_true'], results['mask_data']) acc.append(metrixs['accuracy']) precision.append(metrixs['precision']) recall.append(metrixs['recall']) f_value.append(metrixs['f_value']) acc = np.concatenate(acc) precision = np.concatenate(precision) recall = np.concatenate(recall) f_value = np.concatenate(f_value) print('shape: ', acc.shape) print('accuracy: ', acc.mean()) print('precision: ', precision.mean()) print('recall: ', recall.mean()) print('f_value: ', f_value.mean())
def clean_cargo_cache(self, force=False, show_size=False, keep=None): def get_size(path): if os.path.isfile(path): return os.path.getsize(path) / (1024 * 1024.0) total_size = 0 for dirpath, dirnames, filenames in os.walk(path): for f in filenames: fp = os.path.join(dirpath, f) total_size += os.path.getsize(fp) return total_size / (1024 * 1024.0) removing_anything = False packages = { 'crates': {}, 'git': {}, } import toml if os.environ.get("CARGO_HOME", ""): cargo_dir = os.environ.get("CARGO_HOME") else: home_dir = os.path.expanduser("~") cargo_dir = path.join(home_dir, ".cargo") if not os.path.isdir(cargo_dir): return cargo_file = open(path.join(self.context.topdir, "Cargo.lock")) content = toml.load(cargo_file) for package in content.get("package", []): source = package.get("source", "") version = package["version"] if source == u"registry+https://github.com/rust-lang/crates.io-index": crate_name = "{}-{}".format(package["name"], version) if not packages["crates"].get(crate_name, False): packages["crates"][package["name"]] = { "current": [], "exist": [], } packages["crates"][package["name"]]["current"].append( crate_name) elif source.startswith("git+"): name = source.split("#")[0].split("/")[-1].replace(".git", "") branch = "" crate_name = "{}-{}".format(package["name"], source.split("#")[1]) crate_branch = name.split("?") if len(crate_branch) > 1: branch = crate_branch[1].replace("branch=", "") name = crate_branch[0] if not packages["git"].get(name, False): packages["git"][name] = { "current": [], "exist": [], } packages["git"][name]["current"].append( source.split("#")[1][:7]) if branch: packages["git"][name]["current"].append(branch) crates_dir = path.join(cargo_dir, "registry") crates_cache_dir = "" crates_src_dir = "" if os.path.isdir(path.join(crates_dir, "cache")): for p in os.listdir(path.join(crates_dir, "cache")): crates_cache_dir = path.join(crates_dir, "cache", p) crates_src_dir = path.join(crates_dir, "src", p) git_dir = path.join(cargo_dir, "git") git_db_dir = path.join(git_dir, "db") git_checkout_dir = path.join(git_dir, "checkouts") if os.path.isdir(git_db_dir): git_db_list = filter(lambda f: not f.startswith('.'), os.listdir(git_db_dir)) else: git_db_list = [] if os.path.isdir(git_checkout_dir): git_checkout_list = os.listdir(git_checkout_dir) else: git_checkout_list = [] for d in list(set(git_db_list + git_checkout_list)): crate_name = d.replace("-{}".format(d.split("-")[-1]), "") if not packages["git"].get(crate_name, False): packages["git"][crate_name] = { "current": [], "exist": [], } if os.path.isdir(path.join(git_checkout_dir, d)): with cd(path.join(git_checkout_dir, d)): git_crate_hash = glob.glob('*') if not git_crate_hash or not os.path.isdir( path.join(git_db_dir, d)): packages["git"][crate_name]["exist"].append(("del", d, "")) continue for d2 in git_crate_hash: dep_path = path.join(git_checkout_dir, d, d2) if os.path.isdir(dep_path): packages["git"][crate_name]["exist"].append( (path.getmtime(dep_path), d, d2)) elif os.path.isdir(path.join(git_db_dir, d)): packages["git"][crate_name]["exist"].append(("del", d, "")) for d in os.listdir(crates_src_dir): crate_name = re.sub(r"\-\d+(\.\d+){1,3}.+", "", d) if not packages["crates"].get(crate_name, False): packages["crates"][crate_name] = { "current": [], "exist": [], } packages["crates"][crate_name]["exist"].append(d) total_size = 0 for packages_type in ["git", "crates"]: sorted_packages = sorted(packages[packages_type]) for crate_name in sorted_packages: crate_count = 0 existed_crates = packages[packages_type][crate_name]["exist"] for exist in sorted(existed_crates, reverse=True): current_crate = packages[packages_type][crate_name][ "current"] size = 0 exist_name = path.join( exist[1], exist[2]) if packages_type == "git" else exist exist_item = exist[2] if packages_type == "git" else exist if exist_item not in current_crate: crate_count += 1 if int(crate_count) >= int(keep) or not current_crate or \ exist[0] == "del" or exist[2] == "master": removing_anything = True crate_paths = [] if packages_type == "git": exist_checkout_path = path.join( git_checkout_dir, exist[1]) exist_db_path = path.join(git_db_dir, exist[1]) exist_path = path.join(git_checkout_dir, exist_name) if exist[0] == "del": if os.path.isdir(exist_checkout_path): crate_paths.append(exist_checkout_path) if os.path.isdir(exist_db_path): crate_paths.append(exist_db_path) crate_count += -1 else: crate_paths.append(exist_path) exist_checkout_list = glob.glob( path.join(exist_checkout_path, '*')) if len(exist_checkout_list) <= 1: crate_paths.append(exist_checkout_path) if os.path.isdir(exist_db_path): crate_paths.append(exist_db_path) else: crate_paths.append( path.join(crates_cache_dir, "{}.crate".format(exist))) crate_paths.append( path.join(crates_src_dir, exist)) size = sum( get_size(p) for p in crate_paths) if show_size else 0 total_size += size print_msg = (exist_name, " ({}MB)".format( round(size, 2)) if show_size else "", cargo_dir) if force: print("Removing `{}`{} package from {}".format( *print_msg)) for crate_path in crate_paths: if os.path.exists(crate_path): try: delete(crate_path) except: print(traceback.format_exc()) print("Delete %s failed!" % crate_path) else: print("Would remove `{}`{} package from {}". format(*print_msg)) if removing_anything and show_size: print("\nTotal size of {} MB".format(round(total_size, 2))) if not removing_anything: print("Nothing to remove.") elif not force: print("\nNothing done. " "Run `./mach clean-cargo-cache -f` to actually remove.")
import toml from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import WandbLogger from experiments.circular_padding.resnet_circular_padding import * from models.custom_resnet import * from src import * from src.data.data_module import MusicDataModule config = toml.load('carnatic.toml') fcd = FullChromaDataset(json_path=config['data']['metadata'], data_folder=config['data']['chroma_folder'], include_mbids=json.load( open(config['data']['limit_songs'])), carnatic=True, compression=40) train, fcd_not_train = fcd.greedy_split(train_size=0.70) val, test = fcd_not_train.greedy_split(test_size=0.5) train = ChromaChunkDataset(train, chunk_size=100, augmentation=transpose_chromagram, stride=10) data = MusicDataModule(train, val, batch_size=32) import sys if sys.argv[1] == '34': model = ResNet34Circular(num_classes=max(fcd.y) + 1)
def update_cargo(self, params=None, package=None, all_packages=None, dry_run=None): if not params: params = [] if dry_run: import toml import httplib import colorama cargo_file = open(path.join(self.context.topdir, "Cargo.lock")) content = toml.load(cargo_file) packages = {} outdated_packages = 0 conn = httplib.HTTPSConnection("crates.io") for package in content.get("package", []): if "replace" in package: continue source = package.get("source", "") if source == r"registry+https://github.com/rust-lang/crates.io-index": version = package["version"] name = package["name"] if not packages.get(name, "") or packages[name] > version: packages[name] = package["version"] conn.request('GET', '/api/v1/crates/{}/versions'.format(package["name"])) r = conn.getresponse() json_content = json.load(r) for v in json_content.get("versions"): if not v.get("yanked"): max_version = v.get("num") break if version != max_version: outdated_packages += 1 version_major, version_minor = (version.split("."))[:2] max_major, max_minor = (max_version.split("."))[:2] if version_major == max_major and version_minor == max_minor and "alpha" not in version: msg = "minor update" msg_color = "\033[93m" else: msg = "update, which may contain breaking changes" msg_color = "\033[91m" colorama.init() print("{}Outdated package `{}`, available {}\033[0m".format(msg_color, name, msg), "\n\tCurrent version: {}".format(version), "\n\t Latest version: {}".format(max_version)) conn.close() print("\nFound {} outdated packages from crates.io".format(outdated_packages)) elif package: params += ["-p", package] elif all_packages: params = [] else: print("Please choose package to update with the --package (-p) ") print("flag or update all packages with --all-packages (-a) flag") sys.exit(1) if params or all_packages: self.ensure_bootstrapped() with cd(self.context.topdir): call(["cargo", "update"] + params, env=self.build_env())
"%s_lograt_replo.gr" % outprefix) ipod_utils.write_grfile(G_LOCS, max_logmat, "%s_lograt_rephi.gr" % outprefix) numpy.save("%s_lograt_bs.npy" % outprefix, bs_lograt) numpy.save("%s_lograt.npy" % outprefix, orig_lograt) numpy.save("%s_lograt_replo.npy" % outprefix, min_logmat) numpy.save("%s_lograt_rephi.npy" % outprefix, max_logmat) # now actually implement the full processing of the bootstrap replicates if __name__ == "__main__": conf_file = sys.argv[1] conf_dict = toml.load(conf_file) # figure out some global parameters bs_suffix = conf_dict['general']['bs_suffix'] bs_dir = conf_dict['general']['bs_dirname'] orig_suffix = conf_dict['general']['orig_suffix'] out_prefix = os.path.join(conf_dict['general']['output_path'], conf_dict['general']['out_prefix']) # make missing path if needed if not (os.path.isdir(conf_dict['general']['output_path'])): os.mkdir(conf_dict['general']['output_path']) # get the spline normalization from the input data only # we will use this repeatedly in the analysis below
logger = logging.getLogger('website') logger.setLevel(logging.INFO) # Filter warnings warnings.filterwarnings('ignore', category=RuntimeWarning) # Parse arguments parser = argparse.ArgumentParser() parser.add_argument("config_file", help="The configuration for the webserver.") parser.add_argument("--host", type=str, default='0.0.0.0', help="The host IP to run the webserver on.") parser.add_argument("--port", type=int, default=8080, help="The port to run the webserver on.") args = parser.parse_args() # Read config with open(args.config_file) as a: config = toml.load(a) # Create website object - don't start based on argv app = Application(loop=asyncio.get_event_loop()) app['static_root_url'] = '/static' session_setup(app, ECS(os.urandom(32), max_age=1000000)) # Encrypted cookies # session_setup(app, SimpleCookieStorage(max_age=1000000)) # Simple cookies DEBUG ONLY jinja_setup(app, loader=FileSystemLoader(os.getcwd() + '/website/templates')) app.router.add_routes(website.frontend_routes) app.router.add_routes(website.backend_routes) app.router.add_static('/static', os.getcwd() + '/website/static', append_version=True) # Add our connections app['database'] = utils.DatabaseConnection utils.DatabaseConnection.logger = logger.getChild("db")
def wait_for_materialize_views(args: argparse.Namespace) -> None: """Record the current table status of all views installed in Materialize.""" start_time = time.monotonic() # Create a dictionary mapping view names (as calculated from the filename) to expected contents view_snapshots: typing.Dict[str, str] = { p.stem: p.read_text().strip() for p in pathlib.Path(args.snapshot_dir).glob("*.sql") } source_offsets: typing.Dict[str, int] = { p.stem: int(p.read_text().strip()) for p in pathlib.Path(args.snapshot_dir).glob("*.offset") } # Create a dictionary mapping view names to source name and offset view_sources: typing.Dict[str, SourceInfo] = {} with open(os.path.join(args.snapshot_dir, "config.toml")) as fd: conf = toml.load(fd) if len(conf["sources"]) != 1: print(f"ERROR: Expected just one source block: {conf['sources']}") sys.exit(1) source_info = conf["sources"][0] topic_prefix: str = source_info["topic_namespace"] source_names: typing.List[str] = source_info["names"] for query_info in conf["queries"]: # Ignore views not in this snapshot (they likely have multiple sources...) view: str = query_info["name"] if view not in view_snapshots: continue sources: typing.List[str] = query_info["sources"] if len(query_info["sources"]) != 1: print( f"ERROR: Expected just one source for view {view}: {query_info['sources']}" ) sys.exit(1) source_name: str = query_info["sources"][0] if source_name not in source_name: print( f"ERROR: No matching source {source_name} for view {view}: {source_names}" ) sys.exit(1) topic_name = f"{topic_prefix}{source_name}" if topic_name not in source_offsets: print( f"ERROR: Missing offset information for source {topic_name}: {source_offsets}" ) sys.exit(1) view_sources[view] = SourceInfo(topic_name, source_offsets[topic_name]) with psycopg2.connect( f"postgresql://materialize@{args.host}:{args.port}/materialize" ) as conn: conn.autocommit = True installed_views = set(view_names(conn)) # Verify that we have snapshots for all views installed captured_views = set(view_snapshots.keys()) if not captured_views.issuperset(installed_views): missing_views = installed_views.difference(captured_views) print(f"ERROR: Missing final state for views: {missing_views}") print(f" Have: {captured_views}") sys.exit(1) print("Recording time required until each view matches its snapshot") pending_views = installed_views with psycopg2.connect( f"postgresql://materialize@{args.host}:{args.port}/materialize" ) as conn: conn.autocommit = True while pending_views: views_to_remove = [] time_taken = time.monotonic() - start_time for view in pending_views: with conn.cursor() as cursor: # Determine if the source is at the desired offset and identify the # mz_logical_timestamp associated with the offset timestamp = source_at_offset(cursor, view_sources[view]) if not timestamp: continue # Get the contents of the view at the desired timestamp, where an empty result # implies that the desired timestamp is not yet incorporated into the view try: contents = view_contents(cursor, view, timestamp) if not contents: continue except ViewNotReady: continue views_to_remove.append(view) expected = view_snapshots[view] if contents == expected: print( f"PASSED: {time_taken:>6.1f}s: {view} (result={contents})" ) else: print( f"FAILED: {time_taken:>6.1f}s: {view} ({contents} != {expected})" ) for view in views_to_remove: pending_views.remove(view) if pending_views: time.sleep(0.1)
"sql_uri": 'postgres://localhost/pollbot', "connection_count": 20, "overflow_count": 10, }, 'logging': { "sentry_enabled": False, "sentry_token": "", "log_level": logging.INFO, "debug": False, }, 'webhook': { "enabled": False, "domain": "https://localhost", "token": "pollbot", "cert_path": '/path/to/cert.pem', "port": 7000, } } config_path = os.path.expanduser('~/.config/ultimate_pollbot.toml') if not os.path.exists(config_path): with open(config_path, "w") as file_descriptor: toml.dump(default_config, file_descriptor) print( "Please adjust the configuration file at '~/.config/ultimate_pollbot.toml'" ) sys.exit(1) else: config = toml.load(config_path)
# Variables for this script INTERVAL = 10 # Seconds between sending readings ConfPathFile = os.path.join(HOME_DIR, AUTH_FILE) LogPathFile = os.path.join(CSVPath, LOG_FILE) logging.basicConfig(filename=LogPathFile, level=logging.DEBUG) CurrentTime = datetime.datetime.now().isoformat() logging.debug(CrLf + '***** Starting at: {a}'.format(a=CurrentTime) + ' *****') # Cayenne authentication info. This should be obtained from the Cayenne Dashboard, # and the details should be put into the file listed above. # Read the Cayenne configuration stuff into a dictionary ConfigDict = toml.load(ConfPathFile) CayenneParam = ConfigDict.get('cayenne') # print (CayenneParam) # The callback for when a message is received from Cayenne. def on_message(client, userData, message): # based on https://developers.mydevices.com/cayenne/docs/cayenne-mqtt-api/#cayenne-mqtt-api-mqtt-messaging-topics-send-actuator-updated-value # global COUNTER print("message received: ", str(message.payload.decode("utf-8"))) def on_connect(client, userData, flags, rc): print("Connected with result code " + str(rc))
elif m > 1: elapsed_time.append(str(m) + ' minutes') if s == 1: elapsed_time.append(str(s) + ' second') elif s > 1: elapsed_time.append(str(s) + ' seconds') else: elapsed_time.append('less than a second') return ', '.join(elapsed_time) port_scan_profiles_config_file = 'port-scan-profiles.toml' with open(os.path.join(rootdir, 'autorecon_config', port_scan_profiles_config_file), 'r') as p: try: port_scan_profiles_config = toml.load(p) if len(port_scan_profiles_config) == 0: fail('There do not appear to be any port scan profiles configured in the {port_scan_profiles_config_file} config file.') except toml.decoder.TomlDecodeError as e: fail('Error: Couldn\'t parse {port_scan_profiles_config_file} config file. Check syntax and duplicate tags.') with open(os.path.join(rootdir, 'autorecon_config', 'service-scans.toml'), 'r') as c: try: service_scans_config = toml.load(c) except toml.decoder.TomlDecodeError as e: fail('Error: Couldn\'t parse service-scans.toml config file. Check syntax and duplicate tags.') with open(os.path.join(rootdir, 'autorecon_config', 'global-patterns.toml'), 'r') as p: try:
from subprocess import PIPE, Popen pprint = PrettyPrinter(1).pprint EMSCRIPTEN_REPO = "https://github.com/emscripten-core/emscripten/" DOCKER_REGISTRY = "registry.hub.docker.com" # TODO: remove DOCKER_REPO = "trzeci/emscripten" QUEUE_FILE = "queue.txt" LOG_COMPILATION = "build.log" ROOT = os.path.realpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")) config = toml.load(open("config.toml", "r")) def get_username(): try: import pwd return pwd.getpwuid(os.getuid())[0] except: return os.getlogin() return def format_values(sth, mapping): def handle_object(obj): for k, v in obj.items(): obj[k] = format_values(v, mapping)
def parse_toml(toml_file): data = toml.load(toml_file) return data