def __init__(self, input_file=None, output_file=None, dfxml_file=None, report_file=None, commit=False, ignore_patterns=[], key=None, rules=[]): # Validate configuration from schema import Schema, Optional, Or, Use, And, SchemaError schema = Schema({ 'input_file': Use(lambda f: open(f, 'r'), error='Cannot read the input file'), Optional('output_file'): Or( None, Use(lambda f: open(f, 'w'), error='Cannot write to the output file')), Optional('dfxml_file'): Or(None, Use(lambda f: open(f, 'r'), error='Cannot read DFXML file')), Optional('report_file'): Or(None, lambda f: open(f, 'w'), error='Cannot write to the report file'), 'commit': Or(True, False), 'ignore_patterns': Use(lambda f: re.compile(convert_fileglob_to_re('|'.join(f))), error='Cannot compile unified ignore regex'), 'key': Or(None, str), 'rules': And([(redact_rule, redact_action)], lambda f: len(f) > 0) }) try: kwargs = { 'input_file': input_file, 'output_file': output_file, 'dfxml_file': dfxml_file, 'report_file': report_file, 'commit': commit, 'ignore_patterns': ignore_patterns, 'key': key, 'rules': rules } self.conf = schema.validate(kwargs) except SchemaError as e: logging.warning('The redact configuration did not validate:') exit(e) if self.conf['commit'] and 'output_file' not in self.conf.keys(): logging.error('An output file is required when COMMIT is on.') exit(1) # TODO Check input and output are not same file logging.debug('Configuration:\n%s' % self.conf) # Print rules logging.debug( json.dumps(map( lambda x, y: (x.line, x.__class__.__name__, y.__class__. __name__, x.lgpattern if hasattr(x, 'lgpattern') else ''), self.conf['rules']), indent=4)) self.input_file = self.conf['input_file'] from os import path self.image_size = path.getsize(self.input_file.name) self.output_file = self.conf['output_file'] self.report_file = self.conf['report_file'] self.dfxml_file = self.conf['dfxml_file'] self.commit = self.conf['commit'] self.configure_report_logger()
from app.views.log import find from app.models.log import Log import unittest import uuid log = Log( user_id=1, product_id='product_id', company_id='company_id', phone_number=5511999999999, value=10.00 ) log_validate = Schema({ 'id': Use(str), 'user_id': Use(int), 'product_id': Use(str), 'company_id': Use(str), 'phone_number': Use(int), 'value': Use(float), 'created': Or(str, None) }) logs_validate = Schema([{ 'id': Use(str), 'user_id': Use(int), 'product_id': Use(str), 'company_id': Use(str), 'phone_number': Use(int), 'value': Use(float),
'destination': And(str, len), 'action': And(str, len), Optional('username'): And(str, len), Optional('password'): And(str, len) }, 'influxdb': { 'host': And(str, len), 'port': And(int, port_range), Optional('username'): And(str, len), Optional('password'): And(str, len), 'database': And(str, len), Optional('ssl'): bool }, Optional("base64decode"): { 'source': And(str, len, Use(str_or_jsonPath)), 'target': And(str, len) }, 'points': [{ 'measurement': And(str, len, Use(str_or_jsonPath)), 'topic': And(str, len), Optional('httpcontent'): {str: And(str, len, Use(str_or_jsonPath))}, Optional('fields'): Or({str: And(str, len, Use(str_or_jsonPath))}, And(str, len, Use(str_or_jsonPath))), Optional('tags'): {str: And(str, len, Use(str_or_jsonPath))}, Optional('database'): And(str, len) }] }) def load_config(config_filename): with open(config_filename, 'r') as f:
__all__ = [ 'schema_general_settings', 'schema_derivative_couplings', 'schema_absorption_spectrum', 'schema_electron_transfer'] from numbers import Real from schema import (And, Optional, Schema, Use) schema_general_settings = Schema({ # "Library to distribute the computation" Optional("runner", default="multiprocessing"): And(str, Use(str.lower), lambda s: s in ("multiprocessing", "mpi")), # "default quantum package used" Optional("package_name", default="cp2k"): str, # project Optional("project_name", default="namd"): str, # "Basis set to carry out the quantum chemistry simulation" "basis_name": str, # Working directory Optional("scratch_path", default="/tmp"): str, # path to the HDF5 to store the results "path_hdf5": str, # path to xyz trajectory of the Molecular dynamics
def bitc_util_parser(argv=sys.argv[1:]): __usage__ = """ Bayesian analysis of ITC data. Uses MicroCal .itc files, or custom format .yml files for analysing experiments. Usage: ITC.py <datafiles>... [-w <workdir> | --workdir=<workdir>] [-n <name> | --name=<name>] [-q <file> | --heats=<file>] [-i <ins> | --instrument=<ins> ] [-v | -vv | -vvv] [-r <file> | --report=<file>] [ -l <logfile> | --log=<logfile>] ITC.py mcmc <datafiles>... (-m <model> | --model=<model>) [-w <workdir> | --workdir=<workdir>] [ -r <receptor> | --receptor=<receptor>] [-n <name> | --name=<name>] [-q <file> | --heats=<file>] [-i <ins> | --instrument=<ins> ] [ -l <logfile> | --log=<logfile>] [-v | -vv | -vvv] [--report=<file>] [options] ITC.py (-h | --help) ITC.py --license ITC.py --version Options: -h, --help Show this screen --version Show version --license Show license -l <logfile>, --log=<logfile> File to write logs to. Will be placed in workdir. -v, Verbose output level. Multiple flags increase verbosity. <datafiles> Datafile(s) to perform the analysis on, .itc, .yml -w <workdir>, --workdir=<workdir> Directory for output files [default: ./] -r <receptor> | --receptor=<receptor> The name of the receptor for a Competitive Binding model. -n <name>, --name=<name> Name for the experiment. Will be used for output files. Defaults to inputfile name. -i <ins>, --instrument=<ins> The name of the instrument used for the experiment. Overrides .itc file instrument. -q <file>, --heats=<file> Origin format integrated heats file. (From NITPIC use .dat file) -m <model>, --model=<model> Model to use for mcmc sampling [default: TwoComponent] --nfit=<n> No. of iteration for maximum a posteriori fit [default: 20000] --niters=<n> No. of iterations for mcmc sampling [default: 6000] --nburn=<n> No. of Burn-in iterations for mcmc sampling [default: 1000] --nthin=<n> Thinning period for mcmc sampling [default: 5] --report=<file> Output file with summary in markdown """ arguments = docopt(__usage__, argv=argv, version='ITC.py, pre-alpha') schema = Schema({ '--heats': Or(None, And(str, os.path.isfile, Use(os.path.abspath))), # str, verify that it exists '--help': bool, # True or False are accepted '--license': bool, # True or False are accepted # integer between 0 and 3 '-v': And(int, lambda n: 0 <= n <= 3), # str and found in this dict '--model': And(str, lambda m: m in known_models), '--nfit': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--nburn': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--niters': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--nthin': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--name': Or(None, And(str, len)), # Not an empty string '--instrument': Or(None, And(str, lambda m: m in known_instruments)), # None, or str and found in this dict '--version': bool, # True or False are accepted '--receptor': Or(None, str), # str or None '--workdir': str, # str # list and ensure it contains existing files '<datafiles>': And( list, lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles], Use(lambda inpfiles: [os.path.abspath(inpfile) for inpfile in inpfiles])), 'mcmc': bool, # True or False are accepted '--report': Or(None, Use(lambda f: open(f, 'w'))), # Don't use, or open file with writing permissions '--log': Or(None, str), # Don't use, or str }) return schema.validate(arguments)
from schema import Schema, And, Use group_schema = Schema({ "group": And(str) }) member_schema = Schema({ "number": And(str, Use(str), lambda s: len(s) == 12) }) members_schema = Schema({ "numbers": [And(str, Use(str), lambda s: len(s) == 12)] }) membership_schema = Schema({ "ids": [And(int)] }) message_schema = Schema({ "groups": [And(Use(int), lambda n: n > 0)], "message": And(str) })
from models import * import json from base64 import b64encode from schema import Schema, And, Use, Optional import lxml.etree as xml import re import sys vul_schema = Schema( { 'cwe': And(Use(int)), 'name': basestring, 'tool': basestring, 'severity': And(Use(int), lambda n: 0 <= n <= 3), 'description': basestring, 'target_name': basestring, Optional('observation'): basestring, Optional('remediation'): basestring, }, ignore_extra_keys=False) def parse_zap_json_file(zap_file, target, session, uri): with open(zap_file, 'r') as zapfile: zap_data = json.loads(zapfile.read()) alerts = None pre_alerts = zap_data['Report']['Sites'] if isinstance(pre_alerts, list): for pre in pre_alerts: if uri in pre['Host']: alerts = pre
def get_linkset_spec_schema(ets_ids): matching_methods_info = get_matching_methods() transformers_info = get_transformers() return Schema({ 'id': Use(int), 'label': And(str, len), Optional('description', default=None): Or(str, None), Optional('use_counter', default=True): bool, 'sources': [EntityTypeSelection(ets_ids)], 'targets': [EntityTypeSelection(ets_ids)], 'methods': And(LogicBox(Schema({ 'method': { 'name': And(str, Use(str.lower), lambda m: m in matching_methods_info.keys()), 'config': And(dict, MatchingMethodConfig(ets_ids)), }, Optional('sim_method', default={'name': None, 'config': {}, 'normalized': False}): { Optional('name', default=None): Or(None, And(str, Use(str.lower), lambda m: m in matching_methods_info.keys())), Optional('config', default={}): And(dict, MatchingMethodConfig(ets_ids)), Optional('normalized', default=False): bool, }, Optional('fuzzy', default={'t_norm': 'minimum_t_norm', 's_norm': 'maximum_s_norm', 'threshold': 0}): { Optional('t_norm', default='minimum_t_norm'): lambda s: s in ('minimum_t_norm', 'product_t_norm', 'lukasiewicz_t_norm', 'drastic_t_norm', 'nilpotent_minimum', 'hamacher_product'), Optional('s_norm', default='maximum_s_norm'): lambda s: s in ('maximum_s_norm', 'probabilistic_sum', 'bounded_sum', 'drastic_s_norm', 'nilpotent_maximum', 'einstein_sum'), Optional('threshold', default=0): Or(float, Use(lambda t: 0)), }, Optional('list_matching', default={'threshold': 0, 'is_percentage': False}): { Optional('threshold', default=0): int, Optional('is_percentage', default=False): bool, }, 'sources': { 'properties': { EntityTypeSelection(ets_ids): [{ 'property': And(Use(filter_property), len), Optional('property_transformer_first', default=False): bool, Optional('transformers', default=list): [{ 'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()), 'parameters': dict }], }] }, Optional('transformers', default=list): [{ 'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()), 'parameters': dict }], }, 'targets': { 'properties': { EntityTypeSelection(ets_ids): [{ 'property': And(Use(filter_property), len), Optional('property_transformer_first', default=False): bool, Optional('transformers', default=list): [{ 'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()), 'parameters': dict }], }] }, Optional('transformers', default=list): [{ 'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()), 'parameters': dict }], } }, ignore_extra_keys=True), name='conditions', types=( 'and', 'or', 'minimum_t_norm', 'product_t_norm', 'lukasiewicz_t_norm', 'drastic_t_norm', 'nilpotent_minimum', 'hamacher_product', 'maximum_s_norm', 'probabilistic_sum', 'bounded_sum', 'drastic_s_norm', 'nilpotent_maximum', 'einstein_sum' ), elements_schema=Schema({ 'type': str, 'conditions': list, Optional('threshold', default=0): Or(float, Use(lambda t: 0)), }, ignore_extra_keys=True)), dict), }, ignore_extra_keys=True)
class CrawlCmd(HoaxyCommand): """ Usage: hoaxy crawl --fetch-url (--update | --archive) [--where-expr=<w> --order-by=<o> --limit=<l>] hoaxy crawl --fetch-html [--where-expr=<w> --order-by=<o> --limit=<l>] hoaxy crawl --parse-article [--where-expr=<w> --order-by=<o> --limit=<l>] This command provides crawling functions, including fetching URLs from sites, fetching HTMLs of URLs, and parsing HTML pages into structured article data. When crawling, you can specify which entity to fetch: --fetch-url Fetch URL, either news update or archive. --fetch-html Fetch HTML of URL. --parse-article Parse HTML page into structured article data. At this moment, we use webparse provided by https://mercury.postlight.com/web-parser/ We suggest you add `--limit` option when doing cronjobs to avoid overload. When fetching URL, you can specify which kind of URLs to fetch by option: --updated Fetch news update (the newest articles). --archive Fetch news archive (the whole site articles). Be careful to use this option, it is time consuming. The inputs for crawling processes (domains or URLs) are queried from DB. By default you do not need to consider the internal SQL query. However, if you want to specify your own data collection for crawling in some cases, you can use these options: --where-expr=<w> The WHERE expression for SQL query. --order-by=<o> The sorted order of `id` column, by default ascending [default: asc] --limit=<l> The limit expression, by default return all For --fetch-url, the default raw SQL query is on table `site`: site.is_enabled is True and site.is_alive is True For --fetch-html the default raw SQL query is on table `url` url.status_code=U_DEFAULT For --parse-article the default raw SQL query is on table `url` url.status_code=U_HTML_SUCCESS AND url.site_id IS NOT NULL Please check database.models to see detail. Other options are: -h --help Show help. More information: (1) The spiders are built base on scrapy package (https://scrapy.org/) (2) To distinguish URL records from these three phrase, we use `status_code` column, and will update the column for each action. Please check the Url model to see the table of status_code (3) Parse a HTML page into article is a difficult task. We don't implement it ourselves. Instead we use the service from https://mercury.postlight.com/. However, you can also use other parser, e.g., python-goose, https://github.com/grangier/python-goose, or even implement your own. Examples (`||` represents continue of commands, you can ignore when using): 1. Most often you would like to fetch the updates of all enabled sites, probably used as cronjob: hoaxy crawl --fetch-url --update 2. If you would like to fetch the URLs of old posts (archive): hoaxy crawl --fetch-url --archive Note: this crawling is quite time-consuming. 3. Also you may need structured article instead of HTML documents: hoaxy crawl --parse-article 4. When automating by crontab, you may need --limit option to limit the number of entries to process. This is very useful when you already have a huge number of URLs in the database, and you would like to consume them each time a block. hoaxy crawl --fetch-html --limit=20000 5. Occasionally, you can specify the where expression to crawl only on these entries: hoaxy crawl --parse-article --where-expr='url.html is null' """ name = 'crawl' short_description = 'Crawl news sites' args_schema = Schema({ '--order-by': Or(None, Use(str.lower), lambda s: s in ('asc', 'desc'), error='must be asc or desc'), '--limit': Or(None, Use(int)), object: object }) @classmethod def fetch_url(cls, session, msites, platform_id, purpose): """Actual method to do fetch url action. Parameters ---------- msites : list a list of Site model class, contains info to build spiders. platform_id : int id of platform, bind fetched url with this id. purpose : {'update', 'archive'} indicate which url to fetch. """ settings = Settings(cls.conf['crawl']['scrapy']) settings.set('ITEM_PIPELINES', {'hoaxy.crawl.pipelines.UrlPipeline': 300}) process = CrawlerProcess(settings) sll = cls.conf['logging']['loggers']['scrapy']['level'] logging.getLogger('scrapy').setLevel(logging.getLevelName(sll)) for ms in msites: for sm in build_spiders_iter(ms, purpose): sm['kwargs']['session'] = session sm['kwargs']['platform_id'] = platform_id process.crawl(sm['cls'], *sm['args'], **sm['kwargs']) process.start() @classmethod def fetch_html(cls, session, url_tuples): """Actual method to do fetch html action. Parameters ---------- session : object a SQLAlchemy session object. url_tuples : list a list of url tuple (id, raw, status_code). """ settings = Settings(cls.conf['crawl']['scrapy']) settings.set('ITEM_PIPELINES', {'hoaxy.crawl.pipelines.HtmlPipeline': 300}) process = CrawlerProcess(settings) sll = cls.conf['logging']['loggers']['scrapy']['level'] logging.getLogger('scrapy').setLevel(logging.getLevelName(sll)) logger.warning('Number of url to fetch html is: %s', len(url_tuples)) process.crawl( HtmlSpider, session=session, url_tuples=url_tuples, excluded_domains=cls.conf['crawl']['excluded_domains']) process.start() @classmethod def parse_article(cls, session, url_tuples): """Actual method to do parse to article action. Parameters ---------- session : object a SQLAlchemy session object. url_tuples : list a list of url tuple (id, created_at, date_published, canonical, site_id) """ settings = Settings(cls.conf['crawl']['scrapy']) settings.set('ITEM_PIPELINES', {'hoaxy.crawl.pipelines.ArticlePipeline': 300}) process = CrawlerProcess(settings) sll = cls.conf['logging']['loggers']['scrapy']['level'] logging.getLogger('scrapy').setLevel(logging.getLevelName(sll)) logger.info('Number of url to parse is: %s', len(url_tuples)) process.crawl( ArticleParserSpider, session=session, url_tuples=url_tuples, api_key=cls.conf['crawl']['article_parser']['webparser_api_key'], ) process.start() @classmethod def run(cls, args): """Overriding method as the entry point of this command.""" try: args = cls.args_schema.validate(args) except SchemaError as e: raise SystemExit(e) session = Session(expire_on_commit=False) # session = Session() where_expr = args['--where-expr'] ob_expr = args.get('--order-by', 'asc') limit = args['--limit'] # --fetch-url if args['--fetch-url'] is True: configure_logging( 'crawl.fetch-url', console_level=args['--console-log-level'], file_level='WARNING') purpose = 'update' if args['--update'] is True else 'archive' if where_expr is None: where_expr = [text(DEFAULT_WHERE_EXPR_FETCH_URL)] else: where_expr = [text(where_expr)] ob_expr = Site.id.asc() if ob_expr == 'asc' else Site.id.desc() msites = get_msites( session, f_expr=where_expr, ob_expr=ob_expr, limit=limit) if len(msites) == 0: logger.warning("None sites you queried found in DB!") raise SystemExit(2) platform_id = get_platform_id(session, name=N_PLATFORM_WEB) # detach msites and mplatform from session, # since they definitely would not be modified in session for ms in msites: session.expunge(ms) logger.warning('Starting crawling process to fetch URL update ...') cls.fetch_url(session, msites, platform_id, purpose) elif args['--fetch-html'] is True: configure_logging( 'crawl.fetch-html', console_level=args['--console-log-level'], file_level='WARNING') if not session.query(Site.id).count() > 0: raise SystemExit('Your site table is empty!') q = session.query(Url.id, Url.raw) if where_expr is None: where_expr = [text(DEFAULT_WHERE_EXPR_FETCH_HTML)] else: where_expr = [text(where_expr)] ob_expr = Url.id.asc() if ob_expr == 'asc' else Url.id.desc() q = q.filter(*where_expr).order_by(ob_expr) if limit is not None: q = q.limit(limit) logger.info( q.statement.compile(compile_kwargs={"literal_binds": True})) url_tuples = q.all() if not url_tuples: logger.warning('No such URLs in DB!') raise SystemExit(2) logger.warning('Staring crawling process to fetch HTML ...') cls.fetch_html(session, url_tuples) # --parse-article elif args['--parse-article'] is True: configure_logging( 'crawl.parse-article', console_level=args['--console-log-level'], file_level='WARNING') q = session.query(Url.id, Url.created_at, Url.date_published, Url.canonical, Url.site_id) if where_expr is None: where_expr = [text(DEFAULT_WHERE_EXPR_PARSE_ARTICLE)] else: where_expr = [text(where_expr)] ob_expr = Url.id.asc() if ob_expr == 'asc' else Url.id.desc() q = q.filter(*where_expr).order_by(ob_expr) if limit is not None: q = q.limit(limit) logger.info( q.statement.compile(compile_kwargs={"literal_binds": True})) url_tuples = q.all() if not url_tuples: logger.warning('No URLs found from DB!') raise SystemExit(2) logger.warning('Starting crawling process to parse article ...') cls.parse_article(session, url_tuples) session.close()
class ServerConfiguration(Configuration): VALIDATORS = { "description": Use(str), "ip": Use(str), "port": Use(int), "api_path": Use(str), "uri": Use(str), "allow_drop_all": Use(bool), "logging": { "level": And( Use(str), lambda l: l in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")), "file": Use(str), "use_console": Use(bool), "backup_count": And(Use(int), lambda n: n > 0), "max_size": And(Use(int), lambda b: b > 16), "format": Use(str), "datefmt": Use(str) } }
from schema import Schema, And, Or, Use, Optional, Literal from .tools import HasKey, Nullable, Url, List SiteConfigSchemaObject = { "server_name": Use(str), Optional("web_upstream", default="localhost:8069"): str, Optional("poll_upstream", default="localhost:8072"): str, Optional("proxy_pass"): Use(str), # Or default=None ? Optional("disable_longpolling", default=False): bool, Optional("httpaccess"): bool, Optional("httpsaccess"): bool, Optional("posbox", default=False): bool, Optional("proxy_http_11"): bool, Optional("cache_statics"): bool, Optional("disable_cache_zone"): bool, Optional("header_upgrade"): bool, Optional("header_connection"): bool, Optional("header_host"): bool, Optional("masquerade"): str, Optional("redirect"): str, Optional("allow_tls_v1"): Nullable(bool), Optional("enable_hsts"): bool, Optional("certificate_folder"): str, Optional("ssl_certificate"): str, Optional("ssl_certificate_key"): str, Optional("ssl_trusted_certificate"): bool, Optional("disable_stapling"): bool, Optional("ip_allow", default=[]): List(str), Optional("ip_deny", default=[]): List(str), }
class NodeConfiguration(Configuration): VALIDATORS = { "api_key": And(Use(str), len), "server_url": Use(str), "port": Or(Use(int), None), "task_dir": Use(str), "databases": { Use(str): Use(str) }, "api_path": Use(str), "logging": { "level": And( Use(str), lambda l: l in ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")), "file": Use(str), "use_console": Use(bool), "backup_count": And(Use(int), lambda n: n > 0), "max_size": And(Use(int), lambda b: b > 16), "format": Use(str), "datefmt": Use(str) }, "encryption": { "enabled": bool, Optional("private_key"): Use(str) } }
"channel": "#seamless-thursday", "username": "******", "text": text, "icon_emoji": ":fatbot:", 'link_names': 1 } def post_message(message): if message: return json.dumps(payload(message)) return message @app.route('/', methods=['POST']) def main(): post = [ s.strip() for s in request.form['text'].lower().strip().split(':', 2) ] user = request.form['user_name'] return post_message(orderbot(user, post)) if __name__ == '__main__': orderbot = OrderBot() args = Schema({ '--host': Use(str), '--port': Use(int), '--debug': Use(bool) }).validate(docopt(__doc__)) app.run(host=args['--host'], port=args['--port'], debug=args['--debug'])
--use_regular <v> use regularizers [default: 0]. --regular_c <v> use regular_c [default: 0.0]. -h --help Show this screen and exit. """.format(f=__file__) ''' TODO: pretrained word embeddings visualise word embeddings visualise filters ''' from docopt import docopt from schema import Schema, And, Or, Use s = Schema({ '--n_epoch': Use(int), '--alpha': Use(float), '--width1': Use(int), '--width2': Use(int), '--feat_map_n_1': Use(int), '--feat_map_n_final': Use(int), '--feat_map_n_final': Use(int), '--k_top': Use(int), '--dropout_rate0': Use(float), '--dropout_rate1': Use(float), '--dropout_rate2': Use(float), '--activation': Use(str), '--learn': Use(str), '--skip': Use(int), '--use_regular': Use(int), '--regular_c': Use(float),
}, 'PRODUCTION': { 'MODULES': [ 'loaders/loader.py', 'loaders/lister.py', 'loaders/extractor.py', 'transformers/filename_transformer.py', 'transformers/meta_functions.py', 'transformers/data_transformer.py', 'transformers/wrapper.py', 'filters/selection_functions.py' ], 'LOADERS': ['loaders/json_loader.py', 'loaders/yaml_loader.py'] } } _SCHEMA_CONF = Schema({ Optional('TEMPLATE'): { Optional('BACKEND'): And(str, Use(_importModule)), Optional('DIR'): And(list, _isListOfStr) }, Optional('PRODUCTION'): { Optional('MODULES'): And(list, _isListOfStr, Use(_importModules)), Optional('LOADERS'): And(list, _isListOfStr, Use(_importModules)) } }) class ConfigHandler: def __init__(self, configFile, configPathProvided=False): if configPathProvided: self.config = loadYAML(configFile) else: try:
def test_dict_keys(self): assert Schema({str: int}).validate( {'a': 1, 'b': 2}) == {'a': 1, 'b': 2} self.assertRaises(SchemaError, Schema({str: int}).validate, {1: 1, 'b': 2}) assert Schema({Use(str): Use(int)}).validate( {1: 3.14, 3.14: 1}) == {'1': 3, '3.14': 1}
def make_answer_json(answer_code, body): ans_json = {'code': answer_code, 'body': body} # form answer as json return ans_json def check_schema(inp_conf_schema, dict_object): try: inp_conf_schema.validate(dict_object) return True except SchemaError: return False save_token_request_schema = Schema({ 'request': And(Use(str)), 'body': { 'auth': And(Use(str)), 'push': And(Use(str)), 'univer': And(Use(str)), 'fak': And(Use(str)), 'state': And(Use(str)) } }) def check_token_save_request_struct(dict_obj): check_res = check_schema(inp_conf_schema=save_token_request_schema, dict_object=dict_obj) return check_res
def test_schema_repr(self): # what about repr with `error`s? schema = Schema([Or(None, And(str, Use(float)))]) repr_ = "Schema([Or(None, And(<type 'str'>, Use(<type 'float'>)))])" # in Python 3 repr contains <class 'str'>, not <type 'str'> assert repr(schema).replace('class', 'type') == repr_
# consider valid if all 7 mandatory fields are present if sum(is_refkey.values()) == 7: valid_passport += 1 #============================== ## Part 2 # Create validation schema from schema import Schema, And, Use, Regex, Optional, SchemaError hcl_expr = "^#{1}[0-9a-f]{6}$" # https://regex101.com/r/g51QAR/1 hgt_expr = "^(1[5-8][0-9]|19[0-3])cm|(59|6[0-9]|7[0-6])in$" pid_expr = "^[0-9]{9}$" ecl_list = ('amb','blu','brn','gry','grn','hzl','oth') schema = Schema({'byr': And(Use(int), lambda n: 1920 <= n <= 2002), 'iyr': And(Use(int), lambda n: 2010 <= n <= 2020), 'eyr': And(Use(int), lambda n: 2020 <= n <= 2030), 'hgt': And(str, Regex(hgt_expr)), 'hcl': And(str, Regex(hcl_expr)), 'ecl': And(str, lambda s: s in ecl_list), 'pid': And(str, Regex(pid_expr)), Optional('cid'): And(str, len), Optional('name'): And(str, len) }) # Try on test dict try: validated = schema.validate(testd) print("data validated") except SchemaError as error:
def test_and(self): assert And(int, lambda n: 0 < n < 5).validate(3) == 3 self.assertRaises(SchemaError, And(int, lambda n: 0 < n < 5).validate, 3.33) assert And(Use(int), lambda n: 0 < n < 5).validate(3.33) == 3 self.assertRaises(SchemaError, And(Use(int), lambda n: 0 < n < 5).validate, '3.33')
def main(argv, session=None): args = docopt(__doc__, argv=argv) # Validate args. s = Schema({ six.text_type: Use(bool), '<query>': Use(lambda x: ' '.join(x)), '--parameters': Use(lambda x: get_args_dict(x, query_string=True)), '--header': Or(None, And(Use(get_args_dict), dict), error='--header must be formatted as --header="key:value"'), '--sort': list, '--field': list, '--timeout': Use(lambda x: float(x[0]), error='--timeout must be integer or float.') }) try: args = s.validate(args) except SchemaError as exc: print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr) sys.exit(1) # Support comma separated values. fields = list(chain.from_iterable([x.split(',') for x in args['--field']])) sorts = list(chain.from_iterable([x.split(',') for x in args['--sort']])) r_kwargs = dict( headers=args['--header'], timeout=args['--timeout'], ) search = session.search_items(args['<query>'], fields=fields, sorts=sorts, params=args['--parameters'], full_text_search=args['--fts'], request_kwargs=r_kwargs) try: if args['--num-found']: print('{0}'.format(search.num_found)) sys.exit(0) for result in search: if args['--itemlist']: print(result.get('identifier', '')) else: j = json.dumps(result) print(j) except ValueError as e: print('error: {0}'.format(e), file=sys.stderr) except ConnectTimeout as exc: print( 'error: Request timed out. Increase the --timeout and try again.', file=sys.stderr) sys.exit(1) except AuthenticationError as exc: print('error: {}'.format(exc), file=sys.stderr) sys.exit(1)
def get_options(cls): return { ConfigOption('report_tags'): [Use(tagging.validate_tag_value)], ConfigOption('report_tags_all'): [Use(tagging.validate_tag_value)] }
def integrate_parser(argv=sys.argv[1:]): __usage__ = """ Integrate ITC data using Gaussian process regression. Uses MicroCal .itc files, or custom format .yml files for analysing experiments. Usage: bitc_integrate.py <datafiles>... [-w <workdir> | --workdir=<workdir>] [-v | -vv | -vvv] [options] bitc_integrate.py (-h | --help) bitc_integrate.py --license bitc_integrate.py --version Options: -h, --help Show this screen --version Show version --license Show license -l <logfile>, --log=<logfile> File to write logs to. Will be placed in workdir. -v, Verbose output level. Multiple flags increase verbosity. <datafiles> Datafile(s) to perform the analysis on, .itc, .yml -w <workdir>, --workdir=<workdir> Directory for output files [default: ./] -n <name>, --name=<name> Name for the experiment. Will be used for output files. Defaults to input file name. -i <ins>, --instrument=<ins> The name of the instrument used for the experiment. Overrides .itc file instrument. -f <frac>, --fraction=<frac> The fraction of the injection to fit, measured from the end [default: 0.2] --theta0=<theta0> The parameters in the autocorrelation model. [default: 5.0] --nugget=<nugget> Size of nugget effect to allow smooth predictions from noisy data. [default: 1.0] --plot Generate plots of the baseline fit """ arguments = docopt(__usage__, argv=argv, version='bitc_integrate.py, pre-alpha') schema = Schema({ '--help': bool, # True or False are accepted '--license': bool, # True or False are accepted # integer between 0 and 3 '-v': And(int, lambda n: 0 <= n <= 3), # Float greater than 0 '--fraction': And(Use(float), lambda n: 0 < n <= 1.0), '--nugget': And(Use(float), lambda n: n > 0), '--theta0': And(Use(float), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--name': Or(None, And(str, len)), # Not an empty string '--instrument': Or(None, And(str, lambda m: m in known_instruments)), # None, or str and found in this dict '--version': bool, # True or False are accepted '--plot': bool, # True or False are accepted '--workdir': str, # str # list and ensure it contains existing files '<datafiles>': And( list, lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles], Use(lambda inpfiles: [os.path.abspath(inpfile) for inpfile in inpfiles])), # Don't use, or open file with writing permissions '--log': Or(None, str), # Don't use, or str }) return schema.validate(arguments)
value = sum((cpf[num] * ((i+1) - num) for num in range(0, i))) digit = ((value * 10) % 11) % 10 if digit != cpf[i]: raise SchemaError(f"value ('{number}') is not a valid CPF number!") return True def TO_DATE(format): def fnc(date_text): try: return dt.strptime(date_text, format) except ValueError: raise SchemaError(f"date format is invalid! value:('{date_text}')") return fnc conf_schema = Schema({ 'nome': And(Use(str),LEN_H_THEN(0)), 'cpf': And(Use(str),VALIDCPF), 'dataNascimento':And(Use(str),Use(TO_DATE("%d/%m/%Y"))), 'conta':{ 'limiteSaqueDiario': Or(int, float), 'flagAtivo':bool, 'tipoConta':int } }) conf = { 'nome': "a", 'cpf': "08639614440", 'dataNascimento':"01/01/1991", 'conta':{ 'limiteSaqueDiario':0,
def bitc_mcmc_parser(argv=sys.argv[1:]): __usage__ = """Analyze ITC data using Markov chain Monte Carlo (MCMC). Uses MicroCal .itc files, or custom format .yml files for modeling experiments. When running the program you can select one of two options: competitive A competitive binding model. Requires multiple experiments to be specified. twocomponent A twocomponent binding model. Analyzes only a single experiment enantiomer A enantiomer binding model Usage: bitc_mcmc.py twocomponent <datafile> <heatsfile> [-v | -vv | -vvv] [--cc=<c_cell>] [--cs=<c_syringe> ] [--dc=<dc_cell>] [--ds=<dc_syringe>] [options] bitc_mcmc.py competitive (<datafile> <heatsfile>)... (-r <receptor> | --receptor <receptor>) [-v | -vv | -vvv] [options] bitc_mcmc.py racemicmixture <datafile> <heatsfile> [-v | -vv | -vvv] [--cc=<c_cell>] [--cs=<c_syringe> ] [--dc=<dc_cell>] [--ds=<dc_syringe>] [options] bitc_mcmc.py enantiomer <datafile> <heatsfile> [-v | -vv | -vvv] [--cc=<c_cell>] [--cs=<c_syringe> ] [--dc=<dc_cell>] [--ds=<dc_syringe>] [options] bitc_mcmc.py (-h | --help) bitc_mcmc.py --license bitc_mcmc.py --version Options: -h, --help Show this screen --version Show version --license Show license -l <logfile>, --log=<logfile> File to write logs to. Will be placed in workdir. --cc <c_cell> Concentration of component in cell in mM. Defaults to value in input file --cs <c_syringe> Concentration of component in syringe in mM. Defaults to value in input file --dc <dc_cell> Relative uncertainty in cell concentration [default: 0.1] --ds <dc_syringe> Relative uncertainty in syringe concentration [default: 0.1] -v, Verbose output level. Multiple flags increase verbosity. -w <workdir>, --workdir <workdir> Directory for output files [default: ./] -r <receptor> | --receptor <receptor> The name of the receptor for a competitive binding model. -n <name>, --name <name> Name for the experiment. Will be used for output files. Defaults to inputfile name. -i <ins>, --instrument <ins> The name of the instrument used for the experiment. Overrides .itc file instrument. --nfit=<n> No. of iteration for maximum a posteriori fit [default: 20000] --niters=<n> No. of iterations for mcmc sampling [default: 2000000] --nburn=<n> No. of Burn-in iterations for mcmc sampling [default: 500000] --nthin=<n> Thinning period for mcmc sampling [default: 500] --dummy_itc_file The input itc file is a dummy one --uniform_cell_concentration Use uniform prior for cell concentration if True, use log normal if False --uniform_syringe_concentration Use uniform prior for syringe concentration if True, use log normal if False --concentration_range_factor <c_facctor> To define range of uniform prior. It will be between stated_value/this_factor and stated_value*this_factor [default: 10.] """ arguments = docopt(__usage__, argv=argv, version='bitc_mcmc.py, pre-alpha') schema = Schema({ '--help': bool, # True or False are accepted '--license': bool, # True or False are accepted # integer between 0 and 3 '-v': And(int, lambda n: 0 <= n <= 3), # str and found in this dict 'twocomponent': bool, 'enantiomer': bool, 'racemicmixture': bool, 'competitive': bool, '--nfit': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--nburn': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--niters': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--nthin': And(Use(int), lambda n: n > 0), # Convert str to int, make sure that it is larger than 0 '--name': Or(None, And(str, len)), # Not an empty string '--instrument': Or(None, And(str, lambda m: m in known_instruments)), # None, or str and found in this dict '--version': bool, # True or False are accepted '--receptor': Or(None, str), # str or None '--workdir': str, # str # str and ensure file exists # list and ensure it contains existing files '<datafile>': And( list, lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles], Use(lambda inpfiles: [os.path.abspath(inpfile) for inpfile in inpfiles])), # list and ensure it contains existing files '<heatsfile>': And( list, lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles], Use(lambda inpfiles: [os.path.abspath(inpfile) for inpfile in inpfiles])), # Don't use, or open file with writing permissions '--log': Or(None, str), # Don't use, or str '--cc': Or(None, And(Use(float), lambda n: n > 0.0)), # Not specified, or a float greater than 0 '--cs': Or(None, And(Use(float), lambda n: n > 0.0)), # Not specified, or a float '--dc': And(Use(float), lambda n: n > 0.0), # a float greater than 0 '--ds': And(Use(float), lambda n: n > 0.0), # a float greater than 0 '--dummy_itc_file': bool, '--uniform_cell_concentration': bool, '--uniform_syringe_concentration': bool, '--concentration_range_factor': Use(float), }) return schema.validate(arguments)
def get_options(cls): return { 'name': str, ConfigOption('logger_level', default=logger.TEST_INFO): int, ConfigOption('file_log_level', default=logger.DEBUG): Or(int, None), ConfigOption('runpath', default=default_runpath, block_propagation=False): Or(None, str, lambda x: callable(x)), ConfigOption('path_cleanup', default=True): bool, ConfigOption('all_tasks_local', default=False): bool, ConfigOption('shuffle', default=[]): list, # list of string choices ConfigOption('shuffle_seed', default=float(random.randint(1, 9999))): float, ConfigOption('exporters', default=None): Use(get_exporters), ConfigOption('stdout_style', default=defaults.STDOUT_STYLE, block_propagation=False): Style, ConfigOption('report_dir', default=defaults.REPORT_DIR, block_propagation=False): str, ConfigOption('xml_dir', default=None, block_propagation=False): Or(str, None), ConfigOption('pdf_path', default=None, block_propagation=False): Or(str, None), ConfigOption('json_path', default=None, block_propagation=False): Or(str, None), ConfigOption('pdf_style', default=defaults.PDF_STYLE, block_propagation=False): Style, ConfigOption('report_tags', default=[], block_propagation=False): [Use(tagging.validate_tag_value)], ConfigOption('report_tags_all', default=[], block_propagation=False): [Use(tagging.validate_tag_value)], ConfigOption('merge_scheduled_parts', default=False): bool, ConfigOption('browse', default=None): Or(None, bool), ConfigOption('ui_port', default=None): Or(None, int), ConfigOption('web_server_startup_timeout', default=defaults.WEB_SERVER_TIMEOUT): int, ConfigOption('test_filter', default=filtering.Filter(), block_propagation=False): filtering.BaseFilter, ConfigOption('test_sorter', default=ordering.NoopSorter(), block_propagation=False): ordering.BaseSorter, # Test lister is None by default, otherwise Testplan would # list tests, not run them ConfigOption('test_lister', default=None, block_propagation=False): Or(None, listing.BaseLister), ConfigOption('verbose', default=False, block_propagation=False): bool, ConfigOption('debug', default=False, block_propagation=False): bool, ConfigOption('timeout', default=None): Or(None, And(Or(int, float), lambda t: t >= 0)), ConfigOption('interactive_handler', default=TestRunnerIHandler): object, ConfigOption('extra_deps', default=[]): list }
def main(argv, session): args = docopt(__doc__, argv=argv) # Validation error messages. destdir_msg = '--destdir must be a valid path to a directory.' itemlist_msg = '--itemlist must be a valid path to an existing file.' # Validate args. s = Schema({ str: Use(bool), '--destdir': Or([], And(Use(lambda d: d[0]), dir_exists), error=destdir_msg), '--format': list, '--glob': Use(lambda l: l[0] if l else None), '<file>': list, '--search': Or(str, None), '--itemlist': Or(None, And(lambda f: os.path.isfile(f)), error=itemlist_msg), '<identifier>': Or(str, None), '--retries': Use(lambda x: x[0]), '--search-parameters': Use(lambda x: get_args_dict(x, query_string=True)), '--on-the-fly': Use(bool), '--no-change-timestamp': Use(bool), '--download-history': Use(bool), '--parameters': Use(lambda x: get_args_dict(x, query_string=True)), }) # Filenames should be unicode literals. Support PY2 and PY3. if six.PY2: args['<file>'] = [f.decode('utf-8') for f in args['<file>']] try: args = s.validate(args) if args['--glob']: if args['--format']: raise (SchemaError( None, '--glob and --format cannot be used together.')) except SchemaError as exc: print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)), file=sys.stderr) sys.exit(1) retries = int(args['--retries']) if args['--itemlist']: with open(args['--itemlist']) as fp: ids = [x.strip() for x in fp] total_ids = len(ids) elif args['--search']: try: _search = session.search_items(args['--search'], params=args['--search-parameters']) total_ids = _search.num_found if total_ids == 0: print('error: the query "{0}" returned no results'.format( args['--search']), file=sys.stderr) sys.exit(1) ids = _search except ValueError as e: print('error: {0}'.format(e), file=sys.stderr) sys.exit(1) # Download specific files. if args['<identifier>'] and args['<identifier>'] != '-': if '/' in args['<identifier>']: identifier = args['<identifier>'].split('/')[0] files = ['/'.join(args['<identifier>'].split('/')[1:])] else: identifier = args['<identifier>'] files = args['<file>'] file_with_files_to_download = 'to_download.txt' if not files and os.path.exists(file_with_files_to_download): with open(file_with_files_to_download, 'rt', encoding='utf-8') as f: files = f.read().splitlines() total_ids = 1 ids = [identifier] elif args['<identifier>'] == '-': total_ids = 1 ids = sys.stdin files = None else: files = None errors = list() for i, identifier in enumerate(ids): if args['--stdout']: item = session.get_item(identifier) f = list(item.get_files(args['<file>'])) try: assert len(f) == 1 except AssertionError: print('error: {0}/{1} does not exist!'.format( identifier, args['<file>'][0]), file=sys.stderr) sys.exit(1) if six.PY2: stdout_buf = sys.stdout else: stdout_buf = sys.stdout.buffer f[0].download(retries=args['--retries'], fileobj=stdout_buf, params=args['--parameters']) sys.exit(0) try: identifier = identifier.strip() except AttributeError: identifier = identifier.get('identifier') if total_ids > 1: item_index = '{0}/{1}'.format((i + 1), total_ids) else: item_index = None try: item = session.get_item(identifier) except Exception as exc: print('{0}: failed to retrieve item metadata - errors'.format( identifier), file=sys.stderr) if 'You are attempting to make an HTTPS' in str(exc): print('\n{0}'.format(exc), file=sys.stderr) sys.exit(1) else: continue # Otherwise, download the entire item. ignore_history_dir = True if not args['--download-history'] else False _errors = item.download( files=files, formats=args['--format'], glob_pattern=args['--glob'], dry_run=args['--dry-run'], verbose=args['--verbose'], silent=args['--silent'], ignore_existing=args['--ignore-existing'], checksum=args['--checksum'], destdir=args['--destdir'], no_directory=args['--no-directories'], retries=retries, item_index=item_index, ignore_errors=True, on_the_fly=args['--on-the-fly'], no_change_timestamp=args['--no-change-timestamp'], params=args['--parameters'], ignore_history_dir=ignore_history_dir, ) if _errors: errors.append(_errors) if errors: # TODO: add option for a summary/report. sys.exit(1) else: sys.exit(0)
def validate_method(method, is_challenge): '''Validate method configuration passed as a JSON file.''' # Define a dictionary schema # TODO would be nice to not copy-paste for multiple datasets schema = Schema({ Optional('metadata'): { 'publish_anonymously': bool, 'authors': str, 'contact_email': str, 'method_name': str, 'method_description': str, # 'descriptor_type': str, # 'descriptor_size': And(int, lambda v: v >= 1), Optional('link_to_website'): str, Optional('link_to_pdf'): str, }, 'config_common': { 'json_label': str, 'keypoint': And(Use(str), lambda v: '_' not in v), 'descriptor': And(Use(str), lambda v: '_' not in v), 'num_keypoints': And(int, lambda v: v > 1), }, Optional('config_phototourism_stereo'): { Optional('use_custom_matches'): bool, Optional('custom_matches_name'): str, Optional('matcher'): { 'method': And(str, lambda v: v in ['nn']), 'distance': And(str, lambda v: v.lower() in ['l1', 'l2', 'hamming']), 'flann': bool, 'num_nn': And(int, lambda v: v >= 1), 'filtering': { 'type': And( str, lambda v: v.lower() in [ 'none', 'snn_ratio_pairwise', 'snn_ratio_vs_last', 'fginn_ratio_pairwise' ]), Optional('threshold'): And(Use(float), lambda v: 0 < v <= 1), Optional('fginn_radius'): And(Use(float), lambda v: 0 < v < 100.), }, Optional('descriptor_distance_filter'): { 'threshold': And(Use(float), lambda v: v > 0), }, 'symmetric': { 'enabled': And(bool), Optional('reduce'): And(str, lambda v: v.lower() in ['both', 'any']), }, }, Optional('outlier_filter'): { 'method': And(Use(str), lambda v: v.lower() in ['none', 'cne-bp-nd']), }, Optional('geom'): { 'method': And( str, lambda v: v.lower() in [ 'cv2-ransac-f', 'cv2-ransac-e', 'cv2-lmeds-f', 'cv2-lmeds-e', 'cv2-7pt', 'cv2-8pt', 'cv2-patched-ransac-f', 'cmp-degensac-f', 'cmp-degensac-f-laf', 'cmp-gc-ransac-f', 'cmp-degensac-f-laf', 'cmp-gc-ransac-f', 'cmp-magsac-f', 'cmp-gc-ransac-e', 'skimage-ransac-f', 'intel-dfe-f' ]), Optional('threshold'): And(Use(float), lambda v: v > 0), Optional('confidence'): And(Use(float), lambda v: v > 0), Optional('max_iter'): And(Use(int), lambda v: v > 0), Optional('postprocess'): And(Use(bool), lambda v: v is not None), Optional('error_type'): And(Use(str), lambda v: v.lower() in ['sampson', 'symm_epipolar']), Optional('degeneracy_check'): bool, } }, Optional('config_phototourism_multiview'): { Optional('use_custom_matches'): bool, Optional('custom_matches_name'): str, Optional('matcher'): { 'method': And(str, lambda v: v in ['nn']), 'distance': And(str, lambda v: v.lower() in ['l1', 'l2', 'hamming']), 'flann': bool, 'num_nn': And(int, lambda v: v >= 1), 'filtering': { 'type': And( str, lambda v: v.lower() in [ 'none', 'snn_ratio_pairwise', 'snn_ratio_vs_last', 'fginn_ratio_pairwise' ]), Optional('threshold'): And(Use(float), lambda v: 0 < v <= 1), Optional('fginn_radius'): And(Use(float), lambda v: 0 < v < 100.), }, Optional('descriptor_distance_filter'): { 'threshold': And(Use(float), lambda v: v > 0), }, 'symmetric': { 'enabled': And(bool), Optional('reduce'): And(str, lambda v: v.lower() in ['both', 'any']), }, }, Optional('outlier_filter'): { 'method': And(Use(str), lambda v: v.lower() in ['none', 'cne-bp-nd']), }, Optional('colmap'): {}, }, Optional('config_phototourism_relocalization'): {}, }) schema.validate(method) # Check for metadata for challenge entries if is_challenge and not method['metadata']: raise ValueError('Must specify metadata') # Check what we are running do_pt_stereo = False if 'config_phototourism_stereo' not in method \ else bool(method['config_phototourism_stereo']) do_pt_multiview = False if 'config_phototourism_multiview' not in method \ else bool(method['config_phototourism_multiview']) do_pt_relocalization = False if 'config_phototourism_relocalization' not \ in method else bool(method['config_phototourism_relocalization']) if do_pt_stereo: print('Running: Phototourism, stereo track') if do_pt_multiview: print('Running: Phototourism, multiview track') if do_pt_relocalization: print('Running: Phototourism, relocalization track') if not any([do_pt_stereo, do_pt_multiview, do_pt_relocalization]): raise ValueError('No tasks were specified') # Check for incorrect, missing, or redundant options for dataset in ['phototourism']: for task in ['stereo', 'multiview', 'relocalization']: cur_key = 'config_{}_{}'.format(dataset, task) if cur_key not in method: print('Key "{}" is empty -> skipping check'.format(cur_key)) continue else: print('Validating key "{}"'.format(cur_key)) # If dict is not empty, use_custom_matches should exist if method[cur_key] and ( 'use_custom_matches' not in method[cur_key]): raise ValueError('Must specify use_custom_matches') # If using custom matches, we cannot specify further options if method[cur_key] and ('use_custom_matches' in method[cur_key]) \ and method[cur_key]['use_custom_matches']: if 'matcher' in method[cur_key] or 'outlier_filter' in method[cur_key]: raise ValueError('Cannot specify a matcher or outlier filter with ' 'use_custom_matches=True') # Matcher and filter if 'matcher' in method[cur_key]: matcher = method[cur_key]['matcher'] if matcher['symmetric']['enabled'] and 'reduce' not in \ matcher['symmetric']: raise ValueError( '[{}/{}] Must specify "reduce" if "symmetric" is enabled' .format(dataset, task)) # Check for redundant settings with custom matches cur_config = method['config_{}_stereo'.format(dataset)] if cur_config['use_custom_matches']: if 'matcher' in cur_config or 'outlier_filter' in cur_config \ or 'geom' in cur_config: raise ValueError( '[{}/stereo] Found redundant settings with use_custom_matches=True' .format(dataset)) else: if 'matcher' not in cur_config or 'outlier_filter' not in \ cur_config or 'geom' not in cur_config: raise ValueError( '[{}/stereo] Missing required settings with use_custom_matches=False' .format(dataset)) if cur_config['use_custom_matches']: if 'matcher' in cur_config or 'outlier_filter' in cur_config \ or 'geom' in cur_config: raise ValueError( '[{}/stereo] Found redundant settings with use_custom_matches=True' ) # For stereo, check also geom if task == 'stereo' and \ 'config_{}_stereo'.format(dataset) in method and \ 'geom' in method['config_{}_stereo'.format(dataset)]: geom = method['config_{}_stereo'.format(dataset)]['geom'] # Threshold for RANSAC if geom['method'].lower() in [ 'cv2-ransac-f', 'cv2-ransac-e', 'cv2-patched-ransac-f', 'cmp-degensac-f', 'cmp-gc-ransac-f', 'cmp-gc-ransac-e', 'cmp-degensac-f-laf', 'cmp-magsac-f', 'skimage-ransac-f', 'intel-dfe-f' ]: if 'threshold' not in geom: raise ValueError( '[{}] Must specify a threshold for this method'. format(dataset)) else: if 'threshold' in geom: raise ValueError( '[{}] Cannot specify a threshold for this method'. format(dataset)) # Degeneracy check for RANSAC if geom['method'].lower() in [ 'cmp-degensac-f', 'cmp-degensac-f-laf' ]: if 'degeneracy_check' not in geom: raise ValueError( '[{}] Must indicate degeneracy check for this method' .format(dataset)) if 'error_type' not in geom: raise ValueError( '[{}] Must indicate error type for this method'. format(dataset)) else: if 'degeneracy_check' in geom: raise ValueError( '[{}] Cannot apply degeneracy check to this method' .format(dataset)) if 'error_type' in geom: raise ValueError( '[{}] Cannot indicate error type for this method'. format(dataset)) # Confidence for RANSAC/LMEDS if geom['method'].lower() in [ 'cv2-ransac-f', 'cv2-patched-ransac-f', 'cv2-ransac-e', 'cv2-lmeds-f', 'cv2-lmeds-e', 'cmp-degensac-f', 'cmp-degensac-f-laf', 'cmp-gc-ransac-f', 'cmp-gc-ransac-e', 'skimage-ransac-f', 'cmp-magsac-f', ]: if 'confidence' not in geom: raise ValueError( '[{}] Must specify a confidence value for OpenCV or DEGENSAC' .format(dataset)) else: if 'confidence' in geom: raise ValueError( '[{}] Cannot specify a confidence value for this method' .format(dataset)) # Maximum number of RANSAC iterations if geom['method'].lower() in [ 'cv2-patched-ransac-f', 'cmp-degensac-f', 'cmp-degensac-f-laf', 'cmp-gc-ransac-f', 'cmp-gc-ransac-e', 'skimage-ransac-f', 'cmp-magsac-f', ]: if 'max_iter' not in geom: raise ValueError( '[{}] Must indicate max_iter for this method'. format(dataset)) else: if 'max_iter' in geom: raise ValueError( '[{}] Cannot indicate max_iter for this method'. format(dataset)) # DFE-specific if geom['method'].lower() in ['intel-dfe-f']: if 'postprocess' not in geom: raise ValueError( '[{}] Must specify a postprocess flag for DFE'. format(dataset)) else: if 'postprocess' in geom: raise ValueError( '[{}] Cannot specify a postprocess flag for this method' .format(dataset))
def main(path_result): swmm_obj = pickle_swmm(path_result) swmm_obj.sys_out() pickle_uzf(path_result).uzf_arrays() pickle_ext(path_result).ts_sums() return swmm_obj if __name__ == '__main__': start = time.time() arguments = docopt(__doc__) typecheck = Schema( { 'PATH': os.path.exists, '--fmt': Use(int), '--var': Or(Use(int), str) }, ignore_extra_keys=True) PATH_result = op.abspath(typecheck.validate(arguments)['PATH']) args = typecheck.validate(arguments) ### 1 CPU swmm_obj = main(PATH_result) scenarios, path_picks = swmm_obj.scenarios, swmm_obj.path_picks ts_hr = swmm_obj.ts_hr if args['--fmt']: print('Formatting Pickles') PickleFmt.main(PATH_result)
def main(): ch = logging.StreamHandler() ch.setLevel(logging.INFO) fh = logging.FileHandler('dht.log') fh.setLevel(logging.DEBUG) logging.basicConfig( handlers=[ch, fh], format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) logger = logging.getLogger(__name__) logger.info('dht') parser = argparse.ArgumentParser( description= 'Publishes humidity and temperature data from Adafruit DHT series sensors' ) parser.add_argument('-c', '--config', help='path to config file', default='config.ini') args = parser.parse_args() config = configparser.ConfigParser() config.read(args.config) if not config.read(args.config): logger.warn('failed to read config file, path={}'.format(args.config)) config_schema = Schema( { 'dht': { 'type': Use(int), 'pin': Use(int), Optional('read_interval', default=60): Use(int) }, Optional('mqtt'): { Optional('broker_host', default='localhost'): str, Optional('broker_port', default='1883'): Use(int) } }, ignore_extra_keys=True) config_schema.validate(config._sections) location = config.get('mqtt', 'location', fallback='unknown') humidity_topic = '/'.join((location, 'humidity')) temperature_topic = '/'.join((location, 'temperature')) # Create sensor sensor_type = config.getint('dht', 'type') pin = config.getint('dht', 'pin') sensor = DhtSensor(sensor_type, pin) def on_connect(client, userdata, flags, rc): logger.info('connected to mqtt broker') # Connect to MQTT broker client = mqtt.Client() client.on_connect = on_connect host = config.get('mqtt', 'broker_host', fallback='localhost') port = config.getint('mqtt', 'broker_port', fallback=1883) client.connect(host, port) client.loop_start() # Publish sensor readings read_interval = config.getint('dht', 'read_interval', fallback=60) logger.debug('read_interval={}'.format(read_interval)) scheduler = sched.scheduler(time.time, time.sleep) def read(): scheduler.enter(read_interval, 1, read) reading = sensor.read_retry() if not reading.is_valid(): return logger.debug( 'publishing humidity={} and temperature={} for location={}'.format( reading.humidity, reading.temperature, location)) client.publish(humidity_topic, reading.humidity) client.publish(temperature_topic, reading.temperature) read() scheduler.run()