def __init__(self,
input_file=None,
output_file=None,
dfxml_file=None,
report_file=None,
commit=False,
ignore_patterns=[],
key=None,
rules=[]):
# Validate configuration
from schema import Schema, Optional, Or, Use, And, SchemaError
schema = Schema({
'input_file':
Use(lambda f: open(f, 'r'), error='Cannot read the input file'),
Optional('output_file'):
Or(
None,
Use(lambda f: open(f, 'w'),
error='Cannot write to the output file')),
Optional('dfxml_file'):
Or(None, Use(lambda f: open(f, 'r'),
error='Cannot read DFXML file')),
Optional('report_file'):
Or(None,
lambda f: open(f, 'w'),
error='Cannot write to the report file'),
'commit':
Or(True, False),
'ignore_patterns':
Use(lambda f: re.compile(convert_fileglob_to_re('|'.join(f))),
error='Cannot compile unified ignore regex'),
'key':
Or(None, str),
'rules':
And([(redact_rule, redact_action)], lambda f: len(f) > 0)
})
try:
kwargs = {
'input_file': input_file,
'output_file': output_file,
'dfxml_file': dfxml_file,
'report_file': report_file,
'commit': commit,
'ignore_patterns': ignore_patterns,
'key': key,
'rules': rules
}
self.conf = schema.validate(kwargs)
except SchemaError as e:
logging.warning('The redact configuration did not validate:')
exit(e)
if self.conf['commit'] and 'output_file' not in self.conf.keys():
logging.error('An output file is required when COMMIT is on.')
exit(1)
# TODO Check input and output are not same file
logging.debug('Configuration:\n%s' % self.conf)
# Print rules
logging.debug(
json.dumps(map(
lambda x, y: (x.line, x.__class__.__name__, y.__class__.
__name__, x.lgpattern
if hasattr(x, 'lgpattern') else ''),
self.conf['rules']),
indent=4))
self.input_file = self.conf['input_file']
from os import path
self.image_size = path.getsize(self.input_file.name)
self.output_file = self.conf['output_file']
self.report_file = self.conf['report_file']
self.dfxml_file = self.conf['dfxml_file']
self.commit = self.conf['commit']
self.configure_report_logger()
from app.views.log import find
from app.models.log import Log
import unittest
import uuid
log = Log(
user_id=1,
product_id='product_id',
company_id='company_id',
phone_number=5511999999999,
value=10.00
)
log_validate = Schema({
'id': Use(str),
'user_id': Use(int),
'product_id': Use(str),
'company_id': Use(str),
'phone_number': Use(int),
'value': Use(float),
'created': Or(str, None)
})
logs_validate = Schema([{
'id': Use(str),
'user_id': Use(int),
'product_id': Use(str),
'company_id': Use(str),
'phone_number': Use(int),
'value': Use(float),
'destination': And(str, len),
'action': And(str, len),
Optional('username'): And(str, len),
Optional('password'): And(str, len)
},
'influxdb': {
'host': And(str, len),
'port': And(int, port_range),
Optional('username'): And(str, len),
Optional('password'): And(str, len),
'database': And(str, len),
Optional('ssl'): bool
},
Optional("base64decode"): {
'source': And(str, len, Use(str_or_jsonPath)),
'target': And(str, len)
},
'points': [{
'measurement': And(str, len, Use(str_or_jsonPath)),
'topic': And(str, len),
Optional('httpcontent'): {str: And(str, len, Use(str_or_jsonPath))},
Optional('fields'): Or({str: And(str, len, Use(str_or_jsonPath))}, And(str, len, Use(str_or_jsonPath))),
Optional('tags'): {str: And(str, len, Use(str_or_jsonPath))},
Optional('database'): And(str, len)
}]
})
def load_config(config_filename):
with open(config_filename, 'r') as f:
__all__ = [
'schema_general_settings', 'schema_derivative_couplings',
'schema_absorption_spectrum', 'schema_electron_transfer']
from numbers import Real
from schema import (And, Optional, Schema, Use)
schema_general_settings = Schema({
# "Library to distribute the computation"
Optional("runner", default="multiprocessing"):
And(str, Use(str.lower),
lambda s: s in ("multiprocessing", "mpi")),
# "default quantum package used"
Optional("package_name", default="cp2k"): str,
# project
Optional("project_name", default="namd"): str,
# "Basis set to carry out the quantum chemistry simulation"
"basis_name": str,
# Working directory
Optional("scratch_path", default="/tmp"): str,
# path to the HDF5 to store the results
"path_hdf5": str,
# path to xyz trajectory of the Molecular dynamics
def bitc_util_parser(argv=sys.argv[1:]):
__usage__ = """
Bayesian analysis of ITC data. Uses MicroCal .itc files, or custom format .yml files for analysing experiments.
Usage:
ITC.py <datafiles>... [-w <workdir> | --workdir=<workdir>] [-n <name> | --name=<name>] [-q <file> | --heats=<file>] [-i <ins> | --instrument=<ins> ] [-v | -vv | -vvv] [-r <file> | --report=<file>] [ -l <logfile> | --log=<logfile>]
ITC.py mcmc <datafiles>... (-m <model> | --model=<model>) [-w <workdir> | --workdir=<workdir>] [ -r <receptor> | --receptor=<receptor>] [-n <name> | --name=<name>] [-q <file> | --heats=<file>] [-i <ins> | --instrument=<ins> ] [ -l <logfile> | --log=<logfile>] [-v | -vv | -vvv] [--report=<file>] [options]
ITC.py (-h | --help)
ITC.py --license
ITC.py --version
Options:
-h, --help Show this screen
--version Show version
--license Show license
-l <logfile>, --log=<logfile> File to write logs to. Will be placed in workdir.
-v, Verbose output level. Multiple flags increase verbosity.
<datafiles> Datafile(s) to perform the analysis on, .itc, .yml
-w <workdir>, --workdir=<workdir> Directory for output files [default: ./]
-r <receptor> | --receptor=<receptor> The name of the receptor for a Competitive Binding model.
-n <name>, --name=<name> Name for the experiment. Will be used for output files. Defaults to inputfile name.
-i <ins>, --instrument=<ins> The name of the instrument used for the experiment. Overrides .itc file instrument.
-q <file>, --heats=<file> Origin format integrated heats file. (From NITPIC use .dat file)
-m <model>, --model=<model> Model to use for mcmc sampling [default: TwoComponent]
--nfit=<n> No. of iteration for maximum a posteriori fit [default: 20000]
--niters=<n> No. of iterations for mcmc sampling [default: 6000]
--nburn=<n> No. of Burn-in iterations for mcmc sampling [default: 1000]
--nthin=<n> Thinning period for mcmc sampling [default: 5]
--report=<file> Output file with summary in markdown
"""
arguments = docopt(__usage__, argv=argv, version='ITC.py, pre-alpha')
schema = Schema({
'--heats':
Or(None, And(str, os.path.isfile,
Use(os.path.abspath))), # str, verify that it exists
'--help':
bool, # True or False are accepted
'--license':
bool, # True or False are accepted
# integer between 0 and 3
'-v':
And(int, lambda n: 0 <= n <= 3),
# str and found in this dict
'--model':
And(str, lambda m: m in known_models),
'--nfit':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--nburn':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--niters':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--nthin':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--name':
Or(None, And(str, len)), # Not an empty string
'--instrument':
Or(None, And(str, lambda m: m in known_instruments)),
# None, or str and found in this dict
'--version':
bool, # True or False are accepted
'--receptor':
Or(None, str), # str or None
'--workdir':
str, # str
# list and ensure it contains existing files
'<datafiles>':
And(
list,
lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles],
Use(lambda inpfiles:
[os.path.abspath(inpfile) for inpfile in inpfiles])),
'mcmc':
bool, # True or False are accepted
'--report':
Or(None, Use(lambda f: open(f, 'w'))),
# Don't use, or open file with writing permissions
'--log':
Or(None, str), # Don't use, or str
})
return schema.validate(arguments)
from schema import Schema, And, Use
group_schema = Schema({
"group": And(str)
})
member_schema = Schema({
"number": And(str, Use(str), lambda s: len(s) == 12)
})
members_schema = Schema({
"numbers": [And(str, Use(str), lambda s: len(s) == 12)]
})
membership_schema = Schema({
"ids": [And(int)]
})
message_schema = Schema({
"groups": [And(Use(int), lambda n: n > 0)],
"message": And(str)
})
from models import *
import json
from base64 import b64encode
from schema import Schema, And, Use, Optional
import lxml.etree as xml
import re
import sys
vul_schema = Schema(
{
'cwe': And(Use(int)),
'name': basestring,
'tool': basestring,
'severity': And(Use(int), lambda n: 0 <= n <= 3),
'description': basestring,
'target_name': basestring,
Optional('observation'): basestring,
Optional('remediation'): basestring,
},
ignore_extra_keys=False)
def parse_zap_json_file(zap_file, target, session, uri):
with open(zap_file, 'r') as zapfile:
zap_data = json.loads(zapfile.read())
alerts = None
pre_alerts = zap_data['Report']['Sites']
if isinstance(pre_alerts, list):
for pre in pre_alerts:
if uri in pre['Host']:
alerts = pre
def get_linkset_spec_schema(ets_ids):
matching_methods_info = get_matching_methods()
transformers_info = get_transformers()
return Schema({
'id': Use(int),
'label': And(str, len),
Optional('description', default=None): Or(str, None),
Optional('use_counter', default=True): bool,
'sources': [EntityTypeSelection(ets_ids)],
'targets': [EntityTypeSelection(ets_ids)],
'methods': And(LogicBox(Schema({
'method': {
'name': And(str, Use(str.lower), lambda m: m in matching_methods_info.keys()),
'config': And(dict, MatchingMethodConfig(ets_ids)),
},
Optional('sim_method', default={'name': None, 'config': {}, 'normalized': False}): {
Optional('name', default=None):
Or(None, And(str, Use(str.lower), lambda m: m in matching_methods_info.keys())),
Optional('config', default={}): And(dict, MatchingMethodConfig(ets_ids)),
Optional('normalized', default=False): bool,
},
Optional('fuzzy', default={'t_norm': 'minimum_t_norm', 's_norm': 'maximum_s_norm', 'threshold': 0}): {
Optional('t_norm', default='minimum_t_norm'):
lambda s: s in ('minimum_t_norm', 'product_t_norm', 'lukasiewicz_t_norm',
'drastic_t_norm', 'nilpotent_minimum', 'hamacher_product'),
Optional('s_norm', default='maximum_s_norm'):
lambda s: s in ('maximum_s_norm', 'probabilistic_sum', 'bounded_sum',
'drastic_s_norm', 'nilpotent_maximum', 'einstein_sum'),
Optional('threshold', default=0): Or(float, Use(lambda t: 0)),
},
Optional('list_matching', default={'threshold': 0, 'is_percentage': False}): {
Optional('threshold', default=0): int,
Optional('is_percentage', default=False): bool,
},
'sources': {
'properties': {
EntityTypeSelection(ets_ids): [{
'property': And(Use(filter_property), len),
Optional('property_transformer_first', default=False): bool,
Optional('transformers', default=list): [{
'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()),
'parameters': dict
}],
}]
},
Optional('transformers', default=list): [{
'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()),
'parameters': dict
}],
},
'targets': {
'properties': {
EntityTypeSelection(ets_ids): [{
'property': And(Use(filter_property), len),
Optional('property_transformer_first', default=False): bool,
Optional('transformers', default=list): [{
'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()),
'parameters': dict
}],
}]
},
Optional('transformers', default=list): [{
'name': And(str, Use(str.lower), lambda n: n in transformers_info.keys()),
'parameters': dict
}],
}
}, ignore_extra_keys=True), name='conditions', types=(
'and', 'or', 'minimum_t_norm', 'product_t_norm', 'lukasiewicz_t_norm', 'drastic_t_norm',
'nilpotent_minimum', 'hamacher_product', 'maximum_s_norm', 'probabilistic_sum',
'bounded_sum', 'drastic_s_norm', 'nilpotent_maximum', 'einstein_sum'
), elements_schema=Schema({
'type': str,
'conditions': list,
Optional('threshold', default=0): Or(float, Use(lambda t: 0)),
}, ignore_extra_keys=True)), dict),
}, ignore_extra_keys=True)
class CrawlCmd(HoaxyCommand):
"""
Usage:
hoaxy crawl --fetch-url (--update | --archive)
[--where-expr=<w> --order-by=<o> --limit=<l>]
hoaxy crawl --fetch-html [--where-expr=<w> --order-by=<o> --limit=<l>]
hoaxy crawl --parse-article [--where-expr=<w> --order-by=<o> --limit=<l>]
This command provides crawling functions, including fetching URLs from sites,
fetching HTMLs of URLs, and parsing HTML pages into structured article data.
When crawling, you can specify which entity to fetch:
--fetch-url Fetch URL, either news update or archive.
--fetch-html Fetch HTML of URL.
--parse-article Parse HTML page into structured article data.
At this moment, we use webparse provided by
https://mercury.postlight.com/web-parser/
We suggest you add `--limit` option when doing cronjobs to avoid overload.
When fetching URL, you can specify which kind of URLs to fetch by option:
--updated Fetch news update (the newest articles).
--archive Fetch news archive (the whole site articles). Be
careful to use this option, it is time consuming.
The inputs for crawling processes (domains or URLs) are queried from DB. By
default you do not need to consider the internal SQL query. However,
if you want to specify your own data collection for crawling in some cases,
you can use these options:
--where-expr=<w> The WHERE expression for SQL query.
--order-by=<o> The sorted order of `id` column, by default ascending
[default: asc]
--limit=<l> The limit expression, by default return all
For --fetch-url, the default raw SQL query is on table `site`:
site.is_enabled is True and site.is_alive is True
For --fetch-html the default raw SQL query is on table `url`
url.status_code=U_DEFAULT
For --parse-article the default raw SQL query is on table `url`
url.status_code=U_HTML_SUCCESS AND url.site_id IS NOT NULL
Please check database.models to see detail.
Other options are:
-h --help Show help.
More information:
(1) The spiders are built base on scrapy package (https://scrapy.org/)
(2) To distinguish URL records from these three phrase, we use `status_code`
column, and will update the column for each action. Please check the Url
model to see the table of status_code
(3) Parse a HTML page into article is a difficult task. We don't implement it
ourselves. Instead we use the service from https://mercury.postlight.com/.
However, you can also use other parser, e.g., python-goose,
https://github.com/grangier/python-goose, or even implement your own.
Examples (`||` represents continue of commands, you can ignore when using):
1. Most often you would like to fetch the updates of all enabled sites,
probably used as cronjob:
hoaxy crawl --fetch-url --update
2. If you would like to fetch the URLs of old posts (archive):
hoaxy crawl --fetch-url --archive
Note: this crawling is quite time-consuming.
3. Also you may need structured article instead of HTML documents:
hoaxy crawl --parse-article
4. When automating by crontab, you may need --limit option to limit
the number of entries to process. This is very useful when you
already have a huge number of URLs in the database, and you would
like to consume them each time a block.
hoaxy crawl --fetch-html --limit=20000
5. Occasionally, you can specify the where expression to crawl only on
these entries:
hoaxy crawl --parse-article --where-expr='url.html is null'
"""
name = 'crawl'
short_description = 'Crawl news sites'
args_schema = Schema({
'--order-by':
Or(None,
Use(str.lower),
lambda s: s in ('asc', 'desc'),
error='must be asc or desc'),
'--limit':
Or(None, Use(int)),
object:
object
})
@classmethod
def fetch_url(cls, session, msites, platform_id, purpose):
"""Actual method to do fetch url action.
Parameters
----------
msites : list
a list of Site model class, contains info to build spiders.
platform_id : int
id of platform, bind fetched url with this id.
purpose : {'update', 'archive'}
indicate which url to fetch.
"""
settings = Settings(cls.conf['crawl']['scrapy'])
settings.set('ITEM_PIPELINES',
{'hoaxy.crawl.pipelines.UrlPipeline': 300})
process = CrawlerProcess(settings)
sll = cls.conf['logging']['loggers']['scrapy']['level']
logging.getLogger('scrapy').setLevel(logging.getLevelName(sll))
for ms in msites:
for sm in build_spiders_iter(ms, purpose):
sm['kwargs']['session'] = session
sm['kwargs']['platform_id'] = platform_id
process.crawl(sm['cls'], *sm['args'], **sm['kwargs'])
process.start()
@classmethod
def fetch_html(cls, session, url_tuples):
"""Actual method to do fetch html action.
Parameters
----------
session : object
a SQLAlchemy session object.
url_tuples : list
a list of url tuple (id, raw, status_code).
"""
settings = Settings(cls.conf['crawl']['scrapy'])
settings.set('ITEM_PIPELINES',
{'hoaxy.crawl.pipelines.HtmlPipeline': 300})
process = CrawlerProcess(settings)
sll = cls.conf['logging']['loggers']['scrapy']['level']
logging.getLogger('scrapy').setLevel(logging.getLevelName(sll))
logger.warning('Number of url to fetch html is: %s', len(url_tuples))
process.crawl(
HtmlSpider,
session=session,
url_tuples=url_tuples,
excluded_domains=cls.conf['crawl']['excluded_domains'])
process.start()
@classmethod
def parse_article(cls, session, url_tuples):
"""Actual method to do parse to article action.
Parameters
----------
session : object
a SQLAlchemy session object.
url_tuples : list
a list of url tuple (id, created_at, date_published,
canonical, site_id)
"""
settings = Settings(cls.conf['crawl']['scrapy'])
settings.set('ITEM_PIPELINES',
{'hoaxy.crawl.pipelines.ArticlePipeline': 300})
process = CrawlerProcess(settings)
sll = cls.conf['logging']['loggers']['scrapy']['level']
logging.getLogger('scrapy').setLevel(logging.getLevelName(sll))
logger.info('Number of url to parse is: %s', len(url_tuples))
process.crawl(
ArticleParserSpider,
session=session,
url_tuples=url_tuples,
api_key=cls.conf['crawl']['article_parser']['webparser_api_key'],
)
process.start()
@classmethod
def run(cls, args):
"""Overriding method as the entry point of this command."""
try:
args = cls.args_schema.validate(args)
except SchemaError as e:
raise SystemExit(e)
session = Session(expire_on_commit=False)
# session = Session()
where_expr = args['--where-expr']
ob_expr = args.get('--order-by', 'asc')
limit = args['--limit']
# --fetch-url
if args['--fetch-url'] is True:
configure_logging(
'crawl.fetch-url',
console_level=args['--console-log-level'],
file_level='WARNING')
purpose = 'update' if args['--update'] is True else 'archive'
if where_expr is None:
where_expr = [text(DEFAULT_WHERE_EXPR_FETCH_URL)]
else:
where_expr = [text(where_expr)]
ob_expr = Site.id.asc() if ob_expr == 'asc' else Site.id.desc()
msites = get_msites(
session, f_expr=where_expr, ob_expr=ob_expr, limit=limit)
if len(msites) == 0:
logger.warning("None sites you queried found in DB!")
raise SystemExit(2)
platform_id = get_platform_id(session, name=N_PLATFORM_WEB)
# detach msites and mplatform from session,
# since they definitely would not be modified in session
for ms in msites:
session.expunge(ms)
logger.warning('Starting crawling process to fetch URL update ...')
cls.fetch_url(session, msites, platform_id, purpose)
elif args['--fetch-html'] is True:
configure_logging(
'crawl.fetch-html',
console_level=args['--console-log-level'],
file_level='WARNING')
if not session.query(Site.id).count() > 0:
raise SystemExit('Your site table is empty!')
q = session.query(Url.id, Url.raw)
if where_expr is None:
where_expr = [text(DEFAULT_WHERE_EXPR_FETCH_HTML)]
else:
where_expr = [text(where_expr)]
ob_expr = Url.id.asc() if ob_expr == 'asc' else Url.id.desc()
q = q.filter(*where_expr).order_by(ob_expr)
if limit is not None:
q = q.limit(limit)
logger.info(
q.statement.compile(compile_kwargs={"literal_binds": True}))
url_tuples = q.all()
if not url_tuples:
logger.warning('No such URLs in DB!')
raise SystemExit(2)
logger.warning('Staring crawling process to fetch HTML ...')
cls.fetch_html(session, url_tuples)
# --parse-article
elif args['--parse-article'] is True:
configure_logging(
'crawl.parse-article',
console_level=args['--console-log-level'],
file_level='WARNING')
q = session.query(Url.id, Url.created_at, Url.date_published,
Url.canonical, Url.site_id)
if where_expr is None:
where_expr = [text(DEFAULT_WHERE_EXPR_PARSE_ARTICLE)]
else:
where_expr = [text(where_expr)]
ob_expr = Url.id.asc() if ob_expr == 'asc' else Url.id.desc()
q = q.filter(*where_expr).order_by(ob_expr)
if limit is not None:
q = q.limit(limit)
logger.info(
q.statement.compile(compile_kwargs={"literal_binds": True}))
url_tuples = q.all()
if not url_tuples:
logger.warning('No URLs found from DB!')
raise SystemExit(2)
logger.warning('Starting crawling process to parse article ...')
cls.parse_article(session, url_tuples)
session.close()
class ServerConfiguration(Configuration):
VALIDATORS = {
"description": Use(str),
"ip": Use(str),
"port": Use(int),
"api_path": Use(str),
"uri": Use(str),
"allow_drop_all": Use(bool),
"logging": {
"level":
And(
Use(str), lambda l: l in
("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")),
"file":
Use(str),
"use_console":
Use(bool),
"backup_count":
And(Use(int), lambda n: n > 0),
"max_size":
And(Use(int), lambda b: b > 16),
"format":
Use(str),
"datefmt":
Use(str)
}
}
from schema import Schema, And, Or, Use, Optional, Literal
from .tools import HasKey, Nullable, Url, List
SiteConfigSchemaObject = {
"server_name": Use(str),
Optional("web_upstream", default="localhost:8069"): str,
Optional("poll_upstream", default="localhost:8072"): str,
Optional("proxy_pass"): Use(str), # Or default=None ?
Optional("disable_longpolling", default=False): bool,
Optional("httpaccess"): bool,
Optional("httpsaccess"): bool,
Optional("posbox", default=False): bool,
Optional("proxy_http_11"): bool,
Optional("cache_statics"): bool,
Optional("disable_cache_zone"): bool,
Optional("header_upgrade"): bool,
Optional("header_connection"): bool,
Optional("header_host"): bool,
Optional("masquerade"): str,
Optional("redirect"): str,
Optional("allow_tls_v1"): Nullable(bool),
Optional("enable_hsts"): bool,
Optional("certificate_folder"): str,
Optional("ssl_certificate"): str,
Optional("ssl_certificate_key"): str,
Optional("ssl_trusted_certificate"): bool,
Optional("disable_stapling"): bool,
Optional("ip_allow", default=[]): List(str),
Optional("ip_deny", default=[]): List(str),
}
class NodeConfiguration(Configuration):
VALIDATORS = {
"api_key": And(Use(str), len),
"server_url": Use(str),
"port": Or(Use(int), None),
"task_dir": Use(str),
"databases": {
Use(str): Use(str)
},
"api_path": Use(str),
"logging": {
"level":
And(
Use(str), lambda l: l in
("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")),
"file":
Use(str),
"use_console":
Use(bool),
"backup_count":
And(Use(int), lambda n: n > 0),
"max_size":
And(Use(int), lambda b: b > 16),
"format":
Use(str),
"datefmt":
Use(str)
},
"encryption": {
"enabled": bool,
Optional("private_key"): Use(str)
}
}
"channel": "#seamless-thursday",
"username": "******",
"text": text,
"icon_emoji": ":fatbot:",
'link_names': 1
}
def post_message(message):
if message:
return json.dumps(payload(message))
return message
@app.route('/', methods=['POST'])
def main():
post = [
s.strip() for s in request.form['text'].lower().strip().split(':', 2)
]
user = request.form['user_name']
return post_message(orderbot(user, post))
if __name__ == '__main__':
orderbot = OrderBot()
args = Schema({
'--host': Use(str),
'--port': Use(int),
'--debug': Use(bool)
}).validate(docopt(__doc__))
app.run(host=args['--host'], port=args['--port'], debug=args['--debug'])
--use_regular <v> use regularizers [default: 0].
--regular_c <v> use regular_c [default: 0.0].
-h --help Show this screen and exit.
""".format(f=__file__)
'''
TODO:
pretrained word embeddings
visualise word embeddings
visualise filters
'''
from docopt import docopt
from schema import Schema, And, Or, Use
s = Schema({
'--n_epoch': Use(int),
'--alpha': Use(float),
'--width1': Use(int),
'--width2': Use(int),
'--feat_map_n_1': Use(int),
'--feat_map_n_final': Use(int),
'--feat_map_n_final': Use(int),
'--k_top': Use(int),
'--dropout_rate0': Use(float),
'--dropout_rate1': Use(float),
'--dropout_rate2': Use(float),
'--activation': Use(str),
'--learn': Use(str),
'--skip': Use(int),
'--use_regular': Use(int),
'--regular_c': Use(float),
},
'PRODUCTION': {
'MODULES': [
'loaders/loader.py', 'loaders/lister.py', 'loaders/extractor.py',
'transformers/filename_transformer.py',
'transformers/meta_functions.py',
'transformers/data_transformer.py', 'transformers/wrapper.py',
'filters/selection_functions.py'
],
'LOADERS': ['loaders/json_loader.py', 'loaders/yaml_loader.py']
}
}
_SCHEMA_CONF = Schema({
Optional('TEMPLATE'): {
Optional('BACKEND'): And(str, Use(_importModule)),
Optional('DIR'): And(list, _isListOfStr)
},
Optional('PRODUCTION'): {
Optional('MODULES'): And(list, _isListOfStr, Use(_importModules)),
Optional('LOADERS'): And(list, _isListOfStr, Use(_importModules))
}
})
class ConfigHandler:
def __init__(self, configFile, configPathProvided=False):
if configPathProvided:
self.config = loadYAML(configFile)
else:
try:
def test_dict_keys(self):
assert Schema({str: int}).validate(
{'a': 1, 'b': 2}) == {'a': 1, 'b': 2}
self.assertRaises(SchemaError, Schema({str: int}).validate, {1: 1, 'b': 2})
assert Schema({Use(str): Use(int)}).validate(
{1: 3.14, 3.14: 1}) == {'1': 3, '3.14': 1}
def make_answer_json(answer_code, body):
ans_json = {'code': answer_code,
'body': body} # form answer as json
return ans_json
def check_schema(inp_conf_schema, dict_object):
try:
inp_conf_schema.validate(dict_object)
return True
except SchemaError:
return False
save_token_request_schema = Schema({
'request': And(Use(str)),
'body': {
'auth': And(Use(str)),
'push': And(Use(str)),
'univer': And(Use(str)),
'fak': And(Use(str)),
'state': And(Use(str))
}
})
def check_token_save_request_struct(dict_obj):
check_res = check_schema(inp_conf_schema=save_token_request_schema,
dict_object=dict_obj)
return check_res
def test_schema_repr(self): # what about repr with `error`s?
schema = Schema([Or(None, And(str, Use(float)))])
repr_ = "Schema([Or(None, And(<type 'str'>, Use(<type 'float'>)))])"
# in Python 3 repr contains <class 'str'>, not <type 'str'>
assert repr(schema).replace('class', 'type') == repr_
# consider valid if all 7 mandatory fields are present
if sum(is_refkey.values()) == 7:
valid_passport += 1
#==============================
## Part 2
# Create validation schema
from schema import Schema, And, Use, Regex, Optional, SchemaError
hcl_expr = "^#{1}[0-9a-f]{6}$" # https://regex101.com/r/g51QAR/1
hgt_expr = "^(1[5-8][0-9]|19[0-3])cm|(59|6[0-9]|7[0-6])in$"
pid_expr = "^[0-9]{9}$"
ecl_list = ('amb','blu','brn','gry','grn','hzl','oth')
schema = Schema({'byr': And(Use(int), lambda n: 1920 <= n <= 2002),
'iyr': And(Use(int), lambda n: 2010 <= n <= 2020),
'eyr': And(Use(int), lambda n: 2020 <= n <= 2030),
'hgt': And(str, Regex(hgt_expr)),
'hcl': And(str, Regex(hcl_expr)),
'ecl': And(str, lambda s: s in ecl_list),
'pid': And(str, Regex(pid_expr)),
Optional('cid'): And(str, len),
Optional('name'): And(str, len)
})
# Try on test dict
try:
validated = schema.validate(testd)
print("data validated")
except SchemaError as error:
def test_and(self):
assert And(int, lambda n: 0 < n < 5).validate(3) == 3
self.assertRaises(SchemaError, And(int, lambda n: 0 < n < 5).validate, 3.33)
assert And(Use(int), lambda n: 0 < n < 5).validate(3.33) == 3
self.assertRaises(SchemaError, And(Use(int), lambda n: 0 < n < 5).validate, '3.33')
def main(argv, session=None):
args = docopt(__doc__, argv=argv)
# Validate args.
s = Schema({
six.text_type:
Use(bool),
'<query>':
Use(lambda x: ' '.join(x)),
'--parameters':
Use(lambda x: get_args_dict(x, query_string=True)),
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--sort':
list,
'--field':
list,
'--timeout':
Use(lambda x: float(x[0]), error='--timeout must be integer or float.')
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Support comma separated values.
fields = list(chain.from_iterable([x.split(',') for x in args['--field']]))
sorts = list(chain.from_iterable([x.split(',') for x in args['--sort']]))
r_kwargs = dict(
headers=args['--header'],
timeout=args['--timeout'],
)
search = session.search_items(args['<query>'],
fields=fields,
sorts=sorts,
params=args['--parameters'],
full_text_search=args['--fts'],
request_kwargs=r_kwargs)
try:
if args['--num-found']:
print('{0}'.format(search.num_found))
sys.exit(0)
for result in search:
if args['--itemlist']:
print(result.get('identifier', ''))
else:
j = json.dumps(result)
print(j)
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
except ConnectTimeout as exc:
print(
'error: Request timed out. Increase the --timeout and try again.',
file=sys.stderr)
sys.exit(1)
except AuthenticationError as exc:
print('error: {}'.format(exc), file=sys.stderr)
sys.exit(1)
def get_options(cls):
return {
ConfigOption('report_tags'): [Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all'): [Use(tagging.validate_tag_value)]
}
def integrate_parser(argv=sys.argv[1:]):
__usage__ = """
Integrate ITC data using Gaussian process regression. Uses MicroCal .itc files, or custom format .yml files for analysing experiments.
Usage:
bitc_integrate.py <datafiles>... [-w <workdir> | --workdir=<workdir>] [-v | -vv | -vvv] [options]
bitc_integrate.py (-h | --help)
bitc_integrate.py --license
bitc_integrate.py --version
Options:
-h, --help Show this screen
--version Show version
--license Show license
-l <logfile>, --log=<logfile> File to write logs to. Will be placed in workdir.
-v, Verbose output level. Multiple flags increase verbosity.
<datafiles> Datafile(s) to perform the analysis on, .itc, .yml
-w <workdir>, --workdir=<workdir> Directory for output files [default: ./]
-n <name>, --name=<name> Name for the experiment. Will be used for output files. Defaults to input file name.
-i <ins>, --instrument=<ins> The name of the instrument used for the experiment. Overrides .itc file instrument.
-f <frac>, --fraction=<frac> The fraction of the injection to fit, measured from the end [default: 0.2]
--theta0=<theta0> The parameters in the autocorrelation model. [default: 5.0]
--nugget=<nugget> Size of nugget effect to allow smooth predictions from noisy data. [default: 1.0]
--plot Generate plots of the baseline fit
"""
arguments = docopt(__usage__,
argv=argv,
version='bitc_integrate.py, pre-alpha')
schema = Schema({
'--help':
bool, # True or False are accepted
'--license':
bool, # True or False are accepted
# integer between 0 and 3
'-v':
And(int, lambda n: 0 <= n <= 3),
# Float greater than 0
'--fraction':
And(Use(float), lambda n: 0 < n <= 1.0),
'--nugget':
And(Use(float), lambda n: n > 0),
'--theta0':
And(Use(float), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--name':
Or(None, And(str, len)), # Not an empty string
'--instrument':
Or(None, And(str, lambda m: m in known_instruments)),
# None, or str and found in this dict
'--version':
bool, # True or False are accepted
'--plot':
bool, # True or False are accepted
'--workdir':
str, # str
# list and ensure it contains existing files
'<datafiles>':
And(
list,
lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles],
Use(lambda inpfiles:
[os.path.abspath(inpfile) for inpfile in inpfiles])),
# Don't use, or open file with writing permissions
'--log':
Or(None, str), # Don't use, or str
})
return schema.validate(arguments)
value = sum((cpf[num] * ((i+1) - num) for num in range(0, i)))
digit = ((value * 10) % 11) % 10
if digit != cpf[i]:
raise SchemaError(f"value ('{number}') is not a valid CPF number!")
return True
def TO_DATE(format):
def fnc(date_text):
try:
return dt.strptime(date_text, format)
except ValueError:
raise SchemaError(f"date format is invalid! value:('{date_text}')")
return fnc
conf_schema = Schema({
'nome': And(Use(str),LEN_H_THEN(0)),
'cpf': And(Use(str),VALIDCPF),
'dataNascimento':And(Use(str),Use(TO_DATE("%d/%m/%Y"))),
'conta':{
'limiteSaqueDiario': Or(int, float),
'flagAtivo':bool,
'tipoConta':int
}
})
conf = {
'nome': "a",
'cpf': "08639614440",
'dataNascimento':"01/01/1991",
'conta':{
'limiteSaqueDiario':0,
def bitc_mcmc_parser(argv=sys.argv[1:]):
__usage__ = """Analyze ITC data using Markov chain Monte Carlo (MCMC). Uses MicroCal .itc files, or custom format .yml files for modeling experiments.
When running the program you can select one of two options:
competitive
A competitive binding model. Requires multiple experiments to be specified.
twocomponent
A twocomponent binding model. Analyzes only a single experiment
enantiomer
A enantiomer binding model
Usage:
bitc_mcmc.py twocomponent <datafile> <heatsfile> [-v | -vv | -vvv] [--cc=<c_cell>] [--cs=<c_syringe> ] [--dc=<dc_cell>] [--ds=<dc_syringe>] [options]
bitc_mcmc.py competitive (<datafile> <heatsfile>)... (-r <receptor> | --receptor <receptor>) [-v | -vv | -vvv] [options]
bitc_mcmc.py racemicmixture <datafile> <heatsfile> [-v | -vv | -vvv] [--cc=<c_cell>] [--cs=<c_syringe> ] [--dc=<dc_cell>] [--ds=<dc_syringe>] [options]
bitc_mcmc.py enantiomer <datafile> <heatsfile> [-v | -vv | -vvv] [--cc=<c_cell>] [--cs=<c_syringe> ] [--dc=<dc_cell>] [--ds=<dc_syringe>] [options]
bitc_mcmc.py (-h | --help)
bitc_mcmc.py --license
bitc_mcmc.py --version
Options:
-h, --help Show this screen
--version Show version
--license Show license
-l <logfile>, --log=<logfile> File to write logs to. Will be placed in workdir.
--cc <c_cell> Concentration of component in cell in mM. Defaults to value in input file
--cs <c_syringe> Concentration of component in syringe in mM. Defaults to value in input file
--dc <dc_cell> Relative uncertainty in cell concentration [default: 0.1]
--ds <dc_syringe> Relative uncertainty in syringe concentration [default: 0.1]
-v, Verbose output level. Multiple flags increase verbosity.
-w <workdir>, --workdir <workdir> Directory for output files [default: ./]
-r <receptor> | --receptor <receptor> The name of the receptor for a competitive binding model.
-n <name>, --name <name> Name for the experiment. Will be used for output files. Defaults to inputfile name.
-i <ins>, --instrument <ins> The name of the instrument used for the experiment. Overrides .itc file instrument.
--nfit=<n> No. of iteration for maximum a posteriori fit [default: 20000]
--niters=<n> No. of iterations for mcmc sampling [default: 2000000]
--nburn=<n> No. of Burn-in iterations for mcmc sampling [default: 500000]
--nthin=<n> Thinning period for mcmc sampling [default: 500]
--dummy_itc_file The input itc file is a dummy one
--uniform_cell_concentration Use uniform prior for cell concentration if True, use log normal if False
--uniform_syringe_concentration Use uniform prior for syringe concentration if True, use log normal if False
--concentration_range_factor <c_facctor> To define range of uniform prior. It will be between stated_value/this_factor and stated_value*this_factor [default: 10.]
"""
arguments = docopt(__usage__, argv=argv, version='bitc_mcmc.py, pre-alpha')
schema = Schema({
'--help':
bool, # True or False are accepted
'--license':
bool, # True or False are accepted
# integer between 0 and 3
'-v':
And(int, lambda n: 0 <= n <= 3),
# str and found in this dict
'twocomponent':
bool,
'enantiomer':
bool,
'racemicmixture':
bool,
'competitive':
bool,
'--nfit':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--nburn':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--niters':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--nthin':
And(Use(int), lambda n: n > 0),
# Convert str to int, make sure that it is larger than 0
'--name':
Or(None, And(str, len)), # Not an empty string
'--instrument':
Or(None, And(str, lambda m: m in known_instruments)),
# None, or str and found in this dict
'--version':
bool, # True or False are accepted
'--receptor':
Or(None, str), # str or None
'--workdir':
str, # str
# str and ensure file exists
# list and ensure it contains existing files
'<datafile>':
And(
list,
lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles],
Use(lambda inpfiles:
[os.path.abspath(inpfile) for inpfile in inpfiles])),
# list and ensure it contains existing files
'<heatsfile>':
And(
list,
lambda inpfiles: [os.path.isfile(inpfile) for inpfile in inpfiles],
Use(lambda inpfiles:
[os.path.abspath(inpfile) for inpfile in inpfiles])),
# Don't use, or open file with writing permissions
'--log':
Or(None, str), # Don't use, or str
'--cc':
Or(None,
And(Use(float),
lambda n: n > 0.0)), # Not specified, or a float greater than 0
'--cs':
Or(None, And(Use(float),
lambda n: n > 0.0)), # Not specified, or a float
'--dc':
And(Use(float), lambda n: n > 0.0), # a float greater than 0
'--ds':
And(Use(float), lambda n: n > 0.0), # a float greater than 0
'--dummy_itc_file':
bool,
'--uniform_cell_concentration':
bool,
'--uniform_syringe_concentration':
bool,
'--concentration_range_factor':
Use(float),
})
return schema.validate(arguments)
def get_options(cls):
return {
'name':
str,
ConfigOption('logger_level', default=logger.TEST_INFO):
int,
ConfigOption('file_log_level', default=logger.DEBUG):
Or(int, None),
ConfigOption('runpath',
default=default_runpath,
block_propagation=False):
Or(None, str, lambda x: callable(x)),
ConfigOption('path_cleanup', default=True):
bool,
ConfigOption('all_tasks_local', default=False):
bool,
ConfigOption('shuffle', default=[]):
list, # list of string choices
ConfigOption('shuffle_seed',
default=float(random.randint(1, 9999))):
float,
ConfigOption('exporters', default=None):
Use(get_exporters),
ConfigOption('stdout_style',
default=defaults.STDOUT_STYLE,
block_propagation=False):
Style,
ConfigOption('report_dir',
default=defaults.REPORT_DIR,
block_propagation=False):
str,
ConfigOption('xml_dir', default=None, block_propagation=False):
Or(str, None),
ConfigOption('pdf_path', default=None, block_propagation=False):
Or(str, None),
ConfigOption('json_path', default=None, block_propagation=False):
Or(str, None),
ConfigOption('pdf_style',
default=defaults.PDF_STYLE,
block_propagation=False):
Style,
ConfigOption('report_tags', default=[], block_propagation=False):
[Use(tagging.validate_tag_value)],
ConfigOption('report_tags_all',
default=[],
block_propagation=False):
[Use(tagging.validate_tag_value)],
ConfigOption('merge_scheduled_parts', default=False):
bool,
ConfigOption('browse', default=None):
Or(None, bool),
ConfigOption('ui_port', default=None):
Or(None, int),
ConfigOption('web_server_startup_timeout',
default=defaults.WEB_SERVER_TIMEOUT):
int,
ConfigOption('test_filter',
default=filtering.Filter(),
block_propagation=False):
filtering.BaseFilter,
ConfigOption('test_sorter',
default=ordering.NoopSorter(),
block_propagation=False):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption('test_lister', default=None, block_propagation=False):
Or(None, listing.BaseLister),
ConfigOption('verbose', default=False, block_propagation=False):
bool,
ConfigOption('debug', default=False, block_propagation=False):
bool,
ConfigOption('timeout', default=None):
Or(None, And(Or(int, float), lambda t: t >= 0)),
ConfigOption('interactive_handler', default=TestRunnerIHandler):
object,
ConfigOption('extra_deps', default=[]):
list
}
def main(argv, session):
args = docopt(__doc__, argv=argv)
# Validation error messages.
destdir_msg = '--destdir must be a valid path to a directory.'
itemlist_msg = '--itemlist must be a valid path to an existing file.'
# Validate args.
s = Schema({
str:
Use(bool),
'--destdir':
Or([], And(Use(lambda d: d[0]), dir_exists), error=destdir_msg),
'--format':
list,
'--glob':
Use(lambda l: l[0] if l else None),
'<file>':
list,
'--search':
Or(str, None),
'--itemlist':
Or(None, And(lambda f: os.path.isfile(f)), error=itemlist_msg),
'<identifier>':
Or(str, None),
'--retries':
Use(lambda x: x[0]),
'--search-parameters':
Use(lambda x: get_args_dict(x, query_string=True)),
'--on-the-fly':
Use(bool),
'--no-change-timestamp':
Use(bool),
'--download-history':
Use(bool),
'--parameters':
Use(lambda x: get_args_dict(x, query_string=True)),
})
# Filenames should be unicode literals. Support PY2 and PY3.
if six.PY2:
args['<file>'] = [f.decode('utf-8') for f in args['<file>']]
try:
args = s.validate(args)
if args['--glob']:
if args['--format']:
raise (SchemaError(
None, '--glob and --format cannot be used together.'))
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
retries = int(args['--retries'])
if args['--itemlist']:
with open(args['--itemlist']) as fp:
ids = [x.strip() for x in fp]
total_ids = len(ids)
elif args['--search']:
try:
_search = session.search_items(args['--search'],
params=args['--search-parameters'])
total_ids = _search.num_found
if total_ids == 0:
print('error: the query "{0}" returned no results'.format(
args['--search']),
file=sys.stderr)
sys.exit(1)
ids = _search
except ValueError as e:
print('error: {0}'.format(e), file=sys.stderr)
sys.exit(1)
# Download specific files.
if args['<identifier>'] and args['<identifier>'] != '-':
if '/' in args['<identifier>']:
identifier = args['<identifier>'].split('/')[0]
files = ['/'.join(args['<identifier>'].split('/')[1:])]
else:
identifier = args['<identifier>']
files = args['<file>']
file_with_files_to_download = 'to_download.txt'
if not files and os.path.exists(file_with_files_to_download):
with open(file_with_files_to_download, 'rt',
encoding='utf-8') as f:
files = f.read().splitlines()
total_ids = 1
ids = [identifier]
elif args['<identifier>'] == '-':
total_ids = 1
ids = sys.stdin
files = None
else:
files = None
errors = list()
for i, identifier in enumerate(ids):
if args['--stdout']:
item = session.get_item(identifier)
f = list(item.get_files(args['<file>']))
try:
assert len(f) == 1
except AssertionError:
print('error: {0}/{1} does not exist!'.format(
identifier, args['<file>'][0]),
file=sys.stderr)
sys.exit(1)
if six.PY2:
stdout_buf = sys.stdout
else:
stdout_buf = sys.stdout.buffer
f[0].download(retries=args['--retries'],
fileobj=stdout_buf,
params=args['--parameters'])
sys.exit(0)
try:
identifier = identifier.strip()
except AttributeError:
identifier = identifier.get('identifier')
if total_ids > 1:
item_index = '{0}/{1}'.format((i + 1), total_ids)
else:
item_index = None
try:
item = session.get_item(identifier)
except Exception as exc:
print('{0}: failed to retrieve item metadata - errors'.format(
identifier),
file=sys.stderr)
if 'You are attempting to make an HTTPS' in str(exc):
print('\n{0}'.format(exc), file=sys.stderr)
sys.exit(1)
else:
continue
# Otherwise, download the entire item.
ignore_history_dir = True if not args['--download-history'] else False
_errors = item.download(
files=files,
formats=args['--format'],
glob_pattern=args['--glob'],
dry_run=args['--dry-run'],
verbose=args['--verbose'],
silent=args['--silent'],
ignore_existing=args['--ignore-existing'],
checksum=args['--checksum'],
destdir=args['--destdir'],
no_directory=args['--no-directories'],
retries=retries,
item_index=item_index,
ignore_errors=True,
on_the_fly=args['--on-the-fly'],
no_change_timestamp=args['--no-change-timestamp'],
params=args['--parameters'],
ignore_history_dir=ignore_history_dir,
)
if _errors:
errors.append(_errors)
if errors:
# TODO: add option for a summary/report.
sys.exit(1)
else:
sys.exit(0)
def validate_method(method, is_challenge):
'''Validate method configuration passed as a JSON file.'''
# Define a dictionary schema
# TODO would be nice to not copy-paste for multiple datasets
schema = Schema({
Optional('metadata'): {
'publish_anonymously':
bool,
'authors':
str,
'contact_email':
str,
'method_name':
str,
'method_description':
str,
# 'descriptor_type': str,
# 'descriptor_size': And(int, lambda v: v >= 1),
Optional('link_to_website'):
str,
Optional('link_to_pdf'):
str,
},
'config_common': {
'json_label': str,
'keypoint': And(Use(str), lambda v: '_' not in v),
'descriptor': And(Use(str), lambda v: '_' not in v),
'num_keypoints': And(int, lambda v: v > 1),
},
Optional('config_phototourism_stereo'): {
Optional('use_custom_matches'): bool,
Optional('custom_matches_name'): str,
Optional('matcher'): {
'method':
And(str, lambda v: v in ['nn']),
'distance':
And(str, lambda v: v.lower() in ['l1', 'l2', 'hamming']),
'flann':
bool,
'num_nn':
And(int, lambda v: v >= 1),
'filtering': {
'type':
And(
str, lambda v: v.lower() in [
'none', 'snn_ratio_pairwise', 'snn_ratio_vs_last',
'fginn_ratio_pairwise'
]),
Optional('threshold'):
And(Use(float), lambda v: 0 < v <= 1),
Optional('fginn_radius'):
And(Use(float), lambda v: 0 < v < 100.),
},
Optional('descriptor_distance_filter'): {
'threshold': And(Use(float), lambda v: v > 0),
},
'symmetric': {
'enabled':
And(bool),
Optional('reduce'):
And(str, lambda v: v.lower() in ['both', 'any']),
},
},
Optional('outlier_filter'): {
'method':
And(Use(str), lambda v: v.lower() in ['none', 'cne-bp-nd']),
},
Optional('geom'): {
'method':
And(
str, lambda v: v.lower() in [
'cv2-ransac-f', 'cv2-ransac-e', 'cv2-lmeds-f',
'cv2-lmeds-e', 'cv2-7pt', 'cv2-8pt',
'cv2-patched-ransac-f', 'cmp-degensac-f',
'cmp-degensac-f-laf', 'cmp-gc-ransac-f',
'cmp-degensac-f-laf', 'cmp-gc-ransac-f',
'cmp-magsac-f', 'cmp-gc-ransac-e', 'skimage-ransac-f',
'intel-dfe-f'
]),
Optional('threshold'):
And(Use(float), lambda v: v > 0),
Optional('confidence'):
And(Use(float), lambda v: v > 0),
Optional('max_iter'):
And(Use(int), lambda v: v > 0),
Optional('postprocess'):
And(Use(bool), lambda v: v is not None),
Optional('error_type'):
And(Use(str),
lambda v: v.lower() in ['sampson', 'symm_epipolar']),
Optional('degeneracy_check'):
bool,
}
},
Optional('config_phototourism_multiview'): {
Optional('use_custom_matches'): bool,
Optional('custom_matches_name'): str,
Optional('matcher'): {
'method':
And(str, lambda v: v in ['nn']),
'distance':
And(str, lambda v: v.lower() in ['l1', 'l2', 'hamming']),
'flann':
bool,
'num_nn':
And(int, lambda v: v >= 1),
'filtering': {
'type':
And(
str, lambda v: v.lower() in [
'none', 'snn_ratio_pairwise', 'snn_ratio_vs_last',
'fginn_ratio_pairwise'
]),
Optional('threshold'):
And(Use(float), lambda v: 0 < v <= 1),
Optional('fginn_radius'):
And(Use(float), lambda v: 0 < v < 100.),
},
Optional('descriptor_distance_filter'): {
'threshold': And(Use(float), lambda v: v > 0),
},
'symmetric': {
'enabled':
And(bool),
Optional('reduce'):
And(str, lambda v: v.lower() in ['both', 'any']),
},
},
Optional('outlier_filter'): {
'method':
And(Use(str), lambda v: v.lower() in ['none', 'cne-bp-nd']),
},
Optional('colmap'): {},
},
Optional('config_phototourism_relocalization'): {},
})
schema.validate(method)
# Check for metadata for challenge entries
if is_challenge and not method['metadata']:
raise ValueError('Must specify metadata')
# Check what we are running
do_pt_stereo = False if 'config_phototourism_stereo' not in method \
else bool(method['config_phototourism_stereo'])
do_pt_multiview = False if 'config_phototourism_multiview' not in method \
else bool(method['config_phototourism_multiview'])
do_pt_relocalization = False if 'config_phototourism_relocalization' not \
in method else bool(method['config_phototourism_relocalization'])
if do_pt_stereo:
print('Running: Phototourism, stereo track')
if do_pt_multiview:
print('Running: Phototourism, multiview track')
if do_pt_relocalization:
print('Running: Phototourism, relocalization track')
if not any([do_pt_stereo, do_pt_multiview, do_pt_relocalization]):
raise ValueError('No tasks were specified')
# Check for incorrect, missing, or redundant options
for dataset in ['phototourism']:
for task in ['stereo', 'multiview', 'relocalization']:
cur_key = 'config_{}_{}'.format(dataset, task)
if cur_key not in method:
print('Key "{}" is empty -> skipping check'.format(cur_key))
continue
else:
print('Validating key "{}"'.format(cur_key))
# If dict is not empty, use_custom_matches should exist
if method[cur_key] and (
'use_custom_matches' not in method[cur_key]):
raise ValueError('Must specify use_custom_matches')
# If using custom matches, we cannot specify further options
if method[cur_key] and ('use_custom_matches' in method[cur_key]) \
and method[cur_key]['use_custom_matches']:
if 'matcher' in method[cur_key] or 'outlier_filter' in method[cur_key]:
raise ValueError('Cannot specify a matcher or outlier filter with '
'use_custom_matches=True')
# Matcher and filter
if 'matcher' in method[cur_key]:
matcher = method[cur_key]['matcher']
if matcher['symmetric']['enabled'] and 'reduce' not in \
matcher['symmetric']:
raise ValueError(
'[{}/{}] Must specify "reduce" if "symmetric" is enabled'
.format(dataset, task))
# Check for redundant settings with custom matches
cur_config = method['config_{}_stereo'.format(dataset)]
if cur_config['use_custom_matches']:
if 'matcher' in cur_config or 'outlier_filter' in cur_config \
or 'geom' in cur_config:
raise ValueError(
'[{}/stereo] Found redundant settings with use_custom_matches=True'
.format(dataset))
else:
if 'matcher' not in cur_config or 'outlier_filter' not in \
cur_config or 'geom' not in cur_config:
raise ValueError(
'[{}/stereo] Missing required settings with use_custom_matches=False'
.format(dataset))
if cur_config['use_custom_matches']:
if 'matcher' in cur_config or 'outlier_filter' in cur_config \
or 'geom' in cur_config:
raise ValueError(
'[{}/stereo] Found redundant settings with use_custom_matches=True'
)
# For stereo, check also geom
if task == 'stereo' and \
'config_{}_stereo'.format(dataset) in method and \
'geom' in method['config_{}_stereo'.format(dataset)]:
geom = method['config_{}_stereo'.format(dataset)]['geom']
# Threshold for RANSAC
if geom['method'].lower() in [
'cv2-ransac-f', 'cv2-ransac-e', 'cv2-patched-ransac-f',
'cmp-degensac-f', 'cmp-gc-ransac-f', 'cmp-gc-ransac-e',
'cmp-degensac-f-laf', 'cmp-magsac-f',
'skimage-ransac-f', 'intel-dfe-f'
]:
if 'threshold' not in geom:
raise ValueError(
'[{}] Must specify a threshold for this method'.
format(dataset))
else:
if 'threshold' in geom:
raise ValueError(
'[{}] Cannot specify a threshold for this method'.
format(dataset))
# Degeneracy check for RANSAC
if geom['method'].lower() in [
'cmp-degensac-f', 'cmp-degensac-f-laf'
]:
if 'degeneracy_check' not in geom:
raise ValueError(
'[{}] Must indicate degeneracy check for this method'
.format(dataset))
if 'error_type' not in geom:
raise ValueError(
'[{}] Must indicate error type for this method'.
format(dataset))
else:
if 'degeneracy_check' in geom:
raise ValueError(
'[{}] Cannot apply degeneracy check to this method'
.format(dataset))
if 'error_type' in geom:
raise ValueError(
'[{}] Cannot indicate error type for this method'.
format(dataset))
# Confidence for RANSAC/LMEDS
if geom['method'].lower() in [
'cv2-ransac-f',
'cv2-patched-ransac-f',
'cv2-ransac-e',
'cv2-lmeds-f',
'cv2-lmeds-e',
'cmp-degensac-f',
'cmp-degensac-f-laf',
'cmp-gc-ransac-f',
'cmp-gc-ransac-e',
'skimage-ransac-f',
'cmp-magsac-f',
]:
if 'confidence' not in geom:
raise ValueError(
'[{}] Must specify a confidence value for OpenCV or DEGENSAC'
.format(dataset))
else:
if 'confidence' in geom:
raise ValueError(
'[{}] Cannot specify a confidence value for this method'
.format(dataset))
# Maximum number of RANSAC iterations
if geom['method'].lower() in [
'cv2-patched-ransac-f',
'cmp-degensac-f',
'cmp-degensac-f-laf',
'cmp-gc-ransac-f',
'cmp-gc-ransac-e',
'skimage-ransac-f',
'cmp-magsac-f',
]:
if 'max_iter' not in geom:
raise ValueError(
'[{}] Must indicate max_iter for this method'.
format(dataset))
else:
if 'max_iter' in geom:
raise ValueError(
'[{}] Cannot indicate max_iter for this method'.
format(dataset))
# DFE-specific
if geom['method'].lower() in ['intel-dfe-f']:
if 'postprocess' not in geom:
raise ValueError(
'[{}] Must specify a postprocess flag for DFE'.
format(dataset))
else:
if 'postprocess' in geom:
raise ValueError(
'[{}] Cannot specify a postprocess flag for this method'
.format(dataset))
def main(path_result):
swmm_obj = pickle_swmm(path_result)
swmm_obj.sys_out()
pickle_uzf(path_result).uzf_arrays()
pickle_ext(path_result).ts_sums()
return swmm_obj
if __name__ == '__main__':
start = time.time()
arguments = docopt(__doc__)
typecheck = Schema(
{
'PATH': os.path.exists,
'--fmt': Use(int),
'--var': Or(Use(int), str)
},
ignore_extra_keys=True)
PATH_result = op.abspath(typecheck.validate(arguments)['PATH'])
args = typecheck.validate(arguments)
### 1 CPU
swmm_obj = main(PATH_result)
scenarios, path_picks = swmm_obj.scenarios, swmm_obj.path_picks
ts_hr = swmm_obj.ts_hr
if args['--fmt']:
print('Formatting Pickles')
PickleFmt.main(PATH_result)
def main():
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
fh = logging.FileHandler('dht.log')
fh.setLevel(logging.DEBUG)
logging.basicConfig(
handlers=[ch, fh],
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.info('dht')
parser = argparse.ArgumentParser(
description=
'Publishes humidity and temperature data from Adafruit DHT series sensors'
)
parser.add_argument('-c',
'--config',
help='path to config file',
default='config.ini')
args = parser.parse_args()
config = configparser.ConfigParser()
config.read(args.config)
if not config.read(args.config):
logger.warn('failed to read config file, path={}'.format(args.config))
config_schema = Schema(
{
'dht': {
'type': Use(int),
'pin': Use(int),
Optional('read_interval', default=60): Use(int)
},
Optional('mqtt'): {
Optional('broker_host', default='localhost'): str,
Optional('broker_port', default='1883'): Use(int)
}
},
ignore_extra_keys=True)
config_schema.validate(config._sections)
location = config.get('mqtt', 'location', fallback='unknown')
humidity_topic = '/'.join((location, 'humidity'))
temperature_topic = '/'.join((location, 'temperature'))
# Create sensor
sensor_type = config.getint('dht', 'type')
pin = config.getint('dht', 'pin')
sensor = DhtSensor(sensor_type, pin)
def on_connect(client, userdata, flags, rc):
logger.info('connected to mqtt broker')
# Connect to MQTT broker
client = mqtt.Client()
client.on_connect = on_connect
host = config.get('mqtt', 'broker_host', fallback='localhost')
port = config.getint('mqtt', 'broker_port', fallback=1883)
client.connect(host, port)
client.loop_start()
# Publish sensor readings
read_interval = config.getint('dht', 'read_interval', fallback=60)
logger.debug('read_interval={}'.format(read_interval))
scheduler = sched.scheduler(time.time, time.sleep)
def read():
scheduler.enter(read_interval, 1, read)
reading = sensor.read_retry()
if not reading.is_valid():
return
logger.debug(
'publishing humidity={} and temperature={} for location={}'.format(
reading.humidity, reading.temperature, location))
client.publish(humidity_topic, reading.humidity)
client.publish(temperature_topic, reading.temperature)
read()
scheduler.run()