def _validate_vmci(self, endpoint):
iface, port = self._validate_port_pair(
endpoint, "interface", wildcard_ok=self.side is self.Side.BIND)
if iface == "*":
if self.side is not self.Side.BIND:
raise V.ValidationError(
"wildcard interface not valid in this context")
elif not iface.isdigit():
raise V.ValidationError(f"interface {iface!r} must be an integer")
def validate(self, value, adapt=True):
if not value:
raise v.ValidationError('No email')
addr = parseaddr(value)[1]
hostname = addr[addr.find('@') + 1:]
try:
dns.resolver.query(hostname, 'MX')
except dns.exception.DNSException:
raise v.ValidationError('No MX record for %s' % hostname)
return value
def _validate_tcp_endpoint(self, endpoint, wildcard_ok=False):
host, port = self._validate_port_pair(endpoint,
"host",
wildcard_ok=wildcard_ok)
if host == "*":
if not wildcard_ok:
raise V.ValidationError(
"wildcard host not valid in this context")
elif not DNS_OR_IP_REGEX.match(host):
raise V.ValidationError(f"host {host!r} does not look like a "
"valid hostname nor an IP address")
def _validate_port_pair(self, endpoint, rest_name, wildcard_ok=False):
rest, sep, port = endpoint.rpartition(":")
if not sep:
raise V.ValidationError(f"endpoint {endpoint} does not follow the "
f"format of {rest_name}:port")
if port == "*":
if not wildcard_ok:
# XXX: what context?
raise V.ValidationError(
"wildcard port not valid in this context")
elif not port.isdigit():
raise V.ValidationError(f"port {port!r} must be an integer")
return rest, port
def _validate_pgm(self, endpoint):
rest, port = self._validate_port_pair(endpoint, "interface;multicast")
iface, sep, multicast = rest.rpartition(";")
if not sep:
raise V.ValidationError("missing semicolon separator")
# XXX: is it worth validating iface?
try:
multicast = ipaddress.IPv4Address(multicast)
except ipaddress.AddressValueError as e:
# XXX: what address is causing the error?
raise V.ValidationError(str(e)) from None
if not multicast.is_multicast:
raise V.ValidationError(f"{str(multicast)!r} is not a multicast "
"address")
def validate(self, value, adapt=True):
super().validate(value, adapt=adapt)
transports = {
"tcp": self._validate_tcp,
"ipc": self._validate_ipc,
"inproc": self._validate_inproc,
"pgm": self._validate_pgm,
"epgm": self._validate_pgm,
"vmci": self._validate_vmci,
}
transport, sep, endpoint = value.partition("://")
if not sep:
raise V.ValidationError("missing transport://")
if transport not in transports:
raise V.ValidationError(f"invalid transport {transport!r}")
transports[transport](endpoint)
return value
def _validate_tcp(self, endpoint):
source_endpoint, sep, endpoint = endpoint.rpartition(";")
if sep:
if self.side is not self.Side.CONNECT:
raise V.ValidationError(
"must not specify source in bindable endpoint")
self._validate_tcp_endpoint(source_endpoint, wildcard_ok=True)
self._validate_tcp_endpoint(endpoint,
wildcard_ok=self.side is self.Side.BIND)
def _validate_inproc(self, endpoint):
if not endpoint:
raise V.ValidationError("inproc name must not be empty")
if len(endpoint) > 256:
raise V.ValidationError("inproc name must not be longer than 256 "
"characters")
def _validate_ipc(self, endpoint):
if "\0" in endpoint:
raise V.ValidationError("paths must not contain NUL bytes")
def __call__(self, validated):
# Selects
# -------
# need to add all the groupbys
for g in self._groupby:
found = False
for s in self._selects:
# |v| | v |
# matches "extract(..) as day" and "gph.productid as product"
if s.startswith(g) or s.endswith(g):
found = True
break
if not found:
self._selects[g] = (g, None, None, None)
if self._aggs and set(self._aggs) & set(self._where.keys()) and self._groupby:
# --------------
# New Main Query
# --------------
With = Query()
# select all the columns (by as)
[With.select(col) for col in self._selects.keys()]
With.into(True)
With.tables("from _data")
[With.where(col, *self._where.pop(col)) for col in self._aggs if col in self._where]
# ~~groupby~~ no need
if self._sortby:
With.sortby(*self._sortby)
# need to select sorts in next query
for sort in self._sortby:
if sort not in self._selects.keys():
self._selects[sort] = (sort, None, None, None)
self._sortby = []
# only need to pop these when sorting methods present
limit = validated.pop('limit') if 'limit' in validated else None
offset = validated.pop('offset') if 'offset' in validated else None
# ----------------------
# Change Self Properties
# ----------------------
With._with = self._with
self._with = []
self.into(False)
# remove sorting from **this** query
# ---------
# With Self
# ---------
self._aggs = []
With.with_(self(validated), as_="_data")
validated['limit'] = limit
validated['offset'] = offset
return With(validated)
elif str(validated.get('limit', '')).endswith('%'):
"""
### Making the with query below
```psql
with _limited as (select total from orders where total > 10)
select total from _limited order by total asc limit (select round(count(*)*.1) from _limited)
```
"""
# column, agg, as, distinct
# only AGG: peice 3: select sum(column) from _ll
_l = Query()
_l._selects = dict(map(lambda d: (d[0], (d[1][2] or d[1][0], d[1][1], d[1][2], None)), self._selects.items()))
_l.tables("from _ll")
# only ORDER BY and LIMIT: peice 2
_ll = Query()
# select table.column as as_column
_ll._selects = dict(map(lambda d: (d[0], (d[1][2] or d[1][0], None, None, None)), self._selects.items()))
_ll.tables("from _l")
_ll.into(False)
[_ll.agg(a) for a in self._aggs]
if self._sortby:
_ll.sortby(*self._sortby)
# need to select sorts in next query
for sort in self._sortby:
if sort not in self._selects.keys():
self._selects[sort] = sort
self._sortby = []
self._selects = dict(map(lambda d: (d[0], (d[1][0], None, d[1][2], d[1][3])), self._selects.items()))
self._aggs = []
self.into(False)
limit = validated.pop('limit')
_l.with_(self(validated), "_l")
validated['limit'] = "(select round(count(*)*%s) from _l)" % (float(limit[:-1])/100)
_l.with_(_ll(validated), "_ll")
validated.pop('limit')
return _l(validated)
else:
query = self._query
elements = {}
# With
# ----
elements['with'] = ("with %s "%', '.join(self._with)) if self._with else ""
# Select
# ---------------
# add sortby to select (only needed for DISTINCT)
if self._sortby:
for sort in self._sortby:
if sort not in self._selects.keys():
self._selects[sort] = (sort, None, None, None)
elements["select"] = ', '.join(map(self._column,
sorted(self._selects.values(),
key=lambda a: (not a[3], a[0]))))
# Into
# ----
elements['into'] = "__into__" if self._into else ""
# Tables
# ------
_from = None
for table in self._tables:
if table.startswith('from') or table == "":
_from = table
if _from is None:
raise valideer.ValidationError("Not enough arguments supplied to formulate a tables for query")
newtables = self._tables[self._tables.index(_from):]
oldtables = self._tables[:self._tables.index(_from)]
newtables.extend([t for t in oldtables if not t.startswith('from')])
self._tables = unique(newtables)
elements["tables"] = " ".join(self._tables)
# Group By
# --------
elements['groupby'] = (" group by " + ','.join(map(lambda g: ('"%s"'%g) if g == 'group' else g,
self._groupby))) if self._groupby else ""
# Sort By
# -------
# http://www.postgresql.org/docs/8.1/static/queries-order.html
# Each column specification may be followed by an optional ASC or DESC to set the sort direction to ascending or descending.
# ASC order is the default
elements['sortby'] = (" order by %s %s" % (",".join(self._sortby), validated.get('dir', 'asc'))) if self._sortby else ""
# Limit
# -----
limit = validated.get('limit')
elements['limit'] = (" limit %s" % validated['limit']) if limit else ""
# Offset
# ------
elements['offset'] = (" offset %s" % validated['offset']) if validated.get('offset') else ""
# Where
# -----
# custom ?where=this|that
if validated.get('where'):
# valideer produces the tuple below
keys, string = validated.pop("where")
try:
self._where.setdefault('_', []).append(string % self._algebra(self._where))
except KeyError as e:
raise valideer.ValidationError("Missing argument `%s` needed in provided where clause" % str(e)[1:-1], str(e)[1:-1])
[self._where.pop(key) for key in keys if key in self._where]
wheres = list(itertools.chain(*self._where.values()))
if self.debug: wheres = sorted(wheres)
elements['where'] = (" where " + ' and '.join(wheres)) if wheres else ""
# ----------
# Format SQL
# ----------
try:
# return (query % elements) % validated
try:
return (query % elements) % validated
# except ValueError:
# return (self.escape.sub('%%', query) % elements) % validated
except KeyError:
validated.update(elements)
return ((query % validated) % validated) % validated
except Exception as e:
raise# SyntaxError("failed to create query", e)
def _harvest_validate(self, userkwargs):
"""Validate and Plant user provided arguments
- Go through and plants the seedlings
for any user arguments provided.
- Validate the arguments, cleaning and adapting (valideer wise)
- Extract negatives "!" arguments
"""
# the valideer to parse the
# user arguemnts when watering
parser = {}
userkwargs.update(self.network_kwargs)
# a simple set of original provided argument keys (used in IGNORES)
original_kwargs = set(
map(lambda k: k.split('_')[1] if k.find('_') > -1 else k,
userkwargs.keys()))
# list of columns that are required from seeds
requires = []
# -------------
# Clean up Aggs
# -------------
for key in userkwargs.keys():
# agg example: "avg_total", "max_tax"
if key.find('_') > 0:
agg, base = tuple(key.split('_'))
if base in userkwargs:
if type(userkwargs[base]) is not list:
userkwargs[base] = [(None, userkwargs[base])]
userkwargs[base].append((agg, userkwargs.pop(key)))
else:
userkwargs[base] = [(agg, userkwargs.pop(key))]
# -----------------
# Process Arguments
# -----------------
for key, seed in self.arguments.iteritems():
# --------------
# Argument Alias
# --------------
if seed.get('alias') and key in userkwargs:
# pop the value form the user kwargs (to change the key later)
value = userkwargs.pop(
key) if key in userkwargs else NotImplemented
# for duplicate keys
oldkey = key + ""
# change the key
key = seed.get('alias')
# change the seed
seed = get(self.arguments, seed.get('alias'))
# set the new key:value
if value is not NotImplemented:
if key in userkwargs:
raise valideer.ValidationError(
"Argument alias already specified for `%s` via `%s`"
% (oldkey, key), oldkey)
userkwargs[key] = value
# can provide multiple arguments
if key.endswith('[]'):
multi = True
key = key[:-2]
else:
multi = False
# get value(s) from user
if key in userkwargs:
value = userkwargs.pop(key)
elif seed.get('copy'):
value = userkwargs.get(seed.get('copy'))
else:
value = seed.get('default')
# no argument provided, lets continue)
if value is None or value == []:
if seed.get('required'):
raise valideer.ValidationError(
"missing required property: %s" % key, key)
else:
continue
# add requires
requires.extend(array(get(seed, 'requires', [])))
# -----------
# Inheritance
# -----------
# not permited from arguements yet. would need to happen above the ""PROCESS ARGUMENT"" block
# self._inherit(*array(get(seed, 'inherit', [])))
if type(value) is list and type(value[0]) is tuple:
# complex
for v in value:
ud, pd = self._harvest_args(key, seed, v, multi)
userkwargs.update(ud)
parser.update(pd)
else:
ud, pd = self._harvest_args(key, seed, value, multi)
userkwargs.update(ud)
parser.update(pd)
# ------------
# Ignored Keys
# ------------
for seed in self.seeds:
ignores = set(array(get(seed, 'ignore')))
if ignores:
if ignores & original_kwargs:
if not get(seed, 'silent'):
additionals = ignores & original_kwargs
raise valideer.ValidationError(
"additional properties: %s" %
",".join(additionals), additionals)
[userkwargs.pop(key) for key in ignores if key in userkwargs]
# -------------------------
# Custom Operators (part 1)
# -------------------------
operators = {}
for key, value in userkwargs.items():
rk = key
agg = None
if key.find('_') > -1:
agg, rk = tuple(key.split('_'))
seed = self.arguments.get(rk, self.arguments.get(rk + '[]'))
if seed:
if type(value) is list:
operators[key] = []
# need to remove the operator for validating
new_values = []
for v in value:
operator, v = self._operator(
v,
*seed.get('column', "").rsplit("::", 1))
new_values.append(v)
operators[key].append((agg,
operator) if agg else operator)
userkwargs[key] = new_values
else:
operator, value = self._operator(
value,
*seed.get('column', "").rsplit("::", 1))
operators[key] = (agg, operator) if agg else operator
userkwargs[key] = value
# -----------------
# Plant Sort Method
# -----------------
if 'sortby' in userkwargs:
seed = self.arguments.get(
userkwargs['sortby'].lower(),
self.arguments.get(userkwargs['sortby'].lower() + '[]'))
if seed:
seed['id'] = str(userkwargs['sortby'].lower())
for r in set(requires):
if userkwargs.get(r) is None:
raise valideer.ValidationError(
"required property not set: %s" % r, r)
# --------
# Validate
# --------
parser = valideer.parse(parser, additional_properties=False)
validated = parser.validate(userkwargs, adapt=self.navigator.adapter())
validated.update(self.network_kwargs)
# operators validated
# --------------------------- | --------------------------------
# { {
# "type": ["!", "!"], "type": ['a', 'b'],
# "total": "<", "total": "50",
# "tax": ("avg, ">"), "tax": "1",
# "time": None "time": "2014"
# } }
return operators, validated
def _harvest_args(self, key, seed, value, multi):
agg = value[0] if type(value) is tuple else None
value = value[1] if type(value) is tuple else value
# adapt to proper variable types
# if value and multi and type(value) not in (list, tuple):
# value = [str(value).lower()]
# if type(value) in (int, long, float, bool):
# value = str(value).lower()
if multi and type(value) not in (list, tuple):
multi = False
# plant this arguments seed
key = key if not agg else ("%s_%s" % (agg, key))
seed["id"] = key
self.plant(seed)
# set the value
# userkwargs[key] = value
# add to the parser
# parse key "+" if required
pkey = ("+" + key) if seed.get('required') else key
# validation methods
# 1) choose
if 'options' in seed:
replace_with = []
if seed.get('*') and value in ('*', 'all'):
map(self.plant, seed['options'].values())
else:
for _value in array(value):
found = False
for pattern, _seed in seed['options'].iteritems():
if re.match(pattern, _value):
found = True
# replace the value provided
replace_with.append(_seed.get('value', _value))
# add the seed
self.plant(_seed)
# add the parser, though we have already parsed properly
break
# no seed found: invalid
if not found:
raise valideer.ValidationError("Invalid value", key)
if type(value) in (list, tuple):
value = replace_with
validate = valideer.HomogeneousSequence(valideer.String())
else:
value = replace_with[0] if replace_with else ""
validate = valideer.String()
elif 'validator' in seed:
validator = seed.get('validator')
if validator == 'currency':
# somehow change currency to respect stores... (future)
validator = "float"
validate = valideer.HomogeneousSequence(
validator) if multi else validator
else:
raise EnvironmentError("Missing validation method for " + key)
return {key: value}, {pkey: validate}