def lookup(self,param):
log.debug("param = %s" % str(param))
if param in self.data:
record = self.data.get(param)
return {"address":record}
else:
return {"address":{"message":"unknown address"}}
def lookup(self, param):
log.debug("param = %s" % str(param))
if param in self.data:
record = self.data.get(param)
return {"address": record}
else:
return {"address": {"message": "unknown address"}}
def wrapsafe(callf,rawarg):
try:
return callf(rawarg)
except Exception as e:
log.debug("exception = %s" % e)
# print_tb(e.__traceback__)
return errmsg('internal error')
def get_summary(self,_bbl,_bin):
'''Full ownership summary (Taxbill,DHRC,HPD) for a BBL+BIN pair.'''
log.debug("bbl = %d, bin = %d" % (_bbl,_bin))
query = "select * from hard.property_summary where bbl = %d and (bin = %d or bin is null)";
r = self.fetchone(query,_bbl,_bin)
log.debug("r = %s" % str(r))
return make_summary(r) if r is not None else None
def get_summary(self, _bbl, _bin):
'''Full ownership summary (Taxbill,DHRC,HPD) for a BBL+BIN pair.'''
log.debug("bbl = %d, bin = %d" % (_bbl, _bin))
query = "select * from hard.property_summary where bbl = %d and (bin = %d or bin is null)"
r = self.fetchone(query, _bbl, _bin)
log.debug("r = %s" % str(r))
return make_summary(r) if r is not None else None
def extract_param(query_string):
param = split_query(query_string.decode('utf-8'))
log.debug("param = %s" % param)
fields = 'houseNumber','street','borough'
messy = {k:param.get(k) for k in fields}
clean = {k:mapcgi(messy[k]) for k in messy}
log.debug("clean = %s" % clean)
return NYCGeoAddress(**clean)
def phase_count(data, n=1):
with Timer() as t:
for i in range(n):
data = phase(data)
t.click()
if t.time_for_report():
log.debug(f"{i+1}/{n}, " f"ETA: {t.eta(n)}")
return data
def resolve_lookup(address):
q = address.replace('+',' ').strip()
log.debug("q = %s" % str(q))
if q is None:
return errmsg('invalid query string')
else:
response = agent.get_lookup(q)
return jsonify(response)
def extract_param(query_string):
param = split_query(query_string.decode('utf-8'))
log.debug("param = %s" % param)
fields = 'houseNumber','street','borough'
messy = {k:param.get(k) for k in fields}
clean = {k:mapcgi(messy[k]) for k in messy}
log.debug("clean = %s" % clean)
return NYCGeoAddress(**clean)
def api_fetch(prefix):
if prefix != 'address.json':
return errmsg('invalid service base')
try:
log.debug("query_string = %s" % request.query_string)
return resolve_query(request.query_string)
except Exception as e:
log.debug("exception = %s" % e)
return errmsg('internal error')
def fetch_tiny_plus(self,rawaddr):
log.debug("rawaddr = '%s'" % rawaddr)
status,response = self.fetch(rawaddr)
log.debug("status = %s" % status)
# log.debug("response = %s" % response)
if response is None:
return status,None,None
else:
tinyrec = make_tiny(response)
return status,tinyrec,response
def api_fetch(prefix):
if prefix != 'address.json':
return errmsg('invalid service base')
try:
log.debug("query_string = %s" % request.query_string)
return resolve_query(request.query_string)
except Exception as e:
log.info("exception = %s" % e)
log.exception(e)
return errmsg('internal error')
def phase_count(data, n=1):
with Timer() as t:
for i in range(n):
data = phase(data)
t.click()
if t.time_for_report():
log.debug(
f"{i+1}/{n}, Average: {t.average_per_lap():.2f} seconds, ETA: {t.eta(n - i):.2f} seconds, Average {t.average_per_second():.2f} per second."
)
return data
def run(command: str,
*,
env: T.Optional[T.Dict[str,
str]] = None) -> subprocess.CompletedProcess:
""" Wrapper for running commands """
log.debug(f"Running: {command}")
return subprocess.run(shlex.split(command),
env=env,
capture_output=True,
check=False,
text=True)
def get_baselot(self,bbl):
"""
This method is currently more of a helper accessor for 'get_taxlot',
as it isn't currently (directly) attached to any endpoint. It feches
a minified 'plutoid' struct for the given BBL (assumed to be a 'bank' BBL),
so that it can be attached a child to the 'condo' struct in the outgoing
taxlot struct proper.
Currently it's only intended usage is to be called via from the hybrid
agent if it detects that the incoming BBL is a bank bbl.
"""
log.debug("bbl = %s")
if bbl is None:
return None
query = "select * from deco.baselot where bbl = %s"
r = self.fetchone(query,bbl)
log.debug("r = %s" % r)
return adjust_baselot(r)
def resolve(callf,query):
try:
param = split_query(query)
except ValueError as e:
return errmsg('invalid query string')
try:
log.debug("invoke ..")
r = callf(param)
log.debug("got dict with %d keys." % len(r))
except Exception as e:
log.debug("exception = %s" % e)
return errmsg('internal error')
log.debug(":: return dict with %d keys." % len(r))
return jsonify(r,sort_keys=True)
def main():
global THROW,LOUD
args = parse_args()
LOUD = args.loud
THROW = args.throw
log.info('hi')
log.debug('yow')
nycgeopath = "config/nycgeo-live.json"
geoconf = json.loads(open(nycgeopath,"r").read())
geoclient = SimpleGeoClient(**geoconf)
print(f'agent = {geoclient}')
if args.rawaddr:
status, keytup, response = do_single(geoclient,args.rawaddr)
print(f'status = {status}, keytup = {keytup}')
else:
infile = args.infile
print(f'slurp from {infile} ..')
inrecs = ioany.read_recs(infile)
inrecs = islice(inrecs,args.limit)
do_multi(geoclient,inrecs,capture=args.capture)
print('done')
def phase(input_data):
output_length = len(input_data)
output_data = [0 for _ in range(output_length)]
with Timer() as at:
for a in range(output_length):
build = [0 for _ in range(output_length)]
for b in range(output_length):
v = input_data[b]
p = pattern(b + 1, a + 1)
build[b] = v * p
b += 1
output_data[a] = abs(sum(build)) % 10
a += 1
at.click()
if at.time_for_report():
log.debug(f"{a+1}/{output_length}, "
f"Average: {at.average_per_lap():.2f} seconds, "
f"ETA: {at.human_eta(output_length - a)}, "
f"Average {at.average_per_second():.2f} per second.")
return output_data
def get_taxlot(self,bbl):
"""
The 'taxlot' struct is a catch-all container for "everything we know
about the taxlot for a given BBL." The final struct will have variously
different members, depending on whether we're in Pluto, ACRIS, or both;
whether we're a condo baselot or unit, etc.
But it always has a 'meta' member which provides a few high-level flags
(like 'is_bank', 'is_resi', etc) as well as reflecting the BBL back as well,
to make response handling a bit easier.
"""
log.debug("bbl = %s")
if bbl is None:
return None
# A silly switch we sometimes activate to force a high-level
# exception to be triggered.
# if bbl % 2:
# raise ValueError("odd-numbered BBLs not allowed")
query = "select * from deco.taxlot where bbl = %s"
r = self.fetchone(query,bbl)
log.debug("r = %s" % r)
return stagger_taxlot(r)
def get_lookup(self,rawaddr):
''' Combined geoclient + ownership summary for a given address'''
log.debug(":: rawaddr = '%s'" % rawaddr)
normaddr = fix_borough_name(rawaddr)
log.debug(":: normaddr = '%s'" % normaddr)
r,status = self.geoclient.fetch_tiny(normaddr)
log.debug(":: status = %s " % status)
log.debug(":: response = %s" % r)
if r is None:
return {"error":"invalid address"}
nycgeo = make_tiny(r)
if 'message' in nycgeo:
return {"nycgeo":nycgeo,"extras":None,"error":nycgeo.get('message')}
else:
extras = self.dataclient.get_summary(nycgeo['bbl'],nycgeo['bin'])
return {"nycgeo":nycgeo,"extras":extras}
def fetch_tiny(self, rawaddr):
log.debug("rawaddr = '%s'" % rawaddr)
response, status = self.fetch(rawaddr)
log.debug("status = %s" % status)
log.debug("response = %s" % response)
if response:
tinyrec = make_tiny(response)
return tinyrec, status
else:
return response, status
def fetch_tiny(self,rawaddr):
log.debug("rawaddr = '%s'" % rawaddr)
response,status = self.fetch(rawaddr)
log.debug("status = %s" % status)
log.debug("response = %s" % response)
if response:
tinyrec = make_tiny(response)
return tinyrec,status
else:
return response,status
def get_buildings(self,_bbl,_bin=None):
log.debug("bbl = %s, bin = %s" % (_bbl,_bin))
if _bbl is None:
return []
if _bin is None:
query = "select * from hard.building where bbl = %s and in_pluto order by bin, doitt_id"
args = (_bbl,)
else:
query = "select * from hard.building where bbl = %s and bin = %s and in_pluto order by doitt_id"
args = (_bbl,_bin)
r = self.fetch_recs(query,*args)
log.debug("fetched type(r) = %s" % type(r))
log.debug("fetched r = %s" % r)
if r:
[inflate_shape(_) for _ in r]
return r
def capture_object(i,o):
outfile = savepath(i)
log.debug(f'capture to {outfile} ..')
ioany.save_json(outfile,o)
def resolve_query(query_string):
param = extract_param(query_string)
log.debug("named = %s" % str(param))
response = agent.lookup(param)
log.debug("response = %s" % response)
return jsonify(response)
def fetch_default(self,param):
base = '/geoclient/v1/address.json'
query = namedtuple2query(param)
log.debug("query = %s" % query)
return self.authget(base,query)
def resolve_query(query_string):
param = extract_param(query_string)
log.debug("named = %s" % str(param))
response = agent.lookup(param)
log.debug("response = %s" % response)
return jsonify(response)
def do_single(geoclient,rawaddr):
log.debug(f"rawaddr = '{rawaddr}' ..")
status, keytup, response = geoclient.fetch_tiny_plus(rawaddr)
log.debug(f'status = {status}, keytup = {keytup}')
return status, keytup, response
#!/usr/bin/env python
import sys, argparse
import simplejson as json
from nycgeo.client import SimpleGeoClient
from common.logging import log
parser = argparse.ArgumentParser()
parser.add_argument("--addr", help="address to parse")
parser.add_argument("--tiny", help="fetch a tiny rec", type=int)
parser.add_argument("--mock", help="use the mock service", type=int)
args = parser.parse_args()
print(args)
log.info("info!")
log.debug("debug!")
if args.mock:
configpath = "config/mockgeo-client.json"
else:
configpath = "config/nycgeo.json"
config = json.loads(open(configpath,"r").read())
if args.addr:
rawaddr = args.addr
else:
rawaddr = "529 West 29th St, Manhattan"
print("rawaddr = [%s]" % rawaddr)
agent = SimpleGeoClient(**config)
def fetch_default(self, param):
base = '/geoclient/v1/address.json'
query = namedtuple2query(param)
log.debug("query = %s" % query)
return self.authget(base, query)
