def __init__(self,
pname=None,
cantools_path=HOME,
web_backend="dez",
refresh_symlinks=False):
if not pname and not refresh_symlinks:
pname = raw_input("project name? ")
log("Initializing %s Project: %s" %
(web_backend, pname or "(refresh)"))
self.pname = pname
self.ctroot = os.path.join(cantools_path, "cantools", "cantools")
if not os.path.exists(self.ctroot):
error(
"failed to establish cantools root",
"We tried: %s" % (self.ctroot, ),
"This directory does not contain cantools.",
"You can download cantools at https://github.com/bubbleboy14/cantools",
"Please pass the path to your local copy of cantools via the -c flag."
)
self.web_backend = web_backend
self.refresh_symlinks = refresh_symlinks
if not refresh_symlinks:
self.build_dirs()
self.make_files()
self.vcignore()
self.generate_symlinks()
log("done! goodbye.", 1)
def make_files(self):
log("generating configuration", 1)
cp((self.web_backend == "gae") and "%s\r\n%s"%(config.init.yaml.gae,
config.init.yaml.core)%(self.pname,) or config.init.yaml.core, "app.yaml")
cp(config.init.ctcfg%(self.web_backend,), "ct.cfg")
log("demo index page", 1)
cp(config.init.html%(self.pname,), os.path.join("html", "index.html"))
def _fetch(self, api, *args): # api is latlng/zip
path = self.apis[api]["path"].format(
*[urllib.quote(str(a).replace(" ", "+")) for a in args])
host = self.apis[api]["host"]
user = self.apis[api]["user"]
prop = self.apis[api]["property"]
keys = config.geo.user[user]
onum = num = self.apis[api]["index"]
sig = self.apis[api]["sig"]
kwargs = {"asjson": True}
result = []
while True:
fullpath = path
if keys[num]:
fullpath += "&%s=%s" % (sig, keys[num])
if user == "google":
kwargs["protocol"] = "https"
raw = fetch(host, fullpath, **kwargs)
result = raw.get(prop, [])
if len(result):
break
log("0-length %s result (got: '%s') - changing user" % (api, raw),
important=True)
num = (num + 1) % len(keys)
if num == onum:
log("exhausted key list -- returning [] :'(")
break
self.apis[api]["index"] = num
return result
def mergeDemographics():
log("Merging Demographics", important=True)
propertyValues()
zmap = getMap()
ztotal = zmap["SF"]["total"]
log("adding fire data to demographics map")
for year, year_data in obj.items():
syear = str(year) # de-unicode
for zcode, zip_data in year_data.items():
if zcode == "total":
for tcount in ztotal.keys():
ztotal[tcount] += zip_data[tcount]
elif zcode not in zmap:
log("zipcode not found in demographics map: %s"%(zcode,), 1)
else: # zipcode
zmap[zcode][syear] = {} # de-unicode
for zk, zv in zip_data.items():
zmap[zcode][syear][str(zk)] = zv
if not syear in zmap["SF"]:
zmap["SF"][syear] = { "fires": 0, "people": 0, "units": 0 }
if "property_values" in obj[year]["total"]:
zmap["SF"][syear]["property_values"] = obj[year]["total"]["property_values"]
for tcount in ztotal.keys():
zmap[zcode]["total"][tcount] += zip_data[tcount]
zmap["SF"][syear][tcount] += zip_data[tcount]
log("calculating # fires, # peeps displaced, # units destroyed per 1000 people by neighborhood")
for area, data in zmap.items():
if "ad_hoc" in data: continue
zav = data["per_1000"] = {}
for key in ztotal.keys():
zav[key] = 1000.0 * data["total"][key] / data["population"]
log("writing demographics.json")
writejson(zmap, path("logs", "json", "demographics"))
pvalsByYear(zmap)
def _fetch(self, api, *args): # api is latlng/zip
path = self.apis[api]["path"].format(*[urllib.quote(str(a).replace(" ", "+")) for a in args])
host = self.apis[api]["host"]
user = self.apis[api]["user"]
prop = self.apis[api]["property"]
keys = config.geo.user[user]
onum = num = self.apis[api]["index"]
sig = self.apis[api]["sig"]
kwargs = { "asjson": True }
result = []
while True:
fullpath = path
if keys[num]:
fullpath += "&%s=%s"%(sig, keys[num])
if user == "google":
kwargs["protocol"] = "https"
raw = fetch(host, fullpath, **kwargs)
result = raw.get(prop, [])
if len(result):
break
log("0-length %s result (got: '%s') - changing user"%(api, raw), important=True)
num = (num + 1) % len(keys)
if num == onum:
log("exhausted key list -- returning [] :'(")
break
self.apis[api]["index"] = num
return result
def on_pm(self, obj):
log("Concierge received private message: %s"%(json.dumps(obj),))
user = obj["user"]
usplit = user.split("_")
message = obj["message"]
action = message["action"]
data = message["data"]
if usplit[0] == "host":
if action == "create":
gname = "%s_%s_%s"%(data, usplit[1], self.counter)
self.counter += 1
if data not in self.games:
self.games[data] = []
self.games[data].append(gname)
self.server.subscribe(gname, self.server.client(user))
self.pub({
"action": "newgame",
"data": gname
})
elif action == "start":
gtype = data.split("_")[0]
if gtype in self.games:
if data in self.games[gtype]:
self.games[gtype].remove(data)
self.pub({
"action": "oldgame",
"data": data
})
else: # player
pass
def content(self, sections=None, depth=0, novars=False, page_breaks=False):
body = self.body(depth, novars, page_breaks)
secs = self.sections and self.secs(sections, depth + 1, novars,
page_breaks) or ""
cont = "%s\r\n\r\n%s" % (body, secs)
log(cont)
return cont
def response():
today = datetime.now()
tomorrow = today + timedelta(1)
yesterday = today - timedelta(1)
mems = Member.query().all()
log("memscan! found %s members" % (len(mems), ))
unregistered = []
expired = []
almost = []
for mem in mems:
ex = mem.expiration
if not ex:
unregistered.append(mem)
elif ex > yesterday:
if ex < today:
expired.append(mem)
elif ex < tomorrow:
almost.append(mem)
log("%s unregistered" % (len(unregistered), ))
log("%s expired" % (len(expired), ))
log("%s almost exired" % (len(almost), ))
for mem in expired:
send_mail(to=mem.email,
subject="your subscription",
body=EMSG % ("just", ))
for mem in almost:
send_mail(to=mem.email,
subject="your subscription",
body=EMSG % ("almost", ))
log("goodbye")
def save(self):
log("creating %s Chemical records"%(len(self.chemicals),))
puts = []
for chem in self.chemicals:
puts.append(Chemical(**chem))
log("saving %s Chemical records"%(len(puts),))
db.put_multi(puts)
def propertyValues():
log("Analyzing Property Values", important=True)
for year in range(2008, 2014):
yobj = obj[year]
tobj = yobj["total"]
tobj["property_values"] = { "all": [] }
log("parsing: %s"%(year,), 1)
xls = getxls(read(path("scrapers", "data", "pvals",
"SanFranciscoPropertyAssessmentRoll%s.xlsx"%(year,))), "%sSecured"%(year,), parse=False)
for r in range(1, xls.nrows):
row = xls.row(r)
pval = row[7].value
zipcode = row[8].value[:5]
if zipcode not in yobj:
yobj[zipcode] = { "fires": 0, "people": 0, "units": 0 }
zobj = yobj[zipcode]
if "property_values" not in zobj:
zobj["property_values"] = { "all": [] }
zobj["property_values"]["all"].append(pval)
tobj["property_values"]["all"].append(pval)
for zcode, zobj in yobj.items():
if "property_values" in zobj:
apv = zobj["property_values"]["all"]
ave = None
log("analyzing: %s"%(zcode,), 2)
for conversion, func in ARR_ANYL:
zobj["property_values"][conversion] = func(apv, ave)
ave = ave or zobj["property_values"][conversion] # average happens first
log("%s: %s"%(conversion, zobj["property_values"][conversion]), 3)
del zobj["property_values"]["all"]
else:
log("%s: missing!"%(zcode,), 2)
def full_scan():
set_log(os.path.join("logs", "txt", "rent_control.txt"))
log("Scanning rent_control.geojson", important=True)
rcd = read(os.path.join("scrapers", "data",
"rent_control.geojson"), isjson=True)["features"]
rcp = [rc["properties"] for rc in rcd if rc["properties"]["address"]]
log("using %s of %s rows (omitting blank entries)"%(len(rcp), len(rcd)), 1)
blds = []
bset = set()
for d in rcp:
addr = d["address"]
if addr not in bset:
building = Building.query(Building.address == addr).get()
if not building:
log("Can't find '%s' -- creating new entry"%(addr,), 2)
building = Building(address=addr)
building.rent_control = True
blds.append(building)
bset.add(addr)
log("saving %s rent-control buildings to db"%(len(blds),), 1)
db.put_multi(blds)
log("goodbye")
close_log()
def full_scan():
set_log(path("logs", "txt", "fires.txt"))
fullScan()
mergeDemographics()
export()
log("goodbye")
close_log()
def savecache(self, pretty=False):
log("saving zipcode / latlng (geo) cache (pretty: %s)" % (pretty, ),
important=True)
if pretty:
writejson(self.cache, zcpath)
else:
write(self.cache, "%s.json" % (zcpath, ), True)
def cache(url):
log("checking cache for: %s" % (url, ))
if not check(url):
log("acquiring", 1)
CACHE[url] = requests.get(url).content
STAMP[url] = datetime.now()
return CACHE[url]
def compress(html):
log("compressing html", 1)
newhtml = html.replace("\n", " ").replace("\t", " ")
while " " in newhtml:
newhtml = newhtml.replace(" ", " ")
newhtml = newhtml.replace("> <", "><")
log("orig: %s. new: %s" % (len(html), len(newhtml)), 2)
return newhtml
def full_scan():
set_log(os.path.join("logs", "txt", "lords.txt"))
scan()
process()
export()
report()
log("goodbye")
close_log()
def deposit(self, numdays=1):
service = self.service.get()
details = "compensating commitment: %s service (%s); estimated %s hours per week; paying for %s days" % (
service.name, service.compensation, self.estimate, numdays)
log(details)
self.membership.get().deposit(
service.compensation * self.estimate * numdays / 7.0, self,
"commitment: %s" % (service.name, ), details)
def _zip(zc, addr):
if not zc.isdigit() or len(zc) != 5:
log("unsuitable zip (%s) -- deriving from address (%s)"%(zc, addr), important=True)
if addr:
zc = addr2zip(addr)
else:
return log("can't derive zip from blank address!!!", important=True)
return zc and getzip(zc).key
def scan(etype):
fname = "%s_1997_2015.xlsx"%(etype,)
log("Scanning %s"%(fname,), important=True)
xls = getxls(read(os.path.join("scrapers", "data", fname)))
puts = []
for row in xls[1:]:
puts.append(SCANROW[etype](row))
return puts
def export():
log("Exporting Results", important=True)
# this part is anything but conventional,
# but justified because these functions take so long
if raw_input("export owners? [N/y]").lower().startswith("y"):
export_owners()
if raw_input("export buildings? [N/y]").lower().startswith("y"):
export_buildings()
export_parcels()
def build_dirs(self):
log("building directories", 1)
mkdir(self.pname)
os.chdir(self.pname)
mkdir("js")
mkdir("css")
mkdir("img")
mkdir("html")
mkdir("logs")
def __init__(self, server, channel, name=None):
Bot.num += 1
self.name = name or (self.__class__.__name__ + str(Bot.num))
self.server = server
self.channels = set()
self._set_defaults()
log("Bot Spawned: '%s'" % (self.name, ), 2)
channel.join(self)
self.server.bots[self.name] = self
def latlng2zip(self, lat, lng):
result = self._fetch("zip", lat, lng)
log("finding zip for lat %s and lng %s"%(lat, lng), 3)
if not len(result):
log("can't find zipcode!!!", important=True)
return None
if config.geo.zip == "google":
return [c["short_name"] for c in result[0]["address_components"] if c["types"][0] == "postal_code"][0]
return result[0]["postalCode"]
def chems(self):
pages = self.pages["index"]
log("scanning %s index pages"%(len(pages),), important=True)
for page in pages:
chemcodes = self.clist(page)
self.chemcodes += chemcodes
log("found %s chem pages"%(len(chemcodes),))
for code in chemcodes:
self.pages["chems"].append(self.acquire(CURL%(code,), CHEMS))
def acquire(self, url, path):
fname = url.split("/").pop()
fpath = os.path.join(path, fname)
if os.path.exists(fpath):
return read(fpath)
log("acquiring: %s"%(url,))
data = fetch(url)
write(data, fpath)
return data
def _ll(addrs):
i = 0
for b in addrs:
b.latitude, b.longitude = address2latlng("%s, san francisco, CA"%(b.address,))
i += 1
if not i % 100:
log("processed %s"%(i,), important=True)
log("saving %s building records..."%(len(addrs),), important=True)
return addrs
def make_files(self):
log("generating configuration", 1)
cp((self.web_backend == "gae") and "%s\r\n%s" %
(config.init.yaml.gae, config.init.yaml.core) % (self.pname, )
or config.init.yaml.core, "app.yaml")
cp(config.init.ctcfg % (self.web_backend, ), "ct.cfg")
log("demo index page", 1)
cp(config.init.html % (self.pname, ),
os.path.join("html", "index.html"))
def build_dirs(self):
log("building directories", 1)
mkdir(self.pname)
os.chdir(self.pname)
mkdir("js")
mkdir("css")
mkdir("img")
mkdir("html")
mkdir("logs")
def getDates(page):
datelines, page = page.split("\n\n", 1)
dates = []
for dateline in datelines.split("\n"):
if "/" in dateline:
ds = [int(d) for d in dateline.split(" ")[-1].split("/")]
dates.append(datetime(ds[2], ds[0], ds[1]))
log("got %s dates"%(len(dates),), 1)
return dates, page
def on_subscribe(self, data):
log("Concierge received subscribe: %s"%(json.dumps(data),))
if data["user"].split("_")[0] != "host":
self.server.pm({
"user": data["user"],
"message": {
"action": "list",
"data": self.games
}
}, self)
def getWPmails():
import MySQLdb
h, u, p, d = read(".c").strip().split("|")
log("extracting email list from WP", 1)
conn = MySQLdb.connect(host=h, user=u, passwd=p, db=d)
cur = conn.cursor()
cur.execute("SELECT user_email FROM wp_users")
rowz = cur.fetchall()
log("found %s recipients"%(len(rowz),), 1)
return [r[0] for r in rowz]
def payRes():
log("payRes!", important=True)
yesterday = datetime.now() - timedelta(1)
for res in Resource.query(Resource.created > yesterday).all():
pod = Pod.query(Pod.resources.contains(res.key.urlsafe())).get()
person = res.editors[0].get()
log("paying %s of '%s' pod for posting '%s' resource" %
(person.firstName, pod.name, resource.name))
pod.deposit(person, ratios.resource, resource,
"resource: %s" % (resource.name, ), resource.description)
def on_subscribe(self, data):
log("Concierge received subscribe: %s" % (json.dumps(data), ))
if data["user"].split("_")[0] != "host":
self.server.pm(
{
"user": data["user"],
"message": {
"action": "list",
"data": self.games
}
}, self)
def symage(path):
ext = magic.from_file(path).split(" ").pop(0).lower()
if ext not in ["png", "jpeg"]:
log("converting %s to png!" % (ext, ))
cmd("convert -append -alpha off %s %s.png" % (path, path))
cmd("mv %s.png %s" % (path, path))
ext = "png"
sname = "%s.%s" % (path.replace("blob", "build"), ext)
if not os.path.exists(sname):
sym("../%s" % (path, ), sname)
return sname
def refmap():
log("compiling back reference map")
rmap = {}
for tname, schema in get_schema().items():
for pname, kinds in schema["_kinds"].items():
reference = "%s.%s"%(tname, pname)
counts[reference] = 0
for kind in [k for k in kinds if k != "*"]: # skip wildcard for now
if kind not in rmap:
rmap[kind] = {}
rmap[kind][reference] = get_keys(kind, reference)
return rmap
def verify(self, person):
if person in self.signers():
if Verification.query(Verification.act == self.key,
Verification.person == person).get():
return log("already verified (%s %s)!" % (self.key, person),
important=True)
log("verification (%s %s) success" %
(self.key.urlsafe(), person.urlsafe()))
Verification(act=self.key, person=person).put()
return self.fulfill()
log("verification attempt (%s %s) failed -- unauthorized" %
(self.key, person))
def address2latlng(self, address):
if self._no_cache(address):
log("finding lat/lng for %s" % (address, ), 3)
results = self._fetch("latlng", address)
if not len(results):
log("no results!!!", 4)
self.cache[address] = {"lat": None, "lng": None}
else:
loc = results[0]['geometry']['location']
self.cache[address] = {"lat": loc["lat"], "lng": loc["lng"]}
self.savecache()
return [self.cache[address]['lat'], self.cache[address]['lng']]
def latlng2zip(self, lat, lng):
result = self._fetch("zip", lat, lng)
log("finding zip for lat %s and lng %s" % (lat, lng), 3)
if not len(result):
log("can't find zipcode!!!", important=True)
return None
if config.geo.zip == "google":
return [
c["short_name"] for c in result[0]["address_components"]
if c["types"][0] == "postal_code"
][0]
return result[0]["postalCode"]
def scrape(self):
self.data["name"] = self.extract("<h1>", t="<")
log("%s %s"%(self.code, self.data["name"]), important=True)
self.data["classification"] = self.classification()
for name, flag in list(CARDS.items()):
self.data[name] = self.card(flag)
for name in TCARDS:
self.data[name] = self.tcard(name)
for name in HCARDS:
self.data[name] = self.tcard(name, False)
for name in BCARDS:
self.data[name.lower().replace("/", " ").replace(" ",
"_")] = self.card(name, False, 'card-body">')
def process():
log("Processing %s Owners (%s w/ SF addresses)"%(counts["owners"],
counts["owners_in_sf"]), important=True)
for owner in owners:
o = owners[owner]
for ptype in ["buildings", "parcels"]:
o[ptype] = len(o[ptype])
counts[ptype] += o[ptype]
compMosts(ptype, owner, o)
compMosts("dwelling_units", owner, o)
for building in buildings:
b = buildings[building]
b["parcels"] = len(b["parcels"])
def refmap():
log("compiling back reference map")
rmap = {}
for tname, schema in get_schema().items():
for pname, kinds in schema["_kinds"].items():
reference = "%s.%s" % (tname, pname)
counts[reference] = 0
for kind in [k for k in kinds
if k != "*"]: # skip wildcard for now
if kind not in rmap:
rmap[kind] = {}
rmap[kind][reference] = get_keys(kind, reference)
return rmap
def export_buildings():
i = 0
blist = []
bitems = buildings.items()
log("processing %s buildings"%(len(bitems),), 1)
for b_id, bdata in bitems:
addr = bdata["address"]
building = Building.query(Building.address == addr).get()
owner = Owner.query(Owner.name == bdata["owner"]).get()
byear = bdata["year"]
btype = bdata["building_type"]
if not building:
log("Can't find building '%s' -- creating new entry"%(addr,), 2)
building = Building(address=addr)
if owner:
building.owner = owner.key
if byear:
building.year = byear
if btype:
building.building_type = btype
if b_id:
building.building_id = b_id
if not building.zipcode:
zc = _zip(bdata["zipcode"], addr)
if zc:
building.zipcode = zc
if not building.latitude or not building.longitude:
building.latitude, building.longitude = address2latlng(building.address)
blist.append(building)
i += 1
if not i % 100:
log("processed %s buildings"%(i,), 2)
log("saving buildings", 1)
db.put_multi(blist)
def __init__(self):
log("initializing niosh scraper")
for p in [NDIR, INDEX, CHEMS]:
if not os.path.isdir(p):
mkdir(p)
self.pages = {
"index": [],
"chems": []
}
self.chemcodes = []
self.chemicals = []
self.download()
self.process()
self.save()
log("scrape complete")
def do_batch(chunk, reference):
log("refreshing %s %s keys" % (len(chunk), reference), 1)
i = 0
rc = []
for item in chunk: # item is single-member tuple
rc.append(refresh_counter(item[0], reference))
i += 1
if not i % 100:
log("processed %s" % (i, ), 3)
counts[reference] += len(chunk)
counts["_counters"] += len(rc)
log("refreshed %s total" % (counts[reference], ), 2)
log("updated %s counters" % (counts["_counters"], ), 2)
put_multi(rc)
log("saved", 2)
def do_batch(chunk, reference):
log("refreshing %s %s keys"%(len(chunk), reference), 1)
i = 0
rc = []
for item in chunk: # item is single-member tuple
rc.append(refresh_counter(item[0], reference))
i += 1
if not i % 100:
log("processed %s"%(i,), 3)
counts[reference] += len(chunk)
counts["_counters"] += len(rc)
log("refreshed %s total"%(counts[reference],), 2)
log("updated %s counters"%(counts["_counters"],), 2)
put_multi(rc)
log("saved", 2)
def doyaml(mode):
log("switching to %s mode"%(mode,))
lines = read(config.yaml.path, lines=True)
f = open(config.yaml.path, 'w')
m = None
for line in lines:
if line.startswith(config.yaml.start):
m = line[len(config.yaml.start):].strip()
elif line.startswith(config.yaml.end):
m = None
elif m == mode:
line = line.strip("#")
elif m:
line = "#%s"%(line.strip("#"),)
f.write(line)
f.close()
def refresh(self, cbatch):
from .util import getContribution
freshies = fetch("api.github.com",
"/repos/%s/%s/contributors" % (self.owner, self.repo),
asjson=True,
protocol="https")
pcount = 0
ccount = 0
for item in freshies:
log("checking for: %s" % (item["login"], ), 1)
contrib = getContribution(self, item["login"])
if contrib:
pcount += 1
ccount += contrib.refresh(item["contributions"], cbatch)
return "%s/%s: %s contributors, %s contributions" % (
self.owner, self.repo, pcount, ccount)
def doyaml(mode):
log("switching to %s mode" % (mode, ))
lines = read(config.yaml.path, lines=True)
f = open(config.yaml.path, 'w')
m = None
for line in lines:
if line.startswith(config.yaml.start):
m = line[len(config.yaml.start):].strip()
elif line.startswith(config.yaml.end):
m = None
elif m == mode:
line = line.strip("#")
elif m:
line = "#%s" % (line.strip("#"), )
f.write(line)
f.close()
def processhtml(html):
html = html.replace("{", "{").replace("}", "}").replace(
"</body>", "%s</body>" % (config.noscript, ))
firststart = start = end = html.find(config.js.flag)
js = []
while start != -1:
start += config.js.offset
end = html.find('"', start)
if end == -1:
error("no closing quote in this file: %s" % (html, ))
js.append(html[start:end].strip("/"))
start = html.find(config.js.flag, end)
log("js: %s" % (js, ), 1)
if start == end:
return html, ""
return html[:firststart] + "{jsspot}" + html[end +
config.js.endoffset:], js
def response():
log("initiating cronscan", important=True)
if config.autoexpire: # in days
log("checking for expired VideoPosts", 1)
n = datetime.now()
cutoff = n - timedelta(int(config.autoexpire))
expired = VideoPost.query(VideoPost.modified < cutoff).all()
log("deactivating %s expired VideoPosts"%(len(expired),), 2)
for vp in expired:
vp.live = False
db.put_multi(expired)
if config.autohls:
vz = VideoPost.query().all()
log("scanning %s VideoPosts for missing hls files"%(len(vz),), 1)
for v in vz:
hlsify(v.video.urlsafe()[1:])
log("cronscan complete")
def address2latlng(self, address):
if self._no_cache(address):
log("finding lat/lng for %s"%(address,), 3)
results = self._fetch("latlng", address)
if not len(results):
log("no results!!!", 4)
self.cache[address] = {
"lat": None,
"lng": None
}
else:
loc = results[0]['geometry']['location']
self.cache[address] = {
"lat": loc["lat"],
"lng": loc["lng"]
}
self.savecache()
return [self.cache[address]['lat'], self.cache[address]['lng']]
def go():
if config.web.server != "dez":
error("ctindex only available for dez projects")
log("indexing foreignkey references throughout database")
import model # load schema
for kind, references in refmap().items():
log("processing table: %s"%(kind,), important=True)
for reference, keys in references.items():
batch(keys, do_batch, reference)
tcount = sum(counts.values()) - counts["_counters"]
log("refreshed %s rows and updated %s counters"%(tcount, counts["_counters"]), important=True)
log("goodbye")
def scan():
csv = getcsv(os.path.join("scrapers", "data", "Buildings.csv"))
log("Scanning Buildings.cvs (%s rows)"%(len(csv),), important=True)
for row in csv[1:]:
oname = row[18]
building_id = row[0]
parcel_id = row[1]
dwelling_units = int(row[3])
if oname not in owners:
log("new owner: %s"%(oname,), 1)
owners[oname] = {
"buildings": set(),
"parcels": set(),
"dwelling_units": 0,
"address": row[19],
"in_sf": "SAN FRANCISCO" in row[19]
}
counts["owners"] += 1
if owners[oname]["in_sf"]:
counts["owners_in_sf"] += 1
log("sf native!", 2)
# owners[oname]["address"] = owners[oname]["address"] or row[19] # probably not necessary...
if building_id not in buildings:
addr = ("%s %s %s"%(row[7], row[9], row[10])).strip()
zc = row[15].strip()
log("new building: %s @ %s (%s)"%(building_id, addr, zc), 1)
buildings[building_id] = {
"owner": oname,
"parcels": set(),
"dwelling_units": 0,
"year": int(row[4]),
"building_type": row[2],
"address": addr,
"zipcode": zc
}
parcels[parcel_id] = {
"building": building_id,
"dwelling_units": dwelling_units,
"from_addr": row[7],
"to_addr": row[8]
}
owner = owners[oname]
owner["buildings"].add(building_id)
owner["parcels"].add(parcel_id)
owner["dwelling_units"] += dwelling_units
building = buildings[building_id]
building["parcels"].add(parcel_id)
building["dwelling_units"] += dwelling_units
counts["dwelling_units"] += dwelling_units
def response():
log("cronswarm (db)", important=True)
if dbcfg.peers:
cutoff = {
"value": datetime.now() - timedelta(seconds=dbcfg.interval),
"comparator": ">="
}
for modname, schema in schemas:
filters = {}
if "modified" in schema:
filters["modified"] = cutoff
elif "date" in schema:
filters["date"] = cutoff
else:
continue
for (host, port, protocol) in dbcfg.peers:
load_model(modname, host, port, db.session, filters, protocol,
config.cache("remote admin password? "), "edit",
blobifier)
log("cronswarm (db) complete")
def __init__(self, pname=None, cantools_path=HOME, web_backend="dez", refresh_symlinks=False):
if not pname and not refresh_symlinks:
pname = raw_input("project name? ")
log("Initializing %s Project: %s"%(web_backend, pname or "(refresh)"))
self.pname = pname
self.ctroot = os.path.join(cantools_path, "cantools", "cantools")
if not os.path.exists(self.ctroot):
error("failed to establish cantools root",
"We tried: %s"%(self.ctroot,),
"This directory does not contain cantools.",
"You can download cantools at https://github.com/bubbleboy14/cantools",
"Please pass the path to your local copy of cantools via the -c flag."
)
self.web_backend = web_backend
self.refresh_symlinks = refresh_symlinks
if not refresh_symlinks:
self.build_dirs()
self.make_files()
self.vcignore()
self.generate_symlinks()
log("done! goodbye.", 1)
def generate_symlinks(self):
log("creating symlinks", 1)
if self.refresh_symlinks:
if not os.path.isdir("js"):
mkdir("js")
if not os.path.isdir("css"):
mkdir("css")
if not os.path.isdir("img"):
mkdir("img")
if not os.path.isdir("logs"):
mkdir("logs")
lj = os.path.join("logs", "json")
if not os.path.isdir(lj):
mkdir(lj)
if self.web_backend == "gae":
sym(self.ctroot, "cantools")
sym(os.path.join(self.ctroot, "CT"), os.path.join("js", "CT"))
sym(os.path.join(self.ctroot, "css", "ct.css"), os.path.join("css", "ct.css"))
sym(os.path.join(self.ctroot, "admin"), "_")
sym(os.path.join(self.ctroot, "admin.py"), "admin.py")
sym(os.path.join(self.ctroot, "_db.py"), "_db.py")
def ll():
bs = Building.query(Building.latitude == None)
log("found %s lat/lng -less buildings"%(bs.count(),))
put_multi(_ll(bs.all()))
log("writing prettified geocache")
savecache(True)
log("done!")
def go():
if config.web.server != "dez":
error("ctindex only available for dez projects")
log("indexing foreignkey references throughout database")
import model # load schema
for kind, references in refmap().items():
log("processing table: %s" % (kind, ), important=True)
for reference, keys in references.items():
batch(keys, do_batch, reference)
tcount = sum(counts.values()) - counts["_counters"]
log("refreshed %s rows and updated %s counters" %
(tcount, counts["_counters"]),
important=True)
log("goodbye")
def getRentControl():
if not len(list(rcset)):
import os
from cantools.util import read, log
log("Parsing Rent Control Data", important=True)
rcd = read(os.path.join("scrapers", "data", "rent_control.geojson"),
isjson=True)["features"]
for z in zmap:
zmap[z]["rent_control"] = 0
rcp = [rc["properties"] for rc in rcd if rc["properties"]["address"]]
log("using %s of %s rows (omitting blank entries)"%(len(rcp), len(rcd)), 1)
for d in rcp:
zc = d["zipcode"]
rcset.add(d["address"])
if not zc in zmap:
zmap[zc] = {
"rent_control": 0,
"ad_hoc": True
}
zmap[zc]["rent_control"] += 1
zmap["SF"]["rent_control"] += 1
return rcset
def pvalsByYear(zmap):
log("writing pval csvs")
for zcode, zdata in zmap.items():
if "property_values" in zdata["2008"]:
log("writing %s.csv"%(zcode,), 1)
write(pvalcsv(zdata), path("logs", "csv", "%s.csv"%(zcode,)))
else:
log("skipping %s"%(zcode,), 1)
