def sync(self, data_source):
logger.info('Starting to synchronize use certainty scores.')
self.update_use_certainty_scores(count=data_source.batch_size)
logger.info('Finished synchronizing use certainty scores.')
def update_use_certainty_scores(self, count=1000):
# Get lots that look like they haven't been updated and that we can
# change
lots = Lot.objects.filter(known_use_locked=False,
known_use_certainty=0).order_by('?')
for lot in lots[:count]:
try:
lot.known_use_certainty = lot.calculate_known_use_certainty()
lot.save()
except Exception:
logger.warn('Caught exception while updating use certainty '
'score for lot %s' % lot)
external_data_sync.register(LotOwnershipSynchronizer)
external_data_sync.register(TaxAccountSynchronizer)
external_data_sync.register(LotsAvailablePropertiesSynchronizer)
external_data_sync.register(LotsLandUseAreaSynchronizer)
external_data_sync.register(ZoningSynchronizer)
external_data_sync.register(WaterDeptSynchronizer)
external_data_sync.register(LotsLILicensesSynchronizer)
external_data_sync.register(LotsLIViolationsSynchronizer)
external_data_sync.register(CityCouncilSynchronizer)
external_data_sync.register(PlanningDistrictSynchronizer)
external_data_sync.register(UseCertaintyScoresSynchronizer)
return parcels[:data_source.batch_size]
return parcels
def wait(self):
"""Wait a bit between parcels / external requests"""
sleep(2 * random())
def sync(self, data_source):
for parcel in self.pick_parcels(data_source):
print parcel
try:
tax_bill_number = load_tax_bill_number(parcel)
except Exception:
print ('Could not find tax_bill_number for parcel with pk %d'
% parcel.pk)
if tax_bill_number:
# TODO try to be more defensive about exceptions here
code_lien_information = load_code_lien_information(tax_bill_number)
print '\t', tax_bill_number
for code_lien in code_lien_information:
print '\t', code_lien
create_or_update_lien(parcel, code_lien)
update_parcel_lien_record(parcel, tax_bill_number)
self.wait()
register(LienSynchronizer)
register(TaxBillNumberSynchronizer)
import external_data_sync
from external_data_sync.synchronizers import Synchronizer
from .adapter import find_licenses
logger = logging.getLogger(__name__)
class LILicensesSynchronizer(Synchronizer):
"""
A Synchronizer that updates L&I license data.
"""
codes = (
'3219', # residential vacancy license
'3634', # commercial vacancy license
)
def sync(self, data_source):
logger.info('Starting to synchronize L&I license data.')
self.update_license_data()
logger.info('Finished synchronizing L&I license data.')
def update_license_data(self):
for code in self.codes:
find_licenses(code, self.data_source.last_synchronized)
external_data_sync.register(LILicensesSynchronizer)
from external_data_sync import register, synchronizers
from .finders import VacantParcelFinder
class VacantParcelSynchronizer(synchronizers.Synchronizer):
def sync(self, data_source):
finder = VacantParcelFinder()
finder.find_lots(batch_size=data_source.batch_size)
register(VacantParcelSynchronizer)
from .adapter import (find_available_properties,
find_no_longer_available_properties)
logger = logging.getLogger(__name__)
class PRAAvailablePropertiesSynchronizer(Synchronizer):
"""
Attempts to synchronize the local database with the available properties
as listed by the Philadelphia Redevelopment Authority.
As the PRA's data does not include timestamps for available properties,
we load all available properties every time we synchronize, marking new
ones (status == 'new and available'), existing ones (status ==
'available'), and properties no longer in the PRA's data (status == 'no
longer available').
"""
def sync(self, data_source):
logger.info('Synchronizing available properties.')
find_available_properties()
logger.info('Done synchronizing available properties.')
logger.info('Synchronizing no-longer available properties.')
find_no_longer_available_properties(data_source.last_synchronized)
logger.info('Done synchronizing no-longer available properties.')
external_data_sync.register(PRAAvailablePropertiesSynchronizer)
import external_data_sync
from external_data_sync.synchronizers import Synchronizer
from .adapter import find_violations
logger = logging.getLogger(__name__)
class LIViolationsSynchronizer(Synchronizer):
"""
A Synchronizer that updates L&I Violation data.
"""
# L&I says these should be the useful codes
codes = ('CP-802', 'PM-102.4/1', 'PM-302.2/4', 'PM-306.0/2', 'PM-306.0/91',
'PM-307.1/21',)
def sync(self, data_source):
logger.info('Starting to synchronize L&I Violation data.')
self.update_violation_data()
logger.info('Finished synchronizing L&I Violation data.')
def update_violation_data(self):
for code in self.codes:
find_violations(code, self.data_source.last_synchronized)
external_data_sync.register(LIViolationsSynchronizer)
logger = logging.getLogger(__name__)
class LIViolationsSynchronizer(Synchronizer):
"""
A Synchronizer that updates L&I Violation data.
"""
# L&I says these should be the useful codes
codes = (
'CP-802',
'PM-102.4/1',
'PM-302.2/4',
'PM-306.0/2',
'PM-306.0/91',
'PM-307.1/21',
)
def sync(self, data_source):
logger.info('Starting to synchronize L&I Violation data.')
self.update_violation_data()
logger.info('Finished synchronizing L&I Violation data.')
def update_violation_data(self):
for code in self.codes:
find_violations(code, self.data_source.last_synchronized)
external_data_sync.register(LIViolationsSynchronizer)
from external_data_sync import register, synchronizers
from .finders import VacantParcelFinder
class VacantParcelSynchronizer(synchronizers.Synchronizer):
def sync(self, data_source):
finder = VacantParcelFinder()
finder.find_lots(batch_size=data_source.batch_size)
register(VacantParcelSynchronizer)
import logging
import external_data_sync
from external_data_sync.synchronizers import Synchronizer
from .adapter import find_land_use_areas
logger = logging.getLogger(__name__)
class LandUseAreaSynchronizer(Synchronizer):
"""Synchronizes LandUseAreas."""
def sync(self, data_source):
logger.info('Synchronizing vacant land use area data.')
find_land_use_areas()
logger.info('Done synchronizing vacant land use area data.')
external_data_sync.register(LandUseAreaSynchronizer)
from external_data_sync import register, synchronizers
from ladata.zoning.models import ZoningDistrict
from .models import Lot
class LotZoningSynchronizer(synchronizers.Synchronizer):
def find_lots(self, batch_size=1000):
return Lot.objects.filter(zoning_district=None)[:batch_size]
def sync(self, data_source):
lots = self.find_lots(batch_size=data_source.batch_size)
for lot in lots:
try:
lot.zoning_district = ZoningDistrict.objects.get(
geometry__contains=lot.centroid, )
lot.save()
except ZoningDistrict.DoesNotExist:
print 'No ZoningDistrict found for lot %d' % lot.pk
except ZoningDistrict.MultipleObjectsReturned:
print 'Multiple ZoningDistricts found for lot %d' % lot.pk
register(LotZoningSynchronizer)
self.update_use_certainty_scores(count=data_source.batch_size)
logger.info('Finished synchronizing use certainty scores.')
def update_use_certainty_scores(self, count=1000):
# Get lots that look like they haven't been updated and that we can
# change
lots = Lot.objects.filter(
known_use_locked=False,
known_use_certainty=0
).order_by('?')
for lot in lots[:count]:
try:
lot.known_use_certainty = lot.calculate_known_use_certainty()
lot.save()
except Exception:
logger.warn('Caught exception while updating use certainty '
'score for lot %s' % lot)
external_data_sync.register(LotOwnershipSynchronizer)
external_data_sync.register(TaxAccountSynchronizer)
external_data_sync.register(LotsAvailablePropertiesSynchronizer)
external_data_sync.register(LotsLandUseAreaSynchronizer)
external_data_sync.register(ZoningSynchronizer)
external_data_sync.register(WaterDeptSynchronizer)
external_data_sync.register(LotsLILicensesSynchronizer)
external_data_sync.register(LotsLIViolationsSynchronizer)
external_data_sync.register(CityCouncilSynchronizer)
external_data_sync.register(PlanningDistrictSynchronizer)
external_data_sync.register(UseCertaintyScoresSynchronizer)
from external_data_sync import register, synchronizers
from ladata.zoning.models import ZoningDistrict
from .models import Lot
class LotZoningSynchronizer(synchronizers.Synchronizer):
def find_lots(self, batch_size=1000):
return Lot.objects.filter(zoning_district=None)[:batch_size]
def sync(self, data_source):
lots = self.find_lots(batch_size=data_source.batch_size)
for lot in lots:
try:
lot.zoning_district = ZoningDistrict.objects.get(
geometry__contains=lot.centroid,
)
lot.save()
except ZoningDistrict.DoesNotExist:
print 'No ZoningDistrict found for lot %d' % lot.pk
except ZoningDistrict.MultipleObjectsReturned:
print 'Multiple ZoningDistricts found for lot %d' % lot.pk
register(LotZoningSynchronizer)