def sync(self, data_source): logger.info('Starting to synchronize use certainty scores.') self.update_use_certainty_scores(count=data_source.batch_size) logger.info('Finished synchronizing use certainty scores.') def update_use_certainty_scores(self, count=1000): # Get lots that look like they haven't been updated and that we can # change lots = Lot.objects.filter(known_use_locked=False, known_use_certainty=0).order_by('?') for lot in lots[:count]: try: lot.known_use_certainty = lot.calculate_known_use_certainty() lot.save() except Exception: logger.warn('Caught exception while updating use certainty ' 'score for lot %s' % lot) external_data_sync.register(LotOwnershipSynchronizer) external_data_sync.register(TaxAccountSynchronizer) external_data_sync.register(LotsAvailablePropertiesSynchronizer) external_data_sync.register(LotsLandUseAreaSynchronizer) external_data_sync.register(ZoningSynchronizer) external_data_sync.register(WaterDeptSynchronizer) external_data_sync.register(LotsLILicensesSynchronizer) external_data_sync.register(LotsLIViolationsSynchronizer) external_data_sync.register(CityCouncilSynchronizer) external_data_sync.register(PlanningDistrictSynchronizer) external_data_sync.register(UseCertaintyScoresSynchronizer)
return parcels[:data_source.batch_size] return parcels def wait(self): """Wait a bit between parcels / external requests""" sleep(2 * random()) def sync(self, data_source): for parcel in self.pick_parcels(data_source): print parcel try: tax_bill_number = load_tax_bill_number(parcel) except Exception: print ('Could not find tax_bill_number for parcel with pk %d' % parcel.pk) if tax_bill_number: # TODO try to be more defensive about exceptions here code_lien_information = load_code_lien_information(tax_bill_number) print '\t', tax_bill_number for code_lien in code_lien_information: print '\t', code_lien create_or_update_lien(parcel, code_lien) update_parcel_lien_record(parcel, tax_bill_number) self.wait() register(LienSynchronizer) register(TaxBillNumberSynchronizer)
import external_data_sync from external_data_sync.synchronizers import Synchronizer from .adapter import find_licenses logger = logging.getLogger(__name__) class LILicensesSynchronizer(Synchronizer): """ A Synchronizer that updates L&I license data. """ codes = ( '3219', # residential vacancy license '3634', # commercial vacancy license ) def sync(self, data_source): logger.info('Starting to synchronize L&I license data.') self.update_license_data() logger.info('Finished synchronizing L&I license data.') def update_license_data(self): for code in self.codes: find_licenses(code, self.data_source.last_synchronized) external_data_sync.register(LILicensesSynchronizer)
from external_data_sync import register, synchronizers from .finders import VacantParcelFinder class VacantParcelSynchronizer(synchronizers.Synchronizer): def sync(self, data_source): finder = VacantParcelFinder() finder.find_lots(batch_size=data_source.batch_size) register(VacantParcelSynchronizer)
from .adapter import (find_available_properties, find_no_longer_available_properties) logger = logging.getLogger(__name__) class PRAAvailablePropertiesSynchronizer(Synchronizer): """ Attempts to synchronize the local database with the available properties as listed by the Philadelphia Redevelopment Authority. As the PRA's data does not include timestamps for available properties, we load all available properties every time we synchronize, marking new ones (status == 'new and available'), existing ones (status == 'available'), and properties no longer in the PRA's data (status == 'no longer available'). """ def sync(self, data_source): logger.info('Synchronizing available properties.') find_available_properties() logger.info('Done synchronizing available properties.') logger.info('Synchronizing no-longer available properties.') find_no_longer_available_properties(data_source.last_synchronized) logger.info('Done synchronizing no-longer available properties.') external_data_sync.register(PRAAvailablePropertiesSynchronizer)
import external_data_sync from external_data_sync.synchronizers import Synchronizer from .adapter import find_violations logger = logging.getLogger(__name__) class LIViolationsSynchronizer(Synchronizer): """ A Synchronizer that updates L&I Violation data. """ # L&I says these should be the useful codes codes = ('CP-802', 'PM-102.4/1', 'PM-302.2/4', 'PM-306.0/2', 'PM-306.0/91', 'PM-307.1/21',) def sync(self, data_source): logger.info('Starting to synchronize L&I Violation data.') self.update_violation_data() logger.info('Finished synchronizing L&I Violation data.') def update_violation_data(self): for code in self.codes: find_violations(code, self.data_source.last_synchronized) external_data_sync.register(LIViolationsSynchronizer)
logger = logging.getLogger(__name__) class LIViolationsSynchronizer(Synchronizer): """ A Synchronizer that updates L&I Violation data. """ # L&I says these should be the useful codes codes = ( 'CP-802', 'PM-102.4/1', 'PM-302.2/4', 'PM-306.0/2', 'PM-306.0/91', 'PM-307.1/21', ) def sync(self, data_source): logger.info('Starting to synchronize L&I Violation data.') self.update_violation_data() logger.info('Finished synchronizing L&I Violation data.') def update_violation_data(self): for code in self.codes: find_violations(code, self.data_source.last_synchronized) external_data_sync.register(LIViolationsSynchronizer)
from external_data_sync import register, synchronizers from .finders import VacantParcelFinder class VacantParcelSynchronizer(synchronizers.Synchronizer): def sync(self, data_source): finder = VacantParcelFinder() finder.find_lots(batch_size=data_source.batch_size) register(VacantParcelSynchronizer)
import logging import external_data_sync from external_data_sync.synchronizers import Synchronizer from .adapter import find_land_use_areas logger = logging.getLogger(__name__) class LandUseAreaSynchronizer(Synchronizer): """Synchronizes LandUseAreas.""" def sync(self, data_source): logger.info('Synchronizing vacant land use area data.') find_land_use_areas() logger.info('Done synchronizing vacant land use area data.') external_data_sync.register(LandUseAreaSynchronizer)
from external_data_sync import register, synchronizers from ladata.zoning.models import ZoningDistrict from .models import Lot class LotZoningSynchronizer(synchronizers.Synchronizer): def find_lots(self, batch_size=1000): return Lot.objects.filter(zoning_district=None)[:batch_size] def sync(self, data_source): lots = self.find_lots(batch_size=data_source.batch_size) for lot in lots: try: lot.zoning_district = ZoningDistrict.objects.get( geometry__contains=lot.centroid, ) lot.save() except ZoningDistrict.DoesNotExist: print 'No ZoningDistrict found for lot %d' % lot.pk except ZoningDistrict.MultipleObjectsReturned: print 'Multiple ZoningDistricts found for lot %d' % lot.pk register(LotZoningSynchronizer)
self.update_use_certainty_scores(count=data_source.batch_size) logger.info('Finished synchronizing use certainty scores.') def update_use_certainty_scores(self, count=1000): # Get lots that look like they haven't been updated and that we can # change lots = Lot.objects.filter( known_use_locked=False, known_use_certainty=0 ).order_by('?') for lot in lots[:count]: try: lot.known_use_certainty = lot.calculate_known_use_certainty() lot.save() except Exception: logger.warn('Caught exception while updating use certainty ' 'score for lot %s' % lot) external_data_sync.register(LotOwnershipSynchronizer) external_data_sync.register(TaxAccountSynchronizer) external_data_sync.register(LotsAvailablePropertiesSynchronizer) external_data_sync.register(LotsLandUseAreaSynchronizer) external_data_sync.register(ZoningSynchronizer) external_data_sync.register(WaterDeptSynchronizer) external_data_sync.register(LotsLILicensesSynchronizer) external_data_sync.register(LotsLIViolationsSynchronizer) external_data_sync.register(CityCouncilSynchronizer) external_data_sync.register(PlanningDistrictSynchronizer) external_data_sync.register(UseCertaintyScoresSynchronizer)
from external_data_sync import register, synchronizers from ladata.zoning.models import ZoningDistrict from .models import Lot class LotZoningSynchronizer(synchronizers.Synchronizer): def find_lots(self, batch_size=1000): return Lot.objects.filter(zoning_district=None)[:batch_size] def sync(self, data_source): lots = self.find_lots(batch_size=data_source.batch_size) for lot in lots: try: lot.zoning_district = ZoningDistrict.objects.get( geometry__contains=lot.centroid, ) lot.save() except ZoningDistrict.DoesNotExist: print 'No ZoningDistrict found for lot %d' % lot.pk except ZoningDistrict.MultipleObjectsReturned: print 'Multiple ZoningDistricts found for lot %d' % lot.pk register(LotZoningSynchronizer)