def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['job']) or is_null(row['permitsino']):
continue
if is_null(row['issuancedate']):
continue
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['job']) or is_null(row['jobs1no']):
continue
if is_null(row['latestactiondate']):
continue
# if does_not_contain_values(["a1", "a2", "dm", "nb"], row["jobtype"]):
# continue
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if not is_null(row['waterdebtonly']):
waterdebtonly = row['waterdebtonly'] == 'YES'
row['waterdebtonly'] = waterdebtonly
if not is_null(row['month']):
month = row['month'].split('/')[0]
year = row['month'].split('/')[1]
row['month'] = month
row['year'] = year
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
try:
if is_null(row['indexno']):
continue
if is_null(row['bbl']):
continue
row['key'] = "{}-{}".format(row['indexno'], row['bbl'])
row['bbl'] = row['bbl'].replace('-', '')
yield row
except Exception as e:
logger.warning(row)
logger.warning(e)
continue
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['bin']):
continue
if is_null(row['lot']):
continue
if is_null(row['block']):
continue
if is_null(row['hhnd']):
continue
if is_null(row['lhnd']):
continue
row['stname'] = clean_number_and_streets(row['stname'],
False,
clean_typos=False)
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['bin']):
continue
if is_null(row['lot']):
continue
if is_null(row['block']):
continue
if is_null(row['hhnd']):
continue
if is_null(row['lhnd']):
continue
row['stname'] = clean_number_and_streets(
row['stname'], False, clean_typos=False)
row['key'] = re.sub(
' ', '', "{}{}-{}{}".format(row['bin'], row['lhnd'], row['hhnd'], row['stname']))
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['documentid']):
continue
# add primary key
row['key'] = "{}-{}".format(row['documentid'], row['bbl'])
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['documentid']):
continue
row['key'] = "{}-{}".format(
row['documentid'],
''.join(e for e in row['name'] if e.isalnum()))
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['ucbbl']):
continue
row['ucbbl'] = str(row['ucbbl'])
row['id'] = row['ucbbl']
year_cursor = self.MANUAL_YEAR
while 'latestuctotals' not in row and year_cursor > 2006:
key = "uc{}".format(year_cursor)
if key in row and (row[key]):
row['latestuctotals'] = row[key]
year_cursor -= 1
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['key']):
continue
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['courtindexnumber']):
continue
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['registrationcontactid']):
continue
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['jobfilingnumber']) or is_null(
row['workpermit']) or is_null(row['issueddate']):
continue
yield row
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['documentid']):
continue
yield row
return gen_rows
def pre_validation_filters(self, gen_rows):
for row in gen_rows:
if is_null(row['ecbviolationnumber']):
continue
yield row