def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['job']) or is_null(row['permitsino']): continue if is_null(row['issuancedate']): continue yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['job']) or is_null(row['jobs1no']): continue if is_null(row['latestactiondate']): continue # if does_not_contain_values(["a1", "a2", "dm", "nb"], row["jobtype"]): # continue yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if not is_null(row['waterdebtonly']): waterdebtonly = row['waterdebtonly'] == 'YES' row['waterdebtonly'] = waterdebtonly if not is_null(row['month']): month = row['month'].split('/')[0] year = row['month'].split('/')[1] row['month'] = month row['year'] = year yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: try: if is_null(row['indexno']): continue if is_null(row['bbl']): continue row['key'] = "{}-{}".format(row['indexno'], row['bbl']) row['bbl'] = row['bbl'].replace('-', '') yield row except Exception as e: logger.warning(row) logger.warning(e) continue
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['bin']): continue if is_null(row['lot']): continue if is_null(row['block']): continue if is_null(row['hhnd']): continue if is_null(row['lhnd']): continue row['stname'] = clean_number_and_streets(row['stname'], False, clean_typos=False) yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['bin']): continue if is_null(row['lot']): continue if is_null(row['block']): continue if is_null(row['hhnd']): continue if is_null(row['lhnd']): continue row['stname'] = clean_number_and_streets( row['stname'], False, clean_typos=False) row['key'] = re.sub( ' ', '', "{}{}-{}{}".format(row['bin'], row['lhnd'], row['hhnd'], row['stname'])) yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['documentid']): continue # add primary key row['key'] = "{}-{}".format(row['documentid'], row['bbl']) yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['documentid']): continue row['key'] = "{}-{}".format( row['documentid'], ''.join(e for e in row['name'] if e.isalnum())) yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['ucbbl']): continue row['ucbbl'] = str(row['ucbbl']) row['id'] = row['ucbbl'] year_cursor = self.MANUAL_YEAR while 'latestuctotals' not in row and year_cursor > 2006: key = "uc{}".format(year_cursor) if key in row and (row[key]): row['latestuctotals'] = row[key] year_cursor -= 1 yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['key']): continue yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['courtindexnumber']): continue yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['registrationcontactid']): continue yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['jobfilingnumber']) or is_null( row['workpermit']) or is_null(row['issueddate']): continue yield row
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['documentid']): continue yield row return gen_rows
def pre_validation_filters(self, gen_rows): for row in gen_rows: if is_null(row['ecbviolationnumber']): continue yield row