Esempio n. 1
0
    def clean_list_record(self, record):
        record['CALL_DATE'] = parse_date(record['CALL_DATE'], '%Y-%m-%d')
        record['OFFENSE_DATE'] = parse_date(record['OFFENSE_DATE'], '%Y-%m-%d')
        record['REPORT_DATE'] = parse_date(record['REPORT_DATE'], '%Y-%m-%d')
        record['COMMON_LOCATION'] = record['COMMON_LOCATION'].strip()
        record['ADDRESS_NBR'] = record['ADDRESS_NBR'].strip()
        record['ADDRESS'] = record['ADDRESS'].strip()

        # The 'NARRATIVE' field includes time and disposition data. Parse that out.
        m = re.search(
            r'^Time: (?P<TIME>\d\d?:\d\d)<br>.*?<br>Disposition: (?P<DISPOSITION>.*)$',
            record.pop('NARRATIVE'))
        record.update(m.groupdict())

        record['TIME'] = parse_time(record['TIME'], '%H:%M')

        # Set location_name. The logic is different depending on the ADDRESS_TYPE.
        address_type = record['ADDRESS_TYPE']
        if address_type == 'PREMISE ADDRESS':
            record['location_name'] = '%s block of %s' % (
                record['ADDRESS_NBR'], clean_address(record['ADDRESS']))
        elif address_type == 'INTERSECTION':
            if '/' in record['ADDRESS']:
                streets = record['ADDRESS'].split('/')
                record['location_name'] = '%s and %s' % (clean_address(
                    streets[0]), clean_address(streets[1]))
            else:
                record['location_name'] = clean_address(record['ADDRESS'])
        elif address_type == 'GEO-OVERRIDE':
            record['location_name'] = clean_address(record['ADDRESS'])
        elif address_type == 'COMMON LOCATION':
            if record['ADDRESS_NBR'] and record['ADDRESS']:
                record['location_name'] = '%s %s' % (
                    record['ADDRESS_NBR'], clean_address(record['ADDRESS']))
            elif record['ADDRESS'] and record['COMMON_LOCATION']:
                record['location_name'] = '%s (%s)' % (clean_address(
                    record['ADDRESS']), clean_address(
                        record['COMMON_LOCATION']))
            elif record['COMMON_LOCATION']:
                record['location_name'] = clean_address(
                    record['COMMON_LOCATION'])
            elif record['ADDRESS']:
                record['location_name'] = clean_address(record['ADDRESS'])
            else:
                record['location_name'] = 'Unknown'
        else:
            record['location_name'] = 'Unknown'

        try:
            d = CUSTOM_CATEGORIES[record['ORIG_CRIMETYPE_NAME']]
        except KeyError:
            d = ('Unknown', 'Unknown')
        record['broad_category'], record['detail_category'] = d

        return record
Esempio n. 2
0
    def clean_list_record(self, record):
        record['CALL_DATE'] = parse_date(record['CALL_DATE'], '%Y-%m-%d')
        record['OFFENSE_DATE'] = parse_date(record['OFFENSE_DATE'], '%Y-%m-%d')
        record['REPORT_DATE'] = parse_date(record['REPORT_DATE'], '%Y-%m-%d')
        record['COMMON_LOCATION'] = record['COMMON_LOCATION'].strip()
        record['ADDRESS_NBR'] = record['ADDRESS_NBR'].strip()
        record['ADDRESS'] = record['ADDRESS'].strip()

        # The 'NARRATIVE' field includes time and disposition data. Parse that out.
        m = re.search(r'^Time: (?P<TIME>\d\d?:\d\d)<br>.*?<br>Disposition: (?P<DISPOSITION>.*)$', record.pop('NARRATIVE'))
        record.update(m.groupdict())

        record['TIME'] = parse_time(record['TIME'], '%H:%M')

        # Set location_name. The logic is different depending on the ADDRESS_TYPE.
        address_type = record['ADDRESS_TYPE']
        if address_type == 'PREMISE ADDRESS':
            record['location_name'] = '%s block of %s' % (record['ADDRESS_NBR'], clean_address(record['ADDRESS']))
        elif address_type == 'INTERSECTION':
            if '/' in record['ADDRESS']:
                streets = record['ADDRESS'].split('/')
                record['location_name'] = '%s and %s' % (clean_address(streets[0]), clean_address(streets[1]))
            else:
                record['location_name'] = clean_address(record['ADDRESS'])
        elif address_type == 'GEO-OVERRIDE':
            record['location_name'] = clean_address(record['ADDRESS'])
        elif address_type == 'COMMON LOCATION':
            if record['ADDRESS_NBR'] and record['ADDRESS']:
                record['location_name'] = '%s %s' % (record['ADDRESS_NBR'], clean_address(record['ADDRESS']))
            elif record['ADDRESS'] and record['COMMON_LOCATION']:
                record['location_name'] = '%s (%s)' % (clean_address(record['ADDRESS']), clean_address(record['COMMON_LOCATION']))
            elif record['COMMON_LOCATION']:
                record['location_name'] = clean_address(record['COMMON_LOCATION'])
            elif record['ADDRESS']:
                record['location_name'] = clean_address(record['ADDRESS'])
            else:
                record['location_name'] = 'Unknown'
        else:
            record['location_name'] = 'Unknown'

        try:
            d = CUSTOM_CATEGORIES[record['ORIG_CRIMETYPE_NAME']]
        except KeyError:
            d = ('Unknown', 'Unknown')
        record['broad_category'], record['detail_category'] = d

        return record
Esempio n. 3
0
 def clean_list_record(self, record):
     record['crime_date'] = parse_date(record['crime_date'], '%m-%d-%Y')
     record['crime_time'] = parse_time(record['crime_time'], '%I:%M:%S %p')
     record['lon'] = float(record['lon'])
     record['lat'] = float(record['lat'])
     return record
Esempio n. 4
0
 def clean_list_record(self, record):
     record['crime_date'] = parse_date(record['crime_date'], '%m-%d-%Y')
     record['crime_time'] = parse_time(record['crime_time'], '%I:%M:%S %p')
     record['lon'] = float(record['lon'])
     record['lat'] = float(record['lat'])
     return record
Esempio n. 5
0
def _item_create(info):
    info = copy.deepcopy(info)
    try:
        assert info.pop('type') == 'Feature'
        props = info['properties']
    except (KeyError, AssertionError):
        raise InvalidNewsItem({'type': 'not a valid GeoJSON Feature'})
    try:
        slug = props.pop('type', None)
        schema = models.Schema.objects.get(slug=slug)
    except (models.Schema.DoesNotExist):
        raise InvalidNewsItem({'type': 'schema %r does not exist' % slug})

    data = {'schema': schema.id}
    for key in ('title', 'description', 'url'):
        data[key] = props.pop(key, '')

    # If there are errors parsing the dates, keep the raw data and let
    # the ModelForm sort it out.
    pub_date = props.pop('pub_date', None)
    if pub_date:
        try:
            data['pub_date'] = normalize_datetime(pyrfc3339.parse(pub_date))
        except Exception:
            data['pub_date'] = pub_date
    else:
        data['pub_date'] = normalize_datetime(datetime.datetime.utcnow())
    item_date = props.pop('item_date', None)
    if item_date:
        try:
            data['item_date'] = parse_date(item_date, '%Y-%m-%d', False)
        except Exception:
            data['item_date'] = item_date
    else:
        try:
            data['item_date'] = data['pub_date'].date()
        except Exception:
            data['item_date'] = None

    data['location'], data['location_name'] = _get_location_info(
        info.get('geometry'), props.pop('location_name', None))
    if not data['location']:
        logger.warn("Saving NewsItem %s with no geometry" % data['title'])
    if not data['location_name']:
        logger.warn("Saving NewsItem %s with no location_name" % data['title'])


    from ebpub.db.forms import NewsItemForm
    form = NewsItemForm(data)
    if form.is_valid():
        item = form.save()
    else:
        raise InvalidNewsItem(form.errors)

    # Everything else goes in .attributes.
    attributes = {}
    for key, val in props.items():
        sf = models.SchemaField.objects.get(schema=schema, name=key)
        if sf.is_many_to_many_lookup():
            lookups = []
            for lookup_name in val:
                lookups.append(
                    models.Lookup.objects.get_or_create_lookup(sf, lookup_name))
            val = ','.join((str(lookup.id) for lookup in lookups))
        elif sf.is_lookup:
            val = models.Lookup.objects.get_or_create_lookup(sf, val)
        elif sf.is_type('date'):
            val = normalize_datetime(parse_date(val, '%Y-%m-%d'))
        elif sf.is_type('time'):
            val = normalize_datetime(parse_time(val, '%H:%M'))
        elif sf.is_type('datetime'):
            val = normalize_datetime(pyrfc3339.parse(datetime))
        attributes[key] = val
    item.attributes = attributes
    return item
Esempio n. 6
0
def _item_create(info):
    info = copy.deepcopy(info)
    try:
        assert info.pop('type') == 'Feature'
        props = info['properties']
    except (KeyError, AssertionError):
        raise InvalidNewsItem({'type': 'not a valid GeoJSON Feature'})
    try:
        slug = props.pop('type', None)
        schema = models.Schema.objects.get(slug=slug)
    except (models.Schema.DoesNotExist):
        raise InvalidNewsItem({'type': 'schema %r does not exist' % slug})

    data = {'schema': schema.id}
    for key in ('title', 'description', 'url'):
        data[key] = props.pop(key, '')

    # If there are errors parsing the dates, keep the raw data and let
    # the ModelForm sort it out.
    pub_date = props.pop('pub_date', None)
    if pub_date:
        try:
            data['pub_date'] = normalize_datetime(pyrfc3339.parse(pub_date))
        except Exception:
            data['pub_date'] = pub_date
    else:
        data['pub_date'] = normalize_datetime(datetime.datetime.utcnow())
    item_date = props.pop('item_date', None)
    if item_date:
        try:
            data['item_date'] = parse_date(item_date, '%Y-%m-%d', False)
        except Exception:
            data['item_date'] = item_date
    else:
        try:
            data['item_date'] = data['pub_date'].date()
        except Exception:
            data['item_date'] = None

    data['location'], data['location_name'] = _get_location_info(
        info.get('geometry'), props.pop('location_name', None))
    if not data['location']:
        logger.warn("Saving NewsItem %s with no geometry" % data['title'])
    if not data['location_name']:
        logger.warn("Saving NewsItem %s with no location_name" % data['title'])

    from ebpub.db.forms import NewsItemForm
    form = NewsItemForm(data)
    if form.is_valid():
        item = form.save()
    else:
        raise InvalidNewsItem(form.errors)

    # Everything else goes in .attributes.
    attributes = {}
    for key, val in props.items():
        sf = models.SchemaField.objects.get(schema=schema, name=key)
        if sf.is_many_to_many_lookup():
            lookups = []
            for lookup_name in val:
                lookups.append(
                    models.Lookup.objects.get_or_create_lookup(
                        sf, lookup_name))
            val = ','.join((str(lookup.id) for lookup in lookups))
        elif sf.is_lookup:
            val = models.Lookup.objects.get_or_create_lookup(sf, val)
        elif sf.is_type('date'):
            val = normalize_datetime(parse_date(val, '%Y-%m-%d'))
        elif sf.is_type('time'):
            val = normalize_datetime(parse_time(val, '%H:%M'))
        elif sf.is_type('datetime'):
            val = normalize_datetime(pyrfc3339.parse(datetime))
        attributes[key] = val
    item.attributes = attributes
    return item
Esempio n. 7
0
 def clean_list_record(self, record):
     record["crime_date"] = parse_date(record["crime_date"], "%m-%d-%Y")
     record["crime_time"] = parse_time(record["crime_time"], "%I:%M:%S %p")
     record["lon"] = float(record["lon"])
     record["lat"] = float(record["lat"])
     return record