def buildRecord(self, item):
record = BaseRecord()
payload = IrisPayload()
location = LocationRecord()
payload.setId(GeneralHelper().default(item['FITXA_ID']))
payload.setArea(GeneralHelper().default(item['AREA']))
payload.setChannel(GeneralHelper().default(item['CANALS_RESPOSTA']))
payload.setDetail(GeneralHelper().default(item['DETALL']))
payload.setElement(GeneralHelper().default(item['ELEMENT']))
payload.setStartDate(TimeHelper().toDate(
GeneralHelper().default(item['ANY_DATA_ALTA']),
GeneralHelper().default(item['MES_DATA_ALTA']),
GeneralHelper().default(item['DIA_DATA_ALTA'])))
payload.setEndDate(TimeHelper().toDate(
GeneralHelper().default(item['ANY_DATA_TANCAMENT']),
GeneralHelper().default(item['MES_DATA_TANCAMENT']),
GeneralHelper().default(item['DIA_DATA_TANCAMENT'])))
payload.setSupport(GeneralHelper().default(item['SUPORT']))
location.setPoint(item['LATITUD'], item['LONGITUD'])
location.setAltitude('0.0')
location.setCity('Barcelona')
location.setDistrict(GeneralHelper().default(item['DISTRICTE']))
location.setNeighbourhood(GeneralHelper().default(item['BARRI']))
location.setStreetName(GeneralHelper().default(item['TIPUS_VIA']) +
' ' + GeneralHelper().default(item['CARRER']))
location.setStreetNumber(GeneralHelper().default(item['NUMERO']))
record.setId(GeneralHelper().default(item['FITXA_ID']))
record.setSource(
collectorCfg['collectors']['odi']['iris']['source_name'])
record.setProvider('')
record.setPublisher('')
record.setType(GeneralHelper().default(item['TIPUS']))
record.setTimestamp(TimeHelper().toDate(
GeneralHelper().default(item['ANY_DATA_ALTA']),
GeneralHelper().default(item['MES_DATA_ALTA']),
GeneralHelper().default(item['DIA_DATA_ALTA'])))
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item, type):
if type == "cityos:potencial_fotovoltaic":
LocationHelper().toWGS84Geometry(item['geometry'], 'EPSG:25831', isNested=True)
payload = CityOSPVPotentialPayload()
payload.setId(item['id'])
payload.setPowTh(item['properties']['POW_TH'])
payload.setEventCode(item['properties']['EventCode'])
payload.setSuitability(item['properties']['SUITABILITY'])
payload.setSumModare(item['properties']['SUM_MODARE'])
payload.setPolygon(item['geometry'])
elif type == "cityos:ptt_carril_bici":
LocationHelper().toWGS84Geometry(item['geometry'], 'EPSG:25831')
payload = CityOSBikeLinesPayload()
payload.setId(item['id'])
payload.setId1(item['properties']['ID1'])
payload.setId2(item['properties']['ID2'])
payload.setEventCode(item['properties']['EventCode'])
payload.setLine(item['geometry'])
location = LocationRecord()
coords = shape(item['geometry']).centroid
longitude, latitude = coords.x, coords.y
location.setPoint(latitude, longitude)
location.setAltitude(GeneralHelper().default(0))
location.setCity('')
district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
location.setStreetName(GeneralHelper().default(''))
location.setStreetNumber(GeneralHelper().default(''))
record = BaseRecord()
record.setId(item['id'])
record.setSource(type.split(':')[0] + '-' + type.split(':')[1])
record.setProvider(type.split(':')[0])
record.setPublisher('')
record.setType(type.split(':')[1])
record.setTimestamp(item['properties']['MD_DATA_PUBLISHED'])
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item):
record = BaseRecord()
payload = DecidimProposalPayload()
location = LocationRecord()
payload.setId(GeneralHelper().default(item['id']))
payload.setTitle(GeneralHelper().default(item['title']))
payload.setVoteCount(GeneralHelper().default(item['voteCount']))
payload.setTotalCommentsCount(GeneralHelper().default(
item['totalCommentsCount']))
if 'category' in item:
payload.setCategory(GeneralHelper().default(
item['category']['name']['translations'][0]['text']))
if 'scope' in item:
coords = shape(LocationHelper().getAreaGeometry(
item['scope']['name']['translations'][0]['text'].replace(
' - ', '-'), 'neighbourhood_group')).centroid
location.setDistrict(GeneralHelper().default(
item['scope']['name']['translations'][0]['text'].replace(
' - ', '-').strip()))
payload.setDistrict(GeneralHelper().default(
item['scope']['name']['translations'][0]['text'].replace(
' - ', '-').strip()))
location.setNeighbourhood(GeneralHelper().default(''))
longitude, latitude = coords.x, coords.y
location.setPoint(latitude, longitude)
location.setAltitude(0.0)
location.setCity('Barcelona')
location.setStreetName(GeneralHelper().default(''))
location.setStreetNumber(GeneralHelper().default(''))
record.setId(GeneralHelper().default(item['reference']))
record.setSource(source_name)
record.setProvider('decidim')
record.setPublisher('bcnnow')
record.setType(source_name)
record.setTimestamp(item['publishedAt'])
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item, community_id):
record = BaseRecord()
payload = IoTPayload()
location = LocationRecord()
item['recordedAt'] = (datetime.strptime(item['recordedAt'], '%Y-%m-%dT%H:%M:%SZ')+timedelta(hours=2)).strftime('%Y-%m-%dT%H:%M:%S')
item['label'] = re.sub('[^0-9a-zA-Z]+', '-', item['label'])
latitude, longitude = GeneralHelper().default(item['latitude']), GeneralHelper().default(item['longitude'])
location.setPoint(latitude, longitude)
location.setAltitude(0.0)
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
payload.setId(GeneralHelper().default(item['label']))
payload.setType(GeneralHelper().default(item['type']))
payload.setName(GeneralHelper().default(item['name']))
payload.setDescription(GeneralHelper().default(item['description']))
payload.setUnit(GeneralHelper().default(item['unit']))
if 'value' in item:
payload.setValue(GeneralHelper().default(item['value']))
elif 'values' in item:
value = 0
values = item['values']
bins = item['bins']
for i in range(len(bins)):
value += bins[i] * values[i+1]
payload.setValue(GeneralHelper().default(value))
payload.setRecordedAt(GeneralHelper().default(item['recordedAt']))
payload.setExposure(GeneralHelper().default(item['exposure']))
record.setId(GeneralHelper().default(item['label']+'.'+item['recordedAt']))
record.setSource(('iot__'+community_id+'__'+item['name'].lower().split()[0]).replace('-','_').replace('.','_'))
record.setProvider('smartcitizen')
record.setPublisher('bcnnow')
record.setType('event')
record.setTimestamp(item['recordedAt'])
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item):
record = BaseRecord()
payload = DecidimMeetingPayload()
location = LocationRecord()
latitude, longitude = GeneralHelper().default(item['coordinates']['latitude']), GeneralHelper().default(item['coordinates']['longitude'])
location.setPoint(latitude, longitude)
location.setAltitude(0.0)
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
location.setStreetName(GeneralHelper().default(item['address'].split(',')[0]))
location.setStreetNumber(GeneralHelper().default(int(item['address'].split(',')[1]) if len(item['address'].split(',')[1].strip()) in [1,2] else '') if len(item['address'].split(','))>1 else '')
payload.setId(GeneralHelper().default(item['reference']))
payload.setAttendeeCount(GeneralHelper().default(item['attendeeCount']))
payload.setTotalCommentsCount(GeneralHelper().default(item['totalCommentsCount']))
payload.setContributionCount(GeneralHelper().default(item['contributionCount']))
payload.setTitle(GeneralHelper().default(item['title']['translations'][1]['text']))
payload.setStartTime(GeneralHelper().default(item['startTime']))
payload.setEndTime(GeneralHelper().default(item['endTime']))
payload.setAddress(GeneralHelper().default(item['address']))
attachments = []
for attachment in item['attachments']:
attachments.append(attachment['url'])
payload.setAttachments(GeneralHelper().default(attachments))
record.setId(GeneralHelper().default(item['reference']))
record.setSource(source_name)
record.setProvider('decidim')
record.setPublisher('bcnnow')
record.setType(source_name)
record.setTimestamp(item['startTime'])
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item, start_date):
record = BaseRecord()
payload = BicingPayload()
location = LocationRecord()
payload.setId(item['id'])
payload.setSlots(item['slots'])
payload.setNearbyStationIDs(
[station for station in item['nearbyStations'].split(' ')])
payload.setBikes(item['bikes'])
payload.setType(item['type'])
payload.setStatus(item['status'])
longitude, latitude = GeneralHelper().default(
item['longitude']), GeneralHelper().default(item['latitude'])
location.setPoint(latitude, longitude)
location.setAltitude(GeneralHelper().default(item['altitude']))
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(
latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
location.setStreetName(GeneralHelper().default(item['streetName']))
location.setStreetNumber(GeneralHelper().default(item['streetNumber']))
record.setId(item['id'] + '.' + start_date)
record.setSource(
collectorCfg['collectors']['odi']['bicing']['source_name'])
record.setProvider('')
record.setPublisher('')
record.setType(
collectorCfg['collectors']['odi']['bicing']['source_name'])
record.setTimestamp(start_date)
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item, type):
record = BaseRecord()
payload = OHBPayload()
location = LocationRecord()
payload.setPeriod(item['PERIODE'])
payload.setValue(item['ind_value'])
if 'Districts' in type:
coords = shape(LocationHelper().getAreaGeometry(
item['NOM'], 'neighbourhood_group')).centroid
location.setDistrict(GeneralHelper().default(item['NOM'].strip()))
location.setNeighbourhood(GeneralHelper().default(''))
else:
coords = shape(LocationHelper().getAreaGeometry(
item['NOM'], 'neighbourhood')).centroid
location.setDistrict(GeneralHelper().default(''))
location.setNeighbourhood(GeneralHelper().default(
item['NOM'].strip()))
longitude, latitude = coords.x, coords.y
location.setPoint(latitude, longitude)
location.setAltitude(0.0)
location.setCity('Barcelona')
location.setStreetName(GeneralHelper().default(''))
location.setStreetNumber(GeneralHelper().default(''))
record.setId(str(item['PERIODE'])[:4] + '_' + item['CODI_OHB'])
record.setSource(collectorCfg['collectors']['OHB']['source_name'])
record.setProvider('')
record.setPublisher('')
record.setType(type.replace('.csv', ''))
record.setTimestamp(str(item['PERIODE'])[:4] + '-01-01T00:00:00Z')
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, observation, provider, sensor):
record = BaseRecord()
payload = SentiloPayload()
location = LocationRecord()
payload.setValue(observation['value'])
payload.setType(sensor['type'])
payload.setDescription(sensor['description'])
payload.setEmplacament(sensor['additionalInfo']['emplaçament']
if 'additionalInfo' in sensor else '')
payload.setUnit(sensor['unit'])
longitude, latitude = sensor['location'].split(
' ')[1], sensor['location'].split(' ')[0]
location.setPoint(latitude, longitude)
location.setAltitude(sensor['additionalInfo']['altitud']
if 'additionalInfo' in sensor else '')
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(
latitude, longitude)
location.setDistrict(district)
location.setNeighbourhood(neighbourhood)
location.setStreetName('')
location.setStreetNumber('')
record.setId(sensor['sensor'] + '.' +
observation['timestamp'].replace(' ', ''))
record.setSource(collectorCfg['collectors']['sentilo']['source_name'])
record.setProvider(provider['provider'])
record.setPublisher(sensor['sensor'])
record.setType(sensor['componentType'])
record.setTimestamp(observation['timestamp'])
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item):
record = BaseRecord()
payload = InsideAirbnbReviewPayload()
location = LocationRecord()
payload.setId(item['id'])
payload.setListingID(item['listing_id'])
payload.setDate(item['date'])
payload.setComments(item['comments'])
payload.setReviewerID(item['reviewer_id'])
payload.setReviewerName(item['reviewer_name'])
longitude, latitude = GeneralHelper().default(
''), GeneralHelper().default('')
location.setPoint(latitude, longitude)
location.setAltitude('0.0')
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(
latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
location.setStreetName(GeneralHelper().default(''))
location.setStreetNumber(GeneralHelper().default(''))
record.setId(item['id'])
record.setSource(
collectorCfg['collectors']['insideairbnb']['source_name'])
record.setProvider('')
record.setPublisher('')
record.setType(
collectorCfg['collectors']['insideairbnb']['review_source_name'])
record.setTimestamp(str(datetime.datetime.now()))
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item, sensor):
record = BaseRecord()
payload = SmartCitizenPayload()
location = LocationRecord()
payload.setId(sensor['id'])
payload.setName(sensor['name'])
payload.setState(sensor['state'])
payload.setValue(item[1])
payload.setDescription(sensor['description'])
payload.setAddedAt(sensor['added_at'])
longitude, latitude = sensor['longitude'], sensor['latitude']
location.setPoint(latitude, longitude)
location.setAltitude('0.0')
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(
latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
location.setStreetName('')
location.setStreetNumber('')
record.setId(str(sensor['id']) + '.' + item[0].replace(' ', ''))
record.setSource(
collectorCfg['collectors']['smartcitizen']['source_name'])
record.setProvider(
re.findall('[A-Z][^A-Z]*', sensor['user_tags'][-1])[0])
record.setPublisher(sensor['name'])
record.setType(re.findall('[A-Z][^A-Z]*', sensor['user_tags'][-1])[1])
record.setTimestamp(item[0].split('T')[0] + 'T' +
item[0].split('T')[1].split(':')[0] + ':00:00Z')
record.setLocation(location)
record.setPayload(payload)
return record
def buildCredentialRecord(self, item):
record = BaseRecord()
payload = DecidimPetitionCredentialPayload()
location = LocationRecord()
if not item['district'] == None and not item['district'] == 'not provided':
coords = shape(LocationHelper().getAreaGeometry(item['district'].replace(' - ','-'), 'neighbourhood_group')).centroid
location.setDistrict(GeneralHelper().default(item['district']))
payload.setDistrict(GeneralHelper().default(item['district']))
location.setNeighbourhood(GeneralHelper().default(''))
longitude, latitude = coords.x, coords.y
location.setPoint(latitude, longitude)
location.setAltitude(0.0)
location.setCity('Barcelona')
location.setStreetName(GeneralHelper().default(''))
location.setStreetNumber(GeneralHelper().default(''))
payload.setId(GeneralHelper().default(item['id']))
payload.setPetitionId(GeneralHelper().default(item['petitionId']))
payload.setAge(GeneralHelper().default(item['age']))
payload.setGender(GeneralHelper().default(item['gender']))
payload.setDistrict(GeneralHelper().default(item['district']))
payload.setStartTime(str(datetime.datetime.now()))
payload.setEndTime(str(datetime.datetime.now()))
record.setId(GeneralHelper().default(item['id']))
record.setSource(source_name+'_credential')
record.setProvider('decidim')
record.setPublisher('bcnnow')
record.setType(source_name)
record.setTimestamp(str(datetime.datetime.now()))
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item):
record = BaseRecord()
payload = InsideAirbnbListingPayload()
location = LocationRecord()
payload.setId(item['id'])
payload.setName(item['name'])
payload.setDescription(item['description'])
payload.setAccess(item['access'])
payload.setAmenities(item['amenities'])
payload.setBathrooms(item['bathrooms'])
payload.setBedrooms(item['bedrooms'])
payload.setBeds(item['beds'])
payload.setBedType(item['bed_type'])
payload.setCalendarLastScraped(item['calendar_last_scraped'])
payload.setCalendarUpdated(item['calendar_updated'])
payload.setCancellationPolicy(item['cancellation_policy'])
payload.setCleaningFee(item['cleaning_fee'])
payload.setExperiencesOffered(item['experiences_offered'])
payload.setExtraPeople(item['extra_people'])
payload.setGuestsIncluded(item['guests_included'])
payload.setHasAvailability(item['has_availability'])
payload.setHostAbout(item['host_about'])
payload.setHostAcceptanceRate(item['host_acceptance_rate'])
payload.setHostHasProfilePic(item['host_has_profile_pic'])
payload.setHostID(item['host_id'])
payload.setListingUrl(item['listing_url'])
payload.setScrapeID(item['scrape_id'])
payload.setLastScraped(item['last_scraped'])
payload.setSummary(item['summary'])
payload.setSpace(item['space'])
payload.setNeighbourhoodOverview(item['neighborhood_overview'])
payload.setNotes(item['notes'])
payload.setTransit(item['transit'])
payload.setInteraction(item['interaction'])
payload.setHouseRules(item['house_rules'])
payload.setThumbnailUrl(item['thumbnail_url'])
payload.setMediumUrl(item['medium_url'])
payload.setPictureUrl(item['picture_url'])
payload.setXLPictureUrl(item['xl_picture_url'])
payload.setHostUrl(item['host_url'])
payload.setHostName(item['host_name'])
payload.setHostSince(item['host_since'])
payload.setHostLocation(item['host_location'])
payload.setHostResponseTime(item['host_response_time'])
payload.setHostResponseRate(item['host_response_rate'])
payload.setHostIsSuperhost(item['host_is_superhost'])
payload.setHostThumbnailUrl(item['host_thumbnail_url'])
payload.setHostPictureUrl(item['host_picture_url'])
payload.setHostNeighbourhood(item['host_neighbourhood'])
payload.setHostVerifications(item['host_verifications'])
payload.setHostHasProfile(item['host_has_profile_pic'])
payload.setHostIdentityVerified(item['host_identity_verified'])
payload.setPropertyType(item['property_type'])
payload.setRoomType(item['room_type'])
payload.setAccomodates(item['accommodates'])
payload.setScrapeID(item['scrape_id'])
payload.setSquareFeet(item['square_feet'])
payload.setPrice(item['price'])
payload.setWeeklyPrice(item['weekly_price'])
payload.setMonthlyPrice(item['monthly_price'])
payload.setSecurityDeposit(item['security_deposit'])
payload.setGuestsIncluded(item['guests_included'])
payload.setMinimumNights(item['minimum_nights'])
payload.setMaximumNights(item['maximum_nights'])
payload.setCalendarUpdated(item['calendar_updated'])
payload.setRequiresLicense(item['requires_license'])
payload.setLicense(item['license'])
payload.setJurisdictionNames(item['jurisdiction_names'])
payload.setInstantBookable(item['instant_bookable'])
payload.setRequireGuestProfilePicture(
item['require_guest_profile_picture'])
payload.setRequireGuestPhoneVerification(
item['require_guest_phone_verification'])
longitude, latitude = item['longitude'], item['latitude']
location.setPoint(latitude, longitude)
location.setAltitude('0.0')
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(
latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
location.setStreetName(GeneralHelper().default(''))
location.setStreetNumber(GeneralHelper().default(''))
record.setId(item['id'])
record.setSource(
collectorCfg['collectors']['insideairbnb']['source_name'])
record.setProvider('')
record.setPublisher('')
record.setType(
collectorCfg['collectors']['insideairbnb']['listing_source_name'])
record.setTimestamp('2017-04-08T00:00:00Z')
record.setLocation(location)
record.setPayload(payload)
return record
def buildRecord(self, item, type=''):
record = BaseRecord()
payload = AsiaEquipmentPayload()
location = LocationRecord()
payload.setId(item['dct:identifier'])
payload.setType(type)
payload.setEmail((
item['v:email']['rdf:Description']['@rdf:about']
if not isinstance(item['v:email'], list) else item['v:email'][0]
['rdf:Description']['@rdf:about']) if 'v:email' in item else '')
payload.setUrl((item['v:url']['@rdf:resource'] if not isinstance(
item['v:url'], list) else item['v:url'][0]['@rdf:resource']
) if 'v:url' in item else '')
payload.setName(item['v:fn'])
payload.setTelephone((
item['v:tel']['rdf:Description']['rdf:value']
if not isinstance(item['v:tel'], list) else item['v:tel'][0]
['rdf:Description']['rdf:value']) if 'v:tel' in item else '')
longitude, latitude = (
item['v:geo']['v:Location']['v:longitude']
if not isinstance(item['v:geo'], list) else
item['v:geo'][0]['v:Location']['v:longitude']), (
item['v:geo']['v:Location']['v:latitude']
if not isinstance(item['v:geo'], list) else
item['v:geo'][0]['v:Location']['v:latitude'])
location.setPoint(latitude, longitude)
location.setAltitude('0.0')
location.setCity('Barcelona')
district, neighbourhood = LocationHelper().getLocationAreas(
latitude, longitude)
location.setDistrict(GeneralHelper().default(district))
location.setNeighbourhood(GeneralHelper().default(neighbourhood))
if 'v:adr' in item:
location.setStreetName((
(item['v:adr']['v:Address']['xv:streetAddress']['xv:Street']
['xv:streetName'] if 'xv:streetName' in item['v:adr']
['v:Address']['xv:streetAddress']['xv:Street'] else ''
) if 'xv:streetAddress' in item['v:adr']['v:Address'] else ''
) if not isinstance(item['v:adr'], list) else ((
item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street']
['xv:streetName'] if 'xv:streetName' in
item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street']
['xv:streetName'] else ''
) if 'xv:streetAddress' in item['v:adr'][0]['v:Address'] else ''))
location.setStreetNumber((
(item['v:adr']['v:Address']['xv:streetAddress']['xv:Street']
['xv:streetNumber'] if 'xv:streetNumber' in item['v:adr']
['v:Address']['xv:streetAddress']['xv:Street'] else ''
) if 'xv:streetAddress' in item['v:adr']['v:Address'] else ''
) if not isinstance(item['v:adr'], list) else ((
item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street']
['xv:streetNumber'] if 'xv:streetNumber' in item['v:adr'][0]
['v:Address']['xv:streetAddress']['xv:Street'] else ''
) if 'xv:streetAddress' in item['v:adr'][0]['v:Address'] else ''))
record.setId(item['dct:identifier'])
record.setSource(
collectorCfg['collectors']['odi']['equipment']['source_name'])
record.setProvider('')
record.setPublisher('')
record.setType(
collectorCfg['collectors']['odi']['equipment']['source_name'])
record.setTimestamp(str(datetime.datetime.now()))
record.setLocation(location)
record.setPayload(payload)
return record