def buildRecord(self, item): record = BaseRecord() payload = IrisPayload() location = LocationRecord() payload.setId(GeneralHelper().default(item['FITXA_ID'])) payload.setArea(GeneralHelper().default(item['AREA'])) payload.setChannel(GeneralHelper().default(item['CANALS_RESPOSTA'])) payload.setDetail(GeneralHelper().default(item['DETALL'])) payload.setElement(GeneralHelper().default(item['ELEMENT'])) payload.setStartDate(TimeHelper().toDate( GeneralHelper().default(item['ANY_DATA_ALTA']), GeneralHelper().default(item['MES_DATA_ALTA']), GeneralHelper().default(item['DIA_DATA_ALTA']))) payload.setEndDate(TimeHelper().toDate( GeneralHelper().default(item['ANY_DATA_TANCAMENT']), GeneralHelper().default(item['MES_DATA_TANCAMENT']), GeneralHelper().default(item['DIA_DATA_TANCAMENT']))) payload.setSupport(GeneralHelper().default(item['SUPORT'])) location.setPoint(item['LATITUD'], item['LONGITUD']) location.setAltitude('0.0') location.setCity('Barcelona') location.setDistrict(GeneralHelper().default(item['DISTRICTE'])) location.setNeighbourhood(GeneralHelper().default(item['BARRI'])) location.setStreetName(GeneralHelper().default(item['TIPUS_VIA']) + ' ' + GeneralHelper().default(item['CARRER'])) location.setStreetNumber(GeneralHelper().default(item['NUMERO'])) record.setId(GeneralHelper().default(item['FITXA_ID'])) record.setSource( collectorCfg['collectors']['odi']['iris']['source_name']) record.setProvider('') record.setPublisher('') record.setType(GeneralHelper().default(item['TIPUS'])) record.setTimestamp(TimeHelper().toDate( GeneralHelper().default(item['ANY_DATA_ALTA']), GeneralHelper().default(item['MES_DATA_ALTA']), GeneralHelper().default(item['DIA_DATA_ALTA']))) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type): if type == "cityos:potencial_fotovoltaic": LocationHelper().toWGS84Geometry(item['geometry'], 'EPSG:25831', isNested=True) payload = CityOSPVPotentialPayload() payload.setId(item['id']) payload.setPowTh(item['properties']['POW_TH']) payload.setEventCode(item['properties']['EventCode']) payload.setSuitability(item['properties']['SUITABILITY']) payload.setSumModare(item['properties']['SUM_MODARE']) payload.setPolygon(item['geometry']) elif type == "cityos:ptt_carril_bici": LocationHelper().toWGS84Geometry(item['geometry'], 'EPSG:25831') payload = CityOSBikeLinesPayload() payload.setId(item['id']) payload.setId1(item['properties']['ID1']) payload.setId2(item['properties']['ID2']) payload.setEventCode(item['properties']['EventCode']) payload.setLine(item['geometry']) location = LocationRecord() coords = shape(item['geometry']).centroid longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(GeneralHelper().default(0)) location.setCity('') district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record = BaseRecord() record.setId(item['id']) record.setSource(type.split(':')[0] + '-' + type.split(':')[1]) record.setProvider(type.split(':')[0]) record.setPublisher('') record.setType(type.split(':')[1]) record.setTimestamp(item['properties']['MD_DATA_PUBLISHED']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = DecidimProposalPayload() location = LocationRecord() payload.setId(GeneralHelper().default(item['id'])) payload.setTitle(GeneralHelper().default(item['title'])) payload.setVoteCount(GeneralHelper().default(item['voteCount'])) payload.setTotalCommentsCount(GeneralHelper().default( item['totalCommentsCount'])) if 'category' in item: payload.setCategory(GeneralHelper().default( item['category']['name']['translations'][0]['text'])) if 'scope' in item: coords = shape(LocationHelper().getAreaGeometry( item['scope']['name']['translations'][0]['text'].replace( ' - ', '-'), 'neighbourhood_group')).centroid location.setDistrict(GeneralHelper().default( item['scope']['name']['translations'][0]['text'].replace( ' - ', '-').strip())) payload.setDistrict(GeneralHelper().default( item['scope']['name']['translations'][0]['text'].replace( ' - ', '-').strip())) location.setNeighbourhood(GeneralHelper().default('')) longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(GeneralHelper().default(item['reference'])) record.setSource(source_name) record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(item['publishedAt']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, community_id): record = BaseRecord() payload = IoTPayload() location = LocationRecord() item['recordedAt'] = (datetime.strptime(item['recordedAt'], '%Y-%m-%dT%H:%M:%SZ')+timedelta(hours=2)).strftime('%Y-%m-%dT%H:%M:%S') item['label'] = re.sub('[^0-9a-zA-Z]+', '-', item['label']) latitude, longitude = GeneralHelper().default(item['latitude']), GeneralHelper().default(item['longitude']) location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) payload.setId(GeneralHelper().default(item['label'])) payload.setType(GeneralHelper().default(item['type'])) payload.setName(GeneralHelper().default(item['name'])) payload.setDescription(GeneralHelper().default(item['description'])) payload.setUnit(GeneralHelper().default(item['unit'])) if 'value' in item: payload.setValue(GeneralHelper().default(item['value'])) elif 'values' in item: value = 0 values = item['values'] bins = item['bins'] for i in range(len(bins)): value += bins[i] * values[i+1] payload.setValue(GeneralHelper().default(value)) payload.setRecordedAt(GeneralHelper().default(item['recordedAt'])) payload.setExposure(GeneralHelper().default(item['exposure'])) record.setId(GeneralHelper().default(item['label']+'.'+item['recordedAt'])) record.setSource(('iot__'+community_id+'__'+item['name'].lower().split()[0]).replace('-','_').replace('.','_')) record.setProvider('smartcitizen') record.setPublisher('bcnnow') record.setType('event') record.setTimestamp(item['recordedAt']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = DecidimMeetingPayload() location = LocationRecord() latitude, longitude = GeneralHelper().default(item['coordinates']['latitude']), GeneralHelper().default(item['coordinates']['longitude']) location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default(item['address'].split(',')[0])) location.setStreetNumber(GeneralHelper().default(int(item['address'].split(',')[1]) if len(item['address'].split(',')[1].strip()) in [1,2] else '') if len(item['address'].split(','))>1 else '') payload.setId(GeneralHelper().default(item['reference'])) payload.setAttendeeCount(GeneralHelper().default(item['attendeeCount'])) payload.setTotalCommentsCount(GeneralHelper().default(item['totalCommentsCount'])) payload.setContributionCount(GeneralHelper().default(item['contributionCount'])) payload.setTitle(GeneralHelper().default(item['title']['translations'][1]['text'])) payload.setStartTime(GeneralHelper().default(item['startTime'])) payload.setEndTime(GeneralHelper().default(item['endTime'])) payload.setAddress(GeneralHelper().default(item['address'])) attachments = [] for attachment in item['attachments']: attachments.append(attachment['url']) payload.setAttachments(GeneralHelper().default(attachments)) record.setId(GeneralHelper().default(item['reference'])) record.setSource(source_name) record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(item['startTime']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, start_date): record = BaseRecord() payload = BicingPayload() location = LocationRecord() payload.setId(item['id']) payload.setSlots(item['slots']) payload.setNearbyStationIDs( [station for station in item['nearbyStations'].split(' ')]) payload.setBikes(item['bikes']) payload.setType(item['type']) payload.setStatus(item['status']) longitude, latitude = GeneralHelper().default( item['longitude']), GeneralHelper().default(item['latitude']) location.setPoint(latitude, longitude) location.setAltitude(GeneralHelper().default(item['altitude'])) location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default(item['streetName'])) location.setStreetNumber(GeneralHelper().default(item['streetNumber'])) record.setId(item['id'] + '.' + start_date) record.setSource( collectorCfg['collectors']['odi']['bicing']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['odi']['bicing']['source_name']) record.setTimestamp(start_date) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type): record = BaseRecord() payload = OHBPayload() location = LocationRecord() payload.setPeriod(item['PERIODE']) payload.setValue(item['ind_value']) if 'Districts' in type: coords = shape(LocationHelper().getAreaGeometry( item['NOM'], 'neighbourhood_group')).centroid location.setDistrict(GeneralHelper().default(item['NOM'].strip())) location.setNeighbourhood(GeneralHelper().default('')) else: coords = shape(LocationHelper().getAreaGeometry( item['NOM'], 'neighbourhood')).centroid location.setDistrict(GeneralHelper().default('')) location.setNeighbourhood(GeneralHelper().default( item['NOM'].strip())) longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(str(item['PERIODE'])[:4] + '_' + item['CODI_OHB']) record.setSource(collectorCfg['collectors']['OHB']['source_name']) record.setProvider('') record.setPublisher('') record.setType(type.replace('.csv', '')) record.setTimestamp(str(item['PERIODE'])[:4] + '-01-01T00:00:00Z') record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, observation, provider, sensor): record = BaseRecord() payload = SentiloPayload() location = LocationRecord() payload.setValue(observation['value']) payload.setType(sensor['type']) payload.setDescription(sensor['description']) payload.setEmplacament(sensor['additionalInfo']['emplaçament'] if 'additionalInfo' in sensor else '') payload.setUnit(sensor['unit']) longitude, latitude = sensor['location'].split( ' ')[1], sensor['location'].split(' ')[0] location.setPoint(latitude, longitude) location.setAltitude(sensor['additionalInfo']['altitud'] if 'additionalInfo' in sensor else '') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(district) location.setNeighbourhood(neighbourhood) location.setStreetName('') location.setStreetNumber('') record.setId(sensor['sensor'] + '.' + observation['timestamp'].replace(' ', '')) record.setSource(collectorCfg['collectors']['sentilo']['source_name']) record.setProvider(provider['provider']) record.setPublisher(sensor['sensor']) record.setType(sensor['componentType']) record.setTimestamp(observation['timestamp']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = InsideAirbnbReviewPayload() location = LocationRecord() payload.setId(item['id']) payload.setListingID(item['listing_id']) payload.setDate(item['date']) payload.setComments(item['comments']) payload.setReviewerID(item['reviewer_id']) payload.setReviewerName(item['reviewer_name']) longitude, latitude = GeneralHelper().default( ''), GeneralHelper().default('') location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(item['id']) record.setSource( collectorCfg['collectors']['insideairbnb']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['insideairbnb']['review_source_name']) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, sensor): record = BaseRecord() payload = SmartCitizenPayload() location = LocationRecord() payload.setId(sensor['id']) payload.setName(sensor['name']) payload.setState(sensor['state']) payload.setValue(item[1]) payload.setDescription(sensor['description']) payload.setAddedAt(sensor['added_at']) longitude, latitude = sensor['longitude'], sensor['latitude'] location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName('') location.setStreetNumber('') record.setId(str(sensor['id']) + '.' + item[0].replace(' ', '')) record.setSource( collectorCfg['collectors']['smartcitizen']['source_name']) record.setProvider( re.findall('[A-Z][^A-Z]*', sensor['user_tags'][-1])[0]) record.setPublisher(sensor['name']) record.setType(re.findall('[A-Z][^A-Z]*', sensor['user_tags'][-1])[1]) record.setTimestamp(item[0].split('T')[0] + 'T' + item[0].split('T')[1].split(':')[0] + ':00:00Z') record.setLocation(location) record.setPayload(payload) return record
def buildCredentialRecord(self, item): record = BaseRecord() payload = DecidimPetitionCredentialPayload() location = LocationRecord() if not item['district'] == None and not item['district'] == 'not provided': coords = shape(LocationHelper().getAreaGeometry(item['district'].replace(' - ','-'), 'neighbourhood_group')).centroid location.setDistrict(GeneralHelper().default(item['district'])) payload.setDistrict(GeneralHelper().default(item['district'])) location.setNeighbourhood(GeneralHelper().default('')) longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) payload.setId(GeneralHelper().default(item['id'])) payload.setPetitionId(GeneralHelper().default(item['petitionId'])) payload.setAge(GeneralHelper().default(item['age'])) payload.setGender(GeneralHelper().default(item['gender'])) payload.setDistrict(GeneralHelper().default(item['district'])) payload.setStartTime(str(datetime.datetime.now())) payload.setEndTime(str(datetime.datetime.now())) record.setId(GeneralHelper().default(item['id'])) record.setSource(source_name+'_credential') record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = InsideAirbnbListingPayload() location = LocationRecord() payload.setId(item['id']) payload.setName(item['name']) payload.setDescription(item['description']) payload.setAccess(item['access']) payload.setAmenities(item['amenities']) payload.setBathrooms(item['bathrooms']) payload.setBedrooms(item['bedrooms']) payload.setBeds(item['beds']) payload.setBedType(item['bed_type']) payload.setCalendarLastScraped(item['calendar_last_scraped']) payload.setCalendarUpdated(item['calendar_updated']) payload.setCancellationPolicy(item['cancellation_policy']) payload.setCleaningFee(item['cleaning_fee']) payload.setExperiencesOffered(item['experiences_offered']) payload.setExtraPeople(item['extra_people']) payload.setGuestsIncluded(item['guests_included']) payload.setHasAvailability(item['has_availability']) payload.setHostAbout(item['host_about']) payload.setHostAcceptanceRate(item['host_acceptance_rate']) payload.setHostHasProfilePic(item['host_has_profile_pic']) payload.setHostID(item['host_id']) payload.setListingUrl(item['listing_url']) payload.setScrapeID(item['scrape_id']) payload.setLastScraped(item['last_scraped']) payload.setSummary(item['summary']) payload.setSpace(item['space']) payload.setNeighbourhoodOverview(item['neighborhood_overview']) payload.setNotes(item['notes']) payload.setTransit(item['transit']) payload.setInteraction(item['interaction']) payload.setHouseRules(item['house_rules']) payload.setThumbnailUrl(item['thumbnail_url']) payload.setMediumUrl(item['medium_url']) payload.setPictureUrl(item['picture_url']) payload.setXLPictureUrl(item['xl_picture_url']) payload.setHostUrl(item['host_url']) payload.setHostName(item['host_name']) payload.setHostSince(item['host_since']) payload.setHostLocation(item['host_location']) payload.setHostResponseTime(item['host_response_time']) payload.setHostResponseRate(item['host_response_rate']) payload.setHostIsSuperhost(item['host_is_superhost']) payload.setHostThumbnailUrl(item['host_thumbnail_url']) payload.setHostPictureUrl(item['host_picture_url']) payload.setHostNeighbourhood(item['host_neighbourhood']) payload.setHostVerifications(item['host_verifications']) payload.setHostHasProfile(item['host_has_profile_pic']) payload.setHostIdentityVerified(item['host_identity_verified']) payload.setPropertyType(item['property_type']) payload.setRoomType(item['room_type']) payload.setAccomodates(item['accommodates']) payload.setScrapeID(item['scrape_id']) payload.setSquareFeet(item['square_feet']) payload.setPrice(item['price']) payload.setWeeklyPrice(item['weekly_price']) payload.setMonthlyPrice(item['monthly_price']) payload.setSecurityDeposit(item['security_deposit']) payload.setGuestsIncluded(item['guests_included']) payload.setMinimumNights(item['minimum_nights']) payload.setMaximumNights(item['maximum_nights']) payload.setCalendarUpdated(item['calendar_updated']) payload.setRequiresLicense(item['requires_license']) payload.setLicense(item['license']) payload.setJurisdictionNames(item['jurisdiction_names']) payload.setInstantBookable(item['instant_bookable']) payload.setRequireGuestProfilePicture( item['require_guest_profile_picture']) payload.setRequireGuestPhoneVerification( item['require_guest_phone_verification']) longitude, latitude = item['longitude'], item['latitude'] location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(item['id']) record.setSource( collectorCfg['collectors']['insideairbnb']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['insideairbnb']['listing_source_name']) record.setTimestamp('2017-04-08T00:00:00Z') record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type=''): record = BaseRecord() payload = AsiaEquipmentPayload() location = LocationRecord() payload.setId(item['dct:identifier']) payload.setType(type) payload.setEmail(( item['v:email']['rdf:Description']['@rdf:about'] if not isinstance(item['v:email'], list) else item['v:email'][0] ['rdf:Description']['@rdf:about']) if 'v:email' in item else '') payload.setUrl((item['v:url']['@rdf:resource'] if not isinstance( item['v:url'], list) else item['v:url'][0]['@rdf:resource'] ) if 'v:url' in item else '') payload.setName(item['v:fn']) payload.setTelephone(( item['v:tel']['rdf:Description']['rdf:value'] if not isinstance(item['v:tel'], list) else item['v:tel'][0] ['rdf:Description']['rdf:value']) if 'v:tel' in item else '') longitude, latitude = ( item['v:geo']['v:Location']['v:longitude'] if not isinstance(item['v:geo'], list) else item['v:geo'][0]['v:Location']['v:longitude']), ( item['v:geo']['v:Location']['v:latitude'] if not isinstance(item['v:geo'], list) else item['v:geo'][0]['v:Location']['v:latitude']) location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) if 'v:adr' in item: location.setStreetName(( (item['v:adr']['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetName'] if 'xv:streetName' in item['v:adr'] ['v:Address']['xv:streetAddress']['xv:Street'] else '' ) if 'xv:streetAddress' in item['v:adr']['v:Address'] else '' ) if not isinstance(item['v:adr'], list) else (( item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetName'] if 'xv:streetName' in item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetName'] else '' ) if 'xv:streetAddress' in item['v:adr'][0]['v:Address'] else '')) location.setStreetNumber(( (item['v:adr']['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetNumber'] if 'xv:streetNumber' in item['v:adr'] ['v:Address']['xv:streetAddress']['xv:Street'] else '' ) if 'xv:streetAddress' in item['v:adr']['v:Address'] else '' ) if not isinstance(item['v:adr'], list) else (( item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetNumber'] if 'xv:streetNumber' in item['v:adr'][0] ['v:Address']['xv:streetAddress']['xv:Street'] else '' ) if 'xv:streetAddress' in item['v:adr'][0]['v:Address'] else '')) record.setId(item['dct:identifier']) record.setSource( collectorCfg['collectors']['odi']['equipment']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['odi']['equipment']['source_name']) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record