def buildSignatureRecord(self, item): record = BaseRecord() payload = DecidimPetitionSignaturePayload() location = LocationRecord() payload.setId(GeneralHelper().default(item['id'])) payload.setPetitionId(GeneralHelper().default(item['petitionId'])) payload.setLabel('signature') payload.setStartTime(str(datetime.datetime.now())) payload.setEndTime(str(datetime.datetime.now())) record.setId(GeneralHelper().default(item['id'])) record.setSource(source_name+'_signature') record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, rID, index): record = BaseRecord() payload = DecidimSurveyPayload() location = LocationRecord() payload.setId( GeneralHelper().default(rID.replace('.json', '-') + index)) payload.setGender(GeneralHelper().default(item['gender'])) payload.setAge(GeneralHelper().default(item['age'])) payload.setCountry(GeneralHelper().default(item['country'])) payload.setContinent(GeneralHelper().default(item['continent'])) payload.setEducation(GeneralHelper().default(item['education'])) if 'work situation' in item: payload.setWorkSituation(GeneralHelper().default( item['work situation'])) payload.setOrganization(GeneralHelper().default(item['organization'])) payload.setCity(GeneralHelper().default(item['city'])) if 'district' in item: payload.setDistrict(GeneralHelper().default(item['district'])) payload.setDevice(GeneralHelper().default(item['device'])) if 'scale' in item: payload.setScale(GeneralHelper().default(item['scale'])) payload.setInterest(GeneralHelper().default(item['interest'])) record.setId( GeneralHelper().default(rID.replace('.json', '-') + index)) record.setSource(collectorCfg['collectors']['decidim']['dddc_survey'] ['source_name']) record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(collectorCfg['collectors']['decidim']['dddc_survey'] ['source_name']) record.setTimestamp(str(datetime.datetime.now()).split('.')[0]) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, observation, provider, sensor): record = BaseRecord() payload = SentiloPayload() location = LocationRecord() payload.setValue(observation['value']) payload.setType(sensor['type']) payload.setDescription(sensor['description']) payload.setEmplacament(sensor['additionalInfo']['emplaçament'] if 'additionalInfo' in sensor else '') payload.setUnit(sensor['unit']) longitude, latitude = sensor['location'].split( ' ')[1], sensor['location'].split(' ')[0] location.setPoint(latitude, longitude) location.setAltitude(sensor['additionalInfo']['altitud'] if 'additionalInfo' in sensor else '') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(district) location.setNeighbourhood(neighbourhood) location.setStreetName('') location.setStreetNumber('') record.setId(sensor['sensor'] + '.' + observation['timestamp'].replace(' ', '')) record.setSource(collectorCfg['collectors']['sentilo']['source_name']) record.setProvider(provider['provider']) record.setPublisher(sensor['sensor']) record.setType(sensor['componentType']) record.setTimestamp(observation['timestamp']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type): record = BaseRecord() payload = OHBPayload2() location = LocationRecord() payload.setPeriod(item['PERIODE']) payload.setValue(item['ind_value']) if 'Districts' in type: coords = shape(LocationHelper().getAreaGeometry(item['NOM'], 'neighbourhood_group')).centroid location.setDistrict(GeneralHelper().default(item['NOM'].strip())) location.setNeighbourhood(GeneralHelper().default('')) else: coords = shape(LocationHelper().getAreaGeometry(item['NOM'], 'neighbourhood')).centroid location.setDistrict(GeneralHelper().default('')) location.setNeighbourhood(GeneralHelper().default(item['NOM'].strip())) longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(str(item['PERIODE'])[:4] + '_' + item['CODI_OHB']) record.setSource(collectorCfg['collectors']['OHB2']['source_name']) record.setProvider('') record.setPublisher('') record.setType(type.replace('.csv', '')) record.setTimestamp(str(item['PERIODE'])[:4] + '-01-01T00:00:00Z') record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = InsideAirbnbCalendarPayload() location = LocationRecord() payload.setListingId(item['listing_id']) payload.setAvailable(item['available']) payload.setPrice(item['price']) longitude, latitude = GeneralHelper().default(''), GeneralHelper().default('') location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(str(item['listing_id']) + '.' + item['date'].replace(' ', '')) record.setSource(collectorCfg['collectors']['insideairbnb']['source_name']) record.setProvider('') record.setPublisher('') record.setType(collectorCfg['collectors']['insideairbnb']['calendar_source_name']) record.setTimestamp(item['date']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = DecidimMeetingPayload() location = LocationRecord() latitude, longitude = GeneralHelper().default( item['coordinates']['latitude']), GeneralHelper().default( item['coordinates']['longitude']) location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default( item['address'].split(',')[0])) location.setStreetNumber(GeneralHelper().default( int(item['address'].split(',')[1]) if len(item['address']. split(',')[1].strip()) in [1, 2] else '') if len(item['address'].split(',')) > 1 else '') payload.setId(GeneralHelper().default(item['reference'])) payload.setAttendeeCount(GeneralHelper().default( item['attendeeCount'])) payload.setTotalCommentsCount(GeneralHelper().default( item['totalCommentsCount'])) payload.setContributionCount(GeneralHelper().default( item['contributionCount'])) payload.setTitle(GeneralHelper().default( item['title']['translations'][1]['text'])) payload.setStartTime(GeneralHelper().default(item['startTime'])) payload.setEndTime(GeneralHelper().default(item['endTime'])) payload.setAddress(GeneralHelper().default(item['address'])) attachments = [] for attachment in item['attachments']: attachments.append(attachment['url']) payload.setAttachments(GeneralHelper().default(attachments)) record.setId(GeneralHelper().default(item['reference'])) record.setSource(source_name) record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(item['startTime']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type): if type == "cityos:potencial_fotovoltaic": LocationHelper().toWGS84Geometry(item['geometry'], 'EPSG:25831', isNested=True) payload = CityOSPVPotentialPayload() payload.setId(item['id']) payload.setPowTh(item['properties']['POW_TH']) payload.setEventCode(item['properties']['EventCode']) payload.setSuitability(item['properties']['SUITABILITY']) payload.setSumModare(item['properties']['SUM_MODARE']) payload.setPolygon(item['geometry']) elif type == "cityos:ptt_carril_bici": LocationHelper().toWGS84Geometry(item['geometry'], 'EPSG:25831') payload = CityOSBikeLinesPayload() payload.setId(item['id']) payload.setId1(item['properties']['ID1']) payload.setId2(item['properties']['ID2']) payload.setEventCode(item['properties']['EventCode']) payload.setLine(item['geometry']) location = LocationRecord() coords = shape(item['geometry']).centroid longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(GeneralHelper().default(0)) location.setCity('') district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record = BaseRecord() record.setId(item['id']) record.setSource(type.split(':')[0] + '-' + type.split(':')[1]) record.setProvider(type.split(':')[0]) record.setPublisher('') record.setType(type.split(':')[1]) record.setTimestamp(item['properties']['MD_DATA_PUBLISHED']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = SentiloPayload() location = LocationRecord() payload.setValue(item['message']) sensor = SentiloCollector().retrieve(item['sensor']) longitude, latitude = sensor['location'][0] if 'location' in sensor else '', sensor['location'][1] if 'location' in sensor else '' location.setLongitude(longitude) location.setLatitude(latitude) location.setAltitude('0.0') location.setCity('Barcelona') location.setDistrict('') location.setNeighbourhood('') location.setStreetName('') location.setStreetNumber('') record.setSource('sentilo') record.setProvider(item['provider']) record.setPublisher(item['publisher']) record.setType(item['type']) record.setTimestamp(item['timestamp']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = AsiaEventPayload() location = LocationRecord() payload.setName(GeneralHelper().default(item['nom'])) payload.setEventType(GeneralHelper().toAsiaType(item['tipus_acte'])) payload.setStartDate(TimeHelper().toDash(item['data']['data_inici'])) payload.setEndDate(TimeHelper().toDash(item['data']['data_fi'])) payload.setEquipmentID(GeneralHelper().default( item['lloc_simple']['id'] if 'id' in item['lloc_simple'] else '')) payload.setState(GeneralHelper().toAsiaState(item['estat'])) payload.setStateCycle(GeneralHelper().toAsiaStateCycle( item['estat_cicle'])) payload.setCategories(GeneralHelper().toClassifications( item['classificacions'])) longitude, latitude = LocationHelper().toWGS84( GeneralHelper().default(item['lloc_simple']['adreca_simple'] ['coordenades']['geocodificacio']['@y']), GeneralHelper().default(item['lloc_simple']['adreca_simple'] ['coordenades']['geocodificacio']['@x'])) location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default( item['lloc_simple']['adreca_simple']['carrer']['#text'] if '#text' in item['lloc_simple']['adreca_simple']['carrer'] else '')) location.setStreetNumber(GeneralHelper().default( item['lloc_simple']['adreca_simple']['numero']['@enter'] if '@enter' in item['lloc_simple']['adreca_simple']['numero'] else '')) record.setId(GeneralHelper().default(item['id'])) record.setSource( collectorCfg['collectors']['odi']['asia']['source_name']) record.setProvider('') record.setPublisher('') record.setType('event') record.setTimestamp(TimeHelper().toDash(item['data']['data_inici'])) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, community_id): record = BaseRecord() payload = IoTPayload() location = LocationRecord() item['recordedAt'] = ( datetime.strptime(item['recordedAt'], '%Y-%m-%dT%H:%M:%SZ') + timedelta(hours=2)).strftime('%Y-%m-%dT%H:%M:%S') item['label'] = re.sub('[^0-9a-zA-Z]+', '-', item['label']) latitude, longitude = GeneralHelper().default( item['latitude']), GeneralHelper().default(item['longitude']) location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) payload.setId(GeneralHelper().default(item['label'])) payload.setType(GeneralHelper().default(item['type'])) payload.setName(GeneralHelper().default(item['name'])) payload.setDescription(GeneralHelper().default(item['description'])) payload.setUnit(GeneralHelper().default(item['unit'])) if 'value' in item: payload.setValue(GeneralHelper().default(item['value'])) elif 'values' in item: value = 0 values = item['values'] bins = item['bins'] for i in range(len(bins)): value += bins[i] * values[i + 1] payload.setValue(GeneralHelper().default(value)) payload.setRecordedAt(GeneralHelper().default(item['recordedAt'])) payload.setExposure(GeneralHelper().default(item['exposure'])) record.setId(GeneralHelper().default(item['label'] + '.' + item['recordedAt'])) record.setSource( ('iot__' + community_id + '__' + item['name'].lower().split()[0]).replace('-', '_').replace('.', '_')) record.setProvider('smartcitizen') record.setPublisher('bcnnow') record.setType('event') record.setTimestamp(item['recordedAt']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = DecidimProposalPayload() location = LocationRecord() payload.setId(GeneralHelper().default(item['id'])) payload.setTitle(GeneralHelper().default(item['title'])) payload.setVoteCount(GeneralHelper().default(item['voteCount'])) payload.setTotalCommentsCount(GeneralHelper().default( item['totalCommentsCount'])) if 'category' in item: payload.setCategory(GeneralHelper().default( item['category']['name']['translations'][0]['text'])) if 'scope' in item: coords = shape(LocationHelper().getAreaGeometry( item['scope']['name']['translations'][0]['text'].replace( ' - ', '-'), 'neighbourhood_group')).centroid location.setDistrict(GeneralHelper().default( item['scope']['name']['translations'][0]['text'].replace( ' - ', '-').strip())) payload.setDistrict(GeneralHelper().default( item['scope']['name']['translations'][0]['text'].replace( ' - ', '-').strip())) location.setNeighbourhood(GeneralHelper().default('')) longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(GeneralHelper().default(item['reference'])) record.setSource(source_name) record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(item['publishedAt']) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = TouristicHousePayload() location = LocationRecord() payload.setExpedient(GeneralHelper().default(item['N_EXPEDIENT'])) if item['LATITUD_Y'] != None: location.setPoint(item['LATITUD_Y'], item['LONGITUD_X']) else: location.setPoint(0.0, 0.0) location.setAltitude('0.0') location.setCity('Barcelona') location.setDistrict(GeneralHelper().default(item['DISTRICTE'])) location.setNeighbourhood(GeneralHelper().default(item['BARRI'])) location.setStreetName(GeneralHelper().default(item['TIPUS_CARRER']) + ' ' + GeneralHelper().default(item['CARRER'])) location.setStreetNumber(GeneralHelper().default(item['TIPUS_NUM'])) record.setId(GeneralHelper().default(item['N_EXPEDIENT'])) record.setSource(collectorCfg['collectors']['odi']['touristic_house'] ['source_name']) record.setProvider('') record.setPublisher('') record.setType(GeneralHelper().default(item['PIS'])) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type=''): record = BaseRecord() payload = AsiaEquipmentPayload() location = LocationRecord() payload.setId(item['dct:identifier']) payload.setType(type) payload.setEmail(( item['v:email']['rdf:Description']['@rdf:about'] if not isinstance(item['v:email'], list) else item['v:email'][0] ['rdf:Description']['@rdf:about']) if 'v:email' in item else '') payload.setUrl((item['v:url']['@rdf:resource'] if not isinstance( item['v:url'], list) else item['v:url'][0]['@rdf:resource'] ) if 'v:url' in item else '') payload.setName(item['v:fn']) payload.setTelephone(( item['v:tel']['rdf:Description']['rdf:value'] if not isinstance(item['v:tel'], list) else item['v:tel'][0] ['rdf:Description']['rdf:value']) if 'v:tel' in item else '') longitude, latitude = ( item['v:geo']['v:Location']['v:longitude'] if not isinstance(item['v:geo'], list) else item['v:geo'][0]['v:Location']['v:longitude']), ( item['v:geo']['v:Location']['v:latitude'] if not isinstance(item['v:geo'], list) else item['v:geo'][0]['v:Location']['v:latitude']) location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) if 'v:adr' in item: location.setStreetName(( (item['v:adr']['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetName'] if 'xv:streetName' in item['v:adr'] ['v:Address']['xv:streetAddress']['xv:Street'] else '' ) if 'xv:streetAddress' in item['v:adr']['v:Address'] else '' ) if not isinstance(item['v:adr'], list) else (( item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetName'] if 'xv:streetName' in item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetName'] else '' ) if 'xv:streetAddress' in item['v:adr'][0]['v:Address'] else '')) location.setStreetNumber(( (item['v:adr']['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetNumber'] if 'xv:streetNumber' in item['v:adr'] ['v:Address']['xv:streetAddress']['xv:Street'] else '' ) if 'xv:streetAddress' in item['v:adr']['v:Address'] else '' ) if not isinstance(item['v:adr'], list) else (( item['v:adr'][0]['v:Address']['xv:streetAddress']['xv:Street'] ['xv:streetNumber'] if 'xv:streetNumber' in item['v:adr'][0] ['v:Address']['xv:streetAddress']['xv:Street'] else '' ) if 'xv:streetAddress' in item['v:adr'][0]['v:Address'] else '')) record.setId(item['dct:identifier']) record.setSource( collectorCfg['collectors']['odi']['equipment']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['odi']['equipment']['source_name']) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record
def buildCredentialRecord(self, item): record = BaseRecord() payload = DecidimPetitionCredentialPayload() location = LocationRecord() if not item['district'] == None and not item['district'] == 'not provided': coords = shape(LocationHelper().getAreaGeometry(item['district'].replace(' - ','-'), 'neighbourhood_group')).centroid location.setDistrict(GeneralHelper().default(item['district'])) payload.setDistrict(GeneralHelper().default(item['district'])) location.setNeighbourhood(GeneralHelper().default('')) longitude, latitude = coords.x, coords.y location.setPoint(latitude, longitude) location.setAltitude(0.0) location.setCity('Barcelona') location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) payload.setId(GeneralHelper().default(item['id'])) payload.setPetitionId(GeneralHelper().default(item['petitionId'])) payload.setAge(GeneralHelper().default(item['age'])) payload.setGender(GeneralHelper().default(item['gender'])) payload.setDistrict(GeneralHelper().default(item['district'])) payload.setStartTime(str(datetime.datetime.now())) payload.setEndTime(str(datetime.datetime.now())) record.setId(GeneralHelper().default(item['id'])) record.setSource(source_name+'_credential') record.setProvider('decidim') record.setPublisher('bcnnow') record.setType(source_name) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = IrisPayload() location = LocationRecord() payload.setId(GeneralHelper().default(item['FITXA_ID'])) payload.setArea(GeneralHelper().default(item['AREA'])) payload.setChannel(GeneralHelper().default(item['CANALS_RESPOSTA'])) payload.setDetail(GeneralHelper().default(item['DETALL'])) payload.setElement(GeneralHelper().default(item['ELEMENT'])) payload.setStartDate(TimeHelper().toDate( GeneralHelper().default(item['ANY_DATA_ALTA']), GeneralHelper().default(item['MES_DATA_ALTA']), GeneralHelper().default(item['DIA_DATA_ALTA']))) payload.setEndDate(TimeHelper().toDate( GeneralHelper().default(item['ANY_DATA_TANCAMENT']), GeneralHelper().default(item['MES_DATA_TANCAMENT']), GeneralHelper().default(item['DIA_DATA_TANCAMENT']))) payload.setSupport(GeneralHelper().default(item['SUPORT'])) location.setPoint(item['LATITUD'], item['LONGITUD']) location.setAltitude('0.0') location.setCity('Barcelona') location.setDistrict(GeneralHelper().default(item['DISTRICTE'])) location.setNeighbourhood(GeneralHelper().default(item['BARRI'])) location.setStreetName(GeneralHelper().default(item['TIPUS_VIA']) + ' ' + GeneralHelper().default(item['CARRER'])) location.setStreetNumber(GeneralHelper().default(item['NUMERO'])) record.setId(GeneralHelper().default(item['FITXA_ID'])) record.setSource( collectorCfg['collectors']['odi']['iris']['source_name']) record.setProvider('') record.setPublisher('') record.setType(GeneralHelper().default(item['TIPUS'])) record.setTimestamp(TimeHelper().toDate( GeneralHelper().default(item['ANY_DATA_ALTA']), GeneralHelper().default(item['MES_DATA_ALTA']), GeneralHelper().default(item['DIA_DATA_ALTA']))) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, sensor): record = BaseRecord() payload = SmartCitizenPayload() location = LocationRecord() payload.setId(sensor['id']) payload.setName(sensor['name']) payload.setState(sensor['state']) payload.setValue(item[1]) payload.setDescription(sensor['description']) payload.setAddedAt(sensor['added_at']) longitude, latitude = sensor['longitude'], sensor['latitude'] location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName('') location.setStreetNumber('') record.setId(str(sensor['id']) + '.' + item[0].replace(' ', '')) record.setSource( collectorCfg['collectors']['smartcitizen']['source_name']) record.setProvider( re.findall('[A-Z][^A-Z]*', sensor['user_tags'][-1])[0]) record.setPublisher(sensor['name']) record.setType(re.findall('[A-Z][^A-Z]*', sensor['user_tags'][-1])[1]) record.setTimestamp(item[0].split('T')[0] + 'T' + item[0].split('T')[1].split(':')[0] + ':00:00Z') record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, start_date): record = BaseRecord() payload = BicingPayload() location = LocationRecord() payload.setId(item['id']) payload.setSlots(item['slots']) payload.setNearbyStationIDs([station for station in item['nearbyStations'].split(' ')]) payload.setBikes(item['bikes']) payload.setType(item['type']) payload.setStatus(item['status']) longitude, latitude = GeneralHelper().default(item['longitude']), GeneralHelper().default(item['latitude']) location.setPoint(latitude, longitude) location.setAltitude(GeneralHelper().default(item['altitude'])) location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas(latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default(item['streetName'])) location.setStreetNumber(GeneralHelper().default(item['streetNumber'])) record.setId(item['id'] + '.' + start_date) record.setSource(collectorCfg['collectors']['odi']['bicing']['source_name']) record.setProvider('') record.setPublisher('') record.setType(collectorCfg['collectors']['odi']['bicing']['source_name']) record.setTimestamp(start_date) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = InsideAirbnbReviewPayload() location = LocationRecord() payload.setId(item['id']) payload.setListingID(item['listing_id']) payload.setDate(item['date']) payload.setComments(item['comments']) payload.setReviewerID(item['reviewer_id']) payload.setReviewerName(item['reviewer_name']) longitude, latitude = GeneralHelper().default( ''), GeneralHelper().default('') location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(item['id']) record.setSource( collectorCfg['collectors']['insideairbnb']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['insideairbnb']['review_source_name']) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item): record = BaseRecord() payload = InsideAirbnbListingPayload() location = LocationRecord() payload.setId(item['id']) payload.setName(item['name']) payload.setDescription(item['description']) payload.setAccess(item['access']) payload.setAmenities(item['amenities']) payload.setBathrooms(item['bathrooms']) payload.setBedrooms(item['bedrooms']) payload.setBeds(item['beds']) payload.setBedType(item['bed_type']) payload.setCalendarLastScraped(item['calendar_last_scraped']) payload.setCalendarUpdated(item['calendar_updated']) payload.setCancellationPolicy(item['cancellation_policy']) payload.setCleaningFee(item['cleaning_fee']) payload.setExperiencesOffered(item['experiences_offered']) payload.setExtraPeople(item['extra_people']) payload.setGuestsIncluded(item['guests_included']) payload.setHasAvailability(item['has_availability']) payload.setHostAbout(item['host_about']) payload.setHostAcceptanceRate(item['host_acceptance_rate']) payload.setHostHasProfilePic(item['host_has_profile_pic']) payload.setHostID(item['host_id']) payload.setListingUrl(item['listing_url']) payload.setScrapeID(item['scrape_id']) payload.setLastScraped(item['last_scraped']) payload.setSummary(item['summary']) payload.setSpace(item['space']) payload.setNeighbourhoodOverview(item['neighborhood_overview']) payload.setNotes(item['notes']) payload.setTransit(item['transit']) payload.setInteraction(item['interaction']) payload.setHouseRules(item['house_rules']) payload.setThumbnailUrl(item['thumbnail_url']) payload.setMediumUrl(item['medium_url']) payload.setPictureUrl(item['picture_url']) payload.setXLPictureUrl(item['xl_picture_url']) payload.setHostUrl(item['host_url']) payload.setHostName(item['host_name']) payload.setHostSince(item['host_since']) payload.setHostLocation(item['host_location']) payload.setHostResponseTime(item['host_response_time']) payload.setHostResponseRate(item['host_response_rate']) payload.setHostIsSuperhost(item['host_is_superhost']) payload.setHostThumbnailUrl(item['host_thumbnail_url']) payload.setHostPictureUrl(item['host_picture_url']) payload.setHostNeighbourhood(item['host_neighbourhood']) payload.setHostVerifications(item['host_verifications']) payload.setHostHasProfile(item['host_has_profile_pic']) payload.setHostIdentityVerified(item['host_identity_verified']) payload.setPropertyType(item['property_type']) payload.setRoomType(item['room_type']) payload.setAccomodates(item['accommodates']) payload.setScrapeID(item['scrape_id']) payload.setSquareFeet(item['square_feet']) payload.setPrice(item['price']) payload.setWeeklyPrice(item['weekly_price']) payload.setMonthlyPrice(item['monthly_price']) payload.setSecurityDeposit(item['security_deposit']) payload.setGuestsIncluded(item['guests_included']) payload.setMinimumNights(item['minimum_nights']) payload.setMaximumNights(item['maximum_nights']) payload.setCalendarUpdated(item['calendar_updated']) payload.setRequiresLicense(item['requires_license']) payload.setLicense(item['license']) payload.setJurisdictionNames(item['jurisdiction_names']) payload.setInstantBookable(item['instant_bookable']) payload.setRequireGuestProfilePicture( item['require_guest_profile_picture']) payload.setRequireGuestPhoneVerification( item['require_guest_phone_verification']) longitude, latitude = item['longitude'], item['latitude'] location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(GeneralHelper().default('')) location.setStreetNumber(GeneralHelper().default('')) record.setId(item['id']) record.setSource( collectorCfg['collectors']['insideairbnb']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['insideairbnb']['listing_source_name']) record.setTimestamp('2017-04-08T00:00:00Z') record.setLocation(location) record.setPayload(payload) return record
def buildRecord(self, item, type=''): record = BaseRecord() payload = PointsInterestPayload() location = LocationRecord() payload.setId(item['id'][0]['#text']) payload.setName(item['name']['#text']) payload.setType(item['type']['#text']) payload.setUrl(item['code_url']['#text'] if 'code_url' in item else '') payload.setShortDescription(item['excerpt']['#text']) payload.setLongDescription(item['content']['#text']) payload.setAssociations(GeneralHelper().toAssociations( item['code2']['item'])) item['addresses']['item'] = item['addresses']['item'] if not isinstance( item['addresses']['item'], list) else item['addresses']['item'][0] longitude, latitude = item['addresses']['item']['gmapy'], item[ 'addresses']['item']['gmapx'] location.setPoint(latitude, longitude) location.setAltitude('0.0') location.setCity('Barcelona') district, neighbourhood = LocationHelper().getLocationAreas( latitude, longitude) location.setDistrict(GeneralHelper().default(district)) location.setNeighbourhood(GeneralHelper().default(neighbourhood)) location.setStreetName(item['addresses']['item']['address'].replace( ' ' + (item['addresses']['item']['streetnum'] if 'streetnum' in item['addresses']['item'] else ''), '')) location.setStreetNumber( item['addresses']['item']['streetnum'] if 'streetnum' in item['addresses']['item'] else '') record.setId(item['id'][0]['#text']) record.setSource( collectorCfg['collectors']['odi']['pointsinterest']['source_name']) record.setProvider('') record.setPublisher('') record.setType( collectorCfg['collectors']['odi']['pointsinterest']['source_name']) record.setTimestamp(str(datetime.datetime.now())) record.setLocation(location) record.setPayload(payload) return record