def test_main(self): """ Add, Find, Edit and Delete Naaya Geo Point """ addNyGeoPoint(self._portal().info, id='doc1', title='doc1', lang='en', geo_location=Geo('13', '14', 'here!')) addNyGeoPoint(self._portal().info, id='doc1_fr', title='doc1_fr', lang='fr', geo_location=Geo('15', '16', 'there')) meta = self._portal().getCatalogedObjectsCheckView(meta_type=['Naaya GeoPoint']) #Get added NyGeoPoint for x in meta: if x.getLocalProperty('title', 'en') == 'doc1': meta = x if x.getLocalProperty('title', 'fr') == 'doc1_fr': meta_fr = x self.assertEqual(meta.getLocalProperty('title', 'en'), 'doc1') self.assertEqual(meta_fr.getLocalProperty('title', 'fr'), 'doc1_fr') self.assertEqual(meta.geo_location, Geo('13', '14', 'here!')) #Change NyGeoPoint title meta.saveProperties(title='doc1_edited', lang='en', geo_location=Geo("123.01", "234.30")) meta_fr.saveProperties(title='doc1_fr_edited', lang='fr', geo_location=Geo("123.01", "234.30")) self.assertEqual(meta.getLocalProperty('title', 'en'), 'doc1_edited') self.assertEqual(meta_fr.getLocalProperty('title', 'fr'), 'doc1_fr_edited') #delete NyGeoPoint self._portal().info.manage_delObjects([meta.getId(),]) self._portal().info.manage_delObjects([meta_fr.getId(),]) brains = self._portal().getCatalogedObjectsCheckView(meta_type=['Naaya GeoPoint']) self.assertEqual(len(brains), 0)
def minimap(self): if self.geo_location not in (None, Geo()): simplepoints = [{'lat': self.geo_location.lat, 'lon': self.geo_location.lon}] elif self.aq_parent.geo_location not in (None, Geo()): simplepoints = [{'lat': self.aq_parent.geo_location.lat, 'lon': self.aq_parent.geo_location.lon}] else: return "" json_simplepoints = json.dumps(simplepoints, default=json_encode) return self._minimap_template(points=json_simplepoints)
def do_geocoding(address): """ """ #if self.portal_map.current_engine == 'yahoo': # coordinates = geocoding.yahoo_geocode(address) #elif self.portal_map.current_engine == 'google': coordinates = geocoding.location_geocode(address) if coordinates != None: lat, lon = coordinates return Geo(lat, lon, address) return (None)
def afterSetUp(self): self.portal.manage_install_pluggableitem('Naaya GeoPoint') from naaya.content.geopoint.geopoint_item import addNyGeoPoint addNyGeoPoint(self.portal.info, id='ver_geopoint', title='ver_geopoint', submitted=1, geo_location=Geo('13', '13')) import transaction transaction.commit()
def minimap(self): if self.geo_location not in (Geo(), None): simplepoints = [{ 'lat': self.geo_location.lat, 'lon': self.geo_location.lon }] json_simplepoints = json.dumps(simplepoints, default=json_encode) return self._minimap_template(points=json_simplepoints) simplepoints = [{'lat': photo.geo_location.lat, 'lon': photo.geo_location.lon}\ for photo in self.objectValues("Naaya Photo") if photo.geo_location is not None] if simplepoints != []: json_simplepoints = json.dumps(simplepoints, default=json_encode) return self._minimap_template(points=json_simplepoints) else: return ""
def test_import(self): geo_csv_data = ( "Title,Geo Loc - lat,Geo Loc - lon,Geo Loc - address,Geo Type\n" "doc_one,,,,\n" "doc_two,13.45,22.60,,Test symbol one\n" "doc_three,8,9,somewhere else,Test symbol two\n" "doc_four,,,Bucharest,Test symbol two\n") try: do_import_object(self, 'Naaya Document', geo_csv_data, 'imported') except: raise self.fail('Should not raise exception') self.failUnlessEqual(len(self.portal.imported.objectIds()), 4) doc_one = self.portal.imported._getOb('doc_one') self.failUnlessEqual(doc_one.title, 'doc_one') self.failIf(hasattr(doc_one, 'test_geo_loc')) self.failIf(hasattr(doc_one, 'test_geo_type')) doc_two = self.portal.imported._getOb('doc_two') self.failUnlessEqual(doc_two.test_geo_loc, Geo('13.45', '22.60')) self.failUnlessEqual(doc_two.test_geo_type, 'sym1') doc_three = self.portal.imported._getOb('doc_three') self.failUnlessEqual(doc_three.test_geo_loc, Geo('8', '9', 'somewhere else')) self.failUnlessEqual(doc_three.test_geo_type, 'sym2') doc_four = self.portal.imported._getOb('doc_four') correct = Geo('44.434295', '26.102965', 'Bucharest') self.failUnless( -1 < (doc_four.test_geo_loc.lat - correct.lat) * 100 < 1) self.failUnless( -1 < (doc_four.test_geo_loc.lon - correct.lon) * 100 < 1) self.failUnlessEqual(doc_four.test_geo_loc.address, correct.address) self.failUnlessEqual(doc_four.test_geo_type, 'sym2')
def test_export_geolocation(self): toexport = self.portal['toexport'] addNyContact(toexport, id='contact1', title='Contact 1', contributor='contributor') toexport['contact1'].test_geo_loc = Geo('13.45', '22.60', 'somewhere') csv_output = toexport.csv_export.export(meta_type='Naaya Contact') csv_lines = csv_output.strip().split('\n') self.assertEqual(len(csv_lines), 2) # header line and one object csv_data = dict(zip(*(line.split(',') for line in csv_lines))) self.assertEqual(csv_data['Geo Loc - lat'], '13.45') self.assertEqual(csv_data['Geo Loc - lon'], '22.60') self.assertEqual(csv_data['Geo Loc - address'], 'somewhere')
def afterSetUp(self): self.portal.manage_install_pluggableitem('Naaya GeoPoint') from Products.Naaya.NyFolder import addNyFolder from naaya.content.geopoint.geopoint_item import addNyGeoPoint addNyFolder(self.portal, 'myfolder', contributor='contributor', submitted=1) FolderMetaTypes(self.portal.myfolder).add('Naaya GeoPoint') addNyGeoPoint(self.portal.myfolder, id='mygeopoint', title='My geopoint', submitted=1, contributor='contributor', geo_location=Geo('13', '13')) import transaction transaction.commit()
def set_address(obj): """ Set the postal address based on geo address and viceversa """ # convert postaladdress to geolocation if obj.postaladdress and not obj.geo_location: try: lat, lon = geocoding.location_geocode(obj.postaladdress) obj.geo_location = Geo(lat, lon, obj.postaladdress) except: pass return # convert geolocation to postal address lang = 'en' v = obj.getLocalAttribute("postaladdress", lang) if obj.geo_location and not v: obj.set_localpropvalue('postaladdress', lang, obj.geo_location.address) return
def geolocate_queue(site): for site_path in list(site.geolocation_queue): try: obj = site.unrestrictedTraverse(site_path) except KeyError: # the object is not there anymore site.geolocation_queue.remove(site_path) site._p_changed = True LOG.debug('object not found %s' % site_path) transaction.commit() continue lat = obj.geo_location.lat lon = obj.geo_location.lon address = obj.geo_location.address if address and not (lat and lon): try: # Google also restricts requests per second, so we need # to be careful for this, too time.sleep(2) lat, lon = geocoding.location_geocode(address) if lat and lon: obj.geo_location = Geo(lat, lon, address) site.geolocation_queue.remove(site_path) site._p_changed = True LOG.info('coodrdinates %s and %s found for %s' % (lat, lon, address)) transaction.commit() except GeocoderServiceError, e: if 'ZERO_RESULTS' in e.message: LOG.info('coodrdinates not found for %s' % address) site.geolocation_queue.remove(site_path) else: LOG.info(e) site.previous_geolocation = datetime.now() site._p_changed = True break else: LOG.info('object already geolocated %s' % site_path) site.geolocation_queue.remove(site_path) site._p_changed = True transaction.commit()
def getDatamodel(self, form): """Get datamodel from form""" lat = form.get(self.getWidgetId() + '.lat', None) lon = form.get(self.getWidgetId() + '.lon', None) address = form.get(self.getWidgetId() + '.address', '') if not (lat and lon): coordinates = geocoding.location_geocode(address) if coordinates is not None: lat, lon = coordinates if not lat: lat = None if not lon: lon = None if lat is None and lon is None and address == '': return None try: return Geo(lat, lon, address) except ValueError: raise WidgetError('Invalid geo values for "%s"' % self.title)
def add_object(self, parent): parent.getSite().manage_install_pluggableitem('Naaya GeoPoint') kwargs = {'title': 'My geopoint', 'geo_location': Geo('13', '13')} ob = parent[addNyGeoPoint(parent, **kwargs)] ob.approveThis() return ob
def set_geo_location(self, lat, lon): ''' set lat and lon properties to the Geo object (if missing...)''' from Products.NaayaCore.SchemaTool.widgets.geo import Geo target_answer = self.target_answer() current_address = target_answer.w_location.address target_answer.w_location = Geo(lat=lat, lon=lon, address=current_address)
def saveProperties(self, REQUEST=None, **kwargs): """ """ if not self.checkPermissionEditObject(): raise EXCEPTION_NOTAUTHORIZED, EXCEPTION_NOTAUTHORIZED_MSG if self.hasVersion(): obj = self.version if self.checkout_user != self.REQUEST.AUTHENTICATED_USER.getUserName( ): raise EXCEPTION_NOTAUTHORIZED, EXCEPTION_NOTAUTHORIZED_MSG else: obj = self if REQUEST is not None: schema_raw_data = dict(REQUEST.form) else: schema_raw_data = kwargs _lang = schema_raw_data.pop('_lang', schema_raw_data.pop('lang', None)) _releasedate = self.process_releasedate( schema_raw_data.pop('releasedate', ''), obj.releasedate) self.last_modification = time.localtime() #geo-location: 'geo_location' should always be removed from the schema_raw_data #because the form should contain 'geo_location.lat' type of data if schema_raw_data.has_key('geo_location'): schema_raw_data.pop('geo_location') _city = schema_raw_data.get('organisation_city', None) _country = schema_raw_data.get('organisation_country', None) _address = '' if _city or _country: _address = _city + ', ' + _country if _address: old_geo_location = self.geo_location not in ( None, Geo()) and self.geo_location.address != _address no_geo_location = self.geo_location in (None, Geo()) if old_geo_location or no_geo_location: _lat, _lon = self.do_geocoding(_address) else: _lat, _lon = self.geo_location.lat, self.geo_location.lon schema_raw_data['geo_location.lat'] = _lat schema_raw_data['geo_location.lon'] = _lon schema_raw_data['geo_location.address'] = _address form_errors = self.process_submitted_form( schema_raw_data, _lang, _override_releasedate=_releasedate) if form_errors: if REQUEST is not None: self._prepare_error_response(REQUEST, form_errors, schema_raw_data) REQUEST.RESPONSE.redirect('%s/edit_html?lang=%s' % (self.absolute_url(), _lang)) return else: raise ValueError( form_errors.popitem()[1]) # pick a random error if self.discussion: self.open_for_comments() else: self.close_for_comments() self._p_changed = 1 self.recatalogNyObject(self) #log date contributor = self.REQUEST.AUTHENTICATED_USER.getUserName() auth_tool = self.getAuthenticationTool() auth_tool.changeLastPost(contributor) notify(NyContentObjectEditEvent(self, contributor)) if REQUEST: self.setSessionInfoTrans(MESSAGE_SAVEDCHANGES, date=self.utGetTodayDate()) REQUEST.RESPONSE.redirect('%s/edit_html?lang=%s' % (self.absolute_url(), _lang))
'Warnings: could not find a valid address ' 'for row ${record_number}: ${error}', { 'record_number': record_number + 1, # account for header 'error': str(e) }) warnings.append(msg) address = properties.pop(self.geo_fields['address']) ob_id = add_object(location_obj, _send_notifications=False, **properties) ob = location_obj._getOb(ob_id) if address: setattr(ob, self.geo_fields['address'].split('.')[0], Geo(address=address)) #user = self.REQUEST.AUTHENTICATED_USER.getUserName() #notify(NyContentObjectEditEvent(ob, user)) if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: extra_props_messages = adapter.handle_columns( extra_properties) if extra_props_messages: errors.append(extra_props_messages) obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis(_send_notifications=False) except UnicodeDecodeError, e: raise except Exception, e:
class CSVImporterTask(Persistent): """ An import task for rows from Excel/CSV files """ payload = None template = None error = None status = 'unfinished' def __init__(self, *args, **kwargs): super(CSVImporterTask, self).__init__(*args, **kwargs) self.warnings = PersistentList() self.errors = PersistentList() def on_failure(self, *args, **kwargs): self.status = 'failed' def on_success(self, *args, **kwargs): self.status = 'finished' def run(self, *args, **kwargs): self.payload = kwargs['payload'] self.template = kwargs['template'] self.rec_id = kwargs['rec_id'] site = getSite() user = getSecurityManager().getUser() acl_users = site.acl_users user = user.__of__(acl_users) user_id = kwargs['user_id'] request = BaseRequest() request.response = Response() request.AUTHENTICATED_USER = user request['PARENTS'] = [site] request['URL'] = kwargs['url'] request.steps = [] request.cookies = {} request.form = {} import_location = site.unrestrictedTraverse(kwargs['import_path']) import_location.REQUEST = request site.REQUEST = request geo_fields = kwargs['geo_fields'] self.prop_map = kwargs['properties'] meta_type = kwargs['meta_type'] header = self.template row = self.payload record_number = self.rec_id content_type = import_location.getSite().get_pluggable_item(meta_type) add_object = content_type['add_method'] properties = {} extra_properties = {} address = None for column, value in zip(header, row): if value == '': continue if column not in self.prop_map: extra_properties[column] = value continue key = self.prop_map[column]['column'] widget = self.prop_map[column]['widget'] widget = widget.__of__(import_location) convert = widget.convert_from_user_string properties[key] = convert(value) try: properties = do_geocoding(geo_fields, properties) except GeocoderServiceError, e: msg = ( 'Warnings: could not find a valid address ' 'for row ${record_number}: ${error}', { 'record_number': record_number + 1, # account for header 'error': str(e) }) self.warnings.append(msg) address = properties.pop(geo_fields['address']) ob_id = add_object(import_location, _send_notifications=False, **properties) ob = import_location._getOb(ob_id) if address: setattr(ob, geo_fields['address'].split('.')[0], Geo(address=address)) notify(NyContentObjectEditEvent(ob, user_id)) if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: extra_props_messages = adapter.handle_columns(extra_properties) if extra_props_messages: self.errors.append(extra_props_messages) #obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis(_send_notifications=False) del import_location.REQUEST del site.REQUEST
def _update_ob(self, portal, ob, schema_tool): ob_info = '(%s)' % ob.__class__.__name__ + '/'.join( ob.getPhysicalPath()) schema = schema_tool.getSchemaForMetatype(ob.meta_type) if schema is None: self.log.debug(ob_info + ' skipping (no schema)') return if 'geo_location-property' in schema.objectIds(): missing_schema_prop = False else: missing_schema_prop = True if missing_schema_prop: self.log.debug(ob_info + ' skipping (geo_location not in schema)') return found_lat, lat = fetch_and_remove(ob, 'latitude') found_lon, lon = fetch_and_remove(ob, 'longitude') found_address, address = fetch_and_remove(ob, 'address') found_geo_location = ('geo_location' in ob.__dict__) if found_geo_location: self.log.debug(ob_info + ' skipping (already has geo_location value)') return def normalize_coord(found, value): if not found: return None elif isinstance(value, float): if value: return str(value) else: return None elif isinstance(value, basestring): try: if Decimal(value): return value else: return None except: return None else: return None lat = normalize_coord(found_lat, lat) lon = normalize_coord(found_lon, lon) if not (found_lat or found_lon or found_address): self.log.debug(ob_info + ' skipping (no old-style geo data)') return geo = Geo(lat, lon, address) if geo in (Geo(0, 0), Geo()): geo = None self.log.debug(ob_info + ' saving value ' + repr(geo)) ob.geo_location = geo portal.catalogNyObject(ob)
class CSVImportTool(Implicit, Item): title = "Import from structured file" security = ClassSecurityInfo() geo_fields = {} def __init__(self, id): self.id = id security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'template') def template(self, meta_type, file_type, as_attachment=False, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionPublishObjects(): raise Unauthorized schema = self.getSite().getSchemaTool().getSchemaForMetatype(meta_type) if schema is None: raise ValueError('Schema for meta-type "%s" not found' % meta_type) columns = [] for widget in schema.listWidgets(): if widget.multiple_form_values: for subname in widget.multiple_form_values: columns.append(widget.title + ' - ' + subname) else: columns.append(widget.title) if file_type == 'CSV': ret = generate_csv(columns, [[]]) content_type = 'text/csv; charset=utf-8' filename = schema.title_or_id() + ' bulk upload.csv' elif file_type == 'Excel': ret = generate_excel(columns, [[]]) content_type = 'application/vnd.ms-excel' filename = schema.title_or_id() + ' bulk upload.xls' else: raise ValueError('unknown file format %r' % file_type) if REQUEST is not None: set_response_attachment(REQUEST.RESPONSE, filename, content_type, len(ret)) return ret security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'do_geocoding') def do_geocoding(self, properties): lat = properties.get(self.geo_fields['lat'], '') lon = properties.get(self.geo_fields['lon'], '') address = properties.get(self.geo_fields['address'], '') if lat.strip() == '' and lon.strip() == '' and address: coordinates = geocoding.geocode(self.portal_map, address) if coordinates != None: lat, lon = coordinates properties[self.geo_fields['lat']] = lat properties[self.geo_fields['lon']] = lon return properties security.declareProtected(PERMISSION_PUBLISH_OBJECTS, 'do_import') def do_import(self, meta_type, file_type, data, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionPublishObjects(): raise Unauthorized errors = [] warnings = [] site = self.getSite() schema = site.getSchemaTool().getSchemaForMetatype(meta_type) if schema is None: raise ValueError('Schema for meta-type not found: "%s"' % meta_type) content_type = site.get_pluggable_item(meta_type) add_object = content_type['add_method'] location_obj = self.getParentNode() # build a list of property names based on the object schema # TODO: extract this loop into a separate function prop_map = {} for widget in schema.listWidgets(): prop_name = widget.prop_name() if widget.multiple_form_values: for subname in widget.multiple_form_values: prop_subname = prop_name + '.' + subname prop_map[widget.title + ' - ' + subname] = { 'column': prop_subname, 'convert': widget.convert_from_user_string, } if isinstance(widget, GeoWidget): for subname in widget.multiple_form_values: self.geo_fields[subname] = prop_name + '.' + subname else: prop_map[widget.title] = { 'column': prop_name, 'convert': widget.convert_from_user_string, } if file_type == 'Excel': try: wb = xlrd.open_workbook(file_contents=data.read()) ws = wb.sheets()[0] header = ws.row_values(0) rows = [] for i in range(ws.nrows)[1:]: rows.append(ws.row_values(i)) except xlrd.XLRDError: msg = 'Invalid Excel file' if REQUEST is None: raise ValueError(msg) else: errors.append(msg) rows = [] elif file_type == 'CSV': rows = UnicodeReader(data) try: header = rows.next() except StopIteration: msg = 'Invalid CSV file' if REQUEST is None: raise ValueError(msg) else: errors.append(msg) rows = [] except UnicodeDecodeError, e: if REQUEST is None: raise else: errors.append('CSV file is not utf-8 encoded') else: raise ValueError('unknown file format %r' % file_type) record_number = 0 obj_ids = [] try: for row in rows: try: record_number += 1 #TODO: extract this block into a separate function properties = {} extra_properties = {} address = None for column, value in zip(header, row): if value == '': continue if column not in prop_map: extra_properties[column] = value continue key = prop_map[column]['column'] convert = prop_map[column]['convert'] properties[key] = convert(value) address = properties.pop(self.geo_fields['address']) ob_id = add_object(location_obj, _send_notifications=False, **properties) ob = location_obj._getOb(ob_id) if address: setattr(ob, self.geo_fields['address'].split('.')[0], Geo(address=address)) #user = self.REQUEST.AUTHENTICATED_USER.getUserName() #notify(NyContentObjectEditEvent(ob, user)) if getattr(site, 'geolocation_queue', None): site.geolocation_queue.append( '/' + site.getId() + '/' + path_in_site(ob)) else: site.geolocation_queue = ['/' + site.getId() + '/' + path_in_site(ob)] if extra_properties: adapter = ICSVImportExtraColumns(ob, None) if adapter is not None: extra_props_messages = adapter.handle_columns( extra_properties) if extra_props_messages: errors.append(extra_props_messages) obj_ids.append(ob.getId()) ob.submitThis() ob.approveThis(_send_notifications=False) except UnicodeDecodeError, e: raise except Exception, e: self.log_current_error() msg = ('Error while importing from file, row ${record_number}: ${error}', {'record_number': record_number + 1, # account for header 'error': str(e)}) if REQUEST is None: raise ValueError(msg) else: errors.append(msg)
def test_add(self): self.browser_do_login('contributor', 'contributor') self.browser.go('http://localhost/portal/myfolder/geopoint_add_html') self.failUnless('<h1>Submit GeoPoint</h1>' in self.browser.get_html()) form = self.browser.get_form('frmAdd') expected_controls = set([ 'lang', 'title:utf8:ustring', 'description:utf8:ustring', 'coverage:utf8:ustring', 'keywords:utf8:ustring', 'releasedate', 'discussion:boolean', 'geo_location.lat:utf8:ustring', 'geo_location.lon:utf8:ustring', 'geo_location.address:utf8:ustring', 'geo_type:utf8:ustring', 'url:utf8:ustring', 'pointer:utf8:ustring', ]) found_controls = set(c.name for c in form.controls) self.failUnless( expected_controls.issubset(found_controls), 'Missing form controls: %s' % repr(expected_controls - found_controls)) self.browser.clicked(form, self.browser.get_form_field(form, 'title')) form['title:utf8:ustring'] = 'test_geopoint' form['description:utf8:ustring'] = 'test_geopoint_description' form['coverage:utf8:ustring'] = 'test_geopoint_coverage' form['keywords:utf8:ustring'] = 'keyw1, keyw2' form['geo_location.lat:utf8:ustring'] = '12.587142' form['geo_location.lon:utf8:ustring'] = '55.681004' form[ 'geo_location.address:utf8:ustring'] = 'Kongens Nytorv 6, 1050 Copenhagen K, Denmark' #form['geo_type:utf8:ustring'] = '' form['url:utf8:ustring'] = 'http://www.eea.europa.eu' form['pointer:utf8:ustring'] = 'portal/info/contact' self.browser.submit() html = self.browser.get_html() self.failUnless( 'The administrator will analyze your request and you will be notified about the result shortly.' in html) geopoint = self.portal.myfolder.test_geopoint self.failUnlessEqual(geopoint.title, 'test_geopoint') self.failUnlessEqual( geopoint.geo_location, Geo('12.587142', '55.681004', 'Kongens Nytorv 6, 1050 Copenhagen K, Denmark')) self.failUnlessEqual(geopoint.url, 'http://www.eea.europa.eu') geopoint.approveThis() self.browser.go('http://localhost/portal/myfolder/test_geopoint') html = self.browser.get_html() self.failUnless( re.search(r'<h1>.*test_geopoint.*</h1>', html, re.DOTALL)) self.failUnless('test_geopoint_description' in html) self.failUnless('test_geopoint_coverage' in html) self.failUnless('keyw1, keyw2' in html) self.browser_do_logout()