def get_remote_item(self, path): remote_url = self.remote_url + self.remote_path if not remote_url.endswith('/'): remote_url += '/' if path.startswith('/'): path = path[1:] url = urllib2.urlparse.urljoin(remote_url, urllib.quote(path)) # remote = xmlrpclib.Server( # url, # BasicAuth(self.remote_username, self.remote_password), # ) # XMLRPC seems to be causing unexplained Faults where urllib works remote = Urllibrpc(url, self.remote_username, self.remote_password) try: item = remote.get_item() except UrllibrpcException as e: logger.error("Failed reading url '%s' with error code %s." % (e.url, e.code)) return None, [] try: subitems = remote.get_children() except UrllibrpcException as e: logger.error("Failed reading url '%s' with error code %s." % (e.url, e.code)) return item, [] return item, subitems
def __iter__(self): for item in self.previous: pathkey = self.pathkey(*item.keys())[0] permskey = self.permskey(*item.keys())[0] if not pathkey or not permskey or \ permskey not in item: # not enough info yield item; continue obj = self.context.unrestrictedTraverse( item[pathkey].lstrip('/'), None) if obj is None: # path doesn't exist yield item; continue if IRoleManager.providedBy(obj): for perm, perm_dict in item[permskey].items(): try: obj.manage_permission(perm, roles=perm_dict['roles'], acquire=perm_dict['acquire']) except ValueError: #raise Exception('Error setting the perm "%s"' % perm) logger.error('Error setting the perm "%s" on %s' % (perm, item[pathkey])) yield item
def get_remote_item(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) try: f = urllib2.urlopen(item_url) item_json = f.read() except urllib2.URLError, e: logger.error("Failed reading item from %s. %s" % (item_url, str(e))) return None
def get_items(self, path, depth=0): if path and path[-1] == '/': path = path[:-1] if self.remote_crawl_depth == -1 or depth <= self.remote_crawl_depth: item, subitems = self.get_remote_item(path) if item is None: logger.warn(':: Skipping -> %s. No remote data.' % path) return if item.startswith('ERROR'): logger.error("Could not get item '%s' from remote. Got %s." % (path, item)) return item = simplejson.loads(item) logger.info(':: Crawling %s' % item['_path']) if self.local_path: item['_path'] = self.local_path + item['_path'][len(self.remote_path):] # item['_path'] is relative to domain root. we need relative to plone root # remote_url = self.remote_url # _,_,remote_path,_,_,_ = urlparse.urlparse(remote_url) # item['_path'] = item['_path'][len(remote_path):] # if item['_path'].startswith('/'): # item['_path'] = item['_path'][1:] if item['_type'] == "Plone Site": pass else: yield item if subitems.startswith('ERROR'): logger.error("Could not get subitems for '%s'. Got %s." % (path, subitems)) return for subitem_id in simplejson.loads(subitems): subitem_path = path + '/' + subitem_id if subitem_path[len(self.remote_path):] in self.remote_skip_path: logger.info(':: Skipping -> ' + subitem_path) continue if self.remote_catalog_query: if subitem_path not in self.remote_ok_path: logger.info(':: Skipping (2) -> ' + subitem_path) continue for subitem in self.get_items(subitem_path, depth+1): yield subitem
def get_ok_path(self): catalog_query = '?catalog_query=%s' % urllib.quote(encodestring(self.remote_catalog_query)) url = urllib2.urlparse.urljoin(self.remote_url, urllib.quote(self.remote_catalog_path)) + catalog_query remote = Urllibrpc(url, self.remote_username, self.remote_password) try: items = remote.get_catalog_results() except UrllibrpcException, e: logger.error("Failed reading url '%s' with error code %s." % (e.url, e.code)) return set([])
def _load_path(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) resp = requests.get( item_url, auth=(self.remote_username, self.remote_password), timeout=600) content_type = resp.headers['content-type'] if content_type != 'application/json': logger.error('Could not get : {}'.format(resp.url)) return None else: item = resp.json() return item
def _load_path(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) try: f = urllib2.urlopen(item_url) item_json = f.read() except urllib2.URLError as e: logger.error("Failed reading item from %s. %s" % (item_url, str(e))) return None try: item = json.loads(item_json) except json.JSONDecodeError: logger.error("Could not decode item from %s." % item_url) return None return item
def _load_path(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) try: f = urllib2.urlopen(item_url) item_json = f.read() except urllib2.URLError as e: logger.error( "Failed reading item from %s. %s" % (item_url, str(e))) return None try: item = simplejson.loads(item_json) except simplejson.JSONDecodeError: logger.error("Could not decode item from %s." % item_url) return None return item
def _load_path(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) try: if not self.session: f = urllib2.urlopen(item_url) item_json = f.read() else: item_json = self.session.get(item_url, verify=False).content #f.read() except urllib2.URLError as e: logger.error( "Failed reading item from %s. %s" % (item_url, str(e))) return None try: item = json.loads(item_json) except json.JSONDecodeError: logger.error("Could not decode item from %s." % item_url) return None return item
def get_items(self, path, depth=0): if path and path[-1] == '/': path = path[:-1] if self.remote_crawl_depth == -1 or depth <= self.remote_crawl_depth: item, subitems = self.get_remote_item(path) if item is None: logger.warn(':: Skipping -> %s. No remote data.' % path) return if item.startswith('ERROR'): logger.error( "Could not get item '%s' from remote. Got %s." % (path, item)) return try: item = json.loads(item) except json.JSONDecodeError: logger.error( "Could not decode item from path '%s' as JSON." % path) return logger.info(':: Crawling %s' % item['_path']) # item['_path'] is relative to domain root. we need relative to # plone root remote_url = self.remote_url _, _, remote_path, _, _, _ = urlparse.urlparse(remote_url) item['_path'] = item['_path'][len(remote_path):] if item['_path'].startswith('/'): item['_path'] = item['_path'][1:] if item['_type'] == "Plone Site": pass else: yield item if subitems.startswith('ERROR'): logger.error( "Could not get subitems for '%s'. Got %s." % (path, subitems)) return for subitem_id in json.loads(subitems): subitem_path = path + '/' + subitem_id if subitem_path[len(self.remote_path):]\ in self.remote_skip_path: logger.info(':: Skipping -> ' + subitem_path) continue for subitem in self.get_items(subitem_path, depth + 1): yield subitem
def get_remote_item(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) resp = requests.get(item_url, params=self.payload, auth=(self.remote_username, self.remote_password)) if resp.status_code == 200: item_json = resp.text else: logger.error("Failed reading item from %s. %s" % (path, resp.status_code)) self.errored.append(path) return None try: item = simplejson.loads(item_json) except simplejson.JSONDecodeError: logger.error("Could not decode item from %s." % item_url) logger.error("Response is %s." % item_json) self.errored.append(path) return None return item
def get_remote_item(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) resp = requests.get(item_url, params=self.payload, auth=(self.remote_username, self.remote_password)) if resp.status_code == 200: item_json = resp.text else: logger.error("Failed reading item from %s. %s" % (path, resp.status_code)) self.errored.append(path) return None try: item = json.loads(item_json) except: logger.error("Could not decode item from %s." % item_url) logger.error("Response is %s." % item_json) self.errored.append(path) return None return item
# ) # XMLRPC seems to be causing unexplained Faults where urllib works remote = Urllibrpc(url, self.remote_username, self.remote_password) try: item = remote.get_item() except UrllibrpcException, e: logger.error("Failed reading url '%s' with error code %s." % (e.url, e.code)) return None, [] try: subitems = remote.get_children() except UrllibrpcException, e: logger.error("Failed reading url '%s' with error code %s." % (e.url, e.code)) return item, [] return item, subitems def get_items(self, path, depth=0): if path and path[-1] == '/': path = path[:-1] if self.remote_crawl_depth == -1 or depth <= self.remote_crawl_depth: item, subitems = self.get_remote_item(path) if item is None: logger.warn(':: Skipping -> %s. No remote data.' % path) return
def __iter__(self): for item in self.previous: pathkey = self.pathkey(*item.keys())[0] propertieskey = self.propertieskey(*item.keys())[0] if not pathkey or not propertieskey or \ propertieskey not in item: # not enough info yield item continue path = safe_unicode(item[pathkey].lstrip('/')).encode('ascii') obj = traverse(self.context, path, None) if obj is None: # path doesn't exist yield item continue if not getattr(aq_base(obj), '_setProperty', False): yield item continue # Bugfix > Set exclude_from_nav (Plone 5) if excludeFromNav (Plone 4) is True try: if item['excludeFromNav']: obj.exclude_from_nav = True except: pass # Bugfix > set start & end date in Event object Plone 4 > Plone 5 # Convert all datetime timezone in UTC+0 to avoid hours change try: start = item['startDate'] start = parser.parse(start).replace( tzinfo=pytz.timezone('UTC')) end = item['endDate'] end = parser.parse(end).replace(tzinfo=pytz.timezone('UTC')) if start and end: obj.start = start obj.end = end except: pass # Bugfix > effective_date and expiration_date field. If keys doesn't exists (e.g. effective_date in Plone 4) # or if var is in CamelCase (e.g. expirationDate in Plone 4) keys = item.keys() if 'effectiveDate' in keys: # Bugfix > Convert string (<type 'unicode'>) in DateTime object effective_date = item['effectiveDate'] if effective_date: effective_date = DateTime(effective_date) obj.effective_date = effective_date if not 'effective_date' in keys and not 'effectiveDate' in keys: # Bugfix > Convert string (<type 'unicode'>) in DateTime object creation_date = item['creation_date'] if creation_date: creation_date = DateTime(creation_date) obj.effective_date = creation_date if 'expirationDate' in keys: # Bugfix > Convert string (<type 'unicode'>) in DateTime object expiration_date = item['expirationDate'] if expiration_date: expiration_date = DateTime(expiration_date) obj.expiration_date = expiration_date # Bugfix > Convert Lineage child site in Subsite Dexterity object # Need to create a new Dexterity object called - Sub Site (subsite) portal_types = self.context.portal_types.listContentTypes() if item['_type'] == 'Folder': if 'collective.lineage.interfaces.IChildSite' in item[ '_directly_provided']: dxt_obj_id = 'subsite' if dxt_obj_id in portal_types: obj.portal_type = dxt_obj_id else: logger.error( "Unable to import a Lineage child site. Please add a new Dexterity Folder type with id 'subsite' and select 1. Folder Addable Constrains 2. Layout support 3. Navigation root in Behavior tab " ) raise for pid, pvalue, ptype in item[propertieskey]: if getattr(aq_base(obj), pid, None) is not None: # if object have a attribute equal to property, do nothing continue # Bugfix > plone default_page must be a string, got (<type 'unicode'>) if pid == 'default_page': pvalue = str(pvalue) try: if obj.hasProperty(pid): obj._updateProperty(pid, pvalue) else: obj._setProperty(pid, pvalue, ptype) except ConflictError: raise except Exception as e: raise Exception('Failed to set property "%s" type "%s"' ' to "%s" at object %s. ERROR: %s' % (pid, ptype, pvalue, str(obj), str(e))) logger.info("object creation %s" % (obj.absolute_url_path())) yield item
class CatalogSourceSection(object): """A source section which creates items from a remote Plone site by querying it's catalog. """ classProvides(ISectionBlueprint) implements(ISection) def __init__(self, transmogrifier, name, options, previous): self.previous = previous self.options = options self.context = transmogrifier.context self.remote_url = self.get_option('remote-url', 'http://localhost:8080') remote_username = self.get_option('remote-username', 'admin') remote_password = self.get_option('remote-password', 'admin') catalog_path = self.get_option('catalog-path', '/Plone/portal_catalog') self.site_path_length = len('/'.join(catalog_path.split('/')[:-1])) catalog_query = self.get_option('catalog-query', None) catalog_query = ' '.join(catalog_query.split()) catalog_query = base64.b64encode(catalog_query) self.remote_skip_paths = self.get_option('remote-skip-paths', '').split() # Install a basic auth handler auth_handler = urllib2.HTTPBasicAuthHandler() auth_handler.add_password(realm='Zope', uri=self.remote_url, user=remote_username, passwd=remote_password) opener = urllib2.build_opener(auth_handler) urllib2.install_opener(opener) req = urllib2.Request('%s%s/get_catalog_results' % (self.remote_url, catalog_path), urllib.urlencode({'catalog_query': catalog_query})) try: f = urllib2.urlopen(req) resp = f.read() except urllib2.URLError: raise self.item_paths = sorted(simplejson.loads(resp)) def get_option(self, name, default): """Get an option from the request if available and fallback to the transmogrifier config. """ request = getattr(self.context, 'REQUEST', None) if request is not None: value = request.form.get('form.widgets.'+name.replace('-', '_'), self.options.get(name, default)) else: value = self.options.get(name, default) if isinstance(value, unicode): value = value.encode('utf8') return value def __iter__(self): for item in self.previous: yield item for path in self.item_paths: skip = False for skip_path in self.remote_skip_paths: if path.startswith(skip_path): skip = True if not skip: item = self.get_remote_item(path) if item: item['_path'] = item['_path'][self.site_path_length:] yield item def get_remote_item(self, path): item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path)) try: f = urllib2.urlopen(item_url) item_json = f.read() except urllib2.URLError, e: logger.error("Failed reading item from %s. %s" % (item_url, str(e))) return None try: item = simplejson.loads(item_json) except simplejson.JSONDecodeError: logger.error("Could not decode item from %s." % item_url) return None return item