def get_remote_item(self, path):
remote_url = self.remote_url + self.remote_path
if not remote_url.endswith('/'):
remote_url += '/'
if path.startswith('/'):
path = path[1:]
url = urllib2.urlparse.urljoin(remote_url, urllib.quote(path))
# remote = xmlrpclib.Server(
# url,
# BasicAuth(self.remote_username, self.remote_password),
# )
# XMLRPC seems to be causing unexplained Faults where urllib works
remote = Urllibrpc(url, self.remote_username, self.remote_password)
try:
item = remote.get_item()
except UrllibrpcException as e:
logger.error("Failed reading url '%s' with error code %s." %
(e.url, e.code))
return None, []
try:
subitems = remote.get_children()
except UrllibrpcException as e:
logger.error("Failed reading url '%s' with error code %s." %
(e.url, e.code))
return item, []
return item, subitems
def __iter__(self):
for item in self.previous:
pathkey = self.pathkey(*item.keys())[0]
permskey = self.permskey(*item.keys())[0]
if not pathkey or not permskey or \
permskey not in item: # not enough info
yield item; continue
obj = self.context.unrestrictedTraverse(
item[pathkey].lstrip('/'), None)
if obj is None: # path doesn't exist
yield item; continue
if IRoleManager.providedBy(obj):
for perm, perm_dict in item[permskey].items():
try:
obj.manage_permission(perm,
roles=perm_dict['roles'],
acquire=perm_dict['acquire'])
except ValueError:
#raise Exception('Error setting the perm "%s"' % perm)
logger.error('Error setting the perm "%s" on %s' % (perm, item[pathkey]))
yield item
def get_remote_item(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
try:
f = urllib2.urlopen(item_url)
item_json = f.read()
except urllib2.URLError, e:
logger.error("Failed reading item from %s. %s" % (item_url, str(e)))
return None
def get_items(self, path, depth=0):
if path and path[-1] == '/':
path = path[:-1]
if self.remote_crawl_depth == -1 or depth <= self.remote_crawl_depth:
item, subitems = self.get_remote_item(path)
if item is None:
logger.warn(':: Skipping -> %s. No remote data.' % path)
return
if item.startswith('ERROR'):
logger.error("Could not get item '%s' from remote. Got %s." % (path, item))
return
item = simplejson.loads(item)
logger.info(':: Crawling %s' % item['_path'])
if self.local_path:
item['_path'] = self.local_path + item['_path'][len(self.remote_path):]
# item['_path'] is relative to domain root. we need relative to plone root
# remote_url = self.remote_url
# _,_,remote_path,_,_,_ = urlparse.urlparse(remote_url)
# item['_path'] = item['_path'][len(remote_path):]
# if item['_path'].startswith('/'):
# item['_path'] = item['_path'][1:]
if item['_type'] == "Plone Site":
pass
else:
yield item
if subitems.startswith('ERROR'):
logger.error("Could not get subitems for '%s'. Got %s." % (path, subitems))
return
for subitem_id in simplejson.loads(subitems):
subitem_path = path + '/' + subitem_id
if subitem_path[len(self.remote_path):] in self.remote_skip_path:
logger.info(':: Skipping -> ' + subitem_path)
continue
if self.remote_catalog_query:
if subitem_path not in self.remote_ok_path:
logger.info(':: Skipping (2) -> ' + subitem_path)
continue
for subitem in self.get_items(subitem_path, depth+1):
yield subitem
def get_ok_path(self):
catalog_query = '?catalog_query=%s' % urllib.quote(encodestring(self.remote_catalog_query))
url = urllib2.urlparse.urljoin(self.remote_url, urllib.quote(self.remote_catalog_path)) + catalog_query
remote = Urllibrpc(url, self.remote_username, self.remote_password)
try:
items = remote.get_catalog_results()
except UrllibrpcException, e:
logger.error("Failed reading url '%s' with error code %s." %
(e.url, e.code))
return set([])
def _load_path(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
resp = requests.get(
item_url,
auth=(self.remote_username, self.remote_password),
timeout=600)
content_type = resp.headers['content-type']
if content_type != 'application/json':
logger.error('Could not get : {}'.format(resp.url))
return None
else:
item = resp.json()
return item
def _load_path(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
try:
f = urllib2.urlopen(item_url)
item_json = f.read()
except urllib2.URLError as e:
logger.error("Failed reading item from %s. %s" %
(item_url, str(e)))
return None
try:
item = json.loads(item_json)
except json.JSONDecodeError:
logger.error("Could not decode item from %s." % item_url)
return None
return item
def _load_path(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
try:
f = urllib2.urlopen(item_url)
item_json = f.read()
except urllib2.URLError as e:
logger.error(
"Failed reading item from %s. %s" %
(item_url, str(e)))
return None
try:
item = simplejson.loads(item_json)
except simplejson.JSONDecodeError:
logger.error("Could not decode item from %s." % item_url)
return None
return item
def _load_path(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
try:
if not self.session:
f = urllib2.urlopen(item_url)
item_json = f.read()
else:
item_json = self.session.get(item_url, verify=False).content #f.read()
except urllib2.URLError as e:
logger.error(
"Failed reading item from %s. %s" %
(item_url, str(e)))
return None
try:
item = json.loads(item_json)
except json.JSONDecodeError:
logger.error("Could not decode item from %s." % item_url)
return None
return item
def get_items(self, path, depth=0):
if path and path[-1] == '/':
path = path[:-1]
if self.remote_crawl_depth == -1 or depth <= self.remote_crawl_depth:
item, subitems = self.get_remote_item(path)
if item is None:
logger.warn(':: Skipping -> %s. No remote data.' % path)
return
if item.startswith('ERROR'):
logger.error(
"Could not get item '%s' from remote. Got %s." %
(path, item))
return
try:
item = json.loads(item)
except json.JSONDecodeError:
logger.error(
"Could not decode item from path '%s' as JSON." % path)
return
logger.info(':: Crawling %s' % item['_path'])
# item['_path'] is relative to domain root. we need relative to
# plone root
remote_url = self.remote_url
_, _, remote_path, _, _, _ = urlparse.urlparse(remote_url)
item['_path'] = item['_path'][len(remote_path):]
if item['_path'].startswith('/'):
item['_path'] = item['_path'][1:]
if item['_type'] == "Plone Site":
pass
else:
yield item
if subitems.startswith('ERROR'):
logger.error(
"Could not get subitems for '%s'. Got %s." %
(path, subitems))
return
for subitem_id in json.loads(subitems):
subitem_path = path + '/' + subitem_id
if subitem_path[len(self.remote_path):]\
in self.remote_skip_path:
logger.info(':: Skipping -> ' + subitem_path)
continue
for subitem in self.get_items(subitem_path, depth + 1):
yield subitem
def get_remote_item(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
resp = requests.get(item_url, params=self.payload, auth=(self.remote_username, self.remote_password))
if resp.status_code == 200:
item_json = resp.text
else:
logger.error("Failed reading item from %s. %s" % (path, resp.status_code))
self.errored.append(path)
return None
try:
item = simplejson.loads(item_json)
except simplejson.JSONDecodeError:
logger.error("Could not decode item from %s." % item_url)
logger.error("Response is %s." % item_json)
self.errored.append(path)
return None
return item
def get_remote_item(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
resp = requests.get(item_url, params=self.payload, auth=(self.remote_username, self.remote_password))
if resp.status_code == 200:
item_json = resp.text
else:
logger.error("Failed reading item from %s. %s" % (path, resp.status_code))
self.errored.append(path)
return None
try:
item = json.loads(item_json)
except:
logger.error("Could not decode item from %s." % item_url)
logger.error("Response is %s." % item_json)
self.errored.append(path)
return None
return item
# )
# XMLRPC seems to be causing unexplained Faults where urllib works
remote = Urllibrpc(url, self.remote_username, self.remote_password)
try:
item = remote.get_item()
except UrllibrpcException, e:
logger.error("Failed reading url '%s' with error code %s." %
(e.url, e.code))
return None, []
try:
subitems = remote.get_children()
except UrllibrpcException, e:
logger.error("Failed reading url '%s' with error code %s." %
(e.url, e.code))
return item, []
return item, subitems
def get_items(self, path, depth=0):
if path and path[-1] == '/':
path = path[:-1]
if self.remote_crawl_depth == -1 or depth <= self.remote_crawl_depth:
item, subitems = self.get_remote_item(path)
if item is None:
logger.warn(':: Skipping -> %s. No remote data.' % path)
return
def __iter__(self):
for item in self.previous:
pathkey = self.pathkey(*item.keys())[0]
propertieskey = self.propertieskey(*item.keys())[0]
if not pathkey or not propertieskey or \
propertieskey not in item:
# not enough info
yield item
continue
path = safe_unicode(item[pathkey].lstrip('/')).encode('ascii')
obj = traverse(self.context, path, None)
if obj is None:
# path doesn't exist
yield item
continue
if not getattr(aq_base(obj), '_setProperty', False):
yield item
continue
# Bugfix > Set exclude_from_nav (Plone 5) if excludeFromNav (Plone 4) is True
try:
if item['excludeFromNav']:
obj.exclude_from_nav = True
except:
pass
# Bugfix > set start & end date in Event object Plone 4 > Plone 5
# Convert all datetime timezone in UTC+0 to avoid hours change
try:
start = item['startDate']
start = parser.parse(start).replace(
tzinfo=pytz.timezone('UTC'))
end = item['endDate']
end = parser.parse(end).replace(tzinfo=pytz.timezone('UTC'))
if start and end:
obj.start = start
obj.end = end
except:
pass
# Bugfix > effective_date and expiration_date field. If keys doesn't exists (e.g. effective_date in Plone 4)
# or if var is in CamelCase (e.g. expirationDate in Plone 4)
keys = item.keys()
if 'effectiveDate' in keys:
# Bugfix > Convert string (<type 'unicode'>) in DateTime object
effective_date = item['effectiveDate']
if effective_date:
effective_date = DateTime(effective_date)
obj.effective_date = effective_date
if not 'effective_date' in keys and not 'effectiveDate' in keys:
# Bugfix > Convert string (<type 'unicode'>) in DateTime object
creation_date = item['creation_date']
if creation_date:
creation_date = DateTime(creation_date)
obj.effective_date = creation_date
if 'expirationDate' in keys:
# Bugfix > Convert string (<type 'unicode'>) in DateTime object
expiration_date = item['expirationDate']
if expiration_date:
expiration_date = DateTime(expiration_date)
obj.expiration_date = expiration_date
# Bugfix > Convert Lineage child site in Subsite Dexterity object
# Need to create a new Dexterity object called - Sub Site (subsite)
portal_types = self.context.portal_types.listContentTypes()
if item['_type'] == 'Folder':
if 'collective.lineage.interfaces.IChildSite' in item[
'_directly_provided']:
dxt_obj_id = 'subsite'
if dxt_obj_id in portal_types:
obj.portal_type = dxt_obj_id
else:
logger.error(
"Unable to import a Lineage child site. Please add a new Dexterity Folder type with id 'subsite' and select 1. Folder Addable Constrains 2. Layout support 3. Navigation root in Behavior tab "
)
raise
for pid, pvalue, ptype in item[propertieskey]:
if getattr(aq_base(obj), pid, None) is not None:
# if object have a attribute equal to property, do nothing
continue
# Bugfix > plone default_page must be a string, got (<type 'unicode'>)
if pid == 'default_page':
pvalue = str(pvalue)
try:
if obj.hasProperty(pid):
obj._updateProperty(pid, pvalue)
else:
obj._setProperty(pid, pvalue, ptype)
except ConflictError:
raise
except Exception as e:
raise Exception('Failed to set property "%s" type "%s"'
' to "%s" at object %s. ERROR: %s' %
(pid, ptype, pvalue, str(obj), str(e)))
logger.info("object creation %s" % (obj.absolute_url_path()))
yield item
class CatalogSourceSection(object):
"""A source section which creates items from a remote Plone site by
querying it's catalog.
"""
classProvides(ISectionBlueprint)
implements(ISection)
def __init__(self, transmogrifier, name, options, previous):
self.previous = previous
self.options = options
self.context = transmogrifier.context
self.remote_url = self.get_option('remote-url',
'http://localhost:8080')
remote_username = self.get_option('remote-username', 'admin')
remote_password = self.get_option('remote-password', 'admin')
catalog_path = self.get_option('catalog-path', '/Plone/portal_catalog')
self.site_path_length = len('/'.join(catalog_path.split('/')[:-1]))
catalog_query = self.get_option('catalog-query', None)
catalog_query = ' '.join(catalog_query.split())
catalog_query = base64.b64encode(catalog_query)
self.remote_skip_paths = self.get_option('remote-skip-paths',
'').split()
# Install a basic auth handler
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(realm='Zope',
uri=self.remote_url,
user=remote_username,
passwd=remote_password)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
req = urllib2.Request('%s%s/get_catalog_results' % (self.remote_url,
catalog_path), urllib.urlencode({'catalog_query': catalog_query}))
try:
f = urllib2.urlopen(req)
resp = f.read()
except urllib2.URLError:
raise
self.item_paths = sorted(simplejson.loads(resp))
def get_option(self, name, default):
"""Get an option from the request if available and fallback to the
transmogrifier config.
"""
request = getattr(self.context, 'REQUEST', None)
if request is not None:
value = request.form.get('form.widgets.'+name.replace('-', '_'),
self.options.get(name, default))
else:
value = self.options.get(name, default)
if isinstance(value, unicode):
value = value.encode('utf8')
return value
def __iter__(self):
for item in self.previous:
yield item
for path in self.item_paths:
skip = False
for skip_path in self.remote_skip_paths:
if path.startswith(skip_path):
skip = True
if not skip:
item = self.get_remote_item(path)
if item:
item['_path'] = item['_path'][self.site_path_length:]
yield item
def get_remote_item(self, path):
item_url = '%s%s/get_item' % (self.remote_url, urllib.quote(path))
try:
f = urllib2.urlopen(item_url)
item_json = f.read()
except urllib2.URLError, e:
logger.error("Failed reading item from %s. %s" % (item_url, str(e)))
return None
try:
item = simplejson.loads(item_json)
except simplejson.JSONDecodeError:
logger.error("Could not decode item from %s." % item_url)
return None
return item
