def verify(self):
"""
Verifies if the credentials are valid to start a new session.
:return:
"""
self.session = self.get_session()
# try to get the version of the remote JSON API
version = self.get_version()
if not version or not version.get('version'):
message = _("Please install senaite.jsonapi on the source system")
return False, message
# try to get the current logged in user
user = self.get_authenticated_user()
if not user or user.get("authenticated") is False:
message = _("Wrong username/password")
return False, message
# remember the credentials in the storage
storage = self.get_storage()
for k, v in self.credentials.iteritems():
storage["credentials"][k] = v
for k, v in self.config.iteritems():
storage["configuration"][k] = v
storage["last_fetch_time"] = DateTime()
message = "Data fetched and saved: {}".format(self.domain_name)
return True, message
def import_users(self, domain):
"""Import the users from the storage identified by domain
"""
logger.info("*** IMPORT USERS {} ***".format(domain))
storage = self.get_storage(domain=domain)
userstore = storage["users"]
for username, userdata in userstore.items():
if ploneapi.user.get(username):
logger.info("Skipping existing user {}".format(username))
continue
email = userdata.get("email", "")
roles = userdata.get("roles", ())
# TODO handle groups
# groups = userdata.get("groups", groups=groups)())
logger.info("Creating user {}".format(username))
message = _("Created new user {} with password {}".format(
username, username))
# create new user with the same password as the username
ploneapi.user.create(
email=email,
username=username,
password=username,
roles=roles,
)
self.add_status_message(message, "info")
logger.info(message)
def __call__(self):
protect.CheckAuthenticator(self.request.form)
self.portal = api.get_portal()
self.request.set('disable_plone.rightcolumn', 1)
self.request.set('disable_border', 1)
# Handle form submit
form = self.request.form
if not form.get("dataform", False):
return self.template()
domain_name = form.get("domain_name", None)
# Handle "Clear this Storage" action
if form.get("clear_storage", False):
del self.storage[domain_name]
delete_soup(self.portal, domain_name)
message = _("Cleared Storage {}".format(domain_name))
self.add_status_message(message, "info")
return self.template()
# Get the necessary data for the domain
storage = self.get_storage(domain_name)
credentials = storage["credentials"]
config = storage["configuration"]
# Handle "Import" action
if form.get("import", False):
step = ImportStep(credentials, config)
# Handle "Update" action
else:
fetch_time = form.get("mod_date_limit", None) or \
storage.get("last_fetch_time", None)
if not fetch_time:
message = 'Cannot get last fetched time, please re-run ' \
'the Fetch step.'
self.add_status_message(message, "error")
return self.template()
if isinstance(fetch_time, str):
try:
fetch_time = DateTime(fetch_time)
except:
message = 'Please enter a valid Date & Time'
self.add_status_message(message, "error")
return self.template()
step = UpdateStep(credentials, config, fetch_time)
step.run()
return self.template()
def _import_users(self):
"""Import the users from the storage identified by domain
"""
if not self.import_users:
return
logger.info("*** Importing Users: {} ***".format(self.domain_name))
for user in self.yield_items("users"):
username = user.get("username")
if ploneapi.user.get(username):
logger.debug("Skipping existing user {}".format(username))
continue
email = user.get("email", "")
if not email:
email = "{}@example.com".format(username)
roles = user.get("roles", ())
groups = user.get("groups", ())
logger.debug("Creating user {}".format(username))
message = _("Created new user {} with password {}".format(
username, username))
# create new user with the same password as the username
ploneapi.user.create(
email=email,
username=username,
password=username,
roles=roles,
)
for group in groups:
# Try to add the user to the group if group exists.
try:
ploneapi.group.add_user(groupname=group, username=username)
except KeyError:
continue
logger.debug(message)
logger.info("*** Users Were Imported: {} ***".format(self.domain_name))
def __call__(self):
protect.CheckAuthenticator(self.request.form)
self.portal = api.get_portal()
self.request.set('disable_plone.rightcolumn', 1)
self.request.set('disable_border', 1)
# Handle form submit
form = self.request.form
fetchform = form.get("fetchform", False)
dataform = form.get("dataform", False)
if not any([fetchform, dataform]):
return self.template()
# remember the form field values
url = form.get("url", "")
if not url.startswith("http"):
url = "http://{}".format(url)
self.url = url
self.username = form.get("ac_name", None)
self.password = form.get("ac_password", None)
# Handle "Import" action
if form.get("import", False):
domain = form.get("domain", None)
self.import_registry_records(domain)
self.import_users(domain)
self.import_data(domain)
logger.info("*** END OF DATA IMPORT {} ***".format(domain))
return self.template()
# Handle "Clear this Storage" action
if form.get("clear_storage", False):
domain = form.get("domain", None)
del self.storage[domain]
message = _("Cleared Storage {}".format(domain))
self.add_status_message(message, "info")
return self.template()
# Handle "Clear all Storages" action
if form.get("clear", False):
self.flush_storage()
message = _("Cleared Data Storage")
self.add_status_message(message, "info")
return self.template()
# Handle "Fetch" action
if form.get("fetch", False):
# check if all mandatory fields have values
if not all([self.url, self.username, self.password]):
message = _("Please fill in all required fields")
self.add_status_message(message, "error")
return self.template()
# initialize the session
self.session = self.get_session(self.username, self.password)
# remember the credentials in the storage
storage = self.get_storage(self.url)
storage["credentials"]["username"] = self.username
storage["credentials"]["password"] = self.password
# try to get the version of the remote JSON API
version = self.get_version()
if not version or not version.get('version'):
message = _(
"Please install senaite.jsonapi on the source system")
self.add_status_message(message, "error")
return self.template()
# try to get the current logged in user
user = self.get_authenticated_user()
if not user or user.get("authenticated") is False:
message = _("Wrong username/password")
self.add_status_message(message, "error")
return self.template()
domain = self.url
# Fetch all users from the source
self.fetch_users(domain)
# Start the fetch process beginning from the portal object
self.fetch_data(domain, uid="0")
# Fetch registry records that contain the word bika or senaite
self.fetch_registry_records(domain, keys=["bika", "senaite"])
logger.info("*** FETCHING DATA FINISHED {} ***".format(domain))
# always render the template
return self.template()
def import_data(self, domain):
"""Import the data from the storage identified by domain
"""
logger.info("*** IMPORT DATA {} ***".format(domain))
storage = self.get_storage(domain=domain)
datastore = storage["data"]
indexstore = storage["index"]
uidmap = storage["uidmap"]
credentials = storage["credentials"]
# At some points api cannot retrieve objects by UID in the end of
# creation process. Thus we keep them in an dictionary to access easily.
objmap = {}
# We will create objects from top to bottom, but will update from bottom
# to up.
ordered_uids = []
# initialize a new session with the stored credentials for later requests
username = credentials.get("username")
password = credentials.get("password")
self.session = self.get_session(username, password)
logger.info("Initialized a new session for user {}".format(username))
# Get UIDs grouped by their parent path
ppaths = indexstore.get("by_parent_path")
if ppaths is None:
message = _(
"No parent path info found in the import data. "
"Please install senaite.jsonapi>=1.1.1 on the source instance "
"and clear&refetch this storage")
self.add_status_message(message, "warning")
return
# Import by paths from top to bottom
for ppath in sorted(ppaths):
# nothing to do
if not ppath:
continue
logger.info("Importing items for parent path {}".format(ppath))
uids = ppaths[ppath]
for uid in uids:
ordered_uids.append(uid)
# get the data for this uid
data = datastore[uid]
# check if the object exists in this instance
remote_path = data.get("path")
local_path = self.translate_path(remote_path)
existing = self.portal.unrestrictedTraverse(
str(local_path), None)
if existing:
# remember the UID -> object UID mapping for the update step
uidmap[uid] = api.get_uid(existing)
objmap[uid] = existing
else:
# get the container object by path
container_path = self.translate_path(ppath)
container = self.portal.unrestrictedTraverse(
str(container_path), None)
# create an object slug in this container
obj = self.create_object_slug(container, data)
# remember the UID -> object UID mapping for the update step
uidmap[uid] = api.get_uid(obj)
objmap[uid] = obj
# When creation process is done, commit the transaction to avoid
# ReferenceField relation problems.
transaction.commit()
# UIDs were added from up to bottom. Reverse the list to update objects
# from bottom to up.
ordered_uids.reverse()
# Update all objects with the given data
for uid in ordered_uids:
obj = objmap.get(uid, None)
if obj is None:
logger.warn("Object not found: {} ".format(uid))
continue
logger.info("Update object {} with import data".format(
api.get_path(obj)))
self.update_object_with_data(obj, datastore[uid], domain)
self.reindex_updated_objects()
def __call__(self):
protect.CheckAuthenticator(self.request.form)
self.portal = api.get_portal()
self.request.set('disable_plone.rightcolumn', 1)
self.request.set('disable_border', 1)
# Handle form submit
form = self.request.form
fetchform = form.get("fetchform", False)
if not fetchform:
return self.template()
self.url = form.get("url", "")
if not self.url.startswith("http"):
self.url = "https://{}".format(self.url)
self.domain_name = form.get("domain_name", None)
self.username = form.get("ac_name", None)
self.password = form.get("ac_password", None)
# check if all mandatory fields have values
if not all([self.domain_name, self.url, self.username, self.password]):
message = _("Please fill in all required fields")
self.add_status_message(message, "error")
return self.template()
self.auto_sync = (form.get("auto_sync") == 'on')
self.import_settings = (form.get("import_settings") == 'on')
self.import_users = (form.get("import_users") == 'on')
self.import_registry = (form.get("import_registry") == 'on')
self.remote_prefix = form.get("remote_prefix", None)
self.local_prefix = form.get("local_prefix", None)
self.full_sync_types = utils.filter_content_types(
form.get("full_sync_types"))
self.unwanted_content_types = utils.filter_content_types(
form.get("unwanted_content_types"))
self.read_only_types = utils.filter_content_types(
form.get("read_only_types"))
self.update_only_types = utils.filter_content_types(
form.get("update_only_types"))
self.prefixable_types = utils.filter_content_types(
form.get("prefixable_types"))
# Prefix Validation
if not self.validate_prefix():
return self.template()
credentials = dict(
url=self.url,
domain_name=self.domain_name,
ac_name=self.username,
ac_password=self.password)
config = dict(
auto_sync=self.auto_sync,
import_settings=self.import_settings,
import_users=self.import_users,
import_registry=self.import_registry,
remote_prefix=self.remote_prefix,
local_prefix=self.local_prefix,
full_sync_types=self.full_sync_types,
unwanted_content_types=self.unwanted_content_types,
read_only_types=self.read_only_types,
update_only_types=self.update_only_types,
prefixable_types=self.prefixable_types,
)
fs = FetchStep(credentials, config)
verified, message = fs.verify()
if verified:
fs.run()
self.add_status_message(message, "info")
else:
self.add_status_message(message, "error")
# render the template
return self.template()