def structure_from_csv(self):
"""Import structure/content via csvrepliacta."""
csvfile = self.REQUEST.form['csv_structure']
if not csvfile:
return 'Please provide a csv structure file !'
else:
import transaction
import os
from zope.interface import alsoProvides, noLongerProvides
from Products.csvreplicata.interfaces import ICSVReplicable, \
Icsvreplicata
from pgolf.tma.config import getImportOptionsFromIni
import_settings, default_transition = getImportOptionsFromIni()
import foo
# are we in zopetestcase ?
# if not we have to install temorarly csvreplicata just to import
# structure else it's installed via base_setup
zopetestcase = (('ZOPE_TESTCASE' in os.environ)
or ('ZOPETESTCASE' in os.environ))
portal_quickinstaller = self.portal_quickinstaller
has_csvreplicata = portal_quickinstaller.\
isProductInstalled('csvreplicata')
if not zopetestcase and not has_csvreplicata:
#install temporarly csvreplicata if not used by policy.
portal_quickinstaller.installProduct('csvreplicata')
transaction.savepoint()
#provide ICSVReplicable to plonesite
alsoProvides(self, ICSVReplicable)
csvreplicatatool = self.portal_csvreplicatatool
# save existents user settings
if has_csvreplicata:
old_replicabletypes_settings = csvreplicatatool.replicabletypes
# register Structure replicabletypes
csvreplicatatool.replicabletypes = import_settings
# now import
replicator = Icsvreplicata(self)
replicator.csvimport(csvfile, datetimeformat='%d/%m/%Y',
wf_transition=default_transition)
# restore replicabletypes user settings
if has_csvreplicata :
csvreplicatatool.replicabletypes = old_replicabletypes_settings
#remove ICSVReplicable interface for self
noLongerProvides(self, ICSVReplicable)
#uninistall csvreplicata if not used by policy
if not zopetestcase and not has_csvreplicata:
portal_quickinstaller.uninstallProducts(['csvreplicata'])
status_message = _(u'structure succesfully imported',
default=u'structure succesfully imported')
addStatusMessage(self.REQUEST, status_message)
# and redirect him to the fine location
next_url = self.portal_url()
self.REQUEST.RESPONSE.redirect(next_url)
return self.REQUEST.RESPONSE
def doImport(self):
"""
"""
fileorigin = self.request.get("fileorigin")
if fileorigin == "SERVER":
csvtool = getToolByName(self.context, "portal_csvreplicatatool")
import_dir_path = csvtool.getServerfilepath()
file = open(os.path.join(import_dir_path, self.request.get("serverfile")))
else:
file = self.request.get("csvfile")
replicator = Icsvreplicata(self.context)
datetimeformat = self.request.get("datetimeformat")
vocabularyvalue = self.request.get("vocabularyvalue")
encoding = self.request.get("encoding")
delimiter = self.request.get("delimiter")
stringdelimiter = self.request.get("stringdelimiter")
conflict_winner = self.request.get("conflictrule")
wf_transition = self.request.get("wf_transition")
partial_commit_number = 0
try:
partial_commit_number = int(self.request.get("partial_commit_number"))
except:
pass
if wf_transition == "None":
wf_transition = None
plain_format = False
if self.request.get("is_plain_format", "") == "on":
plain_format = True
importfiles = self.request.get("importfiles")
if importfiles == "Yes":
zip = ZipFile(file)
csvfile = StringIO()
csvfile.write(zip.read("export.csv"))
csvfile.seek(0)
else:
zip = None
csvfile = file
(count_created, count_modified, export_date, errors) = replicator.csvimport(
csvfile,
encoding=encoding,
delimiter=delimiter,
stringdelimiter=stringdelimiter,
datetimeformat=datetimeformat,
conflict_winner=conflict_winner,
wf_transition=wf_transition,
zip=zip,
vocabularyvalue=vocabularyvalue,
plain_format=plain_format,
partial_commit_number=partial_commit_number,
)
if len(errors) == 0:
self.writeMessageOnPage(
["All lines imported, %d object(s) created, %d object(s) modified." % (count_created, count_modified)]
)
self.writeMessageOnPage(
[
"This CSV file has been produced on "
+ str(export_date)
+ ". To avoid inconsistencies, do not import this file anymore. It is preferable to export a new one."
]
)
else:
self.writeMessageOnPage(["%d object(s) created, %d object(s) modified." % (count_created, count_modified)])
self.writeMessageOnPage(errors, error=True)
self.request.RESPONSE.redirect(self.context.absolute_url() + "/@@csvreplicata")
def doExport(self):
"""
"""
replicator = Icsvreplicata(self.context)
encoding = self.request.get("encoding")
depth = int(self.request.get("depth", "1"))
datetimeformat = self.request.get("datetimeformat")
vocabularyvalue = self.request.get("vocabularyvalue")
delimiter = self.request.get("delimiter")
stringdelimiter = self.request.get("stringdelimiter")
wf_states = self.request.get("wf_states", "Any")
selected_exportable_content_types = self.request.get("exportable_content_types")
# handling of selected types
exportable_content_types = selected_exportable_content_types
if "Any" in wf_states:
wf_states = None
exportfiles = self.request.get("exportfiles")
csvtool = getToolByName(self.context, "portal_csvreplicatatool")
tmp = csvtool.getTempPath()
delete_on_exit = False
delete_grand_parent = False
if not tmp:
delete_grand_parent = True
tmp = tempfile.mkdtemp()
tmppath = tempfile.mkdtemp(dir=tmp)
zippath = os.path.join(tmppath, "export.zip")
if exportfiles == "Yes":
# initialize zipfile
zip = ZipFile(zippath, "w", 8)
else:
zip = None
plain_format = False
if self.request.get("is_plain_format", "") == "on":
plain_format = True
csvpath = os.path.join(tmppath, "export.csv")
csvstream = open(csvpath, "w")
csv = replicator.csvexport(
encoding=encoding,
delimiter=delimiter,
stringdelimiter=stringdelimiter,
depth=depth,
datetimeformat=datetimeformat,
wf_states=wf_states,
zip=zip,
vocabularyvalue=vocabularyvalue,
exportable_content_types=exportable_content_types,
stream=csvstream,
plain_format=plain_format,
)
csvstream.flush()
csvstream.close()
streamed = EphemeralStreamIterator(csvpath, delete_parent=True, delete_grand_parent=delete_grand_parent)
if exportfiles == "Yes":
self.request.response.setHeader("Content-type", "application/zip")
self.request.response.setHeader("Content-Disposition", "attachment; filename=export.zip")
zip.write(csvpath, "export.csv")
zip.close()
streamed.file.close()
streamed = EphemeralStreamIterator(zippath, delete_parent=True, delete_grand_parent=delete_grand_parent)
else:
if not encoding:
encoding = "UTF-8"
self.request.response.setHeader("Content-type", "text/csv;charset=" + encoding)
self.request.response.setHeader("Content-Disposition", "attachment; filename=export.csv")
self.request.response.setHeader("Content-Length", str(len(streamed)))
return streamed
def doImport(self):
"""
"""
fileorigin = self.request.get("fileorigin")
if fileorigin == "SERVER":
csvtool = getToolByName(self.context, "portal_csvreplicatatool")
import_dir_path = csvtool.getServerfilepath()
file = open(
os.path.join(import_dir_path, self.request.get("serverfile")))
else:
file = self.request.get("csvfile")
replicator = Icsvreplicata(self.context)
datetimeformat = self.request.get("datetimeformat")
vocabularyvalue = self.request.get("vocabularyvalue")
encoding = self.request.get("encoding")
delimiter = self.request.get("delimiter")
stringdelimiter = self.request.get("stringdelimiter")
conflict_winner = self.request.get("conflictrule")
wf_transition = self.request.get("wf_transition")
partial_commit_number = 0
try:
partial_commit_number = int(
self.request.get("partial_commit_number"))
except:
pass
if wf_transition == "None":
wf_transition = None
plain_format = False
if self.request.get('is_plain_format', '') == 'on':
plain_format = True
importfiles = self.request.get("importfiles")
if importfiles == "Yes":
zip = ZipFile(file)
csvfile = StringIO()
csvfile.write(zip.read("export.csv"))
csvfile.seek(0)
else:
zip = None
csvfile = file
(count_created, count_modified, export_date,
errors) = replicator.csvimport(
csvfile,
encoding=encoding,
delimiter=delimiter,
stringdelimiter=stringdelimiter,
datetimeformat=datetimeformat,
conflict_winner=conflict_winner,
wf_transition=wf_transition,
zip=zip,
vocabularyvalue=vocabularyvalue,
plain_format=plain_format,
partial_commit_number=partial_commit_number,
)
if len(errors) == 0:
self.writeMessageOnPage([
"All lines imported, %d object(s) created, %d object(s) modified."
% (count_created, count_modified)
])
self.writeMessageOnPage([
"This CSV file has been produced on " + str(export_date) +
". To avoid inconsistencies, do not import this file anymore. It is preferable to export a new one."
])
else:
self.writeMessageOnPage([
"%d object(s) created, %d object(s) modified." %
(count_created, count_modified)
])
self.writeMessageOnPage(errors, error=True)
self.request.RESPONSE.redirect(self.context.absolute_url() +
'/@@csvreplicata')
def doExport(self):
"""
"""
replicator = Icsvreplicata(self.context)
encoding = self.request.get('encoding')
depth = int(self.request.get('depth', '1'))
datetimeformat = self.request.get("datetimeformat")
vocabularyvalue = self.request.get("vocabularyvalue")
delimiter = self.request.get("delimiter")
stringdelimiter = self.request.get("stringdelimiter")
wf_states = self.request.get("wf_states", 'Any')
selected_exportable_content_types = self.request.get(
"exportable_content_types")
# handling of selected types
exportable_content_types = selected_exportable_content_types
if 'Any' in wf_states:
wf_states = None
exportfiles = self.request.get("exportfiles")
csvtool = getToolByName(self.context, "portal_csvreplicatatool")
tmp = csvtool.getTempPath()
delete_on_exit = False
delete_grand_parent = False
if not tmp:
delete_grand_parent = True
tmp = tempfile.mkdtemp()
tmppath = tempfile.mkdtemp(dir=tmp)
zippath = os.path.join(tmppath, 'export.zip')
if exportfiles == "Yes":
# initialize zipfile
zip = ZipFile(zippath, "w", 8)
else:
zip = None
plain_format = False
if self.request.get('is_plain_format', '') == 'on':
plain_format = True
csvpath = os.path.join(tmppath, 'export.csv')
csvstream = open(csvpath, 'w')
csv = replicator.csvexport(
encoding=encoding,
delimiter=delimiter,
stringdelimiter=stringdelimiter,
depth=depth,
datetimeformat=datetimeformat,
wf_states=wf_states,
zip=zip,
vocabularyvalue=vocabularyvalue,
exportable_content_types=exportable_content_types,
stream=csvstream,
plain_format=plain_format)
csvstream.flush()
csvstream.close()
streamed = EphemeralStreamIterator(
csvpath,
delete_parent=True,
delete_grand_parent=delete_grand_parent)
if exportfiles == "Yes":
self.request.response.setHeader('Content-type', 'application/zip')
self.request.response.setHeader('Content-Disposition',
"attachment; filename=export.zip")
zip.write(csvpath, "export.csv")
zip.close()
streamed.file.close()
streamed = EphemeralStreamIterator(
zippath,
delete_parent=True,
delete_grand_parent=delete_grand_parent)
else:
if not encoding:
encoding = 'UTF-8'
self.request.response.setHeader('Content-type',
'text/csv;charset=' + encoding)
self.request.response.setHeader('Content-Disposition',
"attachment; filename=export.csv")
self.request.response.setHeader('Content-Length', str(len(streamed)))
return streamed
def importcsvStep(context):
"""This step give you a way to import content in your website with
csvreplicata.
How it works:
read replicata.cfg file to load configuration
read replicata.csv and call csvreplicata with the config.
Config:
[replicable_types]
#List schemata for each types you want to import
#Folder : default, categorization, dates, ownership, settings
#News Item : default
Document: default
[settings]
#list here every csv settings that will be given to replicator.importcsv
encoding:utf-8
delimiter:,
stringdelimiter:"
datetimeformat:%d/%m/%Y
wf_transition = publish
conflict_winner:SERVER
"""
if not 'openDataFile' in dir(context):
return
replicatacfg = context.openDataFile('replicata.cfg')
replicatacsv = context.openDataFile('replicata.csv')
replicatazip = context.openDataFile('replicata.zip')
if replicatazip is None and replicatacsv is None:
return
site = context.getSite()
wftool = getToolByName(site, 'portal_workflow')
csvreplicatatool = getToolByName(site, 'portal_csvreplicatatool')
# save existents user settings
old_replicabletypes_settings = csvreplicatatool.replicabletypes
#get import settings
Config = ConfigParser.ConfigParser()
#defaults system of configparser just sucks, so set values
Config.add_section("replicable_types")
Config.add_section("settings")
Config.set("settings", "encoding","utf-8")
Config.set("settings", "delimiter",";")
Config.set("settings", "stringdelimiter",'"')
Config.set("settings", "datetimeformat","%d/%m/%Y %H:%M:%S")
Config.set("settings", "wf_transition", "publish")
Config.set("settings", "conflict_winner","SERVER")
Config.set("settings", "global_path","")
Config.set("settings", "ignore_content_errors","false")
Config.set("settings", "plain_format","true")
if replicatacfg is not None:
Config.read(replicatacfg.name)
options = Config.items('replicable_types', raw=True)
replicable_types = {}
for option in options:
replicable_types[option[0].title()] = option[1].split(',')
if replicable_types:
csvreplicatatool.replicabletypes = replicable_types
else:
csvreplicatatool.replicabletypes = old_replicabletypes_settings
#build kwargs of csvimport method.
kwargs = {}
wf_transition = Config.get('settings', 'wf_transition')
kwargs['wf_transition'] = wf_transition
datetimeformat = Config.get('settings', 'datetimeformat')
kwargs['datetimeformat'] = datetimeformat
delimiter = Config.get('settings', 'delimiter')
kwargs['delimiter'] = delimiter
stringdelimiter = Config.get('settings', 'stringdelimiter')
kwargs['stringdelimiter'] = stringdelimiter
encoding = Config.get('settings', 'encoding')
kwargs['encoding'] = encoding
conflict_winner = Config.get('settings', 'conflict_winner')
kwargs['conflict_winner'] = conflict_winner
wf_transition = Config.get('settings', 'wf_transition')
kwargs['wf_transition'] = wf_transition
signore_content_errors = Config.get('settings', 'ignore_content_errors')
ignore_content_errors = False
if signore_content_errors:
if signore_content_errors == 'true':
ignore_content_errors = True
kwargs['ignore_content_errors'] = ignore_content_errors
plain_format = Config.get('settings', 'plain_format')
kwargs['plain_format'] = False
if plain_format == 'true':
kwargs['plain_format'] = True
if replicatazip:
from zipfile import ZipFile
kwargs['zip'] = ZipFile(replicatazip.name)
#retrive context:
global_path = Config.get('settings', 'global_path')
try:
folder = site.restrictedTraverse(global_path)
except KeyError:
logger.error('can t find global path')
folder = site
# now import
alsoProvides(folder, ICSVReplicable)
replicator = Icsvreplicata(folder)
replicator.csvimport(replicatacsv, **kwargs)
# restore replicabletypes user settings
csvreplicatatool.replicabletypes = old_replicabletypes_settings