def get_display(my):
widget = DivWdg(id='preview_data')
widget.add_style('padding: 6px')
my.set_as_panel(widget)
widget.add(SpanWdg(), 'warning')
widget.add(HtmlElement.br(2))
my.get_column_preview(widget)
web = WebContainer.get_web()
csv_parser = CsvParser(my.file_path)
if my.encoder:
csv_parser.set_encoder(my.encoder)
try:
csv_parser.parse()
# that can be all kinds of encoding/decoding exception
except Exception, e:
# possibly incompatible encoder selected, use the default instead.
# Let the user pick it.
span = SpanWdg('WARNING: The selected encoder is not compatible with your csv file. Please choose the proper one. Refer to the documentation/tutorial on how to save your csv file with UTF-8 encoding if you have special characters in it.', css='warning')
widget.add(span, 'warning')
return widget
def get_display(my):
widget = DivWdg(id='preview_data')
widget.add_style('padding: 6px')
my.set_as_panel(widget)
widget.add(SpanWdg(), 'warning')
widget.add(HtmlElement.br(2))
my.get_column_preview(widget)
web = WebContainer.get_web()
csv_parser = CsvParser(my.file_path)
if my.encoder:
csv_parser.set_encoder(my.encoder)
try:
csv_parser.parse()
# that can be all kinds of encoding/decoding exception
except Exception, e:
# possibly incompatible encoder selected, use the default instead.
# Let the user pick it.
span = SpanWdg('WARNING: The selected encoder is not compatible with your csv file. Please choose the proper one. Refer to the documentation/tutorial on how to save your csv file with UTF-8 encoding if you have special characters in it.', css='warning')
widget.add(span, 'warning')
return widget
def execute(my):
if not my.initialized:
my.init()
assert my.search_type
assert my.file_path
assert my.columns
csv_parser = CsvParser(my.file_path)
if my.has_title:
csv_parser.set_has_title_row(True)
else:
csv_parser.set_has_title_row(False)
if my.encoder:
csv_parser.set_encoder(my.encoder)
csv_parser.parse()
# get the data and columns
#csv_titles = csv_parser.get_titles()
csv_data = csv_parser.get_data()
# make sure all of the new columns are created
csv_titles = []
for i, column in enumerate(my.columns):
if not column:
new_column = my.new_columns[i]
new_column_type = my.new_column_types[i]
if new_column and new_column not in ['id', 'code'] and\
i in my.enabled_idx:
# create the new column
from pyasm.command import ColumnAddCmd
#col_type = "Name/Code"
cmd = ColumnAddCmd(my.search_type, new_column, new_column_type)
cmd.execute()
# create the sobject for now
sobject = SObjectFactory.create("prod/custom_property")
sobject.set_value("search_type", my.search_type)
sobject.set_value("name", new_column)
sobject.set_value("description", new_column)
sobject.commit()
csv_titles.append( my.new_columns[i] )
else:
csv_titles.append( column )
try:
id_col = csv_titles.index(my.id_col)
# id is special we want it to be identifiable at all times
# but not importable
if my.id_col != 'id' and id_col not in my.enabled_idx:
id_col = -1
except ValueError:
id_col = -1
new_entries = []
updated_entries = []
error_entries = []
error = False
# create entries or update values
for row_count, row in enumerate(csv_data):
sobject = None
# if id_col doesn't exist
is_new_entry = False
if id_col == -1:
sobject = SObjectFactory.create(my.search_type)
is_new_entry = True
else:
id = row[id_col]
if id:
# this essentially updates the current sobject in db
if my.id_col=='code':
sobject = Search.get_by_code(my.search_type, id.strip())
elif my.id_col=='id':
sobject = Search.get_by_id(my.search_type, id.strip())
else:
u_search = Search(my.search_type)
u_search.add_filter(my.id_col, id.strip())
sobject = u_search.get_sobject()
#assert sobject
# in case a previously exported sobject with this code
# or id has been deleted or it is a completely foreign code
# or id, sobject will be None
else: # skip if empty id or code
continue
if not sobject:
sobject = SObjectFactory.create(my.search_type)
is_new_entry = True
new_columns = 0
note = None
for cell_count, cell in enumerate(row):
'''
column_override = my.columns[cell_count]
if column_override:
title = column_override
else:
title = csv_titles[cell_count]
if not title:
continue
'''
# skip if not enabled
if cell_count not in my.enabled_idx:
continue
title = csv_titles[cell_count]
if not title:
continue
# always skip id column
if title == "id":
continue
cell = cell.strip()
# remove control, other characters in unicode
#cell = re.sub(r'\p{Cc}','', cell)
cell = re.sub(r"[\x01-\x08\x0b-\x1f\x7f-\x9f]",'', cell)
if title == "(note)":
note = cell
else:
sobject.set_value(title, cell)
new_columns += 1
if not new_columns:
msg = "No column or only the id column is selected."
raise CommandException(msg)
try:
#sobject.commit(triggers=False)
sobject.commit(triggers=True)
if note:
note_obj = SearchType.create("sthpw/note")
note_obj.set_value("note", note)
note_obj.set_value("process", "publish")
note_obj.set_value("context", "publish")
note_obj.set_user()
note_obj.set_parent(sobject)
note_obj.commit()
except SqlException, e:
msg = "Error creating new entry for row [%s]: %s, %s" % (row_count, str(row), e.__str__() )
# (Maybe not neccessary) must raise SqlException or it would fail silently
if my.test_run:
error = True
error_entries.append(sobject.get_code())
raise SqlException(msg)
else:
if is_new_entry:
new_entries.append(sobject.get_code())
else:
updated_entries.append(sobject.get_code())
def get_column_preview(my, div):
# parse the first fow
csv_parser = CsvParser(my.file_path)
if my.has_title:
csv_parser.set_has_title_row(True)
else:
csv_parser.set_has_title_row(False)
if my.lowercase_title:
csv_parser.set_lowercase_title(True)
if my.encoder:
csv_parser.set_encoder(my.encoder)
try:
csv_parser.parse()
# that can be all kinds of encoding/decoding exception
except Exception, e:
# possibly incompatible encoder selected, use the default instead.
# Let the user pick it.
span = SpanWdg('WARNING: The selected encoder is not compatible with your csv file. Please choose the proper one (e.g. UTF-8). Refer to the documentation/tutorial on how to save your csv file with UTF-8 encoding if you have special characters in it.', css='warning')
div.add(SpanWdg(e.__str__()))
div.add(HtmlElement.br())
div.add(span, 'warning')
return div
def get_preview_wdg(self):
widget = Widget()
web = WebContainer.get_web()
csv_parser = CsvParser(self.file_path)
csv_parser.parse()
csv_titles = csv_parser.get_titles()
csv_data = csv_parser.get_data()
ajax = AjaxCmd()
ajax.set_option("search_type", self.search_type)
ajax.set_option("file_path", self.file_path)
ajax.add_element_name("has_title")
event = WebContainer.get_event_container()
caller = event.get_event_caller(SiteMenuWdg.EVENT_ID)
div = ajax.generate_div()
div.set_post_ajax_script(caller)
widget.add(div)
columns = []
num_columns = len(csv_titles)
ajax.set_option("num_columns", num_columns)
for i in range(0, num_columns):
column = web.get_form_value("column_%s" % i)
if column:
column = csv_titles[i]
else:
column = web.get_form_value("column_new_%s" % i)
columns.append(column)
ajax.add_element_name("column_%s" % i)
ajax.add_element_name("new_column_%s" % i)
ajax.register_cmd("pyasm.command.csv_import_cmd.CsvImportCmd")
import_button = IconButtonWdg("Import", IconWdg.REFRESH, True)
import_button.add_event("onclick",
ajax.get_on_script(show_progress=False))
import_button.add_style("float: right")
widget.add(import_button)
preview_submit = IconSubmitWdg("Preview", IconWdg.REFRESH, True)
preview_submit.add_style("float: right")
widget.add(preview_submit)
sobject_title = self.search_type_obj.get_title()
widget.add("<p>4. Import</p>")
widget.add(HtmlElement.br())
widget.add(
"<p>The following table will be imported into %s (Showing Max: 100)</p>"
% sobject_title)
table = Table(css="table")
table.add_style("width: 100%")
table.add_row()
for i, title in enumerate(columns):
if not title:
title = "<b style='color:red'>*</b>"
table.add_header(title)
for i, row in enumerate(csv_data):
if i > 100:
break
table.add_row()
for j, cell in enumerate(row):
table.add_cell(cell)
widget.add(table)
return widget
def get_first_row_wdg(self):
# read the csv file
self.file_path = ""
div = DivWdg()
div.add(self.get_upload_wdg())
if not self.search_type:
return div
if not self.file_path:
return div
if not self.file_path.endswith(".csv"):
div.add("Uploaded file [%s] is not a csv file" % self.file_path)
return div
if not os.path.exists(self.file_path):
raise Exception("Path '%s' does not exists" % self.file_path)
div.add(HtmlElement.br(2))
div.add(
HtmlElement.
b("The following is taken from first line in the uploaded csv file. Select the appropriate column to match."
))
div.add(HtmlElement.br())
div.add(
HtmlElement.b(
"Make sure you have all the required columns** in the csv."))
option_div = DivWdg()
option_div.add_style("float: left")
option_div.add_style("margin-right: 30px")
option_div.add("<p>3. Parsing Options:</p>")
self.search_type_obj = SearchType.get(self.search_type)
# first row and second row
option_div.add(HtmlElement.br(2))
option_div.add("Use Title Row: ")
title_row_checkbox = FilterCheckboxWdg("has_title")
title_row_checkbox.set_default_checked()
option_div.add(title_row_checkbox)
option_div.add(
HintWdg(
"Set this to use the first row as a title row to match up columns in the database"
))
option_div.add(HtmlElement.br(2))
option_div.add("Sample Data Row: ")
data_row_text = TextWdg("data_row")
data_row_text.set_attr("size", "3")
option_div.add(data_row_text)
option_div.add(
HintWdg(
"Set this as a sample data row to match the columns to the database"
))
option_div.add(HtmlElement.br(2))
div.add(option_div)
self.has_title = title_row_checkbox.is_checked()
# parse the first fow
csv_parser = CsvParser(self.file_path)
if self.has_title:
csv_parser.set_has_title_row(True)
else:
csv_parser.set_has_title_row(False)
csv_parser.parse()
csv_titles = csv_parser.get_titles()
csv_data = csv_parser.get_data()
data_row = data_row_text.get_value()
if not data_row:
data_row = 0
else:
try:
data_row = int(data_row)
except ValueError:
data_row = 0
if data_row >= len(csv_data):
data_row = len(csv_data) - 1
data_row_text.set_value(data_row)
table = Table()
table.set_attr("cellpadding", "10")
table.add_row()
table.add_header("CSV Column Value")
table.add_header("TACTIC Column")
table.add_header("Create New Column")
columns = self.search_type_obj.get_columns()
search_type = self.search_type_obj.get_base_search_type()
sobj = SObjectFactory.create(search_type)
required_columns = sobj.get_required_columns()
row = csv_data[data_row]
labels = []
for column in columns:
if column in required_columns:
label = '%s**' % column
else:
label = column
labels.append(label)
for j, cell in enumerate(row):
table.add_row()
table.add_cell(cell)
column_select = SelectWdg("column_%s" % j)
column_select.add_event(
"onchange",
"if (this.value!='') {set_display_off('new_column_div_%s')} else {set_display_on('new_column_div_%s')}"
% (j, j))
column_select.add_empty_option("-- Select --")
column_select.set_option("values", columns)
column_select.set_option("labels", labels)
# only set the value if it is actually in there
if csv_titles[j] in columns:
column_select.set_option("default", csv_titles[j])
column_select.set_persist_on_submit()
column_select_value = column_select.get_value()
display = column_select.get_buffer_display()
td = table.add_cell(display)
if csv_titles[j] not in columns:
td.add(" <b style='color: red'>*</b>")
# new property
new_column_div = DivWdg()
if column_select_value:
new_column_div.add_style("display", "none")
else:
new_column_div.add_style("display", "block")
new_column_div.set_id("new_column_div_%s" % j)
td = table.add_cell(new_column_div)
text = TextWdg("new_column_%s" % j)
text.set_persist_on_submit()
if self.has_title:
text.set_value(csv_titles[j])
new_column_div.add(" ... or ...")
new_column_div.add(text)
self.num_columns = len(row)
hidden = HiddenWdg("num_columns", self.num_columns)
# need to somehow specify defaults for columns
div.add(table)
div.add("<br/><br/>")
div.add(self.get_preview_wdg())
return div
def get_preview_wdg(my):
widget = Widget()
web = WebContainer.get_web()
csv_parser = CsvParser(my.file_path)
csv_parser.parse()
csv_titles = csv_parser.get_titles()
csv_data = csv_parser.get_data()
ajax = AjaxCmd()
ajax.set_option("search_type", my.search_type)
ajax.set_option("file_path", my.file_path)
ajax.add_element_name("has_title")
event = WebContainer.get_event_container()
caller = event.get_event_caller(SiteMenuWdg.EVENT_ID)
div = ajax.generate_div()
div.set_post_ajax_script(caller)
widget.add(div)
columns = []
num_columns = len(csv_titles)
ajax.set_option("num_columns", num_columns)
for i in range(0, num_columns):
column = web.get_form_value("column_%s" % i)
if column:
column = csv_titles[i]
else:
column = web.get_form_value("column_new_%s" % i)
columns.append(column)
ajax.add_element_name("column_%s" % i)
ajax.add_element_name("new_column_%s" % i)
ajax.register_cmd("pyasm.command.csv_import_cmd.CsvImportCmd")
import_button = IconButtonWdg("Import", IconWdg.REFRESH, True)
import_button.add_event("onclick", ajax.get_on_script(show_progress=False))
import_button.add_style("float: right")
widget.add( import_button )
preview_submit = IconSubmitWdg("Preview", IconWdg.REFRESH, True)
preview_submit.add_style("float: right")
widget.add(preview_submit)
sobject_title = my.search_type_obj.get_title()
widget.add("<p>4. Import</p>")
widget.add(HtmlElement.br())
widget.add("<p>The following table will be imported into %s (Showing Max: 100)</p>" % sobject_title)
table = Table(css="table")
table.add_style("width: 100%")
table.add_row()
for i, title in enumerate(columns):
if not title:
title = "<b style='color:red'>*</b>"
table.add_header(title)
for i, row in enumerate(csv_data):
if i > 100:
break
table.add_row()
for j, cell in enumerate(row):
table.add_cell(cell)
widget.add(table)
return widget
def get_first_row_wdg(my):
# read the csv file
my.file_path = ""
div = DivWdg()
div.add( my.get_upload_wdg() )
if not my.search_type:
return div
if not my.file_path:
return div
if not my.file_path.endswith(".csv"):
div.add( "Uploaded file [%s] is not a csv file"% my.file_path)
return div
if not os.path.exists(my.file_path):
raise Exception("Path '%s' does not exists" % my.file_path)
div.add(HtmlElement.br(2))
div.add( HtmlElement.b("The following is taken from first line in the uploaded csv file. Select the appropriate column to match.") )
div.add(HtmlElement.br())
div.add( HtmlElement.b("Make sure you have all the required columns** in the csv."))
option_div = DivWdg()
option_div.add_style("float: left")
option_div.add_style("margin-right: 30px")
option_div.add("<p>3. Parsing Options:</p>")
my.search_type_obj = SearchType.get(my.search_type)
# first row and second row
option_div.add( HtmlElement.br(2) )
option_div.add("Use Title Row: ")
title_row_checkbox = FilterCheckboxWdg("has_title")
title_row_checkbox.set_default_checked()
option_div.add(title_row_checkbox)
option_div.add( HintWdg("Set this to use the first row as a title row to match up columns in the database") )
option_div.add( HtmlElement.br(2) )
option_div.add("Sample Data Row: ")
data_row_text = TextWdg("data_row")
data_row_text.set_attr("size", "3")
option_div.add(data_row_text)
option_div.add( HintWdg("Set this as a sample data row to match the columns to the database") )
option_div.add( HtmlElement.br(2) )
div.add(option_div)
my.has_title = title_row_checkbox.is_checked()
# parse the first fow
csv_parser = CsvParser(my.file_path)
if my.has_title:
csv_parser.set_has_title_row(True)
else:
csv_parser.set_has_title_row(False)
csv_parser.parse()
csv_titles = csv_parser.get_titles()
csv_data = csv_parser.get_data()
data_row = data_row_text.get_value()
if not data_row:
data_row = 0
else:
try:
data_row = int(data_row)
except ValueError:
data_row = 0
if data_row >= len(csv_data):
data_row = len(csv_data)-1
data_row_text.set_value(data_row)
table = Table()
table.set_attr("cellpadding", "10")
table.add_row()
table.add_header("CSV Column Value")
table.add_header("TACTIC Column")
table.add_header("Create New Column")
columns = my.search_type_obj.get_columns()
search_type = my.search_type_obj.get_base_search_type()
sobj = SObjectFactory.create(search_type)
required_columns = sobj.get_required_columns()
row = csv_data[data_row]
labels = []
for column in columns:
if column in required_columns:
label = '%s**'%column
else:
label = column
labels.append(label)
for j, cell in enumerate(row):
table.add_row()
table.add_cell(cell)
column_select = SelectWdg("column_%s" % j)
column_select.add_event("onchange", "if (this.value!='') {set_display_off('new_column_div_%s')} else {set_display_on('new_column_div_%s')}" % (j,j))
column_select.add_empty_option("-- Select --")
column_select.set_option("values", columns)
column_select.set_option("labels", labels)
# only set the value if it is actually in there
if csv_titles[j] in columns:
column_select.set_option("default", csv_titles[j])
column_select.set_persist_on_submit()
column_select_value = column_select.get_value()
display = column_select.get_buffer_display()
td = table.add_cell( display )
if csv_titles[j] not in columns:
td.add(" <b style='color: red'>*</b>")
# new property
new_column_div = DivWdg()
if column_select_value:
new_column_div.add_style("display", "none")
else:
new_column_div.add_style("display", "block")
new_column_div.set_id("new_column_div_%s" % j)
td = table.add_cell( new_column_div )
text = TextWdg("new_column_%s" % j)
text.set_persist_on_submit()
if my.has_title:
text.set_value(csv_titles[j])
new_column_div.add( " ... or ..." )
new_column_div.add( text )
my.num_columns = len(row)
hidden = HiddenWdg("num_columns", my.num_columns)
# need to somehow specify defaults for columns
div.add(table)
div.add("<br/><br/>")
div.add(my.get_preview_wdg())
return div
def get_column_preview(my, div):
# parse the first fow
csv_parser = CsvParser(my.file_path)
if my.has_title:
csv_parser.set_has_title_row(True)
else:
csv_parser.set_has_title_row(False)
if my.lowercase_title:
csv_parser.set_lowercase_title(True)
if my.encoder:
csv_parser.set_encoder(my.encoder)
try:
csv_parser.parse()
# that can be all kinds of encoding/decoding exception
except Exception, e:
# possibly incompatible encoder selected, use the default instead.
# Let the user pick it.
span = SpanWdg('WARNING: The selected encoder is not compatible with your csv file. Please choose the proper one (e.g. UTF-8). Refer to the documentation/tutorial on how to save your csv file with UTF-8 encoding if you have special characters in it.', css='warning')
div.add(SpanWdg(e.__str__()))
div.add(HtmlElement.br())
div.add(span, 'warning')
return div