def upload(rows=None,
submit_after_import=None,
ignore_encoding_errors=False,
overwrite=False,
ignore_links=False):
"""upload data"""
webnotes.mute_emails = True
webnotes.check_admin_or_system_manager()
# extra input params
import json
params = json.loads(webnotes.form_dict.get("params") or '{}')
if params.get("_submit"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
from webnotes.utils.datautils import read_csv_content_from_uploaded_file
def bad_template():
webnotes.msgprint("Please do not change the rows above '%s'" %
data_keys.data_separator,
raise_exception=1)
def check_data_length():
max_rows = 5000
if not data:
webnotes.msgprint("No data found", raise_exception=True)
elif len(data) > max_rows:
webnotes.msgprint("Please upload only upto %d %ss at a time" % \
(max_rows, doctype), raise_exception=True)
def get_start_row():
for i, row in enumerate(rows):
if row and row[0] == data_keys.data_separator:
return i + 1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0] == key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = filter(lambda x: x in ("", None), columns)
if empty_cols:
if columns[-1 * len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1 * len(empty_cols)]
else:
webnotes.msgprint(_(
"Please make sure that there are no empty columns in the file."
),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(data_keys.doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d: # value in doctype_row
if doctype_row[i] == dt:
# prev column is doctype (in case of parentfield)
doctype_parentfield[dt] = doctype_row[i + 1]
else:
dt = d
doctypes.append(d)
column_idx_to_fieldname[dt] = {}
column_idx_to_fieldtype[dt] = {}
if dt:
column_idx_to_fieldname[dt][i + 1] = rows[row_idx + 2][i +
1]
column_idx_to_fieldtype[dt][i + 1] = rows[row_idx + 4][i +
1]
def get_doclist(start_idx):
if doctypes:
doclist = []
for idx in xrange(start_idx, len(rows)):
if (not len(doclist)) or main_doc_empty(rows[idx]):
for dt in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[dt]:
try:
fieldname = column_idx_to_fieldname[dt][
column_idx]
fieldtype = column_idx_to_fieldtype[dt][
column_idx]
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency"):
d[fieldname] = flt(d[fieldname])
except IndexError, e:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt != doctype:
if not overwrite:
d['parent'] = doclist[0]["name"]
d['parenttype'] = doctype
d['parentfield'] = doctype_parentfield[dt]
doclist.append(d)
else:
break
return doclist
else:
def get_template(doctype=None,
parent_doctype=None,
all_doctypes="No",
with_data="No"):
webnotes.check_admin_or_system_manager()
all_doctypes = all_doctypes == "Yes"
if not parent_doctype:
parent_doctype = doctype
column_start_end = {}
if all_doctypes:
doctype_parentfield = {}
child_doctypes = []
for d in get_table_fields(doctype):
child_doctypes.append(d[0])
doctype_parentfield[d[0]] = d[1]
def add_main_header():
w.writerow(['Data Import Template'])
w.writerow([data_keys.main_table, doctype])
if parent_doctype != doctype:
w.writerow([data_keys.parent_table, parent_doctype])
else:
w.writerow([''])
w.writerow([''])
w.writerow(['Notes:'])
w.writerow(['Please do not change the template headings.'])
w.writerow(['First data column must be blank.'])
w.writerow([
'If you are uploading new records, leave the "name" (ID) column blank.'
])
w.writerow([
'If you are uploading new records, "Naming Series" becomes mandatory, if present.'
])
w.writerow([
'Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.'
])
w.writerow(['For updating, you can update only selective columns.'])
w.writerow([
'You can only upload upto 5000 records in one go. (may be less in some cases)'
])
if key == "parent":
w.writerow([
'"Parent" signifies the parent table in which this row must be added'
])
w.writerow([
'If you are updating, please select "Overwrite" else existing rows will not be deleted.'
])
def build_field_columns(dt):
doctype_dl = webnotes.model.doctype.get(dt)
tablecolumns = filter(None, [
doctype_dl.get_field(f[0])
for f in webnotes.conn.sql('desc `tab%s`' % dt)
])
tablecolumns.sort(lambda a, b: a.idx - b.idx)
if dt == doctype:
column_start_end[dt] = webnotes._dict({"start": 0})
else:
column_start_end[dt] = webnotes._dict({"start": len(columns)})
append_field_column(
webnotes._dict({
"fieldname": "name",
"label": "ID",
"fieldtype": "Data",
"reqd": 1,
"idx": 0,
"info": "Leave blank for new records"
}), True)
for docfield in tablecolumns:
append_field_column(docfield, True)
# all non mandatory fields
for docfield in tablecolumns:
append_field_column(docfield, False)
# append DocType name
tablerow[column_start_end[dt].start + 1] = dt
if dt != doctype:
tablerow[column_start_end[dt].start + 2] = doctype_parentfield[dt]
column_start_end[dt].end = len(columns) + 1
def append_field_column(docfield, mandatory):
if docfield and ((mandatory and docfield.reqd) or not (mandatory or docfield.reqd)) \
and (docfield.fieldname not in ('parenttype', 'trash_reason')) and not docfield.hidden:
tablerow.append("")
fieldrow.append(docfield.fieldname)
labelrow.append(docfield.label)
mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
columns.append(docfield.fieldname)
def append_empty_field_column():
tablerow.append("~")
fieldrow.append("~")
labelrow.append("")
mandatoryrow.append("")
typerow.append("")
inforow.append("")
columns.append("")
def getinforow(docfield):
"""make info comment for options, links etc."""
if docfield.fieldtype == 'Select':
if not docfield.options:
return ''
elif docfield.options.startswith('link:'):
return 'Valid %s' % docfield.options[5:]
else:
return 'One of: %s' % ', '.join(
filter(None, docfield.options.split('\n')))
elif docfield.fieldtype == 'Link':
return 'Valid %s' % docfield.options
elif docfield.fieldtype in ('Int'):
return 'Integer'
elif docfield.fieldtype == "Check":
return "0 or 1"
elif docfield.info:
return docfield.info
else:
return ''
def add_field_headings():
w.writerow(tablerow)
w.writerow(labelrow)
w.writerow(fieldrow)
w.writerow(mandatoryrow)
w.writerow(typerow)
w.writerow(inforow)
w.writerow([data_keys.data_separator])
def add_data():
def add_data_row(row_group, dt, doc, rowidx):
d = doc.copy()
if all_doctypes:
d.name = '"' + d.name + '"'
if len(row_group) < rowidx + 1:
row_group.append([""] * (len(columns) + 1))
row = row_group[rowidx]
for i, c in enumerate(columns[column_start_end[dt].
start:column_start_end[dt].end]):
row[column_start_end[dt].start + i + 1] = d.get(c, "")
if with_data == 'Yes':
data = webnotes.conn.sql("""select * from `tab%s` where docstatus<2""" \
% doctype, as_dict=1)
for doc in data:
# add main table
row_group = []
add_data_row(row_group, doctype, doc, 0)
if all_doctypes:
# add child tables
for child_doctype in child_doctypes:
for ci, child in enumerate(
webnotes.conn.sql("""select * from `tab%s`
where parent=%s order by idx""" % (child_doctype, "%s"),
doc.name,
as_dict=1)):
add_data_row(row_group, child_doctype, child, ci)
for row in row_group:
w.writerow(row)
w = UnicodeWriter()
key = 'parent' if parent_doctype != doctype else 'name'
add_main_header()
w.writerow([''])
tablerow = [data_keys.doctype, ""]
labelrow = ["Column Labels:", "ID"]
fieldrow = [data_keys.columns, key]
mandatoryrow = ['Mandatory:', 'Yes']
typerow = ['Type:', 'Data (text)']
inforow = ['Info:', '']
columns = [key]
build_field_columns(doctype)
if all_doctypes:
for d in child_doctypes:
append_empty_field_column()
build_field_columns(d)
add_field_headings()
add_data()
# write out response as a type csv
webnotes.response['result'] = cstr(w.getvalue())
webnotes.response['type'] = 'csv'
webnotes.response['doctype'] = doctype
def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No"):
webnotes.check_admin_or_system_manager()
all_doctypes = all_doctypes=="Yes"
if not parent_doctype:
parent_doctype = doctype
column_start_end = {}
if all_doctypes:
doctype_parentfield = {}
child_doctypes = []
for d in get_table_fields(doctype):
child_doctypes.append(d[0])
doctype_parentfield[d[0]] = d[1]
def add_main_header():
w.writerow(['Data Import Template'])
w.writerow([data_keys.main_table, doctype])
if parent_doctype != doctype:
w.writerow([data_keys.parent_table, parent_doctype])
else:
w.writerow([''])
w.writerow([''])
w.writerow(['Notes:'])
w.writerow(['Please do not change the template headings.'])
w.writerow(['First data column must be blank.'])
w.writerow(['If you are uploading new records, leave the "name" (ID) column blank.'])
w.writerow(['If you are uploading new records, "Naming Series" becomes mandatory, if present.'])
w.writerow(['Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.'])
w.writerow(['For updating, you can update only selective columns.'])
w.writerow(['You can only upload upto 5000 records in one go. (may be less in some cases)'])
if key == "parent":
w.writerow(['"Parent" signifies the parent table in which this row must be added'])
w.writerow(['If you are updating, please select "Overwrite" else existing rows will not be deleted.'])
def build_field_columns(dt):
doctype_dl = webnotes.model.doctype.get(dt)
tablecolumns = filter(None,
[doctype_dl.get_field(f[0]) for f in webnotes.conn.sql('desc `tab%s`' % dt)])
tablecolumns.sort(lambda a, b: a.idx - b.idx)
if dt==doctype:
column_start_end[dt] = webnotes._dict({"start": 0})
else:
column_start_end[dt] = webnotes._dict({"start": len(columns)})
append_field_column(webnotes._dict({
"fieldname": "name",
"label": "ID",
"fieldtype": "Data",
"reqd": 1,
"idx": 0,
"info": "Leave blank for new records"
}), True)
for docfield in tablecolumns:
append_field_column(docfield, True)
# all non mandatory fields
for docfield in tablecolumns:
append_field_column(docfield, False)
# append DocType name
tablerow[column_start_end[dt].start + 1] = dt
if dt!=doctype:
tablerow[column_start_end[dt].start + 2] = doctype_parentfield[dt]
column_start_end[dt].end = len(columns) + 1
def append_field_column(docfield, mandatory):
if docfield and ((mandatory and docfield.reqd) or not (mandatory or docfield.reqd)) \
and (docfield.fieldname not in ('parenttype', 'trash_reason')) and not docfield.hidden:
tablerow.append("")
fieldrow.append(docfield.fieldname)
labelrow.append(docfield.label)
mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
columns.append(docfield.fieldname)
def append_empty_field_column():
tablerow.append("~")
fieldrow.append("~")
labelrow.append("")
mandatoryrow.append("")
typerow.append("")
inforow.append("")
columns.append("")
def getinforow(docfield):
"""make info comment for options, links etc."""
if docfield.fieldtype == 'Select':
if not docfield.options:
return ''
elif docfield.options.startswith('link:'):
return 'Valid %s' % docfield.options[5:]
else:
return 'One of: %s' % ', '.join(filter(None, docfield.options.split('\n')))
elif docfield.fieldtype == 'Link':
return 'Valid %s' % docfield.options
elif docfield.fieldtype in ('Int'):
return 'Integer'
elif docfield.fieldtype == "Check":
return "0 or 1"
elif docfield.info:
return docfield.info
else:
return ''
def add_field_headings():
w.writerow(tablerow)
w.writerow(labelrow)
w.writerow(fieldrow)
w.writerow(mandatoryrow)
w.writerow(typerow)
w.writerow(inforow)
w.writerow([data_keys.data_separator])
def add_data():
def add_data_row(row_group, dt, doc, rowidx):
d = doc.copy()
if all_doctypes:
d.name = '"'+ d.name+'"'
if len(row_group) < rowidx + 1:
row_group.append([""] * (len(columns) + 1))
row = row_group[rowidx]
for i, c in enumerate(columns[column_start_end[dt].start:column_start_end[dt].end]):
row[column_start_end[dt].start + i + 1] = d.get(c, "")
if with_data=='Yes':
data = webnotes.conn.sql("""select * from `tab%s` where docstatus<2""" \
% doctype, as_dict=1)
for doc in data:
# add main table
row_group = []
add_data_row(row_group, doctype, doc, 0)
if all_doctypes:
# add child tables
for child_doctype in child_doctypes:
for ci, child in enumerate(webnotes.conn.sql("""select * from `tab%s`
where parent=%s order by idx""" % (child_doctype, "%s"), doc.name, as_dict=1)):
add_data_row(row_group, child_doctype, child, ci)
for row in row_group:
w.writerow(row)
w = UnicodeWriter()
key = 'parent' if parent_doctype != doctype else 'name'
add_main_header()
w.writerow([''])
tablerow = [data_keys.doctype, ""]
labelrow = ["Column Labels:", "ID"]
fieldrow = [data_keys.columns, key]
mandatoryrow = ['Mandatory:', 'Yes']
typerow = ['Type:', 'Data (text)']
inforow = ['Info:', '']
columns = [key]
build_field_columns(doctype)
if all_doctypes:
for d in child_doctypes:
append_empty_field_column()
build_field_columns(d)
add_field_headings()
add_data()
# write out response as a type csv
webnotes.response['result'] = cstr(w.getvalue())
webnotes.response['type'] = 'csv'
webnotes.response['doctype'] = doctype
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, overwrite=False, ignore_links=False):
"""upload data"""
webnotes.mute_emails = True
webnotes.check_admin_or_system_manager()
# extra input params
import json
params = json.loads(webnotes.form_dict.get("params") or '{}')
if params.get("_submit"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
from webnotes.utils.datautils import read_csv_content_from_uploaded_file
def bad_template():
webnotes.msgprint("Please do not change the rows above '%s'" % data_keys.data_separator,
raise_exception=1)
def check_data_length():
max_rows = 5000
if not data:
webnotes.msgprint("No data found", raise_exception=True)
elif len(data) > max_rows:
webnotes.msgprint("Please upload only upto %d %ss at a time" % \
(max_rows, doctype), raise_exception=True)
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==data_keys.data_separator:
return i+1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = filter(lambda x: x in ("", None), columns)
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
webnotes.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(data_keys.doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d: # value in doctype_row
if doctype_row[i]==dt:
# prev column is doctype (in case of parentfield)
doctype_parentfield[dt] = doctype_row[i+1]
else:
dt = d
doctypes.append(d)
column_idx_to_fieldname[dt] = {}
column_idx_to_fieldtype[dt] = {}
if dt:
column_idx_to_fieldname[dt][i+1] = rows[row_idx + 2][i+1]
column_idx_to_fieldtype[dt][i+1] = rows[row_idx + 4][i+1]
def get_doclist(start_idx):
if doctypes:
doclist = []
for idx in xrange(start_idx, len(rows)):
if (not len(doclist)) or main_doc_empty(rows[idx]):
for dt in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[dt]:
try:
fieldname = column_idx_to_fieldname[dt][column_idx]
fieldtype = column_idx_to_fieldtype[dt][column_idx]
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency"):
d[fieldname] = flt(d[fieldname])
except IndexError, e:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt != doctype:
if not overwrite:
d['parent'] = doclist[0]["name"]
d['parenttype'] = doctype
d['parentfield'] = doctype_parentfield[dt]
doclist.append(d)
else:
break
return doclist
else:
def get_template():
webnotes.check_admin_or_system_manager()
doctype = webnotes.form_dict.doctype
parenttype = webnotes.form_dict.parent_doctype
all_doctypes = webnotes.form_dict.all_doctypes == "Yes"
with_data = webnotes.form_dict.with_data
column_start_end = {}
if all_doctypes:
doctype_parentfield = {}
child_doctypes = []
for d in get_table_fields(doctype):
child_doctypes.append(d[0])
doctype_parentfield[d[0]] = d[1]
def add_main_header():
w.writerow(["Data Import Template"])
w.writerow([data_keys.main_table, doctype])
if parenttype != doctype:
w.writerow([data_keys.parent_table, parenttype])
else:
w.writerow([""])
w.writerow([""])
w.writerow(["Notes:"])
w.writerow(["Please do not change the template headings."])
w.writerow(["First data column must be blank."])
w.writerow(['If you are uploading new records, leave the "name" (ID) column blank.'])
w.writerow(['If you are uploading new records, "Naming Series" becomes mandatory, if present.'])
w.writerow(
["Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish."]
)
w.writerow(["For updating, you can update only selective columns."])
w.writerow(["You can only upload upto 5000 records in one go. (may be less in some cases)"])
if key == "parent":
w.writerow(['"Parent" signifies the parent table in which this row must be added'])
w.writerow(['If you are updating, please select "Overwrite" else existing rows will not be deleted.'])
def build_field_columns(dt):
doctype_dl = webnotes.model.doctype.get(dt)
tablecolumns = filter(None, [doctype_dl.get_field(f[0]) for f in webnotes.conn.sql("desc `tab%s`" % dt)])
tablecolumns.sort(lambda a, b: a.idx - b.idx)
if dt == doctype:
column_start_end[dt] = webnotes._dict({"start": 0})
if all_doctypes and with_data:
append_field_column(
webnotes._dict(
{
"fieldname": "modified",
"label": "Last Updated On",
"fieldtype": "Data",
"reqd": 1,
"idx": 0,
"info": "Don't change!",
}
),
True,
)
else:
column_start_end[dt] = webnotes._dict({"start": len(columns)})
append_field_column(
webnotes._dict(
{
"fieldname": "name",
"label": "ID",
"fieldtype": "Data",
"reqd": 1,
"idx": 0,
"info": "Leave blank for new records",
}
),
True,
)
for docfield in tablecolumns:
append_field_column(docfield, True)
# all non mandatory fields
for docfield in tablecolumns:
append_field_column(docfield, False)
# append DocType name
tablerow[column_start_end[dt].start + 1] = dt
if dt != doctype:
tablerow[column_start_end[dt].start + 2] = doctype_parentfield[dt]
column_start_end[dt].end = len(columns) + 1
def append_field_column(docfield, mandatory):
if (
docfield
and ((mandatory and docfield.reqd) or not (mandatory or docfield.reqd))
and (docfield.fieldname not in ("parenttype", "trash_reason"))
and not docfield.hidden
):
tablerow.append("")
fieldrow.append(docfield.fieldname)
labelrow.append(docfield.label)
mandatoryrow.append(docfield.reqd and "Yes" or "No")
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
columns.append(docfield.fieldname)
def append_empty_field_column():
tablerow.append("~")
fieldrow.append("~")
labelrow.append("")
mandatoryrow.append("")
typerow.append("")
inforow.append("")
columns.append("")
def getinforow(docfield):
"""make info comment for options, links etc."""
if docfield.fieldtype == "Select":
if not docfield.options:
return ""
elif docfield.options.startswith("link:"):
return "Valid %s" % docfield.options[5:]
else:
return "One of: %s" % ", ".join(filter(None, docfield.options.split("\n")))
elif docfield.fieldtype == "Link":
return "Valid %s" % docfield.options
elif docfield.fieldtype in ("Int"):
return "Integer"
elif docfield.fieldtype == "Check":
return "0 or 1"
elif docfield.info:
return docfield.info
else:
return ""
def add_field_headings():
w.writerow(tablerow)
w.writerow(labelrow)
w.writerow(fieldrow)
w.writerow(mandatoryrow)
w.writerow(typerow)
w.writerow(inforow)
w.writerow([data_keys.data_separator])
def add_data():
def add_data_row(row_group, dt, d, rowidx):
if len(row_group) < rowidx + 1:
row_group.append([""] * (len(columns) + 1))
row = row_group[rowidx]
for i, c in enumerate(columns[column_start_end[dt].start : column_start_end[dt].end]):
row[column_start_end[dt].start + i + 1] = d.get(c, "")
if with_data == "Yes":
data = webnotes.conn.sql("""select * from `tab%s` where docstatus<2""" % doctype, as_dict=1)
for doc in data:
# add main table
row_group = []
if all_doctypes:
doc.modified = '"' + doc.modified + '"'
add_data_row(row_group, doctype, doc, 0)
if all_doctypes:
# add child tables
for child_doctype in child_doctypes:
for ci, child in enumerate(
webnotes.conn.sql(
"""select * from `tab%s`
where parent=%s order by idx"""
% (child_doctype, "%s"),
doc.name,
as_dict=1,
)
):
add_data_row(row_group, child_doctype, child, ci)
for row in row_group:
w.writerow(row)
w = UnicodeWriter()
key = "parent" if parenttype != doctype else "name"
add_main_header()
w.writerow([""])
tablerow = [data_keys.doctype, ""]
labelrow = ["Column Labels:", "ID"]
fieldrow = [data_keys.columns, key]
mandatoryrow = ["Mandatory:", "Yes"]
typerow = ["Type:", "Data (text)"]
inforow = ["Info:", ""]
columns = [key]
build_field_columns(doctype)
if all_doctypes:
for d in child_doctypes:
append_empty_field_column()
build_field_columns(d)
add_field_headings()
add_data()
# write out response as a type csv
webnotes.response["result"] = cstr(w.getvalue())
webnotes.response["type"] = "csv"
webnotes.response["doctype"] = doctype
def get_template():
webnotes.check_admin_or_system_manager()
doctype = webnotes.form_dict['doctype']
parenttype = webnotes.form_dict.get('parent_doctype')
doctype_dl = webnotes.model.doctype.get(doctype)
tablecolumns = filter(None,
[doctype_dl.get_field(f[0]) for f in webnotes.conn.sql('desc `tab%s`' % doctype)])
tablecolumns.sort(lambda a, b: a.idx - b.idx)
def getinforow(docfield):
"""make info comment"""
if docfield.fieldtype == 'Select':
if not docfield.options:
return ''
elif docfield.options.startswith('link:'):
return 'Valid %s' % docfield.options[5:]
else:
return 'One of: %s' % ', '.join(filter(None, docfield.options.split('\n')))
if docfield.fieldtype == 'Link':
return 'Valid %s' % docfield.options
if docfield.fieldtype in ('Int'):
return 'Integer'
if docfield.fieldtype == "Check":
return "0 or 1"
else:
return ''
w = UnicodeWriter()
key = 'name'
w.writerow(['Data Import Template'])
w.writerow([data_keys.main_table, doctype])
if parenttype != doctype:
w.writerow([data_keys.parent_table, parenttype])
key = 'parent'
else:
w.writerow([''])
w.writerow([''])
w.writerow(['Notes:'])
w.writerow(['Please do not change the template headings.'])
w.writerow(['First data column must be blank.'])
w.writerow(['Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.'])
w.writerow(['For updating, you can update only selective columns.'])
w.writerow(['If you are uploading new records, leave the "name" (ID) column blank.'])
w.writerow(['If you are uploading new records, "Naming Series" becomes mandatory, if present.'])
w.writerow(['You can only upload upto 5000 records in one go. (may be less in some cases)'])
if key == "parent":
w.writerow(['"Parent" signifies the parent table in which this row must be added'])
w.writerow(['If you are updating, please select "Overwrite" else existing rows will not be deleted.'])
w.writerow([''])
labelrow = ["Column Labels", "ID"]
fieldrow = [data_keys.columns, key]
mandatoryrow = ['Mandatory:', 'Yes']
typerow = ['Type:', 'Data (text)']
inforow = ['Info:', '']
columns = [key]
def append_row(docfield, mandatory):
if docfield and ((mandatory and docfield.reqd) or not (mandatory or docfield.reqd)) \
and (docfield.fieldname not in ('parenttype', 'trash_reason')) and not docfield.hidden:
fieldrow.append(docfield.fieldname)
labelrow.append(docfield.label)
mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
typerow.append(docfield.fieldtype)
inforow.append(getinforow(docfield))
columns.append(docfield.fieldname)
# get all mandatory fields
for t in tablecolumns:
append_row(t, True)
# all non mandatory fields
for t in tablecolumns:
append_row(t, False)
w.writerow(labelrow)
w.writerow(fieldrow)
w.writerow(mandatoryrow)
w.writerow(typerow)
w.writerow(inforow)
w.writerow([data_keys.data_separator])
if webnotes.form_dict.get('with_data')=='Yes':
data = webnotes.conn.sql("""select * from `tab%s` where docstatus<2""" % doctype, as_dict=1)
for d in data:
row = [""]
for c in columns:
row.append(d.get(c, ""))
w.writerow(row)
# write out response as a type csv
webnotes.response['result'] = cstr(w.getvalue())
webnotes.response['type'] = 'csv'
webnotes.response['doctype'] = doctype
def upload():
"""upload data"""
webnotes.mute_emails = True
webnotes.check_admin_or_system_manager()
from webnotes.utils.datautils import read_csv_content_from_uploaded_file
def bad_template():
webnotes.msgprint("Please do not change the rows above '%s'" % data_keys.data_separator,
raise_exception=1)
def check_data_length():
max_rows = 5000
if not data:
webnotes.msgprint("No data found", raise_exception=True)
elif len(data) > max_rows:
webnotes.msgprint("Please upload only upto %d %ss at a time" % \
(max_rows, doctype), raise_exception=True)
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==data_keys.data_separator:
return i+1
bad_template()
def get_header_row(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row
return []
def filter_empty_columns(columns):
empty_cols = filter(lambda x: x in ("", None), columns)
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
webnotes.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
# extra input params
import json
params = json.loads(webnotes.form_dict.get("params") or '{}')
# header
rows = read_csv_content_from_uploaded_file(params.get("ignore_encoding_errors"))
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
doctype = get_header_row(data_keys.main_table)[1]
columns = filter_empty_columns(get_header_row(data_keys.columns)[1:])
parenttype = get_header_row(data_keys.parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
parentfield = get_parent_field(doctype, parenttype)
# allow limit rows to be uploaded
check_data_length()
webnotes.conn.begin()
overwrite = params.get('overwrite')
doctype_dl = webnotes.model.doctype.get(doctype)
# delete child rows (if parenttype)
if parenttype and overwrite:
delete_child_rows(data, doctype)
ret = []
error = False
parent_list = []
for i, row in enumerate(data):
# bypass empty rows
if not row: continue
row_idx = (i + 1) + start_row
d = webnotes._dict(zip(columns, row[1:]))
d['doctype'] = doctype
try:
webnotes.message_log = []
check_record(d, parenttype, doctype_dl)
if parenttype:
# child doc
doc = Document(doctype)
doc.fields.update(d)
if parenttype:
doc.parenttype = parenttype
doc.parentfield = parentfield
doc.save()
ret.append('Inserted row for %s at #%s' % (getlink(parenttype,
doc.parent), unicode(doc.idx)))
parent_list.append(doc.parent)
else:
ret.append(import_doc(d, doctype, overwrite, row_idx, params.get("_submit")))
except Exception, e:
error = True
err_msg = webnotes.message_log and "<br>".join(webnotes.message_log) or cstr(e)
ret.append('Error for row (#%d) %s : %s' % (row_idx,
len(row)>1 and row[1] or "", err_msg))
webnotes.errprint(webnotes.getTraceback())
