def upload(select_doctype=None, rows=None):
from frappe.utils.datautils import read_csv_content_from_uploaded_file
from frappe.model.rename_doc import rename_doc
if not select_doctype:
select_doctype = frappe.form_dict.select_doctype
if not frappe.has_permission(select_doctype, "write"):
raise frappe.PermissionError
if not rows:
rows = read_csv_content_from_uploaded_file()
if not rows:
frappe.throw(_("Please select a valid csv file with data"))
max_rows = 500
if len(rows) > max_rows:
frappe.throw(_("Maximum {0} rows allowed").format(max_rows))
rename_log = []
for row in rows:
# if row has some content
if len(row) > 1 and row[0] and row[1]:
try:
if rename_doc(select_doctype, row[0], row[1]):
rename_log.append(_("Successful: ") + row[0] + " -> " + row[1])
frappe.db.commit()
else:
rename_log.append(_("Ignored: ") + row[0] + " -> " + row[1])
except Exception, e:
rename_log.append("<span style='color: RED'>" + \
_("Failed: ") + row[0] + " -> " + row[1] + "</span>")
rename_log.append("<span style='margin-left: 20px;'>" + repr(e) + "</span>")
def upload():
if not frappe.has_permission("Attendance", "create"):
raise frappe.PermissionError
from frappe.utils.datautils import read_csv_content_from_uploaded_file
from frappe.modules import scrub
rows = read_csv_content_from_uploaded_file()
if not rows:
msg = [_("Please select a csv file")]
return {"messages": msg, "error": msg}
columns = [scrub(f) for f in rows[4]]
columns[0] = "name"
columns[3] = "att_date"
ret = []
error = False
from frappe.utils.datautils import check_record, import_doc
for i, row in enumerate(rows[5:]):
if not row: continue
row_idx = i + 5
d = frappe._dict(zip(columns, row))
d["doctype"] = "Attendance"
if d.name:
d["docstatus"] = frappe.db.get_value("Attendance", d.name, "docstatus")
try:
check_record(d)
ret.append(import_doc(d, "Attendance", 1, row_idx, submit=True))
except Exception, e:
error = True
ret.append('Error for row (#%d) %s : %s' % (row_idx,
len(row)>1 and row[1] or "", cstr(e)))
frappe.errprint(frappe.get_traceback())
def import_user_permissions():
frappe.only_for("System Manager")
rows = read_csv_content_from_uploaded_file(ignore_encoding=True)
clear_default(parenttype="User Permission")
if rows[0][0]!="User Permissions" and rows[1][0] != "User":
frappe.throw(frappe._("Please upload using the same template as download."))
for row in rows[2:]:
frappe.permissions.add_user_permission(row[1], row[2], row[0])
def upload(select_doctype=None, rows=None):
from frappe.utils.datautils import read_csv_content_from_uploaded_file
from frappe.modules import scrub
from frappe.model.rename_doc import rename_doc
if not select_doctype:
select_doctype = frappe.form_dict.select_doctype
if not frappe.has_permission(select_doctype, "write"):
raise frappe.PermissionError
if not rows:
rows = read_csv_content_from_uploaded_file()
if not rows:
frappe.msgprint(_("Please select a valid csv file with data."))
raise Exception
if len(rows) > 500:
frappe.msgprint(_("Max 500 rows only."))
raise Exception
rename_log = []
for row in rows:
# if row has some content
if len(row) > 1 and row[0] and row[1]:
try:
if rename_doc(select_doctype, row[0], row[1]):
rename_log.append(
_("Successful: ") + row[0] + " -> " + row[1])
frappe.db.commit()
else:
rename_log.append(
_("Ignored: ") + row[0] + " -> " + row[1])
except Exception, e:
rename_log.append("<span style='color: RED'>" + \
_("Failed: ") + row[0] + " -> " + row[1] + "</span>")
rename_log.append("<span style='margin-left: 20px;'>" +
repr(e) + "</span>")
def upload():
if not frappe.has_permission("Attendance", "create"):
raise frappe.PermissionError
from frappe.utils.datautils import read_csv_content_from_uploaded_file
from frappe.modules import scrub
rows = read_csv_content_from_uploaded_file()
if not rows:
msg = [_("Please select a csv file")]
return {"messages": msg, "error": msg}
columns = [scrub(f) for f in rows[4]]
columns[0] = "name"
columns[3] = "att_date"
ret = []
error = False
from frappe.utils.datautils import check_record, import_doc
for i, row in enumerate(rows[5:]):
if not row: continue
row_idx = i + 5
d = frappe._dict(zip(columns, row))
d["doctype"] = "Attendance"
if d.name:
d["docstatus"] = frappe.db.get_value("Attendance", d.name,
"docstatus")
try:
check_record(d)
ret.append(import_doc(d, "Attendance", 1, row_idx, submit=True))
except Exception, e:
error = True
ret.append('Error for row (#%d) %s : %s' %
(row_idx, len(row) > 1 and row[1] or "", cstr(e)))
frappe.errprint(frappe.get_traceback())
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, overwrite=None, ignore_links=False):
"""upload data"""
frappe.flags.mute_emails = True
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("_submit"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
from frappe.utils.datautils import read_csv_content_from_uploaded_file
def bad_template():
frappe.throw(_("Please do not change the rows above {0}").format(data_keys.data_separator))
def check_data_length():
max_rows = 5000
if not data:
frappe.throw(_("No data found"))
elif len(data) > max_rows:
frappe.throw(_("Only allowed {0} rows in one import").format(max_rows))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==data_keys.data_separator:
return i+1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = filter(lambda x: x in ("", None), columns)
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(data_keys.doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d: # value in doctype_row
if doctype_row[i]==dt:
# prev column is doctype (in case of parentfield)
doctype_parentfield[dt] = doctype_row[i+1]
else:
dt = d
doctypes.append(d)
column_idx_to_fieldname[dt] = {}
column_idx_to_fieldtype[dt] = {}
if dt:
column_idx_to_fieldname[dt][i+1] = rows[row_idx + 2][i+1]
column_idx_to_fieldtype[dt][i+1] = rows[row_idx + 4][i+1]
def get_doc(start_idx):
if doctypes:
doc = {}
for idx in xrange(start_idx, len(rows)):
if (not doc) or main_doc_empty(rows[idx]):
for dt in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[dt]:
try:
fieldname = column_idx_to_fieldname[dt][column_idx]
fieldtype = column_idx_to_fieldtype[dt][column_idx]
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
d[fieldname] = parse_date(d[fieldname]) if d[fieldname] else None
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite:
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = doctype_parentfield[dt]
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc
else:
doc = frappe._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc
def main_doc_empty(row):
return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2])))
# header
if not rows:
rows = read_csv_content_from_uploaded_file(ignore_encoding_errors)
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
doctype = get_header_row(data_keys.main_table)[1]
columns = filter_empty_columns(get_header_row(data_keys.columns)[1:])
doctypes = []
doctype_parentfield = {}
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if submit_after_import and not cint(frappe.db.get_value("DocType",
doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(data_keys.parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not frappe.permissions.can_import(parenttype or doctype):
frappe.flags.mute_emails = False
return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
# allow limit rows to be uploaded
check_data_length()
make_column_map()
frappe.db.begin()
if overwrite==None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
if parenttype and overwrite:
delete_child_rows(data, doctype)
ret = []
error = False
for i, row in enumerate(data):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
doc = get_doc(row_idx)
if doc.get("name"):
doc["_new_name_set"] = True
try:
frappe.local.message_log = []
if doc.get("parentfield"):
parent = frappe.get_doc(doc["parenttype"], doc["parentfield"])
parent.append(doc)
parent.save()
ret.append('Inserted row for %s at #%s' % (getlink(parenttype,
doc.parent), unicode(doc.idx)))
else:
if overwrite and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
original.update(doc)
original.ignore_links = ignore_links
original.save()
ret.append('Updated row (#%d) %s' % (row_idx + 1, getlink(original.doctype, original.name)))
else:
doc = frappe.get_doc(doc)
doc.ignore_links = ignore_links
doc.insert()
ret.append('Inserted row (#%d) %s' % (row_idx + 1, getlink(doc.doctype, doc.name)))
if submit_after_import:
doc.submit()
ret.append('Submitted row (#%d) %s' % (row_idx + 1, getlink(doc.doctype, doc.name)))
except Exception, e:
error = True
if doc:
frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
err_msg = frappe.local.message_log and "<br>".join(frappe.local.message_log) or cstr(e)
ret.append('Error for row (#%d) %s : %s' % (row_idx + 1,
len(row)>1 and row[1] or "", err_msg))
frappe.errprint(frappe.get_traceback())
def upload(): from frappe.utils.datautils import read_csv_content_from_uploaded_file return read_csv_content_from_uploaded_file()
def upload(rows=None,
submit_after_import=None,
ignore_encoding_errors=False,
overwrite=None,
ignore_links=False):
"""upload data"""
frappe.flags.mute_emails = True
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("_submit"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
from frappe.utils.datautils import read_csv_content_from_uploaded_file
def bad_template():
frappe.throw(
_("Please do not change the rows above {0}").format(
data_keys.data_separator))
def check_data_length():
max_rows = 5000
if not data:
frappe.throw(_("No data found"))
elif len(data) > max_rows:
frappe.throw(
_("Only allowed {0} rows in one import").format(max_rows))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0] == data_keys.data_separator:
return i + 1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0] == key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = filter(lambda x: x in ("", None), columns)
if empty_cols:
if columns[-1 * len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1 * len(empty_cols)]
else:
frappe.msgprint(_(
"Please make sure that there are no empty columns in the file."
),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(data_keys.doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d: # value in doctype_row
if doctype_row[i] == dt:
# prev column is doctype (in case of parentfield)
doctype_parentfield[dt] = doctype_row[i + 1]
else:
dt = d
doctypes.append(d)
column_idx_to_fieldname[dt] = {}
column_idx_to_fieldtype[dt] = {}
if dt:
column_idx_to_fieldname[dt][i + 1] = rows[row_idx + 2][i +
1]
column_idx_to_fieldtype[dt][i + 1] = rows[row_idx + 4][i +
1]
def get_doc(start_idx):
if doctypes:
doc = {}
for idx in xrange(start_idx, len(rows)):
if (not doc) or main_doc_empty(rows[idx]):
for dt in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[dt]:
try:
fieldname = column_idx_to_fieldname[dt][
column_idx]
fieldtype = column_idx_to_fieldtype[dt][
column_idx]
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
d[fieldname] = parse_date(
d[fieldname]) if d[fieldname] else None
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite:
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = doctype_parentfield[dt]
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc
else:
doc = frappe._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc
def main_doc_empty(row):
return not (row and ((len(row) > 1 and row[1]) or
(len(row) > 2 and row[2])))
# header
if not rows:
rows = read_csv_content_from_uploaded_file(ignore_encoding_errors)
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
doctype = get_header_row(data_keys.main_table)[1]
columns = filter_empty_columns(get_header_row(data_keys.columns)[1:])
doctypes = []
doctype_parentfield = {}
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if submit_after_import and not cint(
frappe.db.get_value("DocType", doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(data_keys.parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not frappe.permissions.can_import(parenttype or doctype):
frappe.flags.mute_emails = False
return {
"messages": [_("Not allowed to Import") + ": " + _(doctype)],
"error": True
}
# allow limit rows to be uploaded
check_data_length()
make_column_map()
frappe.db.begin()
if overwrite == None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
if parenttype and overwrite:
delete_child_rows(data, doctype)
ret = []
error = False
for i, row in enumerate(data):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
doc = get_doc(row_idx)
try:
frappe.local.message_log = []
if doc.get("parentfield"):
parent = frappe.get_doc(doc["parenttype"], doc["parentfield"])
parent.append(doc)
parent.save()
ret.append('Inserted row for %s at #%s' %
(getlink(parenttype, doc.parent), unicode(doc.idx)))
else:
if overwrite and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
original.update(doc)
original.ignore_links = ignore_links
original.save()
ret.append('Updated row (#%d) %s' %
(row_idx + 1,
getlink(original.doctype, original.name)))
else:
doc = frappe.get_doc(doc)
doc.ignore_links = ignore_links
doc.insert()
ret.append('Inserted row (#%d) %s' %
(row_idx + 1, getlink(doc.doctype, doc.name)))
if submit_after_import:
doc.submit()
ret.append('Submitted row (#%d) %s' %
(row_idx + 1, getlink(doc.doctype, doc.name)))
except Exception, e:
error = True
if doc:
frappe.errprint(
doc if isinstance(doc, dict) else doc.as_dict())
err_msg = frappe.local.message_log and "<br>".join(
frappe.local.message_log) or cstr(e)
ret.append('Error for row (#%d) %s : %s' %
(row_idx + 1, len(row) > 1 and row[1] or "", err_msg))
frappe.errprint(frappe.get_traceback())
else:
break
return doclist
else:
d = frappe._dict(zip(columns, rows[start_idx][1:]))
d['doctype'] = doctype
return [d]
def main_doc_empty(row):
return not (row and ((len(row) > 1 and row[1]) or
(len(row) > 2 and row[2])))
# header
if not rows:
rows = read_csv_content_from_uploaded_file(ignore_encoding_errors)
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
doctype = get_header_row(data_keys.main_table)[1]
columns = filter_empty_columns(get_header_row(data_keys.columns)[1:])
doctypes = []
doctype_parentfield = {}
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if submit_after_import and not cint(
frappe.db.get_value("DocType", doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(data_keys.parent_table)
