def prepare_to_notify(self, print_html=None, print_format=None, attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
if print_format:
self.content += self.get_attach_link(print_format)
self.set_incoming_outgoing_accounts()
if not self.sender or cint(self.outgoing_email_account.always_use_account_email_id_as_sender):
self.sender = formataddr([frappe.session.data.full_name or "Notification", self.outgoing_email_account.email_id])
self.attachments = []
if print_html or print_format:
self.attachments.append(frappe.attach_print(self.reference_doctype, self.reference_name,
print_format=print_format, html=print_html))
if attachments:
if isinstance(attachments, basestring):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
self.attachments.append({"fname": file[0], "fcontent": file[1]})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
self.attachments.append(a)
def get_attachments(communications):
for comm in communications:
frappe.db.set_value("Communication", comm.name, "seen", 1)
comm['content'] = re.sub('src="', 'src="' + frappe.utils.get_url(),
comm.get('content'))
file_attachments = []
if comm.get('has_attachment'):
files = frappe.get_list("File",
filters={
"attached_to_doctype": "Communication",
"attached_to_name": comm.get('name')
})
for d in files:
filename, content = get_file(d.name)
file_attachments.append({
"filename":
filename,
"content":
base64.b64encode(content).decode('ascii')
})
file_attachments = json.dumps(file_attachments)
comm.update({"attachments": file_attachments})
return communications
def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
# Only look for completed prepared reports with given filters.
doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name, "filters": json.dumps(filters), "owner": user})
doc = None
if len(doc_list):
if dn:
# Get specified dn
doc = frappe.get_doc("Prepared Report", dn)
else:
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
# Prepared Report data is stored in a GZip compressed JSON file
attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name")
compressed_content = get_file(attached_file_name)[1]
uncompressed_content = gzip_decompress(compressed_content)
data = json.loads(uncompressed_content)
if data:
latest_report_data = {
"columns": json.loads(doc.columns) if doc.columns else data[0],
"result": data
}
latest_report_data.update({
"prepared_report": True,
"doc": doc
})
return latest_report_data
def execute():
frappe.reload_doc('core', 'doctype', 'file_data')
for name, file_name, file_url in frappe.db.sql(
"""select name, file_name, file_url from `tabFile`
where file_name is not null"""):
b = frappe.get_doc('File', name)
old_file_name = b.file_name
b.file_name = os.path.basename(old_file_name)
if old_file_name.startswith('files/') or old_file_name.startswith(
'/files/'):
b.file_url = os.path.normpath('/' + old_file_name)
else:
b.file_url = os.path.normpath('/files/' + old_file_name)
try:
_file_name, content = get_file(name)
b.content_hash = get_content_hash(content)
except IOError:
print('Warning: Error processing ', name)
_file_name = old_file_name
b.content_hash = None
try:
b.save()
except frappe.DuplicateEntryError:
frappe.delete_doc(b.doctype, b.name)
def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
# Only look for completed prepared reports with given filters.
doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name, "filters": json.dumps(filters), "owner": user})
doc = None
if len(doc_list):
if dn:
# Get specified dn
doc = frappe.get_doc("Prepared Report", dn)
else:
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
# Prepared Report data is stored in a GZip compressed JSON file
attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name")
compressed_content = get_file(attached_file_name)[1]
uncompressed_content = gzip_decompress(compressed_content)
data = json.loads(uncompressed_content)
if data:
latest_report_data = {
"columns": json.loads(doc.columns) if doc.columns else data[0],
"result": data
}
latest_report_data.update({
"prepared_report": True,
"doc": doc
})
return latest_report_data
def upload(file):
from frappe.utils.file_manager import get_file
filepath = get_file(file)
pps = read_csv_content(filepath[1])
date = ''
err_list = ''
for pp in pps:
if pp[0] not in ('TamilNadu PetroProducts',
'Consumption Report for the', 'Slno', None, ''):
if pp[2] is not None:
if not frappe.db.exists("Food Item", pp[2]):
err_list += '<ul>Item - <b>%s</b> Not Fount</ul><br>' % pp[
2]
if err_list:
frappe.throw(err_list)
else:
for pp in pps:
if pp[0] not in ('TamilNadu PetroProducts', 'Slno', '', None):
if pp[0] == "Consumption Report for the":
if pp[1]:
try:
date = datetime.strptime(pp[1], '%d/%m/%Y')
except:
date = datetime.strptime(pp[1], '%d-%m-%Y')
if pp[2]:
if frappe.db.exists("Food Item", pp[2]):
zfe = frappe.new_doc("Zecons Food Entry")
zfe.date = date
zfe.code = pp[1]
zfe.item = pp[2]
zfe.rate = pp[3]
zfe.qty = pp[4]
zfe.cntr_value = pp[6]
zfe.save(ignore_permissions=True)
return 'ok'
def prepare_to_notify(self, print_html=None, print_format=None, attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
if print_format:
self.content += self.get_attach_link(print_format)
self.set_incoming_outgoing_accounts()
if not self.sender:
self.sender = formataddr([frappe.session.data.full_name or "Notification", self.outgoing_email_account])
self.attachments = []
if print_html or print_format:
self.attachments.append(frappe.attach_print(self.reference_doctype, self.reference_name,
print_format=print_format, html=print_html))
if attachments:
if isinstance(attachments, basestring):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
self.attachments.append({"fname": file[0], "fcontent": file[1]})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
self.attachments.append(a)
def prepare_message(email, recipient, recipients_list):
message = email.message
if not message:
return ""
if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
email.unsubscribe_method, email.unsubscribe_params)
message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
if email.expose_recipients == "header":
pass
else:
if email.expose_recipients == "footer":
if isinstance(email.show_as_cc, string_types):
email.show_as_cc = email.show_as_cc.split(",")
email_sent_to = [r.recipient for r in recipients_list]
email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
if email_sent_cc:
email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
else:
email_sent_message = _("This email was sent to {0}").format(email_sent_to)
message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
message = message.replace("<!--recipient-->", recipient)
message = (message and message.encode('utf8')) or ''
if not email.attachments:
return message
# On-demand attachments
from email.parser import Parser
msg_obj = Parser().parsestr(message)
attachments = json.loads(email.attachments)
for attachment in attachments:
if attachment.get('fcontent'): continue
fid = attachment.get("fid")
if fid:
fname, fcontent = get_file(fid)
attachment.update({
'fname': fname,
'fcontent': fcontent,
'parent': msg_obj
})
attachment.pop("fid", None)
add_attachment(**attachment)
elif attachment.get("print_format_attachment") == 1:
attachment.pop("print_format_attachment", None)
print_format_file = frappe.attach_print(**attachment)
print_format_file.update({"parent": msg_obj})
add_attachment(**print_format_file)
return msg_obj.as_string()
def prepare_message(email, recipient, recipients_list):
message = email.message
if not message:
return ""
if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
email.unsubscribe_method, email.unsubscribe_params)
message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
if email.expose_recipients == "header":
pass
else:
if email.expose_recipients == "footer":
if isinstance(email.show_as_cc, string_types):
email.show_as_cc = email.show_as_cc.split(",")
email_sent_to = [r.recipient for r in recipients_list]
email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
if email_sent_cc:
email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
else:
email_sent_message = _("This email was sent to {0}").format(email_sent_to)
message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
message = message.replace("<!--recipient-->", recipient)
message = (message and message.encode('utf8')) or ''
if not email.attachments:
return message
# On-demand attachments
from email.parser import Parser
msg_obj = Parser().parsestr(message)
attachments = json.loads(email.attachments)
for attachment in attachments:
if attachment.get('fcontent'): continue
fid = attachment.get("fid")
if fid:
fname, fcontent = get_file(fid)
attachment.update({
'fname': fname,
'fcontent': fcontent,
'parent': msg_obj
})
attachment.pop("fid", None)
add_attachment(**attachment)
elif attachment.get("print_format_attachment") == 1:
attachment.pop("print_format_attachment", None)
print_format_file = frappe.attach_print(**attachment)
print_format_file.update({"parent": msg_obj})
add_attachment(**print_format_file)
return msg_obj.as_string()
def attach_file(self, n):
"""attach a file from the `FileData` table"""
from frappe.utils.file_manager import get_file
res = get_file(n)
if not res:
return
self.add_attachment(res[0], res[1])
def attach_file(self, n): """attach a file from the `FileData` table""" from frappe.utils.file_manager import get_file res = get_file(n) if not res: return self.add_attachment(res[0], res[1])
def reply_to_support_issue(client_issue_id,
subject,
description,
raised_by,
recipients,
frappe_issue_id,
attachments=None):
comm = frappe.get_doc({
"doctype": "Communication",
"subject": subject,
"content": description,
"recipients": recipients,
"sent_or_received": "Sent",
"reference_doctype": "ERPNext Support Issue",
"communication_medium": "Email",
"sender": raised_by,
"reference_name": client_issue_id,
"has_attachment": 1 if attachments else 0
}).insert(ignore_permissions=True)
if isinstance(attachments, string_types):
attachments = json.loads(attachments)
if attachments:
add_attachments("Communication", comm.name, attachments)
add_attachments("ERPNext Support Issue", client_issue_id, attachments)
frappe.db.commit()
file_attachments = []
if attachments:
for a in attachments:
filename, content = get_file(a)
file_attachments.append({
"filename":
filename,
"content":
base64.b64encode(content).decode("ascii")
})
file_attachments = json.dumps(file_attachments)
params = {
"subject": subject,
"recipients": recipients,
"description": description,
"raised_by": raised_by,
"frappe_issue_id": frappe_issue_id,
"attachments": file_attachments
}
last_sync_on = json.loads(
call_remote_method("create_reply_from_customer", params))
frappe.db.set_value("ERPNext Support Issue", client_issue_id,
"last_sync_on",
get_datetime(last_sync_on.get("last_sync_on")))
def prepare_to_notify(doc,
print_html=None,
print_format=None,
attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
view_link = frappe.utils.cint(
frappe.db.get_value("Print Settings", "Print Settings",
"attach_view_link"))
if print_format and view_link:
doc.content += get_attach_link(doc, print_format)
set_incoming_outgoing_accounts(doc)
if not doc.sender:
doc.sender = doc.outgoing_email_account.email_id
if not doc.sender_full_name:
doc.sender_full_name = doc.outgoing_email_account.name or _(
"Notification")
if doc.sender:
# combine for sending to get the format 'Jane <*****@*****.**>'
doc.sender = formataddr([doc.sender_full_name, doc.sender])
doc.attachments = []
if print_html or print_format:
doc.attachments.append({
"print_format_attachment": 1,
"doctype": doc.reference_doctype,
"name": doc.reference_name,
"print_format": print_format,
"html": print_html
})
if attachments:
if isinstance(attachments, string_types):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, string_types):
# is it a filename?
try:
# keep this for error handling
file = get_file(a)
# these attachments will be attached on-demand
# and won't be stored in the message
doc.attachments.append({"fid": a})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
doc.attachments.append(a)
def prepare_to_notify(doc,
print_html=None,
print_format=None,
attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
view_link = frappe.utils.cint(
frappe.db.get_value("Print Settings", "Print Settings",
"attach_view_link"))
if print_format and view_link:
doc.content += get_attach_link(doc, print_format)
set_incoming_outgoing_accounts(doc)
if not doc.sender:
doc.sender = doc.outgoing_email_account.email_id
if not doc.sender_full_name:
doc.sender_full_name = doc.outgoing_email_account.name or _(
"Notification")
if doc.sender:
# combine for sending to get the format 'Jane <*****@*****.**>'
doc.sender = formataddr([doc.sender_full_name, doc.sender])
doc.attachments = []
if print_html or print_format:
doc.attachments.append(
frappe.attach_print(doc.reference_doctype,
doc.reference_name,
print_format=print_format,
html=print_html))
if attachments:
if isinstance(attachments, basestring):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
doc.attachments.append({
"fname": file[0],
"fcontent": file[1]
})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
doc.attachments.append(a)
def update_from_csv(filename):
from frappe.utils.csvutils import read_csv_content
from frappe.utils.file_manager import get_file
_file = frappe.get_doc("File", {"file_name": filename})
filepath = get_file(filename)
pps = read_csv_content(filepath[1])
# print len(pps)
for pp in pps:
# pp1 = datetime.strptime(pp[1], '%d-%m-%Y').strftime('%Y-%m-%d')pp[0]
ss = frappe.get_value("Salary Slip",{"employee":pp[0],"posting_date":'2019-05-02'})
print ss
def mark_attendance(file_url, start_date, end_date):
#below is the method to get file from Frappe File manager
from frappe.utils.file_manager import get_file
#Method to fetch file using get_doc and stored as _file
_file = frappe.get_doc("File", {"file_url": file_url})
#Path in the system
filepath = get_file(file_url)
#CSV Content stored as pps
pps = read_csv_content(filepath[1])
for pp in pps:
frappe.errprint(pp)
holiday_list = frappe.db.get_value("Employee", {"name": pp[0]},
["holiday_list"])
holiday_map = frappe._dict()
# for d in holiday_list:
# if d:
holiday_list = frappe.db.sql(
'''select holiday_date from `tabHoliday`
where parent=%s and holiday_date between %s and %s''',
(holiday_list, start_date, end_date))
holidays = []
for holiday in holiday_list:
holidays.append(holiday[0])
import pandas as pd
total_days = pd.date_range(start_date, end_date).tolist()
day = 1
for days in total_days:
date = days.date()
if date not in holidays:
frappe.errprint(date)
if int(pp[1]) >= day:
attendance = frappe.new_doc("Attendance")
attendance.update({
"employee": pp[0],
"attendance_date": date,
"status": "Absent"
}).save(ignore_permissions=True)
attendance.submit()
frappe.db.commit()
day = day + 1
if float(pp[2]) >= 1:
ts = frappe.new_doc("Timesheet")
ts.employee = pp[0]
ts.append(
"time_logs", {
"activity_type": "Overtime",
"from_time": start_date,
"hours": float(pp[2])
})
ts.save(ignore_permissions=True)
ts.submit()
frappe.db.commit()
def bulk_update_from_csv(filename):
#below is the method to get file from Frappe File manager
from frappe.utils.file_manager import get_file
#Method to fetch file using get_doc and stored as _file
_file = frappe.get_doc("File", {"file_name": filename})
#Path in the system
filepath = get_file(filename)
#CSV Content stored as pps
pps = read_csv_content(filepath[1])
count = 0
def get_prepared_report_result(report, filters, dn="", user=None):
latest_report_data = {}
doc = None
if dn:
# Get specified dn
doc = frappe.get_doc("Prepared Report", dn)
else:
# Only look for completed prepared reports with given filters.
doc_list = frappe.get_all("Prepared Report",
filters={
"status": "Completed",
"filters": json.dumps(filters),
"owner": user,
"report_name": report.report_name
}
)
if doc_list:
# Get latest
doc = frappe.get_doc("Prepared Report", doc_list[0])
if doc:
try:
# Prepared Report data is stored in a GZip compressed JSON file
attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name")
compressed_content = get_file(attached_file_name)[1]
uncompressed_content = gzip_decompress(compressed_content)
data = json.loads(uncompressed_content)
if data:
columns = json.loads(doc.columns) if doc.columns else data[0]
for column in columns:
if isinstance(column, dict):
column["label"] = _(column["label"])
latest_report_data = {
"columns": columns,
"result": data
}
except Exception:
frappe.log_error(frappe.get_traceback())
frappe.delete_doc("Prepared Report", doc.name)
frappe.db.commit()
doc = None
latest_report_data.update({
"prepared_report": True,
"doc": doc
})
return latest_report_data
def read_csv_content_from_attached_file(doc):
fileid = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype,
"attached_to_name":doc.name}, "name")
if not fileid:
msgprint(_("File not attached"))
raise Exception
try:
from frappe.utils.file_manager import get_file
fname, fcontent = get_file(fileid)
return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors'))
except Exception:
frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format'))
def export_package():
"""Export package as JSON."""
package_doc = frappe.get_single("Package Publish Tool")
package = []
for doctype in package_doc.package_details:
filters = []
if doctype.get("filters_json"):
filters = json.loads(doctype.get("filters_json"))
docs = frappe.get_all(doctype.get("document_type"), filters=filters)
length = len(docs)
for idx, doc in enumerate(docs):
frappe.publish_realtime("package", {
"progress": idx,
"total": length,
"message": doctype.get("document_type"),
"prefix": _("Exporting")
},
user=frappe.session.user)
document = frappe.get_doc(doctype.get("document_type"),
doc.name).as_dict()
attachments = []
if doctype.attachments:
filters = {
"attached_to_doctype": document.get("doctype"),
"attached_to_name": document.get("name")
}
for f in frappe.get_list("File", filters=filters):
fname, fcontents = get_file(f.name)
attachments.append({
"fname":
fname,
"content":
base64.b64encode(fcontents).decode('ascii')
})
document.update({
"__attachments": attachments,
"__overwrite": True if doctype.overwrite else False
})
package.append(document)
return post_process(package), package_doc
def read_csv_content_from_attached_file(doc):
fileid = frappe.db.get_value("File Data", {"attached_to_doctype": doc.doctype,
"attached_to_name":doc.name}, "name")
if not fileid:
msgprint(_("File not attached"))
raise Exception
try:
from frappe.utils.file_manager import get_file
fname, fcontent = get_file(fileid)
return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors'))
except Exception:
frappe.msgprint(_("Unable to open attached file. Please try again."))
raise Exception
def read_csv_content_from_attached_file(doc):
fileid = frappe.db.get_value("File Data", {"attached_to_doctype": doc.doctype,
"attached_to_name":doc.name}, "name")
if not fileid:
msgprint("File not attached!")
raise Exception
try:
from frappe.utils.file_manager import get_file
fname, fcontent = get_file(fileid)
return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors'))
except Exception, e:
frappe.msgprint("""Unable to open attached file. Please try again.""")
raise Exception
def sync_erpnext_support_issue(doc=None, method=None):
"""
Sync Issue which arent synced with Server due to FrappeClient request failure
"""
filters = [["status", "=", "Open"], ["frappe_issue_id", "=", ""],
["sync_count", "<", 5]]
for unsynced_issue in frappe.get_list("ERPNext Support Issue",
filters=filters):
erpnext_support_issue = frappe.get_doc("ERPNext Support Issue",
unsynced_issue.name)
comm = frappe.get_doc(
"Communication", {
"reference_doctype": "ERPNext Support Issue",
"reference_name": erpnext_support_issue.name
})
file_attachments = []
if comm.has_attachment:
files = frappe.get_list("File",
filters={
"attached_to_doctype": "Communication",
"attached_to_name": comm.get("name")
})
for d in files:
filename, content = get_file(d.name)
file_attachments.append({
"filename":
filename,
"content":
base64.b64encode(content).decode("ascii")
})
file_attachments = json.dumps(file_attachments)
params = get_params(erpnext_support_issue, file_attachments)
frappe_issue = call_remote_method("create_issue_from_customer", params)
# If FrappeClient request fails, increment sync count
if frappe_issue.get("failed"):
erpnext_support_issue.db_set("sync_count",
erpnext_support_issue.sync_count + 1)
continue
set_corresponding_frappe_values(erpnext_support_issue, frappe_issue)
def execute():
frappe.reload_doc('core', 'doctype', 'file_data')
for name, file_name, file_url in frappe.db.sql(
"""select name, file_name, file_url from `tabFile Data`
where file_name is not null"""):
b = frappe.get_doc('File Data', name)
old_file_name = b.file_name
b.file_name = os.path.basename(old_file_name)
if old_file_name.startswith('files/') or old_file_name.startswith('/files/'):
b.file_url = os.path.normpath('/' + old_file_name)
else:
b.file_url = os.path.normpath('/files/' + old_file_name)
_file_name, content = get_file(name)
b.content_hash = get_content_hash(content)
b.save()
def bulk_update_from_csv(filename):
#below is the method to get file from Frappe File manager
from frappe.utils.file_manager import get_file
#Method to fetch file using get_doc and stored as _file
_file = frappe.get_doc("File", {"file_name": filename})
#Path in the system
filepath = get_file(filename)
#CSV Content stored as pps
pps = read_csv_content(filepath[1])
count = 0
for pp in pps:
cl = frappe.get_doc("Candidate", {'name': pp[0]})
cl.pending_for = 'IDB'
cl.db_update()
frappe.db.commit()
def get_fints_import_file_content(fints_import):
"""Get FinTS Import json file content as json.
:param fints_import: fints_import doc
:type fints_import: fints_import doc
:return: Transaction from file as json object list
"""
if fints_import.file_url:
content = get_file(fints_import.file_url)[1]
# Check content hash for file manipulations
if fints_import.file_hash == get_content_hash(content):
return frappe.json.loads(content, strict=False)
else:
raise ValueError('File hash does not match')
else:
return frappe.json.loads('[]')
def execute():
frappe.reload_doc('core', 'doctype', 'file_data')
for name, file_name, file_url in frappe.db.sql(
"""select name, file_name, file_url from `tabFile Data`
where file_name is not null"""):
b = frappe.get_doc('File Data', name)
old_file_name = b.file_name
b.file_name = os.path.basename(old_file_name)
if old_file_name.startswith('files/') or old_file_name.startswith(
'/files/'):
b.file_url = os.path.normpath('/' + old_file_name)
else:
b.file_url = os.path.normpath('/files/' + old_file_name)
_file_name, content = get_file(name)
b.content_hash = get_content_hash(content)
b.save()
def read_csv_content_from_attached_file(doc):
fileid = frappe.get_all("File", fields = ["name"], filters = {"attached_to_doctype": doc.doctype,
"attached_to_name":doc.name}, order_by="creation desc")
if fileid : fileid = fileid[0].name
if not fileid:
msgprint(_("File not attached"))
raise Exception
try:
from frappe.utils.file_manager import get_file
fname, fcontent = get_file(fileid)
return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors'))
except Exception:
frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format'))
def sync_communication_server(erpnext_support_user, erpnext_support_issues, bench_site):
"""
returns a dict of support issues and associated communications in the format
{
"erpnext_support_issue_name_1": [{Communications}]
"erpnext_support_issue_name_2": [{Communications}]
}
"""
authenticate_erpnext_support_user(erpnext_support_user)
communications = {}
erpnext_support_issues = json.loads(erpnext_support_issues)
for erpnext_support_issue in erpnext_support_issues:
if erpnext_support_issue.get('associated_issue'):
filters = {
'reference_doctype': 'Issue',
'reference_name': erpnext_support_issue.get('associated_issue'),
'communication_medium': 'Email',
'sent_or_received': 'Sent',
'seen': 0
}
fields = ["name", "subject", "content", "recipients", "has_attachment"]
comms = frappe.get_all("Communication", filters=filters, fields=fields, order_by="creation ASC")
for comm in comms:
frappe.db.set_value("Communication", comm.name, "seen", 1)
comm['content'] = re.sub('src="', 'src="' + frappe.utils.get_url(), comm.get('content'))
file_attachments = []
if comm.get('has_attachment'):
files = frappe.get_list("File", filters={"attached_to_doctype": "Communication", "attached_to_name": comm.get('name')})
for d in files:
filename, content = get_file(d.name)
file_attachments.append({
"filename": filename,
"content": base64.b64encode(content).decode('ascii')
})
file_attachments = json.dumps(file_attachments)
comm.update({"attachments": file_attachments})
communications[erpnext_support_issue.get('name')] = comms
communications = json.dumps(communications)
return communications
def prepare_to_notify(doc,
print_html=None,
print_format=None,
attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
if print_format:
doc.content += get_attach_link(doc, print_format)
set_incoming_outgoing_accounts(doc)
if not doc.sender or cint(
doc.outgoing_email_account.always_use_account_email_id_as_sender):
sender_name = (frappe.session.data.full_name
or doc.outgoing_email_account.name or _("Notification"))
sender_email_id = doc.outgoing_email_account.email_id
doc.sender = formataddr([sender_name, sender_email_id])
doc.attachments = []
if print_html or print_format:
doc.attachments.append(
frappe.attach_print(doc.reference_doctype,
doc.reference_name,
print_format=print_format,
html=print_html))
if attachments:
if isinstance(attachments, basestring):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
doc.attachments.append({
"fname": file[0],
"fcontent": file[1]
})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
doc.attachments.append(a)
def queue_all(self):
if not self.get("recipients"):
# in case it is called via worker
self.recipients = self.get_recipients()
self.validate_send()
sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
if not frappe.flags.in_test:
frappe.db.auto_commit_on_many_writes = True
attachments = []
if self.send_attachements:
files = frappe.get_all("File",
fields=["name"],
filters={
"attached_to_doctype": "Newsletter",
"attached_to_name": self.name
},
order_by="creation desc")
for file in files:
try:
file = get_file(file.name)
attachments.append({"fname": file[0], "fcontent": file[1]})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
send(
recipients=self.recipients,
sender=sender,
subject=self.subject,
message=self.message,
reference_doctype=self.doctype,
reference_name=self.name,
add_unsubscribe_link=self.send_unsubscribe_link,
attachments=attachments,
unsubscribe_method=
"/api/method/frappe.email.doctype.newsletter.newsletter.unsubscribe",
unsubscribe_params={"name": self.name},
send_priority=0,
queue_separately=True)
if not frappe.flags.in_test:
frappe.db.auto_commit_on_many_writes = False
def prepare_attachments(g_attachments=None):
attachments = []
if g_attachments:
if isinstance(g_attachments, basestring):
import json
g_attachments = json.loads(g_attachments)
for a in g_attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
attachments.append({"fname": file[0], "fcontent": file[1]})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
attachments.append(a)
return attachments
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
view_link = frappe.utils.cint(frappe.db.get_value("Print Settings", "Print Settings", "attach_view_link"))
if print_format and view_link:
doc.content += get_attach_link(doc, print_format)
set_incoming_outgoing_accounts(doc)
if not doc.sender:
doc.sender = doc.outgoing_email_account.email_id
if not doc.sender_full_name:
doc.sender_full_name = doc.outgoing_email_account.name or _("Notification")
if doc.sender:
# combine for sending to get the format 'Jane <*****@*****.**>'
doc.sender = formataddr([doc.sender_full_name, doc.sender])
doc.attachments = []
if print_html or print_format:
doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype,
"name":doc.reference_name, "print_format":print_format, "html":print_html})
if attachments:
if isinstance(attachments, string_types):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, string_types):
# is it a filename?
try:
# keep this for error handling
file = get_file(a)
# these attachments will be attached on-demand
# and won't be stored in the message
doc.attachments.append({"fid": a})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
doc.attachments.append(a)
def bulk_update_from_csv(filename):
#below is the method to get file from Frappe File manager
from frappe.utils.file_manager import get_file
#Method to fetch file using get_doc and stored as _file
_file = frappe.get_doc("File", {"file_name": filename})
#Path in the system
filepath = get_file(filename)
#CSV Content stored as pps
pps = read_csv_content(filepath[1])
count = 0
for pp in pps[:5]:
mobile = pp[0]
project = pp[1]
position = pp[2]
given_name = pp[3]
customer = pp[4]
print(mobile, project, position, given_name, customer)
def bulk_update_from_csv(filename):
#below is the method to get file from Frappe File manager
from frappe.utils.file_manager import get_file
#Method to fetch file using get_doc and stored as _file
_file = frappe.get_doc("File", {"file_name": filename})
#Path in the system
filepath = get_file(filename)
#CSV Content stored as pps
payloads = read_csv_content(filepath[1])
for payload in payloads:
if payload:
food_order = frappe.new_doc("Food Order No")
food_order.employee = payload[0]
food_order.date = payload[1]
food_order.save(ignore_permissions=True)
frappe.db.commit()
for i in range(cint(payload[2])):
limit = frappe.db.get_value('Food Item',payload[3],"limit")
if frappe.db.count("Daily Food Entry",{"employee":payload[0],"item":payload[3],"date":payload[1]}) > limit:
dfe = frappe.new_doc("Daily Food Entry")
dfe.employee = payload[0]
dfe.item = payload[3]
dfe.date = payload[1]
# dfe.time = payload[4]
dfe.plant = payload[4]
dfe.food_order_no = food_order.name
dfe.price = frappe.db.get_value("Food Item",payload[3],["original_rate"])
dfe.save(ignore_permissions=True)
frappe.db.commit()
else:
dfe = frappe.new_doc("Daily Food Entry")
dfe.employee = payload[0]
dfe.item = payload[3]
dfe.date = payload[1]
# dfe.time = payload[4]
dfe.plant = payload[4]
dfe.food_order_no = food_order.name
dfe.price = frappe.db.get_value("Food Item",payload[3],["subsidy_rate"])
dfe.subsidy = 1
dfe.save(ignore_permissions=True)
frappe.db.commit()
return 'OK'
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
view_link = frappe.utils.cint(frappe.db.get_value("Print Settings", "Print Settings", "attach_view_link"))
if print_format and view_link:
doc.content += get_attach_link(doc, print_format)
set_incoming_outgoing_accounts(doc)
if not doc.sender:
doc.sender = doc.outgoing_email_account.email_id
if not doc.sender_full_name:
doc.sender_full_name = doc.outgoing_email_account.name or _("Notification")
if doc.sender:
# combine for sending to get the format 'Jane <*****@*****.**>'
doc.sender = formataddr([doc.sender_full_name, doc.sender])
doc.attachments = []
if print_html or print_format:
doc.attachments.append(frappe.attach_print(doc.reference_doctype, doc.reference_name,
print_format=print_format, html=print_html))
if attachments:
if isinstance(attachments, basestring):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
doc.attachments.append({"fname": file[0], "fcontent": file[1]})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
doc.attachments.append(a)
def prepare_attachments(g_attachments=None):
attachments = []
if g_attachments:
if isinstance(g_attachments, basestring):
import json
g_attachments = json.loads(g_attachments)
for a in g_attachments:
if isinstance(a, basestring):
# is it a filename?
try:
file = get_file(a)
attachments.append({"fname": file[0], "fcontent": file[1]})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
attachments.append(a)
return attachments
def bulk_update_from_csv(filename):
#below is the method to get file from Frappe File manager
from frappe.utils.file_manager import get_file
#Method to fetch file using get_doc and stored as _file
_file = frappe.get_doc("File", {"file_name": filename})
#Path in the system
filepath = get_file(filename)
#CSV Content stored as pps
pps = read_csv_content(filepath[1])
count = 0
for pp in pps:
ld = frappe.db.exists("Lead", {'name': pp[0]})
if ld:
# items = frappe.get_all("Lead",{'name':pp[0]})
# for item in items:
i = frappe.get_doc('Lead', pp[0])
if not i.contac % t_by:
i.contact_date = pp[1]
print(pp[1])
def execute():
frappe.db.auto_commit_on_many_writes = True
rename_replacing_files()
for name, file_name, file_url in frappe.db.sql(
"""select name, file_name, file_url from `tabFile`
where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""):
b = frappe.get_doc('File', name)
old_file_name = b.file_name
b.file_name = os.path.basename(old_file_name)
if old_file_name.startswith('files/') or old_file_name.startswith('/files/'):
b.file_url = os.path.normpath('/' + old_file_name)
else:
b.file_url = os.path.normpath('/files/' + old_file_name)
try:
_file_name, content = get_file(name)
b.content_hash = get_content_hash(content)
except IOError:
print 'Warning: Error processing ', name
b.content_hash = None
b.flags.ignore_duplicate_entry_error = True
b.save()
frappe.db.auto_commit_on_many_writes = False
def upload_from_json(filename):
# payloads = [{"employee": "300752", "date": "2021-9-11", "qty": "1", "item": "LUNCH MEALS"}]
from frappe.utils.file_manager import get_file
_file = frappe.get_doc("File", {"file_name": filename})
filepath = get_file(filename)
# payloads = read_csv_content(filepath[1])
payloads = json.loads(filepath[1])
for payload in payloads:
payload = payload[0]
if payload:
food_order = frappe.new_doc("Food Order No")
food_order.employee = payload["employee"]
food_order.date = payload["date"]
food_order.save(ignore_permissions=True)
frappe.db.commit()
for i in range(cint(payload['qty'])):
print(payload['item'])
limit = frappe.db.get_value('Food Item',payload['item'],"limit")
if frappe.db.count("Daily Food Entry",{"employee":payload["employee"],"item":payload["item"],"date":payload["date"]}) > limit:
dfe = frappe.new_doc("Daily Food Entry")
dfe.employee = payload["employee"]
dfe.item = payload["item"]
dfe.date = payload["date"]
dfe.time = payload["time"]
dfe.food_order_no = food_order.name
dfe.price = frappe.db.get_value("Food Item",payload["item"],["original_rate"])
dfe.save(ignore_permissions=True)
frappe.db.commit()
else:
dfe = frappe.new_doc("Daily Food Entry")
dfe.employee = payload['employee']
dfe.item = payload['item']
dfe.date = payload["date"]
dfe.time = payload["time"]
dfe.food_order_no = food_order.name
dfe.price = frappe.db.get_value("Food Item",payload["item"],["subsidy_rate"])
dfe.subsidy = 1
dfe.save(ignore_permissions=True)
frappe.db.commit()
def execute():
frappe.db.auto_commit_on_many_writes = True
rename_replacing_files()
for name, file_name, file_url in frappe.db.sql(
"""select name, file_name, file_url from `tabFile`
where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""):
b = frappe.get_doc('File', name)
old_file_name = b.file_name
b.file_name = os.path.basename(old_file_name)
if old_file_name.startswith('files/') or old_file_name.startswith('/files/'):
b.file_url = os.path.normpath('/' + old_file_name)
else:
b.file_url = os.path.normpath('/files/' + old_file_name)
try:
_file_name, content = get_file(name)
b.content_hash = get_content_hash(content)
except IOError:
print('Warning: Error processing ', name)
b.content_hash = None
b.flags.ignore_duplicate_entry_error = True
b.save()
frappe.db.auto_commit_on_many_writes = False
def execute():
frappe.reload_doc('core', 'doctype', 'file_data')
for name, file_name, file_url in frappe.db.sql(
"""select name, file_name, file_url from `tabFile`
where file_name is not null"""):
b = frappe.get_doc('File', name)
old_file_name = b.file_name
b.file_name = os.path.basename(old_file_name)
if old_file_name.startswith('files/') or old_file_name.startswith('/files/'):
b.file_url = os.path.normpath('/' + old_file_name)
else:
b.file_url = os.path.normpath('/files/' + old_file_name)
try:
_file_name, content = get_file(name)
b.content_hash = get_content_hash(content)
except IOError:
print('Warning: Error processing ', name)
_file_name = old_file_name
b.content_hash = None
try:
b.save()
except frappe.DuplicateEntryError:
frappe.delete_doc(b.doctype, b.name)
def prepare_message(email, recipient, recipients_list):
message = email.message
if not message:
return ""
# Parse "Email Account" from "Email Sender"
email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender)
if frappe.conf.use_ssl and email_account.track_email_status:
# Using SSL => Publically available domain => Email Read Reciept Possible
message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode())
else:
# No SSL => No Email Read Reciept
message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode())
if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
email.unsubscribe_method, email.unsubscribe_params)
message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
if email.expose_recipients == "header":
pass
else:
if email.expose_recipients == "footer":
if isinstance(email.show_as_cc, string_types):
email.show_as_cc = email.show_as_cc.split(",")
email_sent_to = [r.recipient for r in recipients_list]
email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
if email_sent_cc:
email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
else:
email_sent_message = _("This email was sent to {0}").format(email_sent_to)
message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
message = message.replace("<!--recipient-->", recipient)
message = (message and message.encode('utf8')) or ''
message = safe_decode(message)
if not email.attachments:
return message
# On-demand attachments
from email.parser import Parser
msg_obj = Parser().parsestr(message)
attachments = json.loads(email.attachments)
for attachment in attachments:
if attachment.get('fcontent'): continue
fid = attachment.get("fid")
if fid:
fname, fcontent = get_file(fid)
attachment.update({
'fname': fname,
'fcontent': fcontent,
'parent': msg_obj
})
attachment.pop("fid", None)
add_attachment(**attachment)
elif attachment.get("print_format_attachment") == 1:
attachment.pop("print_format_attachment", None)
print_format_file = frappe.attach_print(**attachment)
print_format_file.update({"parent": msg_obj})
add_attachment(**print_format_file)
return msg_obj.as_string()
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
skip_errors = True):
"""upload data"""
frappe.flags.in_import = True
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
frappe.flags.mute_emails = no_email
def get_data_keys_definition():
return get_data_keys()
def bad_template():
frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator))
def check_data_length():
max_rows = 5000
if not data:
frappe.throw(_("No data found"))
elif not via_console and len(data) > max_rows:
frappe.throw(_("Only allowed {0} rows in one import").format(max_rows))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==get_data_keys_definition().data_separator:
return i+1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = list(filter(lambda x: x in ("", None), columns))
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d and doctype_row[i] in (None, '' ,'~', '-', 'DocType:'):
dt, parentfield = d, None
# xls format truncates the row, so it may not have more columns
if len(doctype_row) > i+2:
parentfield = doctype_row[i+2]
doctypes.append((dt, parentfield))
column_idx_to_fieldname[(dt, parentfield)] = {}
column_idx_to_fieldtype[(dt, parentfield)] = {}
if dt:
column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1]
column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1]
def get_doc(start_idx):
if doctypes:
doc = {}
for idx in range(start_idx, len(rows)):
if (not doc) or main_doc_empty(rows[idx]):
for dt, parentfield in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[(dt, parentfield)]:
try:
fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency", "Percent"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
if d[fieldname] and isinstance(d[fieldname], string_types):
d[fieldname] = getdate(parse_date(d[fieldname]))
elif fieldtype == "Datetime":
if d[fieldname]:
if " " in d[fieldname]:
_date, _time = d[fieldname].split()
else:
_date, _time = d[fieldname], '00:00:00'
_date = parse_date(d[fieldname])
d[fieldname] = get_datetime(_date + " " + _time)
else:
d[fieldname] = None
elif fieldtype in ("Image", "Attach Image", "Attach"):
# added file to attachments list
attachments.append(d[fieldname])
elif fieldtype in ("Link", "Dynamic Link") and d[fieldname]:
# as fields can be saved in the number format(long type) in data import template
d[fieldname] = cstr(d[fieldname])
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite:
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = parentfield
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc
else:
doc = frappe._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc
def main_doc_empty(row):
return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2])))
users = frappe.db.sql_list("select name from tabUser")
def prepare_for_insert(doc):
# don't block data import if user is not set
# migrating from another system
if not doc.owner in users:
doc.owner = frappe.session.user
if not doc.modified_by in users:
doc.modified_by = frappe.session.user
def is_valid_url(url):
is_valid = False
if url.startswith("/files") or url.startswith("/private/files"):
url = get_url(url)
try:
r = requests.get(url)
is_valid = True if r.status_code == 200 else False
except Exception:
pass
return is_valid
def attach_file_to_doc(doctype, docname, file_url):
# check if attachment is already available
# check if the attachement link is relative or not
if not file_url:
return
if not is_valid_url(file_url):
return
files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and
attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format(
doctype=doctype,
docname=docname,
file_url=file_url
))
if files:
# file is already attached
return
save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
# header
if not rows:
from frappe.utils.file_manager import get_file_doc
file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1)
filename, file_extension = os.path.splitext(file_doc.file_name)
if file_extension == '.xlsx' and from_data_import == 'Yes':
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
rows = read_xlsx_file_from_attached_file(file_id=file_doc.name)
elif file_extension == '.csv':
from frappe.utils.file_manager import get_file
from frappe.utils.csvutils import read_csv_content
fname, fcontent = get_file(file_doc.name)
rows = read_csv_content(fcontent, ignore_encoding_errors)
else:
frappe.throw(_("Unsupported File Format"))
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
doctype = get_header_row(get_data_keys_definition().main_table)[1]
columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:])
doctypes = []
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
attachments = []
if submit_after_import and not cint(frappe.db.get_value("DocType",
doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(get_data_keys_definition().parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not frappe.permissions.can_import(parenttype or doctype):
frappe.flags.mute_emails = False
return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
# allow limit rows to be uploaded
check_data_length()
make_column_map()
if overwrite==None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
parentfield = None
if parenttype:
parentfield = get_parent_field(doctype, parenttype)
if overwrite:
delete_child_rows(data, doctype)
ret = []
def log(msg):
if via_console:
print(msg.encode('utf-8'))
else:
ret.append(msg)
def as_link(doctype, name):
if via_console:
return "{0}: {1}".format(doctype, name)
else:
return getlink(doctype, name)
error = False
total = len(data)
for i, row in enumerate(data):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
# publish task_update
frappe.publish_realtime("data_import_progress", {"progress": [i, total]},
user=frappe.session.user)
try:
doc = get_doc(row_idx)
if pre_process:
pre_process(doc)
if parentfield:
parent = frappe.get_doc(parenttype, doc["parent"])
doc = parent.append(parentfield, doc)
parent.save()
log('Inserted row for %s at #%s' % (as_link(parenttype,
doc.parent),text_type(doc.idx)))
else:
if overwrite and doc["name"] and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
original_name = original.name
original.update(doc)
# preserve original name for case sensitivity
original.name = original_name
original.flags.ignore_links = ignore_links
original.save()
log('Updated row (#%d) %s' % (row_idx + 1, as_link(original.doctype, original.name)))
doc = original
else:
if not update_only:
doc = frappe.get_doc(doc)
prepare_for_insert(doc)
doc.flags.ignore_links = ignore_links
doc.insert()
log('Inserted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
else:
log('Ignored row (#%d) %s' % (row_idx + 1, row[1]))
if attachments:
# check file url and create a File document
for file_url in attachments:
attach_file_to_doc(doc.doctype, doc.name, file_url)
if submit_after_import:
doc.submit()
log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
except Exception as e:
if not skip_errors:
error = True
if doc:
frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
log('Error for row (#%d) %s : %s' % (row_idx + 1,
len(row)>1 and row[1] or "", err_msg))
frappe.errprint(frappe.get_traceback())
finally:
frappe.local.message_log = []
if error:
frappe.db.rollback()
else:
frappe.db.commit()
frappe.flags.mute_emails = False
frappe.flags.in_import = False
return {"messages": ret, "error": error}
def download_attachment(dn):
attachment = get_attachments("Prepared Report", dn)[0]
frappe.local.response.filename = attachment.file_name[:-2]
frappe.local.response.filecontent = gzip_decompress(get_file(attachment.name)[1])
frappe.local.response.type = "binary"
def test_save(self): filename, content = get_file(self.saved_file.name) self.assertEqual(content, self.test_content)
def test_saved_content(self): filename1, content1 = get_file(self.saved_file1.name) self.assertEqual(content1, self.test_content1) filename2, content2 = get_file(self.saved_file2.name) self.assertEqual(content2, self.test_content2)
def test_saved_content(self): filename1, content1 = get_file(self.saved_file1.name) filename2, content2 = get_file(self.saved_file2.name) self.assertEqual(filename1, filename2) self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
skip_errors = True, data_import_doc=None, validate_template=False, user=None):
"""upload data"""
# for translations
if user:
frappe.cache().hdel("lang", user)
frappe.set_user_lang(user)
if data_import_doc and isinstance(data_import_doc, string_types):
data_import_doc = frappe.get_doc("Data Import", data_import_doc)
if data_import_doc and from_data_import == "Yes":
no_email = data_import_doc.no_email
ignore_encoding_errors = data_import_doc.ignore_encoding_errors
update_only = data_import_doc.only_update
submit_after_import = data_import_doc.submit_after_import
overwrite = data_import_doc.overwrite
skip_errors = data_import_doc.skip_errors
else:
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
frappe.flags.in_import = True
frappe.flags.mute_emails = no_email
def get_data_keys_definition():
return get_data_keys()
def bad_template():
frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator))
def check_data_length():
if not data:
frappe.throw(_("No data found in the file. Please reattach the new file with data."))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==get_data_keys_definition().data_separator:
return i+1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = list(filter(lambda x: x in ("", None), columns))
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"):
dt, parentfield = d, None
# xls format truncates the row, so it may not have more columns
if len(doctype_row) > i+2:
parentfield = doctype_row[i+2]
doctypes.append((dt, parentfield))
column_idx_to_fieldname[(dt, parentfield)] = {}
column_idx_to_fieldtype[(dt, parentfield)] = {}
if dt:
column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1]
column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1]
def get_doc(start_idx):
if doctypes:
doc = {}
attachments = []
last_error_row_idx = None
for idx in range(start_idx, len(rows)):
last_error_row_idx = idx # pylint: disable=W0612
if (not doc) or main_doc_empty(rows[idx]):
for dt, parentfield in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[(dt, parentfield)]:
try:
fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
if not fieldname or not rows[idx][column_idx]:
continue
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency", "Percent"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
if d[fieldname] and isinstance(d[fieldname], string_types):
d[fieldname] = getdate(parse_date(d[fieldname]))
elif fieldtype == "Datetime":
if d[fieldname]:
if " " in d[fieldname]:
_date, _time = d[fieldname].split()
else:
_date, _time = d[fieldname], '00:00:00'
_date = parse_date(d[fieldname])
d[fieldname] = get_datetime(_date + " " + _time)
else:
d[fieldname] = None
elif fieldtype in ("Image", "Attach Image", "Attach"):
# added file to attachments list
attachments.append(d[fieldname])
elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]:
# as fields can be saved in the number format(long type) in data import template
d[fieldname] = cstr(d[fieldname])
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite and doc.get("name"):
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = parentfield
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc, attachments, last_error_row_idx
else:
doc = frappe._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc, [], None
# used in testing whether a row is empty or parent row or child row
# checked only 3 first columns since first two columns can be blank for example the case of
# importing the item variant where item code and item name will be blank.
def main_doc_empty(row):
if row:
for i in range(3,0,-1):
if len(row) > i and row[i]:
return False
return True
def validate_naming(doc):
autoname = frappe.get_meta(doctype).autoname
if autoname:
if autoname[0:5] == 'field':
autoname = autoname[6:]
elif autoname == 'naming_series:':
autoname = 'naming_series'
else:
return True
if (autoname not in doc) or (not doc[autoname]):
from frappe.model.base_document import get_controller
if not hasattr(get_controller(doctype), "autoname"):
frappe.throw(_("{0} is a mandatory field".format(autoname)))
return True
users = frappe.db.sql_list("select name from tabUser")
def prepare_for_insert(doc):
# don't block data import if user is not set
# migrating from another system
if not doc.owner in users:
doc.owner = frappe.session.user
if not doc.modified_by in users:
doc.modified_by = frappe.session.user
def is_valid_url(url):
is_valid = False
if url.startswith("/files") or url.startswith("/private/files"):
url = get_url(url)
try:
r = requests.get(url)
is_valid = True if r.status_code == 200 else False
except Exception:
pass
return is_valid
def attach_file_to_doc(doctype, docname, file_url):
# check if attachment is already available
# check if the attachement link is relative or not
if not file_url:
return
if not is_valid_url(file_url):
return
files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and
attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format(
doctype=doctype,
docname=docname,
file_url=file_url
))
if files:
# file is already attached
return
save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
# header
filename, file_extension = ['','']
if not rows:
from frappe.utils.file_manager import get_file # get_file_doc
fname, fcontent = get_file(data_import_doc.import_file)
filename, file_extension = os.path.splitext(fname)
if file_extension == '.xlsx' and from_data_import == 'Yes':
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file)
elif file_extension == '.csv':
from frappe.utils.csvutils import read_csv_content
rows = read_csv_content(fcontent, ignore_encoding_errors)
else:
frappe.throw(_("Unsupported File Format"))
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
try:
doctype = get_header_row(get_data_keys_definition().main_table)[1]
columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:])
except:
frappe.throw(_("Cannot change header content"))
doctypes = []
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if skip_errors:
data_rows_with_error = header
if submit_after_import and not cint(frappe.db.get_value("DocType",
doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(get_data_keys_definition().parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not frappe.permissions.can_import(parenttype or doctype):
frappe.flags.mute_emails = False
return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
# Throw expception in case of the empty data file
check_data_length()
make_column_map()
total = len(data)
if validate_template:
if total:
data_import_doc.total_rows = total
return True
if overwrite==None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
parentfield = None
if parenttype:
parentfield = get_parent_field(doctype, parenttype)
if overwrite:
delete_child_rows(data, doctype)
import_log = []
def log(**kwargs):
if via_console:
print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
else:
import_log.append(kwargs)
def as_link(doctype, name):
if via_console:
return "{0}: {1}".format(doctype, name)
else:
return getlink(doctype, name)
# publish realtime task update
def publish_progress(achieved, reload=False):
if data_import_doc:
frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
error_flag = rollback_flag = False
batch_size = frappe.conf.data_import_batch_size or 1000
for batch_start in range(0, total, batch_size):
batch = data[batch_start:batch_start + batch_size]
for i, row in enumerate(batch):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
publish_progress(i)
try:
doc, attachments, last_error_row_idx = get_doc(row_idx)
validate_naming(doc)
if pre_process:
pre_process(doc)
original = None
if parentfield:
parent = frappe.get_doc(parenttype, doc["parent"])
doc = parent.append(parentfield, doc)
parent.save()
else:
if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
original_name = original.name
original.update(doc)
# preserve original name for case sensitivity
original.name = original_name
original.flags.ignore_links = ignore_links
original.save()
doc = original
else:
if not update_only:
doc = frappe.get_doc(doc)
prepare_for_insert(doc)
doc.flags.ignore_links = ignore_links
doc.insert()
if attachments:
# check file url and create a File document
for file_url in attachments:
attach_file_to_doc(doc.doctype, doc.name, file_url)
if submit_after_import:
doc.submit()
# log errors
if parentfield:
log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
"link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
elif submit_after_import:
log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"})
elif original:
log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
elif not update_only:
log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
else:
log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
"message": "Document updation ignored", "indicator": "orange"})
except Exception as e:
error_flag = True
# build error message
if frappe.local.message_log:
err_msg = "\n".join(['<p class="border-bottom small">{}</p>'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log])
else:
err_msg = '<p class="border-bottom small">{}</p>'.format(cstr(e))
error_trace = frappe.get_traceback()
if error_trace:
error_log_doc = frappe.log_error(error_trace)
error_link = get_url_to_form("Error Log", error_log_doc.name)
else:
error_link = None
log(**{
"row": row_idx + 1,
"title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""),
"message": err_msg,
"indicator": "red",
"link":error_link
})
# data with error to create a new file
# include the errored data in the last row as last_error_row_idx will not be updated for the last row
if skip_errors:
if last_error_row_idx == len(rows)-1:
last_error_row_idx = len(rows)
data_rows_with_error += rows[row_idx:last_error_row_idx]
else:
rollback_flag = True
finally:
frappe.local.message_log = []
start_row += batch_size
if rollback_flag:
frappe.db.rollback()
else:
frappe.db.commit()
frappe.flags.mute_emails = False
frappe.flags.in_import = False
log_message = {"messages": import_log, "error": error_flag}
if data_import_doc:
data_import_doc.log_details = json.dumps(log_message)
import_status = None
if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
import_status = "Partially Successful"
# write the file with the faulty row
from frappe.utils.file_manager import save_file
file_name = 'error_' + filename + file_extension
if file_extension == '.xlsx':
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
file_data = xlsx_file.getvalue()
else:
from frappe.utils.csvutils import to_csv
file_data = to_csv(data_rows_with_error)
error_data_file = save_file(file_name, file_data, "Data Import",
data_import_doc.name, "Home/Attachments")
data_import_doc.error_file = error_data_file.file_url
elif error_flag:
import_status = "Failed"
else:
import_status = "Successful"
data_import_doc.import_status = import_status
data_import_doc.save()
if data_import_doc.import_status in ["Successful", "Partially Successful"]:
data_import_doc.submit()
publish_progress(100, True)
else:
publish_progress(0, True)
frappe.db.commit()
else:
return log_message
