def prepare_to_notify(self, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" if print_format: self.content += self.get_attach_link(print_format) self.set_incoming_outgoing_accounts() if not self.sender or cint(self.outgoing_email_account.always_use_account_email_id_as_sender): self.sender = formataddr([frappe.session.data.full_name or "Notification", self.outgoing_email_account.email_id]) self.attachments = [] if print_html or print_format: self.attachments.append(frappe.attach_print(self.reference_doctype, self.reference_name, print_format=print_format, html=print_html)) if attachments: if isinstance(attachments, basestring): attachments = json.loads(attachments) for a in attachments: if isinstance(a, basestring): # is it a filename? try: file = get_file(a) self.attachments.append({"fname": file[0], "fcontent": file[1]}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: self.attachments.append(a)
def get_attachments(communications): for comm in communications: frappe.db.set_value("Communication", comm.name, "seen", 1) comm['content'] = re.sub('src="', 'src="' + frappe.utils.get_url(), comm.get('content')) file_attachments = [] if comm.get('has_attachment'): files = frappe.get_list("File", filters={ "attached_to_doctype": "Communication", "attached_to_name": comm.get('name') }) for d in files: filename, content = get_file(d.name) file_attachments.append({ "filename": filename, "content": base64.b64encode(content).decode('ascii') }) file_attachments = json.dumps(file_attachments) comm.update({"attachments": file_attachments}) return communications
def get_prepared_report_result(report, filters, dn="", user=None): latest_report_data = {} # Only look for completed prepared reports with given filters. doc_list = frappe.get_all("Prepared Report", filters={"status": "Completed", "report_name": report.name, "filters": json.dumps(filters), "owner": user}) doc = None if len(doc_list): if dn: # Get specified dn doc = frappe.get_doc("Prepared Report", dn) else: # Get latest doc = frappe.get_doc("Prepared Report", doc_list[0]) # Prepared Report data is stored in a GZip compressed JSON file attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name") compressed_content = get_file(attached_file_name)[1] uncompressed_content = gzip_decompress(compressed_content) data = json.loads(uncompressed_content) if data: latest_report_data = { "columns": json.loads(doc.columns) if doc.columns else data[0], "result": data } latest_report_data.update({ "prepared_report": True, "doc": doc }) return latest_report_data
def execute(): frappe.reload_doc('core', 'doctype', 'file_data') for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile` where file_name is not null"""): b = frappe.get_doc('File', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) if old_file_name.startswith('files/') or old_file_name.startswith( '/files/'): b.file_url = os.path.normpath('/' + old_file_name) else: b.file_url = os.path.normpath('/files/' + old_file_name) try: _file_name, content = get_file(name) b.content_hash = get_content_hash(content) except IOError: print('Warning: Error processing ', name) _file_name = old_file_name b.content_hash = None try: b.save() except frappe.DuplicateEntryError: frappe.delete_doc(b.doctype, b.name)
def upload(file): from frappe.utils.file_manager import get_file filepath = get_file(file) pps = read_csv_content(filepath[1]) date = '' err_list = '' for pp in pps: if pp[0] not in ('TamilNadu PetroProducts', 'Consumption Report for the', 'Slno', None, ''): if pp[2] is not None: if not frappe.db.exists("Food Item", pp[2]): err_list += '<ul>Item - <b>%s</b> Not Fount</ul><br>' % pp[ 2] if err_list: frappe.throw(err_list) else: for pp in pps: if pp[0] not in ('TamilNadu PetroProducts', 'Slno', '', None): if pp[0] == "Consumption Report for the": if pp[1]: try: date = datetime.strptime(pp[1], '%d/%m/%Y') except: date = datetime.strptime(pp[1], '%d-%m-%Y') if pp[2]: if frappe.db.exists("Food Item", pp[2]): zfe = frappe.new_doc("Zecons Food Entry") zfe.date = date zfe.code = pp[1] zfe.item = pp[2] zfe.rate = pp[3] zfe.qty = pp[4] zfe.cntr_value = pp[6] zfe.save(ignore_permissions=True) return 'ok'
def prepare_to_notify(self, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" if print_format: self.content += self.get_attach_link(print_format) self.set_incoming_outgoing_accounts() if not self.sender: self.sender = formataddr([frappe.session.data.full_name or "Notification", self.outgoing_email_account]) self.attachments = [] if print_html or print_format: self.attachments.append(frappe.attach_print(self.reference_doctype, self.reference_name, print_format=print_format, html=print_html)) if attachments: if isinstance(attachments, basestring): attachments = json.loads(attachments) for a in attachments: if isinstance(a, basestring): # is it a filename? try: file = get_file(a) self.attachments.append({"fname": file[0], "fcontent": file[1]}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: self.attachments.append(a)
def prepare_message(email, recipient, recipients_list): message = email.message if not message: return "" if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient, email.unsubscribe_method, email.unsubscribe_params) message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode()) if email.expose_recipients == "header": pass else: if email.expose_recipients == "footer": if isinstance(email.show_as_cc, string_types): email.show_as_cc = email.show_as_cc.split(",") email_sent_to = [r.recipient for r in recipients_list] email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc]) email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc]) if email_sent_cc: email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc) else: email_sent_message = _("This email was sent to {0}").format(email_sent_to) message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode()) message = message.replace("<!--recipient-->", recipient) message = (message and message.encode('utf8')) or '' if not email.attachments: return message # On-demand attachments from email.parser import Parser msg_obj = Parser().parsestr(message) attachments = json.loads(email.attachments) for attachment in attachments: if attachment.get('fcontent'): continue fid = attachment.get("fid") if fid: fname, fcontent = get_file(fid) attachment.update({ 'fname': fname, 'fcontent': fcontent, 'parent': msg_obj }) attachment.pop("fid", None) add_attachment(**attachment) elif attachment.get("print_format_attachment") == 1: attachment.pop("print_format_attachment", None) print_format_file = frappe.attach_print(**attachment) print_format_file.update({"parent": msg_obj}) add_attachment(**print_format_file) return msg_obj.as_string()
def attach_file(self, n): """attach a file from the `FileData` table""" from frappe.utils.file_manager import get_file res = get_file(n) if not res: return self.add_attachment(res[0], res[1])
def reply_to_support_issue(client_issue_id, subject, description, raised_by, recipients, frappe_issue_id, attachments=None): comm = frappe.get_doc({ "doctype": "Communication", "subject": subject, "content": description, "recipients": recipients, "sent_or_received": "Sent", "reference_doctype": "ERPNext Support Issue", "communication_medium": "Email", "sender": raised_by, "reference_name": client_issue_id, "has_attachment": 1 if attachments else 0 }).insert(ignore_permissions=True) if isinstance(attachments, string_types): attachments = json.loads(attachments) if attachments: add_attachments("Communication", comm.name, attachments) add_attachments("ERPNext Support Issue", client_issue_id, attachments) frappe.db.commit() file_attachments = [] if attachments: for a in attachments: filename, content = get_file(a) file_attachments.append({ "filename": filename, "content": base64.b64encode(content).decode("ascii") }) file_attachments = json.dumps(file_attachments) params = { "subject": subject, "recipients": recipients, "description": description, "raised_by": raised_by, "frappe_issue_id": frappe_issue_id, "attachments": file_attachments } last_sync_on = json.loads( call_remote_method("create_reply_from_customer", params)) frappe.db.set_value("ERPNext Support Issue", client_issue_id, "last_sync_on", get_datetime(last_sync_on.get("last_sync_on")))
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" view_link = frappe.utils.cint( frappe.db.get_value("Print Settings", "Print Settings", "attach_view_link")) if print_format and view_link: doc.content += get_attach_link(doc, print_format) set_incoming_outgoing_accounts(doc) if not doc.sender: doc.sender = doc.outgoing_email_account.email_id if not doc.sender_full_name: doc.sender_full_name = doc.outgoing_email_account.name or _( "Notification") if doc.sender: # combine for sending to get the format 'Jane <*****@*****.**>' doc.sender = formataddr([doc.sender_full_name, doc.sender]) doc.attachments = [] if print_html or print_format: doc.attachments.append({ "print_format_attachment": 1, "doctype": doc.reference_doctype, "name": doc.reference_name, "print_format": print_format, "html": print_html }) if attachments: if isinstance(attachments, string_types): attachments = json.loads(attachments) for a in attachments: if isinstance(a, string_types): # is it a filename? try: # keep this for error handling file = get_file(a) # these attachments will be attached on-demand # and won't be stored in the message doc.attachments.append({"fid": a}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: doc.attachments.append(a)
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" view_link = frappe.utils.cint( frappe.db.get_value("Print Settings", "Print Settings", "attach_view_link")) if print_format and view_link: doc.content += get_attach_link(doc, print_format) set_incoming_outgoing_accounts(doc) if not doc.sender: doc.sender = doc.outgoing_email_account.email_id if not doc.sender_full_name: doc.sender_full_name = doc.outgoing_email_account.name or _( "Notification") if doc.sender: # combine for sending to get the format 'Jane <*****@*****.**>' doc.sender = formataddr([doc.sender_full_name, doc.sender]) doc.attachments = [] if print_html or print_format: doc.attachments.append( frappe.attach_print(doc.reference_doctype, doc.reference_name, print_format=print_format, html=print_html)) if attachments: if isinstance(attachments, basestring): attachments = json.loads(attachments) for a in attachments: if isinstance(a, basestring): # is it a filename? try: file = get_file(a) doc.attachments.append({ "fname": file[0], "fcontent": file[1] }) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: doc.attachments.append(a)
def update_from_csv(filename): from frappe.utils.csvutils import read_csv_content from frappe.utils.file_manager import get_file _file = frappe.get_doc("File", {"file_name": filename}) filepath = get_file(filename) pps = read_csv_content(filepath[1]) # print len(pps) for pp in pps: # pp1 = datetime.strptime(pp[1], '%d-%m-%Y').strftime('%Y-%m-%d')pp[0] ss = frappe.get_value("Salary Slip",{"employee":pp[0],"posting_date":'2019-05-02'}) print ss
def mark_attendance(file_url, start_date, end_date): #below is the method to get file from Frappe File manager from frappe.utils.file_manager import get_file #Method to fetch file using get_doc and stored as _file _file = frappe.get_doc("File", {"file_url": file_url}) #Path in the system filepath = get_file(file_url) #CSV Content stored as pps pps = read_csv_content(filepath[1]) for pp in pps: frappe.errprint(pp) holiday_list = frappe.db.get_value("Employee", {"name": pp[0]}, ["holiday_list"]) holiday_map = frappe._dict() # for d in holiday_list: # if d: holiday_list = frappe.db.sql( '''select holiday_date from `tabHoliday` where parent=%s and holiday_date between %s and %s''', (holiday_list, start_date, end_date)) holidays = [] for holiday in holiday_list: holidays.append(holiday[0]) import pandas as pd total_days = pd.date_range(start_date, end_date).tolist() day = 1 for days in total_days: date = days.date() if date not in holidays: frappe.errprint(date) if int(pp[1]) >= day: attendance = frappe.new_doc("Attendance") attendance.update({ "employee": pp[0], "attendance_date": date, "status": "Absent" }).save(ignore_permissions=True) attendance.submit() frappe.db.commit() day = day + 1 if float(pp[2]) >= 1: ts = frappe.new_doc("Timesheet") ts.employee = pp[0] ts.append( "time_logs", { "activity_type": "Overtime", "from_time": start_date, "hours": float(pp[2]) }) ts.save(ignore_permissions=True) ts.submit() frappe.db.commit()
def bulk_update_from_csv(filename): #below is the method to get file from Frappe File manager from frappe.utils.file_manager import get_file #Method to fetch file using get_doc and stored as _file _file = frappe.get_doc("File", {"file_name": filename}) #Path in the system filepath = get_file(filename) #CSV Content stored as pps pps = read_csv_content(filepath[1]) count = 0
def get_prepared_report_result(report, filters, dn="", user=None): latest_report_data = {} doc = None if dn: # Get specified dn doc = frappe.get_doc("Prepared Report", dn) else: # Only look for completed prepared reports with given filters. doc_list = frappe.get_all("Prepared Report", filters={ "status": "Completed", "filters": json.dumps(filters), "owner": user, "report_name": report.report_name } ) if doc_list: # Get latest doc = frappe.get_doc("Prepared Report", doc_list[0]) if doc: try: # Prepared Report data is stored in a GZip compressed JSON file attached_file_name = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name") compressed_content = get_file(attached_file_name)[1] uncompressed_content = gzip_decompress(compressed_content) data = json.loads(uncompressed_content) if data: columns = json.loads(doc.columns) if doc.columns else data[0] for column in columns: if isinstance(column, dict): column["label"] = _(column["label"]) latest_report_data = { "columns": columns, "result": data } except Exception: frappe.log_error(frappe.get_traceback()) frappe.delete_doc("Prepared Report", doc.name) frappe.db.commit() doc = None latest_report_data.update({ "prepared_report": True, "doc": doc }) return latest_report_data
def read_csv_content_from_attached_file(doc): fileid = frappe.db.get_value("File", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name") if not fileid: msgprint(_("File not attached")) raise Exception try: from frappe.utils.file_manager import get_file fname, fcontent = get_file(fileid) return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) except Exception: frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format'))
def export_package(): """Export package as JSON.""" package_doc = frappe.get_single("Package Publish Tool") package = [] for doctype in package_doc.package_details: filters = [] if doctype.get("filters_json"): filters = json.loads(doctype.get("filters_json")) docs = frappe.get_all(doctype.get("document_type"), filters=filters) length = len(docs) for idx, doc in enumerate(docs): frappe.publish_realtime("package", { "progress": idx, "total": length, "message": doctype.get("document_type"), "prefix": _("Exporting") }, user=frappe.session.user) document = frappe.get_doc(doctype.get("document_type"), doc.name).as_dict() attachments = [] if doctype.attachments: filters = { "attached_to_doctype": document.get("doctype"), "attached_to_name": document.get("name") } for f in frappe.get_list("File", filters=filters): fname, fcontents = get_file(f.name) attachments.append({ "fname": fname, "content": base64.b64encode(fcontents).decode('ascii') }) document.update({ "__attachments": attachments, "__overwrite": True if doctype.overwrite else False }) package.append(document) return post_process(package), package_doc
def read_csv_content_from_attached_file(doc): fileid = frappe.db.get_value("File Data", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name") if not fileid: msgprint(_("File not attached")) raise Exception try: from frappe.utils.file_manager import get_file fname, fcontent = get_file(fileid) return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) except Exception: frappe.msgprint(_("Unable to open attached file. Please try again.")) raise Exception
def read_csv_content_from_attached_file(doc): fileid = frappe.db.get_value("File Data", {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, "name") if not fileid: msgprint("File not attached!") raise Exception try: from frappe.utils.file_manager import get_file fname, fcontent = get_file(fileid) return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) except Exception, e: frappe.msgprint("""Unable to open attached file. Please try again.""") raise Exception
def sync_erpnext_support_issue(doc=None, method=None): """ Sync Issue which arent synced with Server due to FrappeClient request failure """ filters = [["status", "=", "Open"], ["frappe_issue_id", "=", ""], ["sync_count", "<", 5]] for unsynced_issue in frappe.get_list("ERPNext Support Issue", filters=filters): erpnext_support_issue = frappe.get_doc("ERPNext Support Issue", unsynced_issue.name) comm = frappe.get_doc( "Communication", { "reference_doctype": "ERPNext Support Issue", "reference_name": erpnext_support_issue.name }) file_attachments = [] if comm.has_attachment: files = frappe.get_list("File", filters={ "attached_to_doctype": "Communication", "attached_to_name": comm.get("name") }) for d in files: filename, content = get_file(d.name) file_attachments.append({ "filename": filename, "content": base64.b64encode(content).decode("ascii") }) file_attachments = json.dumps(file_attachments) params = get_params(erpnext_support_issue, file_attachments) frappe_issue = call_remote_method("create_issue_from_customer", params) # If FrappeClient request fails, increment sync count if frappe_issue.get("failed"): erpnext_support_issue.db_set("sync_count", erpnext_support_issue.sync_count + 1) continue set_corresponding_frappe_values(erpnext_support_issue, frappe_issue)
def execute(): frappe.reload_doc('core', 'doctype', 'file_data') for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile Data` where file_name is not null"""): b = frappe.get_doc('File Data', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): b.file_url = os.path.normpath('/' + old_file_name) else: b.file_url = os.path.normpath('/files/' + old_file_name) _file_name, content = get_file(name) b.content_hash = get_content_hash(content) b.save()
def bulk_update_from_csv(filename): #below is the method to get file from Frappe File manager from frappe.utils.file_manager import get_file #Method to fetch file using get_doc and stored as _file _file = frappe.get_doc("File", {"file_name": filename}) #Path in the system filepath = get_file(filename) #CSV Content stored as pps pps = read_csv_content(filepath[1]) count = 0 for pp in pps: cl = frappe.get_doc("Candidate", {'name': pp[0]}) cl.pending_for = 'IDB' cl.db_update() frappe.db.commit()
def get_fints_import_file_content(fints_import): """Get FinTS Import json file content as json. :param fints_import: fints_import doc :type fints_import: fints_import doc :return: Transaction from file as json object list """ if fints_import.file_url: content = get_file(fints_import.file_url)[1] # Check content hash for file manipulations if fints_import.file_hash == get_content_hash(content): return frappe.json.loads(content, strict=False) else: raise ValueError('File hash does not match') else: return frappe.json.loads('[]')
def execute(): frappe.reload_doc('core', 'doctype', 'file_data') for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile Data` where file_name is not null"""): b = frappe.get_doc('File Data', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) if old_file_name.startswith('files/') or old_file_name.startswith( '/files/'): b.file_url = os.path.normpath('/' + old_file_name) else: b.file_url = os.path.normpath('/files/' + old_file_name) _file_name, content = get_file(name) b.content_hash = get_content_hash(content) b.save()
def read_csv_content_from_attached_file(doc): fileid = frappe.get_all("File", fields = ["name"], filters = {"attached_to_doctype": doc.doctype, "attached_to_name":doc.name}, order_by="creation desc") if fileid : fileid = fileid[0].name if not fileid: msgprint(_("File not attached")) raise Exception try: from frappe.utils.file_manager import get_file fname, fcontent = get_file(fileid) return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) except Exception: frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format'))
def sync_communication_server(erpnext_support_user, erpnext_support_issues, bench_site): """ returns a dict of support issues and associated communications in the format { "erpnext_support_issue_name_1": [{Communications}] "erpnext_support_issue_name_2": [{Communications}] } """ authenticate_erpnext_support_user(erpnext_support_user) communications = {} erpnext_support_issues = json.loads(erpnext_support_issues) for erpnext_support_issue in erpnext_support_issues: if erpnext_support_issue.get('associated_issue'): filters = { 'reference_doctype': 'Issue', 'reference_name': erpnext_support_issue.get('associated_issue'), 'communication_medium': 'Email', 'sent_or_received': 'Sent', 'seen': 0 } fields = ["name", "subject", "content", "recipients", "has_attachment"] comms = frappe.get_all("Communication", filters=filters, fields=fields, order_by="creation ASC") for comm in comms: frappe.db.set_value("Communication", comm.name, "seen", 1) comm['content'] = re.sub('src="', 'src="' + frappe.utils.get_url(), comm.get('content')) file_attachments = [] if comm.get('has_attachment'): files = frappe.get_list("File", filters={"attached_to_doctype": "Communication", "attached_to_name": comm.get('name')}) for d in files: filename, content = get_file(d.name) file_attachments.append({ "filename": filename, "content": base64.b64encode(content).decode('ascii') }) file_attachments = json.dumps(file_attachments) comm.update({"attachments": file_attachments}) communications[erpnext_support_issue.get('name')] = comms communications = json.dumps(communications) return communications
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" if print_format: doc.content += get_attach_link(doc, print_format) set_incoming_outgoing_accounts(doc) if not doc.sender or cint( doc.outgoing_email_account.always_use_account_email_id_as_sender): sender_name = (frappe.session.data.full_name or doc.outgoing_email_account.name or _("Notification")) sender_email_id = doc.outgoing_email_account.email_id doc.sender = formataddr([sender_name, sender_email_id]) doc.attachments = [] if print_html or print_format: doc.attachments.append( frappe.attach_print(doc.reference_doctype, doc.reference_name, print_format=print_format, html=print_html)) if attachments: if isinstance(attachments, basestring): attachments = json.loads(attachments) for a in attachments: if isinstance(a, basestring): # is it a filename? try: file = get_file(a) doc.attachments.append({ "fname": file[0], "fcontent": file[1] }) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: doc.attachments.append(a)
def queue_all(self): if not self.get("recipients"): # in case it is called via worker self.recipients = self.get_recipients() self.validate_send() sender = self.send_from or frappe.utils.get_formatted_email(self.owner) if not frappe.flags.in_test: frappe.db.auto_commit_on_many_writes = True attachments = [] if self.send_attachements: files = frappe.get_all("File", fields=["name"], filters={ "attached_to_doctype": "Newsletter", "attached_to_name": self.name }, order_by="creation desc") for file in files: try: file = get_file(file.name) attachments.append({"fname": file[0], "fcontent": file[1]}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) send( recipients=self.recipients, sender=sender, subject=self.subject, message=self.message, reference_doctype=self.doctype, reference_name=self.name, add_unsubscribe_link=self.send_unsubscribe_link, attachments=attachments, unsubscribe_method= "/api/method/frappe.email.doctype.newsletter.newsletter.unsubscribe", unsubscribe_params={"name": self.name}, send_priority=0, queue_separately=True) if not frappe.flags.in_test: frappe.db.auto_commit_on_many_writes = False
def prepare_attachments(g_attachments=None): attachments = [] if g_attachments: if isinstance(g_attachments, basestring): import json g_attachments = json.loads(g_attachments) for a in g_attachments: if isinstance(a, basestring): # is it a filename? try: file = get_file(a) attachments.append({"fname": file[0], "fcontent": file[1]}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: attachments.append(a) return attachments
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" view_link = frappe.utils.cint(frappe.db.get_value("Print Settings", "Print Settings", "attach_view_link")) if print_format and view_link: doc.content += get_attach_link(doc, print_format) set_incoming_outgoing_accounts(doc) if not doc.sender: doc.sender = doc.outgoing_email_account.email_id if not doc.sender_full_name: doc.sender_full_name = doc.outgoing_email_account.name or _("Notification") if doc.sender: # combine for sending to get the format 'Jane <*****@*****.**>' doc.sender = formataddr([doc.sender_full_name, doc.sender]) doc.attachments = [] if print_html or print_format: doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype, "name":doc.reference_name, "print_format":print_format, "html":print_html}) if attachments: if isinstance(attachments, string_types): attachments = json.loads(attachments) for a in attachments: if isinstance(a, string_types): # is it a filename? try: # keep this for error handling file = get_file(a) # these attachments will be attached on-demand # and won't be stored in the message doc.attachments.append({"fid": a}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: doc.attachments.append(a)
def bulk_update_from_csv(filename): #below is the method to get file from Frappe File manager from frappe.utils.file_manager import get_file #Method to fetch file using get_doc and stored as _file _file = frappe.get_doc("File", {"file_name": filename}) #Path in the system filepath = get_file(filename) #CSV Content stored as pps pps = read_csv_content(filepath[1]) count = 0 for pp in pps[:5]: mobile = pp[0] project = pp[1] position = pp[2] given_name = pp[3] customer = pp[4] print(mobile, project, position, given_name, customer)
def bulk_update_from_csv(filename): #below is the method to get file from Frappe File manager from frappe.utils.file_manager import get_file #Method to fetch file using get_doc and stored as _file _file = frappe.get_doc("File", {"file_name": filename}) #Path in the system filepath = get_file(filename) #CSV Content stored as pps payloads = read_csv_content(filepath[1]) for payload in payloads: if payload: food_order = frappe.new_doc("Food Order No") food_order.employee = payload[0] food_order.date = payload[1] food_order.save(ignore_permissions=True) frappe.db.commit() for i in range(cint(payload[2])): limit = frappe.db.get_value('Food Item',payload[3],"limit") if frappe.db.count("Daily Food Entry",{"employee":payload[0],"item":payload[3],"date":payload[1]}) > limit: dfe = frappe.new_doc("Daily Food Entry") dfe.employee = payload[0] dfe.item = payload[3] dfe.date = payload[1] # dfe.time = payload[4] dfe.plant = payload[4] dfe.food_order_no = food_order.name dfe.price = frappe.db.get_value("Food Item",payload[3],["original_rate"]) dfe.save(ignore_permissions=True) frappe.db.commit() else: dfe = frappe.new_doc("Daily Food Entry") dfe.employee = payload[0] dfe.item = payload[3] dfe.date = payload[1] # dfe.time = payload[4] dfe.plant = payload[4] dfe.food_order_no = food_order.name dfe.price = frappe.db.get_value("Food Item",payload[3],["subsidy_rate"]) dfe.subsidy = 1 dfe.save(ignore_permissions=True) frappe.db.commit() return 'OK'
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None): """Prepare to make multipart MIME Email :param print_html: Send given value as HTML attachment. :param print_format: Attach print format of parent document.""" view_link = frappe.utils.cint(frappe.db.get_value("Print Settings", "Print Settings", "attach_view_link")) if print_format and view_link: doc.content += get_attach_link(doc, print_format) set_incoming_outgoing_accounts(doc) if not doc.sender: doc.sender = doc.outgoing_email_account.email_id if not doc.sender_full_name: doc.sender_full_name = doc.outgoing_email_account.name or _("Notification") if doc.sender: # combine for sending to get the format 'Jane <*****@*****.**>' doc.sender = formataddr([doc.sender_full_name, doc.sender]) doc.attachments = [] if print_html or print_format: doc.attachments.append(frappe.attach_print(doc.reference_doctype, doc.reference_name, print_format=print_format, html=print_html)) if attachments: if isinstance(attachments, basestring): attachments = json.loads(attachments) for a in attachments: if isinstance(a, basestring): # is it a filename? try: file = get_file(a) doc.attachments.append({"fname": file[0], "fcontent": file[1]}) except IOError: frappe.throw(_("Unable to find attachment {0}").format(a)) else: doc.attachments.append(a)
def bulk_update_from_csv(filename): #below is the method to get file from Frappe File manager from frappe.utils.file_manager import get_file #Method to fetch file using get_doc and stored as _file _file = frappe.get_doc("File", {"file_name": filename}) #Path in the system filepath = get_file(filename) #CSV Content stored as pps pps = read_csv_content(filepath[1]) count = 0 for pp in pps: ld = frappe.db.exists("Lead", {'name': pp[0]}) if ld: # items = frappe.get_all("Lead",{'name':pp[0]}) # for item in items: i = frappe.get_doc('Lead', pp[0]) if not i.contac % t_by: i.contact_date = pp[1] print(pp[1])
def execute(): frappe.db.auto_commit_on_many_writes = True rename_replacing_files() for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile` where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""): b = frappe.get_doc('File', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): b.file_url = os.path.normpath('/' + old_file_name) else: b.file_url = os.path.normpath('/files/' + old_file_name) try: _file_name, content = get_file(name) b.content_hash = get_content_hash(content) except IOError: print 'Warning: Error processing ', name b.content_hash = None b.flags.ignore_duplicate_entry_error = True b.save() frappe.db.auto_commit_on_many_writes = False
def upload_from_json(filename): # payloads = [{"employee": "300752", "date": "2021-9-11", "qty": "1", "item": "LUNCH MEALS"}] from frappe.utils.file_manager import get_file _file = frappe.get_doc("File", {"file_name": filename}) filepath = get_file(filename) # payloads = read_csv_content(filepath[1]) payloads = json.loads(filepath[1]) for payload in payloads: payload = payload[0] if payload: food_order = frappe.new_doc("Food Order No") food_order.employee = payload["employee"] food_order.date = payload["date"] food_order.save(ignore_permissions=True) frappe.db.commit() for i in range(cint(payload['qty'])): print(payload['item']) limit = frappe.db.get_value('Food Item',payload['item'],"limit") if frappe.db.count("Daily Food Entry",{"employee":payload["employee"],"item":payload["item"],"date":payload["date"]}) > limit: dfe = frappe.new_doc("Daily Food Entry") dfe.employee = payload["employee"] dfe.item = payload["item"] dfe.date = payload["date"] dfe.time = payload["time"] dfe.food_order_no = food_order.name dfe.price = frappe.db.get_value("Food Item",payload["item"],["original_rate"]) dfe.save(ignore_permissions=True) frappe.db.commit() else: dfe = frappe.new_doc("Daily Food Entry") dfe.employee = payload['employee'] dfe.item = payload['item'] dfe.date = payload["date"] dfe.time = payload["time"] dfe.food_order_no = food_order.name dfe.price = frappe.db.get_value("Food Item",payload["item"],["subsidy_rate"]) dfe.subsidy = 1 dfe.save(ignore_permissions=True) frappe.db.commit()
def execute(): frappe.db.auto_commit_on_many_writes = True rename_replacing_files() for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile` where ifnull(file_name, '')!='' and ifnull(content_hash, '')=''"""): b = frappe.get_doc('File', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): b.file_url = os.path.normpath('/' + old_file_name) else: b.file_url = os.path.normpath('/files/' + old_file_name) try: _file_name, content = get_file(name) b.content_hash = get_content_hash(content) except IOError: print('Warning: Error processing ', name) b.content_hash = None b.flags.ignore_duplicate_entry_error = True b.save() frappe.db.auto_commit_on_many_writes = False
def execute(): frappe.reload_doc('core', 'doctype', 'file_data') for name, file_name, file_url in frappe.db.sql( """select name, file_name, file_url from `tabFile` where file_name is not null"""): b = frappe.get_doc('File', name) old_file_name = b.file_name b.file_name = os.path.basename(old_file_name) if old_file_name.startswith('files/') or old_file_name.startswith('/files/'): b.file_url = os.path.normpath('/' + old_file_name) else: b.file_url = os.path.normpath('/files/' + old_file_name) try: _file_name, content = get_file(name) b.content_hash = get_content_hash(content) except IOError: print('Warning: Error processing ', name) _file_name = old_file_name b.content_hash = None try: b.save() except frappe.DuplicateEntryError: frappe.delete_doc(b.doctype, b.name)
def prepare_message(email, recipient, recipients_list): message = email.message if not message: return "" # Parse "Email Account" from "Email Sender" email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender) if frappe.conf.use_ssl and email_account.track_email_status: # Using SSL => Publically available domain => Email Read Reciept Possible message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode()) else: # No SSL => No Email Read Reciept message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode()) if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient, email.unsubscribe_method, email.unsubscribe_params) message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode()) if email.expose_recipients == "header": pass else: if email.expose_recipients == "footer": if isinstance(email.show_as_cc, string_types): email.show_as_cc = email.show_as_cc.split(",") email_sent_to = [r.recipient for r in recipients_list] email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc]) email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc]) if email_sent_cc: email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc) else: email_sent_message = _("This email was sent to {0}").format(email_sent_to) message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode()) message = message.replace("<!--recipient-->", recipient) message = (message and message.encode('utf8')) or '' message = safe_decode(message) if not email.attachments: return message # On-demand attachments from email.parser import Parser msg_obj = Parser().parsestr(message) attachments = json.loads(email.attachments) for attachment in attachments: if attachment.get('fcontent'): continue fid = attachment.get("fid") if fid: fname, fcontent = get_file(fid) attachment.update({ 'fname': fname, 'fcontent': fcontent, 'parent': msg_obj }) attachment.pop("fid", None) add_attachment(**attachment) elif attachment.get("print_format_attachment") == 1: attachment.pop("print_format_attachment", None) print_format_file = frappe.attach_print(**attachment) print_format_file.update({"parent": msg_obj}) add_attachment(**print_format_file) return msg_obj.as_string()
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", skip_errors = True): """upload data""" frappe.flags.in_import = True # extra input params params = json.loads(frappe.form_dict.get("params") or '{}') if params.get("submit_after_import"): submit_after_import = True if params.get("ignore_encoding_errors"): ignore_encoding_errors = True if not params.get("no_email"): no_email = False if params.get('update_only'): update_only = True if params.get('from_data_import'): from_data_import = params.get('from_data_import') if not params.get('skip_errors'): skip_errors = params.get('skip_errors') frappe.flags.mute_emails = no_email def get_data_keys_definition(): return get_data_keys() def bad_template(): frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator)) def check_data_length(): max_rows = 5000 if not data: frappe.throw(_("No data found")) elif not via_console and len(data) > max_rows: frappe.throw(_("Only allowed {0} rows in one import").format(max_rows)) def get_start_row(): for i, row in enumerate(rows): if row and row[0]==get_data_keys_definition().data_separator: return i+1 bad_template() def get_header_row(key): return get_header_row_and_idx(key)[0] def get_header_row_and_idx(key): for i, row in enumerate(header): if row and row[0]==key: return row, i return [], -1 def filter_empty_columns(columns): empty_cols = list(filter(lambda x: x in ("", None), columns)) if empty_cols: if columns[-1*len(empty_cols):] == empty_cols: # filter empty columns if they exist at the end columns = columns[:-1*len(empty_cols)] else: frappe.msgprint(_("Please make sure that there are no empty columns in the file."), raise_exception=1) return columns def make_column_map(): doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype) if row_idx == -1: # old style return dt = None for i, d in enumerate(doctype_row[1:]): if d not in ("~", "-"): if d and doctype_row[i] in (None, '' ,'~', '-', 'DocType:'): dt, parentfield = d, None # xls format truncates the row, so it may not have more columns if len(doctype_row) > i+2: parentfield = doctype_row[i+2] doctypes.append((dt, parentfield)) column_idx_to_fieldname[(dt, parentfield)] = {} column_idx_to_fieldtype[(dt, parentfield)] = {} if dt: column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1] column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1] def get_doc(start_idx): if doctypes: doc = {} for idx in range(start_idx, len(rows)): if (not doc) or main_doc_empty(rows[idx]): for dt, parentfield in doctypes: d = {} for column_idx in column_idx_to_fieldname[(dt, parentfield)]: try: fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx] fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx] d[fieldname] = rows[idx][column_idx] if fieldtype in ("Int", "Check"): d[fieldname] = cint(d[fieldname]) elif fieldtype in ("Float", "Currency", "Percent"): d[fieldname] = flt(d[fieldname]) elif fieldtype == "Date": if d[fieldname] and isinstance(d[fieldname], string_types): d[fieldname] = getdate(parse_date(d[fieldname])) elif fieldtype == "Datetime": if d[fieldname]: if " " in d[fieldname]: _date, _time = d[fieldname].split() else: _date, _time = d[fieldname], '00:00:00' _date = parse_date(d[fieldname]) d[fieldname] = get_datetime(_date + " " + _time) else: d[fieldname] = None elif fieldtype in ("Image", "Attach Image", "Attach"): # added file to attachments list attachments.append(d[fieldname]) elif fieldtype in ("Link", "Dynamic Link") and d[fieldname]: # as fields can be saved in the number format(long type) in data import template d[fieldname] = cstr(d[fieldname]) except IndexError: pass # scrub quotes from name and modified if d.get("name") and d["name"].startswith('"'): d["name"] = d["name"][1:-1] if sum([0 if not val else 1 for val in d.values()]): d['doctype'] = dt if dt == doctype: doc.update(d) else: if not overwrite: d['parent'] = doc["name"] d['parenttype'] = doctype d['parentfield'] = parentfield doc.setdefault(d['parentfield'], []).append(d) else: break return doc else: doc = frappe._dict(zip(columns, rows[start_idx][1:])) doc['doctype'] = doctype return doc def main_doc_empty(row): return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2]))) users = frappe.db.sql_list("select name from tabUser") def prepare_for_insert(doc): # don't block data import if user is not set # migrating from another system if not doc.owner in users: doc.owner = frappe.session.user if not doc.modified_by in users: doc.modified_by = frappe.session.user def is_valid_url(url): is_valid = False if url.startswith("/files") or url.startswith("/private/files"): url = get_url(url) try: r = requests.get(url) is_valid = True if r.status_code == 200 else False except Exception: pass return is_valid def attach_file_to_doc(doctype, docname, file_url): # check if attachment is already available # check if the attachement link is relative or not if not file_url: return if not is_valid_url(file_url): return files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format( doctype=doctype, docname=docname, file_url=file_url )) if files: # file is already attached return save_url(file_url, None, doctype, docname, "Home/Attachments", 0) # header if not rows: from frappe.utils.file_manager import get_file_doc file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1) filename, file_extension = os.path.splitext(file_doc.file_name) if file_extension == '.xlsx' and from_data_import == 'Yes': from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file rows = read_xlsx_file_from_attached_file(file_id=file_doc.name) elif file_extension == '.csv': from frappe.utils.file_manager import get_file from frappe.utils.csvutils import read_csv_content fname, fcontent = get_file(file_doc.name) rows = read_csv_content(fcontent, ignore_encoding_errors) else: frappe.throw(_("Unsupported File Format")) start_row = get_start_row() header = rows[:start_row] data = rows[start_row:] doctype = get_header_row(get_data_keys_definition().main_table)[1] columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:]) doctypes = [] column_idx_to_fieldname = {} column_idx_to_fieldtype = {} attachments = [] if submit_after_import and not cint(frappe.db.get_value("DocType", doctype, "is_submittable")): submit_after_import = False parenttype = get_header_row(get_data_keys_definition().parent_table) if len(parenttype) > 1: parenttype = parenttype[1] # check permissions if not frappe.permissions.can_import(parenttype or doctype): frappe.flags.mute_emails = False return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True} # allow limit rows to be uploaded check_data_length() make_column_map() if overwrite==None: overwrite = params.get('overwrite') # delete child rows (if parenttype) parentfield = None if parenttype: parentfield = get_parent_field(doctype, parenttype) if overwrite: delete_child_rows(data, doctype) ret = [] def log(msg): if via_console: print(msg.encode('utf-8')) else: ret.append(msg) def as_link(doctype, name): if via_console: return "{0}: {1}".format(doctype, name) else: return getlink(doctype, name) error = False total = len(data) for i, row in enumerate(data): # bypass empty rows if main_doc_empty(row): continue row_idx = i + start_row doc = None # publish task_update frappe.publish_realtime("data_import_progress", {"progress": [i, total]}, user=frappe.session.user) try: doc = get_doc(row_idx) if pre_process: pre_process(doc) if parentfield: parent = frappe.get_doc(parenttype, doc["parent"]) doc = parent.append(parentfield, doc) parent.save() log('Inserted row for %s at #%s' % (as_link(parenttype, doc.parent),text_type(doc.idx))) else: if overwrite and doc["name"] and frappe.db.exists(doctype, doc["name"]): original = frappe.get_doc(doctype, doc["name"]) original_name = original.name original.update(doc) # preserve original name for case sensitivity original.name = original_name original.flags.ignore_links = ignore_links original.save() log('Updated row (#%d) %s' % (row_idx + 1, as_link(original.doctype, original.name))) doc = original else: if not update_only: doc = frappe.get_doc(doc) prepare_for_insert(doc) doc.flags.ignore_links = ignore_links doc.insert() log('Inserted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name))) else: log('Ignored row (#%d) %s' % (row_idx + 1, row[1])) if attachments: # check file url and create a File document for file_url in attachments: attach_file_to_doc(doc.doctype, doc.name, file_url) if submit_after_import: doc.submit() log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name))) except Exception as e: if not skip_errors: error = True if doc: frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict()) err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e) log('Error for row (#%d) %s : %s' % (row_idx + 1, len(row)>1 and row[1] or "", err_msg)) frappe.errprint(frappe.get_traceback()) finally: frappe.local.message_log = [] if error: frappe.db.rollback() else: frappe.db.commit() frappe.flags.mute_emails = False frappe.flags.in_import = False return {"messages": ret, "error": error}
def download_attachment(dn): attachment = get_attachments("Prepared Report", dn)[0] frappe.local.response.filename = attachment.file_name[:-2] frappe.local.response.filecontent = gzip_decompress(get_file(attachment.name)[1]) frappe.local.response.type = "binary"
def test_save(self): filename, content = get_file(self.saved_file.name) self.assertEqual(content, self.test_content)
def test_saved_content(self): filename1, content1 = get_file(self.saved_file1.name) self.assertEqual(content1, self.test_content1) filename2, content2 = get_file(self.saved_file2.name) self.assertEqual(content2, self.test_content2)
def test_saved_content(self): filename1, content1 = get_file(self.saved_file1.name) filename2, content2 = get_file(self.saved_file2.name) self.assertEqual(filename1, filename2) self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", skip_errors = True, data_import_doc=None, validate_template=False, user=None): """upload data""" # for translations if user: frappe.cache().hdel("lang", user) frappe.set_user_lang(user) if data_import_doc and isinstance(data_import_doc, string_types): data_import_doc = frappe.get_doc("Data Import", data_import_doc) if data_import_doc and from_data_import == "Yes": no_email = data_import_doc.no_email ignore_encoding_errors = data_import_doc.ignore_encoding_errors update_only = data_import_doc.only_update submit_after_import = data_import_doc.submit_after_import overwrite = data_import_doc.overwrite skip_errors = data_import_doc.skip_errors else: # extra input params params = json.loads(frappe.form_dict.get("params") or '{}') if params.get("submit_after_import"): submit_after_import = True if params.get("ignore_encoding_errors"): ignore_encoding_errors = True if not params.get("no_email"): no_email = False if params.get('update_only'): update_only = True if params.get('from_data_import'): from_data_import = params.get('from_data_import') if not params.get('skip_errors'): skip_errors = params.get('skip_errors') frappe.flags.in_import = True frappe.flags.mute_emails = no_email def get_data_keys_definition(): return get_data_keys() def bad_template(): frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator)) def check_data_length(): if not data: frappe.throw(_("No data found in the file. Please reattach the new file with data.")) def get_start_row(): for i, row in enumerate(rows): if row and row[0]==get_data_keys_definition().data_separator: return i+1 bad_template() def get_header_row(key): return get_header_row_and_idx(key)[0] def get_header_row_and_idx(key): for i, row in enumerate(header): if row and row[0]==key: return row, i return [], -1 def filter_empty_columns(columns): empty_cols = list(filter(lambda x: x in ("", None), columns)) if empty_cols: if columns[-1*len(empty_cols):] == empty_cols: # filter empty columns if they exist at the end columns = columns[:-1*len(empty_cols)] else: frappe.msgprint(_("Please make sure that there are no empty columns in the file."), raise_exception=1) return columns def make_column_map(): doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype) if row_idx == -1: # old style return dt = None for i, d in enumerate(doctype_row[1:]): if d not in ("~", "-"): if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"): dt, parentfield = d, None # xls format truncates the row, so it may not have more columns if len(doctype_row) > i+2: parentfield = doctype_row[i+2] doctypes.append((dt, parentfield)) column_idx_to_fieldname[(dt, parentfield)] = {} column_idx_to_fieldtype[(dt, parentfield)] = {} if dt: column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1] column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1] def get_doc(start_idx): if doctypes: doc = {} attachments = [] last_error_row_idx = None for idx in range(start_idx, len(rows)): last_error_row_idx = idx # pylint: disable=W0612 if (not doc) or main_doc_empty(rows[idx]): for dt, parentfield in doctypes: d = {} for column_idx in column_idx_to_fieldname[(dt, parentfield)]: try: fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx] fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx] if not fieldname or not rows[idx][column_idx]: continue d[fieldname] = rows[idx][column_idx] if fieldtype in ("Int", "Check"): d[fieldname] = cint(d[fieldname]) elif fieldtype in ("Float", "Currency", "Percent"): d[fieldname] = flt(d[fieldname]) elif fieldtype == "Date": if d[fieldname] and isinstance(d[fieldname], string_types): d[fieldname] = getdate(parse_date(d[fieldname])) elif fieldtype == "Datetime": if d[fieldname]: if " " in d[fieldname]: _date, _time = d[fieldname].split() else: _date, _time = d[fieldname], '00:00:00' _date = parse_date(d[fieldname]) d[fieldname] = get_datetime(_date + " " + _time) else: d[fieldname] = None elif fieldtype in ("Image", "Attach Image", "Attach"): # added file to attachments list attachments.append(d[fieldname]) elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]: # as fields can be saved in the number format(long type) in data import template d[fieldname] = cstr(d[fieldname]) except IndexError: pass # scrub quotes from name and modified if d.get("name") and d["name"].startswith('"'): d["name"] = d["name"][1:-1] if sum([0 if not val else 1 for val in d.values()]): d['doctype'] = dt if dt == doctype: doc.update(d) else: if not overwrite and doc.get("name"): d['parent'] = doc["name"] d['parenttype'] = doctype d['parentfield'] = parentfield doc.setdefault(d['parentfield'], []).append(d) else: break return doc, attachments, last_error_row_idx else: doc = frappe._dict(zip(columns, rows[start_idx][1:])) doc['doctype'] = doctype return doc, [], None # used in testing whether a row is empty or parent row or child row # checked only 3 first columns since first two columns can be blank for example the case of # importing the item variant where item code and item name will be blank. def main_doc_empty(row): if row: for i in range(3,0,-1): if len(row) > i and row[i]: return False return True def validate_naming(doc): autoname = frappe.get_meta(doctype).autoname if autoname: if autoname[0:5] == 'field': autoname = autoname[6:] elif autoname == 'naming_series:': autoname = 'naming_series' else: return True if (autoname not in doc) or (not doc[autoname]): from frappe.model.base_document import get_controller if not hasattr(get_controller(doctype), "autoname"): frappe.throw(_("{0} is a mandatory field".format(autoname))) return True users = frappe.db.sql_list("select name from tabUser") def prepare_for_insert(doc): # don't block data import if user is not set # migrating from another system if not doc.owner in users: doc.owner = frappe.session.user if not doc.modified_by in users: doc.modified_by = frappe.session.user def is_valid_url(url): is_valid = False if url.startswith("/files") or url.startswith("/private/files"): url = get_url(url) try: r = requests.get(url) is_valid = True if r.status_code == 200 else False except Exception: pass return is_valid def attach_file_to_doc(doctype, docname, file_url): # check if attachment is already available # check if the attachement link is relative or not if not file_url: return if not is_valid_url(file_url): return files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format( doctype=doctype, docname=docname, file_url=file_url )) if files: # file is already attached return save_url(file_url, None, doctype, docname, "Home/Attachments", 0) # header filename, file_extension = ['',''] if not rows: from frappe.utils.file_manager import get_file # get_file_doc fname, fcontent = get_file(data_import_doc.import_file) filename, file_extension = os.path.splitext(fname) if file_extension == '.xlsx' and from_data_import == 'Yes': from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file) elif file_extension == '.csv': from frappe.utils.csvutils import read_csv_content rows = read_csv_content(fcontent, ignore_encoding_errors) else: frappe.throw(_("Unsupported File Format")) start_row = get_start_row() header = rows[:start_row] data = rows[start_row:] try: doctype = get_header_row(get_data_keys_definition().main_table)[1] columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:]) except: frappe.throw(_("Cannot change header content")) doctypes = [] column_idx_to_fieldname = {} column_idx_to_fieldtype = {} if skip_errors: data_rows_with_error = header if submit_after_import and not cint(frappe.db.get_value("DocType", doctype, "is_submittable")): submit_after_import = False parenttype = get_header_row(get_data_keys_definition().parent_table) if len(parenttype) > 1: parenttype = parenttype[1] # check permissions if not frappe.permissions.can_import(parenttype or doctype): frappe.flags.mute_emails = False return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True} # Throw expception in case of the empty data file check_data_length() make_column_map() total = len(data) if validate_template: if total: data_import_doc.total_rows = total return True if overwrite==None: overwrite = params.get('overwrite') # delete child rows (if parenttype) parentfield = None if parenttype: parentfield = get_parent_field(doctype, parenttype) if overwrite: delete_child_rows(data, doctype) import_log = [] def log(**kwargs): if via_console: print((kwargs.get("title") + kwargs.get("message")).encode('utf-8')) else: import_log.append(kwargs) def as_link(doctype, name): if via_console: return "{0}: {1}".format(doctype, name) else: return getlink(doctype, name) # publish realtime task update def publish_progress(achieved, reload=False): if data_import_doc: frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)), "data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user) error_flag = rollback_flag = False batch_size = frappe.conf.data_import_batch_size or 1000 for batch_start in range(0, total, batch_size): batch = data[batch_start:batch_start + batch_size] for i, row in enumerate(batch): # bypass empty rows if main_doc_empty(row): continue row_idx = i + start_row doc = None publish_progress(i) try: doc, attachments, last_error_row_idx = get_doc(row_idx) validate_naming(doc) if pre_process: pre_process(doc) original = None if parentfield: parent = frappe.get_doc(parenttype, doc["parent"]) doc = parent.append(parentfield, doc) parent.save() else: if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]): original = frappe.get_doc(doctype, doc["name"]) original_name = original.name original.update(doc) # preserve original name for case sensitivity original.name = original_name original.flags.ignore_links = ignore_links original.save() doc = original else: if not update_only: doc = frappe.get_doc(doc) prepare_for_insert(doc) doc.flags.ignore_links = ignore_links doc.insert() if attachments: # check file url and create a File document for file_url in attachments: attach_file_to_doc(doc.doctype, doc.name, file_url) if submit_after_import: doc.submit() # log errors if parentfield: log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)), "link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"}) elif submit_after_import: log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)), "message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"}) elif original: log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)), "message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"}) elif not update_only: log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)), "message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"}) else: log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None, "message": "Document updation ignored", "indicator": "orange"}) except Exception as e: error_flag = True # build error message if frappe.local.message_log: err_msg = "\n".join(['<p class="border-bottom small">{}</p>'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log]) else: err_msg = '<p class="border-bottom small">{}</p>'.format(cstr(e)) error_trace = frappe.get_traceback() if error_trace: error_log_doc = frappe.log_error(error_trace) error_link = get_url_to_form("Error Log", error_log_doc.name) else: error_link = None log(**{ "row": row_idx + 1, "title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""), "message": err_msg, "indicator": "red", "link":error_link }) # data with error to create a new file # include the errored data in the last row as last_error_row_idx will not be updated for the last row if skip_errors: if last_error_row_idx == len(rows)-1: last_error_row_idx = len(rows) data_rows_with_error += rows[row_idx:last_error_row_idx] else: rollback_flag = True finally: frappe.local.message_log = [] start_row += batch_size if rollback_flag: frappe.db.rollback() else: frappe.db.commit() frappe.flags.mute_emails = False frappe.flags.in_import = False log_message = {"messages": import_log, "error": error_flag} if data_import_doc: data_import_doc.log_details = json.dumps(log_message) import_status = None if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error): import_status = "Partially Successful" # write the file with the faulty row from frappe.utils.file_manager import save_file file_name = 'error_' + filename + file_extension if file_extension == '.xlsx': from frappe.utils.xlsxutils import make_xlsx xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template") file_data = xlsx_file.getvalue() else: from frappe.utils.csvutils import to_csv file_data = to_csv(data_rows_with_error) error_data_file = save_file(file_name, file_data, "Data Import", data_import_doc.name, "Home/Attachments") data_import_doc.error_file = error_data_file.file_url elif error_flag: import_status = "Failed" else: import_status = "Successful" data_import_doc.import_status = import_status data_import_doc.save() if data_import_doc.import_status in ["Successful", "Partially Successful"]: data_import_doc.submit() publish_progress(100, True) else: publish_progress(0, True) frappe.db.commit() else: return log_message