def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): "Import CSV using data import" from frappe.core.doctype.data_import import importer from frappe.utils.csvutils import read_csv_content site = get_site(context) if not os.path.exists(path): path = os.path.join('..', path) if not os.path.exists(path): print('Invalid path {0}'.format(path)) sys.exit(1) with open(path, 'r') as csvfile: content = read_csv_content(csvfile.read()) frappe.init(site=site) frappe.connect() try: importer.upload(content, submit_after_import=submit_after_import, no_email=no_email, ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert, via_console=True) frappe.db.commit() except Exception: print(frappe.get_traceback()) frappe.destroy()
def test_import_only_children(self): user_email = "*****@*****.**" if frappe.db.exists("User", user_email): frappe.delete_doc("User", user_email, force=True) frappe.get_doc({"doctype": "User", "email": user_email, "first_name": "Test Import UserRole"}).insert() exporter.export_data("Has Role", "User", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) content.append(["", "*****@*****.**", "Blogger"]) importer.upload(content) user = frappe.get_doc("User", user_email) self.assertTrue(frappe.db.get_value("Has Role", filters={"role": "Blogger", "parent": user_email, "parenttype": "User"})) self.assertTrue(user.get("roles")[0].role, "Blogger") # overwrite exporter.export_data("Has Role", "User", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) content.append(["", "*****@*****.**", "Website Manager"]) importer.upload(content, overwrite=True) user = frappe.get_doc("User", user_email) self.assertEqual(len(user.get("roles")), 1) self.assertTrue(user.get("roles")[0].role, "Website Manager")
def test_import(self): if frappe.db.exists("Blog Category", "test-category"): frappe.delete_doc("Blog Category", "test-category") exporter.get_template("Blog Category", all_doctypes="No", with_data="No") content = read_csv_content(frappe.response.result) content.append(["", "", "test-category", "Test Cateogry"]) importer.upload(content) self.assertTrue( frappe.db.get_value("Blog Category", "test-category", "title"), "Test Category") # export with data exporter.get_template("Blog Category", all_doctypes="No", with_data="Yes") content = read_csv_content(frappe.response.result) # overwrite content[-1][3] = "New Title" importer.upload(content, overwrite=True) self.assertTrue( frappe.db.get_value("Blog Category", "test-category", "title"), "New Title")
def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): "Import CSV using data import" from frappe.core.doctype.data_import import importer from frappe.utils.csvutils import read_csv_content site = get_site(context) if not os.path.exists(path): path = os.path.join('..', path) if not os.path.exists(path): print('Invalid path {0}'.format(path)) sys.exit(1) with open(path, 'r') as csvfile: content = read_csv_content(csvfile.read()) frappe.init(site=site) frappe.connect() try: importer.upload(content, submit_after_import=submit_after_import, no_email=no_email, ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert, via_console=True) frappe.db.commit() except Exception: print(frappe.get_traceback()) frappe.destroy()
def validate(self): if not self.import_file: self.db_set("total_rows", 0) if self.import_status == "In Progress": frappe.throw(_("Can't save the form as data import is in progress.")) # validate the template just after the upload # if there is total_rows in the doc, it means that the template is already validated and error free if self.import_file and not self.total_rows: upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
def test_excel_import(self): if frappe.db.exists("Event", "EV00001"): frappe.delete_doc("Event", "EV00001") exporter.export_data("Event", all_doctypes=True, template=True, file_type="Excel") from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent) content.append(["", "_test", "Private", "05-11-2017 13:51:48", "Event", "0", "0", "", "1", "", "", 0, 0, 0, 0, 0, 0, 0, "blue"]) importer.upload(content) self.assertTrue(frappe.db.get_value("Event", {"subject": "_test"}, "name"))
def validate(self): if not self.import_file: self.db_set("total_rows", 0) if self.import_status == "In Progress": frappe.throw(_("Can't save the form as data import is in progress.")) # validate the template just after the upload # if there is total_rows in the doc, it means that the template is already validated and error free if self.import_file and not self.total_rows: upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
def test_excel_import(self): if frappe.db.exists("Event", "EV00001"): frappe.delete_doc("Event", "EV00001") exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes") from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent) content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"]) importer.upload(content) self.assertTrue(frappe.db.get_value("Event", "EV00001", "subject"), "_test")
def test_excel_import(self): if frappe.db.exists("Event", "EV00001"): frappe.delete_doc("Event", "EV00001") exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes") from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent) content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"]) importer.upload(content) self.assertTrue(frappe.db.get_value("Event", "EV00001", "subject"), "_test")
def test_import_with_children(self): exporter.get_template("Event", all_doctypes="Yes", with_data="No") content = read_csv_content(frappe.response.result) content.append([None] * len(content[-2])) content[-1][2] = "__Test Event with children" content[-1][3] = "Private" content[-1][4] = "2014-01-01 10:00:00.000000" importer.upload(content) ev = frappe.get_doc("Event", {"subject":"__Test Event with children"})
def test_import_with_children(self): #pylint: disable=R0201 if frappe.db.exists("Event", "EV00001"): frappe.delete_doc("Event", "EV00001") exporter.export_data("Event", all_doctypes="Yes", template=True) content = read_csv_content(frappe.response.result) content.append([None] * len(content[-2])) content[-1][1] = "__Test Event with children" content[-1][2] = "Private" content[-1][3] = "2014-01-01 10:00:00.000000" importer.upload(content) frappe.get_doc("Event", {"subject":"__Test Event with children"})
def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True): from frappe.utils.csvutils import read_csv_content print("Importing " + path) with open(path, "r") as infile: upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite, submit_after_import=submit, pre_process=pre_process)
def test_import(self): if frappe.db.exists("Blog Category", "test-category"): frappe.delete_doc("Blog Category", "test-category") exporter.export_data("Blog Category", all_doctypes=True, template=True) content = read_csv_content(frappe.response.result) content.append(["", "test-category", "Test Cateogry"]) importer.upload(content) self.assertTrue(frappe.db.get_value("Blog Category", "test-category", "title"), "Test Category") # export with data exporter.export_data("Blog Category", all_doctypes=True, template=True, with_data=True) content = read_csv_content(frappe.response.result) # overwrite content[-1][3] = "New Title" importer.upload(content, overwrite=True) self.assertTrue(frappe.db.get_value("Blog Category", "test-category", "title"), "New Title")
def upload_file(path, transactiontype, pos_transaction_date, filename, client=None): try: if client == 'yes': path = get_file_path(path) with open(encode(path), 'r') as f: content = f.read() rows = read_csv_content(content) result = importer.upload(rows, submit_after_import=True, update_only=False, ignore_encoding_errors=True, no_email=True) # generate JV name title = result['messages'][0]['title'] st = title.rfind('J') en = title.rfind('<') JV_name = title[st:en] transaction_link = JV_name error_status = result['error'] # failed due to content error if error_status == True: log_name = make_sync_log("Failed", transactiontype, result, '#fff168', pos_transaction_date, None) attachments = [{'fname': filename, 'fcontent': content}] send_email('Failed', transactiontype, result, pos_transaction_date, log_name, attachments) os.remove(path) return log_name #import is successful elif error_status == False: log_name = make_sync_log("Successful", transactiontype, result, '#9deca2', pos_transaction_date, transaction_link) os.remove(path) send_email('Successful', transactiontype, result, pos_transaction_date) return log_name except Exception as e: error = True log_name = make_sync_log("File not found failure", transactiontype, frappe.get_traceback(), '#ff4d4d', None, None) send_email('File not found failure', transactiontype, frappe.get_traceback(), pos_transaction_date, log_name) return log_name if error: frappe.db.rollback() else: frappe.db.commit() return {"error": error}
def get_categories(self): plaid = PlaidController() categories = plaid.get_category() content=[] exporter.get_template("Plaid Category", all_doctypes="No", with_data="No") content = read_csv_content(frappe.response.result) for key,val in categories.iteritems(): if key == "categories": for v in val: if (len(v['hierarchy'])>2): item=["",v['category_id'],v['category_id'], v['group'],v['hierarchy'][0],v['hierarchy'][1],v['hierarchy'][2]] elif(len(v['hierarchy'])>1): item=["",v['category_id'],v['category_id'], v['group'],v['hierarchy'][0],v['hierarchy'][1]] elif(len(v['hierarchy'])>0): item=["",v['category_id'],v['category_id'], v['group'],v['hierarchy'][0]] content.append(item) importer.upload(content,overwrite=True,update_only =False,ignore_encoding_errors=True, no_email=True) frappe.response['type'] = "json" return categories
def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True): from frappe.utils.csvutils import read_csv_content print("Importing " + path) with open(path, "r") as infile: upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite, submit_after_import=submit, pre_process=pre_process)