def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
"Import CSV using data import"
from frappe.core.doctype.data_import import importer
from frappe.utils.csvutils import read_csv_content
site = get_site(context)
if not os.path.exists(path):
path = os.path.join('..', path)
if not os.path.exists(path):
print('Invalid path {0}'.format(path))
sys.exit(1)
with open(path, 'r') as csvfile:
content = read_csv_content(csvfile.read())
frappe.init(site=site)
frappe.connect()
try:
importer.upload(content, submit_after_import=submit_after_import, no_email=no_email,
ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert,
via_console=True)
frappe.db.commit()
except Exception:
print(frappe.get_traceback())
frappe.destroy()
def test_import_only_children(self):
user_email = "*****@*****.**"
if frappe.db.exists("User", user_email):
frappe.delete_doc("User", user_email, force=True)
frappe.get_doc({"doctype": "User", "email": user_email, "first_name": "Test Import UserRole"}).insert()
exporter.export_data("Has Role", "User", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
content.append(["", "*****@*****.**", "Blogger"])
importer.upload(content)
user = frappe.get_doc("User", user_email)
self.assertTrue(frappe.db.get_value("Has Role", filters={"role": "Blogger", "parent": user_email, "parenttype": "User"}))
self.assertTrue(user.get("roles")[0].role, "Blogger")
# overwrite
exporter.export_data("Has Role", "User", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
content.append(["", "*****@*****.**", "Website Manager"])
importer.upload(content, overwrite=True)
user = frappe.get_doc("User", user_email)
self.assertEqual(len(user.get("roles")), 1)
self.assertTrue(user.get("roles")[0].role, "Website Manager")
def test_import(self):
if frappe.db.exists("Blog Category", "test-category"):
frappe.delete_doc("Blog Category", "test-category")
exporter.get_template("Blog Category",
all_doctypes="No",
with_data="No")
content = read_csv_content(frappe.response.result)
content.append(["", "", "test-category", "Test Cateogry"])
importer.upload(content)
self.assertTrue(
frappe.db.get_value("Blog Category", "test-category", "title"),
"Test Category")
# export with data
exporter.get_template("Blog Category",
all_doctypes="No",
with_data="Yes")
content = read_csv_content(frappe.response.result)
# overwrite
content[-1][3] = "New Title"
importer.upload(content, overwrite=True)
self.assertTrue(
frappe.db.get_value("Blog Category", "test-category", "title"),
"New Title")
def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
"Import CSV using data import"
from frappe.core.doctype.data_import import importer
from frappe.utils.csvutils import read_csv_content
site = get_site(context)
if not os.path.exists(path):
path = os.path.join('..', path)
if not os.path.exists(path):
print('Invalid path {0}'.format(path))
sys.exit(1)
with open(path, 'r') as csvfile:
content = read_csv_content(csvfile.read())
frappe.init(site=site)
frappe.connect()
try:
importer.upload(content, submit_after_import=submit_after_import, no_email=no_email,
ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert,
via_console=True)
frappe.db.commit()
except Exception:
print(frappe.get_traceback())
frappe.destroy()
def validate(self):
if not self.import_file:
self.db_set("total_rows", 0)
if self.import_status == "In Progress":
frappe.throw(_("Can't save the form as data import is in progress."))
# validate the template just after the upload
# if there is total_rows in the doc, it means that the template is already validated and error free
if self.import_file and not self.total_rows:
upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
def test_excel_import(self):
if frappe.db.exists("Event", "EV00001"):
frappe.delete_doc("Event", "EV00001")
exporter.export_data("Event", all_doctypes=True, template=True, file_type="Excel")
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent)
content.append(["", "_test", "Private", "05-11-2017 13:51:48", "Event", "0", "0", "", "1", "", "", 0, 0, 0, 0, 0, 0, 0, "blue"])
importer.upload(content)
self.assertTrue(frappe.db.get_value("Event", {"subject": "_test"}, "name"))
def validate(self):
if not self.import_file:
self.db_set("total_rows", 0)
if self.import_status == "In Progress":
frappe.throw(_("Can't save the form as data import is in progress."))
# validate the template just after the upload
# if there is total_rows in the doc, it means that the template is already validated and error free
if self.import_file and not self.total_rows:
upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
def test_excel_import(self):
if frappe.db.exists("Event", "EV00001"):
frappe.delete_doc("Event", "EV00001")
exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes")
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent)
content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"])
importer.upload(content)
self.assertTrue(frappe.db.get_value("Event", "EV00001", "subject"), "_test")
def test_excel_import(self):
if frappe.db.exists("Event", "EV00001"):
frappe.delete_doc("Event", "EV00001")
exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes")
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent)
content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"])
importer.upload(content)
self.assertTrue(frappe.db.get_value("Event", "EV00001", "subject"), "_test")
def test_import_with_children(self):
exporter.get_template("Event", all_doctypes="Yes", with_data="No")
content = read_csv_content(frappe.response.result)
content.append([None] * len(content[-2]))
content[-1][2] = "__Test Event with children"
content[-1][3] = "Private"
content[-1][4] = "2014-01-01 10:00:00.000000"
importer.upload(content)
ev = frappe.get_doc("Event", {"subject":"__Test Event with children"})
def test_import_with_children(self): #pylint: disable=R0201
if frappe.db.exists("Event", "EV00001"):
frappe.delete_doc("Event", "EV00001")
exporter.export_data("Event", all_doctypes="Yes", template=True)
content = read_csv_content(frappe.response.result)
content.append([None] * len(content[-2]))
content[-1][1] = "__Test Event with children"
content[-1][2] = "Private"
content[-1][3] = "2014-01-01 10:00:00.000000"
importer.upload(content)
frappe.get_doc("Event", {"subject":"__Test Event with children"})
def import_file_by_path(path,
ignore_links=False,
overwrite=False,
submit=False,
pre_process=None,
no_email=True):
from frappe.utils.csvutils import read_csv_content
print("Importing " + path)
with open(path, "r") as infile:
upload(rows=read_csv_content(infile.read()),
ignore_links=ignore_links,
no_email=no_email,
overwrite=overwrite,
submit_after_import=submit,
pre_process=pre_process)
def test_import(self):
if frappe.db.exists("Blog Category", "test-category"):
frappe.delete_doc("Blog Category", "test-category")
exporter.export_data("Blog Category", all_doctypes=True, template=True)
content = read_csv_content(frappe.response.result)
content.append(["", "test-category", "Test Cateogry"])
importer.upload(content)
self.assertTrue(frappe.db.get_value("Blog Category", "test-category", "title"), "Test Category")
# export with data
exporter.export_data("Blog Category", all_doctypes=True, template=True, with_data=True)
content = read_csv_content(frappe.response.result)
# overwrite
content[-1][3] = "New Title"
importer.upload(content, overwrite=True)
self.assertTrue(frappe.db.get_value("Blog Category", "test-category", "title"), "New Title")
def upload_file(path,
transactiontype,
pos_transaction_date,
filename,
client=None):
try:
if client == 'yes':
path = get_file_path(path)
with open(encode(path), 'r') as f:
content = f.read()
rows = read_csv_content(content)
result = importer.upload(rows,
submit_after_import=True,
update_only=False,
ignore_encoding_errors=True,
no_email=True)
# generate JV name
title = result['messages'][0]['title']
st = title.rfind('J')
en = title.rfind('<')
JV_name = title[st:en]
transaction_link = JV_name
error_status = result['error']
# failed due to content error
if error_status == True:
log_name = make_sync_log("Failed", transactiontype, result,
'#fff168', pos_transaction_date, None)
attachments = [{'fname': filename, 'fcontent': content}]
send_email('Failed', transactiontype, result, pos_transaction_date,
log_name, attachments)
os.remove(path)
return log_name
#import is successful
elif error_status == False:
log_name = make_sync_log("Successful", transactiontype, result,
'#9deca2', pos_transaction_date,
transaction_link)
os.remove(path)
send_email('Successful', transactiontype, result,
pos_transaction_date)
return log_name
except Exception as e:
error = True
log_name = make_sync_log("File not found failure", transactiontype,
frappe.get_traceback(), '#ff4d4d', None, None)
send_email('File not found failure', transactiontype,
frappe.get_traceback(), pos_transaction_date, log_name)
return log_name
if error:
frappe.db.rollback()
else:
frappe.db.commit()
return {"error": error}
def get_categories(self):
plaid = PlaidController()
categories = plaid.get_category()
content=[]
exporter.get_template("Plaid Category", all_doctypes="No", with_data="No")
content = read_csv_content(frappe.response.result)
for key,val in categories.iteritems():
if key == "categories":
for v in val:
if (len(v['hierarchy'])>2):
item=["",v['category_id'],v['category_id'], v['group'],v['hierarchy'][0],v['hierarchy'][1],v['hierarchy'][2]]
elif(len(v['hierarchy'])>1):
item=["",v['category_id'],v['category_id'], v['group'],v['hierarchy'][0],v['hierarchy'][1]]
elif(len(v['hierarchy'])>0):
item=["",v['category_id'],v['category_id'], v['group'],v['hierarchy'][0]]
content.append(item)
importer.upload(content,overwrite=True,update_only =False,ignore_encoding_errors=True, no_email=True)
frappe.response['type'] = "json"
return categories
def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True):
from frappe.utils.csvutils import read_csv_content
print("Importing " + path)
with open(path, "r") as infile:
upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
submit_after_import=submit, pre_process=pre_process)
