def execute():
""" Create Contact for each User if not present """
frappe.reload_doc('integrations', 'doctype', 'google_contacts')
frappe.reload_doc('contacts', 'doctype', 'contact')
frappe.reload_doc('core', 'doctype', 'dynamic_link')
frappe.reload_doc('communication', 'doctype', 'call_log')
contact_meta = frappe.get_meta("Contact")
if contact_meta.has_field("phone_nos") and contact_meta.has_field(
"email_ids"):
frappe.reload_doc('contacts', 'doctype', 'contact_phone')
frappe.reload_doc('contacts', 'doctype', 'contact_email')
users = frappe.get_all(
'User',
filters={"name": ('not in', 'Administrator, Guest')},
fields=["*"])
for user in users:
if frappe.db.exists("Contact",
{"email_id": user.email}) or frappe.db.exists(
"Contact Email", {"email_id": user.email}):
continue
if user.first_name:
user.first_name = re.sub("[<>]+", '',
frappe.safe_decode(user.first_name))
if user.last_name:
user.last_name = re.sub("[<>]+", '',
frappe.safe_decode(user.last_name))
create_contact(user, ignore_links=True, ignore_mandatory=True)
def populate_matching_vouchers(self):
for entry in self.new_transaction_items:
if (not entry.party or entry.reference_name): continue
print("Finding matching voucher for {0}".format(
frappe.safe_decode(entry.description)))
amount = abs(entry.amount)
invoices = []
vouchers = get_matching_journal_entries(self.from_date,
self.to_date, entry.party,
self.bank_account, amount)
if len(vouchers) == 0: continue
for voucher in vouchers:
added = next((entry.invoice
for entry in self.payment_invoice_items
if entry.invoice == voucher.voucher_no), None)
if (added):
print("Found voucher {0}".format(added))
continue
print("Adding voucher {0} {1} {2}".format(
voucher.voucher_no, voucher.posting_date, voucher.debit))
ent = self.append('payment_invoice_items', {})
ent.invoice_date = voucher.posting_date
ent.invoice_type = "Journal Entry"
ent.invoice = voucher.voucher_no
ent.payment_description = frappe.safe_decode(entry.description)
ent.allocated_amount = max(voucher.debit, voucher.credit)
invoices += [ent.invoice_type + "|" + ent.invoice]
entry.reference_type = "Journal Entry"
entry.mode_of_payment = "Wire Transfer"
entry.reference_name = ent.invoice
#entry.account = entry.party
entry.invoices = ",".join(invoices)
break
def generate_sms_pin():
mobile = frappe.local.form_dict.mobile
newPIN = cint(frappe.local.form_dict.newPIN or "0")
# If mobile needs to automatically the received
hash = frappe.local.form_dict.hash
if not mobile:
frappe.throw("No Mobile Number")
# check cache for pin
pin = frappe.safe_decode(frappe.cache().get("sms:" + mobile))
user = get_linked_user(mobile)
if not pin and user:
# check if available in db
pin = frappe.db.get_value("User", user, "renovation_sms_pin")
if not pin or newPIN:
# generate a pin
pin = frappe.safe_decode(str(get_pin()))
frappe.cache().set("sms:" + mobile, pin)
# save in User doc if mobile linked to any User
if user:
frappe.db.set_value("User", user, "renovation_sms_pin",
pin, update_modified=False)
msg = u"Your verification OTP is: " + pin
if hash:
msg = msg + u". " + hash
sms = send_sms([mobile], msg, success_msg=False)
status = "fail"
if sms and isinstance(sms, list) and mobile in sms:
status = "success"
update_http_response({"status": status, "mobile": mobile})
def execute():
"""Create Contact for each User if not present"""
frappe.reload_doc("integrations", "doctype", "google_contacts")
frappe.reload_doc("contacts", "doctype", "contact")
frappe.reload_doc("core", "doctype", "dynamic_link")
contact_meta = frappe.get_meta("Contact")
if contact_meta.has_field("phone_nos") and contact_meta.has_field(
"email_ids"):
frappe.reload_doc("contacts", "doctype", "contact_phone")
frappe.reload_doc("contacts", "doctype", "contact_email")
users = frappe.get_all(
"User",
filters={"name": ("not in", "Administrator, Guest")},
fields=["*"])
for user in users:
if frappe.db.exists("Contact",
{"email_id": user.email}) or frappe.db.exists(
"Contact Email", {"email_id": user.email}):
continue
if user.first_name:
user.first_name = re.sub("[<>]+", "",
frappe.safe_decode(user.first_name))
if user.last_name:
user.last_name = re.sub("[<>]+", "",
frappe.safe_decode(user.last_name))
create_contact(user, ignore_links=True, ignore_mandatory=True)
def get_desk_assets(build_version):
"""Get desk assets to be loaded for mobile app"""
data = get_context({"for_mobile": True})
assets = [{"type": "js", "data": ""}, {"type": "css", "data": ""}]
if build_version != data["build_version"]:
# new build, send assets
for path in data["include_js"]:
# assets path shouldn't start with /
# as it points to different location altogether
if path.startswith('/assets/'):
path = path.replace('/assets/', 'assets/')
try:
with open(os.path.join(frappe.local.sites_path, path) ,"r") as f:
assets[0]["data"] = assets[0]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8")
except IOError:
pass
for path in data["include_css"]:
if path.startswith('/assets/'):
path = path.replace('/assets/', 'assets/')
try:
with open(os.path.join(frappe.local.sites_path, path) ,"r") as f:
assets[1]["data"] = assets[1]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8")
except IOError:
pass
return {
"build_version": data["build_version"],
"boot": data["boot"],
"assets": assets
}
def migrate():
check_if_admin()
can_migrate()
status = frappe.safe_decode(frappe.cache().get(migration_status_key))
error = frappe.safe_decode(frappe.cache().get(migration_error_key))
if status == migration_status_migrating:
return "Migration in progress"
elif status == migration_status_done:
frappe.cache().set(migration_status_key,
migration_status_done + "_seen")
return {
"status":
"SUCCESS: Last Migration was successfull, Please send a new request to restart migration",
"error": error
}
elif status == migration_status_error:
frappe.cache().set(migration_status_key,
migration_status_error + "_seen")
return {
"status":
"ERROR: Last Migration wasnt successfull, Please send a new request to retry",
"error": error
}
frappe.enqueue("renovation_core.utils.site.start_migration")
return {
"status": "Started migration",
"last_migration": {
"status": status,
"error": error
}
}
def test_category_link(self):
# Make a temporary Blog Post (and a Blog Category)
blog = make_test_blog()
# Visit the blog post page
set_request(path=blog.route)
blog_page_response = render()
blog_page_html = frappe.safe_decode(blog_page_response.get_data())
# On blog post page find link to the category page
soup = BeautifulSoup(blog_page_html, "lxml")
category_page_link = list(
soup.find_all('a', href=re.compile(blog.blog_category)))[0]
category_page_url = category_page_link["href"]
# Visit the category page (by following the link found in above stage)
set_request(path=category_page_url)
category_page_response = render()
category_page_html = frappe.safe_decode(
category_page_response.get_data())
# Category page should contain the blog post title
self.assertIn(blog.title, category_page_html)
# Cleanup afterwords
frappe.delete_doc("Blog Post", blog.name)
frappe.delete_doc("Blog Category", blog.blog_category)
def test_cc_footer(self):
frappe.conf.use_ssl = True
# test if sending with cc's makes it into header
frappe.sendmail(recipients=['*****@*****.**'],
cc=['*****@*****.**'],
sender="*****@*****.**",
reference_doctype='User',
reference_name="Administrator",
subject='Testing Email Queue',
message='This is mail is queued!',
unsubscribe_message="Unsubscribe",
expose_recipients="footer",
now=True)
email_queue = frappe.db.sql(
"""select name from `tabEmail Queue` where status='Sent'""",
as_dict=1)
self.assertEqual(len(email_queue), 1)
queue_recipients = [
r.recipient for r in frappe.db.sql(
"""select recipient from `tabEmail Queue Recipient`
where status='Sent'""",
as_dict=1)
]
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue(
'This email was sent to [email protected] and copied to [email protected]'
in frappe.safe_decode(frappe.flags.sent_mail))
# check for email tracker
self.assertTrue(
'mark_email_as_seen' in frappe.safe_decode(frappe.flags.sent_mail))
frappe.conf.use_ssl = False
def populate_matching_vouchers(self):
for entry in self.new_transaction_items:
if (not entry.party or entry.reference_name): continue
print("Finding matching voucher for {0}".format(frappe.safe_decode(entry.description)))
amount = abs(entry.amount)
invoices = []
vouchers = get_matching_journal_entries(self.from_date, self.to_date, entry.party, self.bank_account, amount)
if len(vouchers) == 0: continue
for voucher in vouchers:
added = next((entry.invoice for entry in self.payment_invoice_items if entry.invoice == voucher.voucher_no), None)
if (added):
print("Found voucher {0}".format(added))
continue
print("Adding voucher {0} {1} {2}".format(voucher.voucher_no, voucher.posting_date, voucher.debit))
ent = self.append('payment_invoice_items', {})
ent.invoice_date = voucher.posting_date
ent.invoice_type = "Journal Entry"
ent.invoice = voucher.voucher_no
ent.payment_description = frappe.safe_decode(entry.description)
ent.allocated_amount = max(voucher.debit, voucher.credit)
invoices += [ent.invoice_type + "|" + ent.invoice]
entry.reference_type = "Journal Entry"
entry.mode_of_payment = "Wire Transfer"
entry.reference_name = ent.invoice
#entry.account = entry.party
entry.invoices = ",".join(invoices)
break
def get_desk_assets(build_version):
"""Get desk assets to be loaded for mobile app"""
data = get_context({"for_mobile": True})
assets = [{"type": "js", "data": ""}, {"type": "css", "data": ""}]
if build_version != data["build_version"]:
# new build, send assets
for path in data["include_js"]:
# assets path shouldn't start with /
# as it points to different location altogether
if path.startswith('/assets/'):
path = path.replace('/assets/', 'assets/')
try:
with open(os.path.join(frappe.local.sites_path, path) ,"r") as f:
assets[0]["data"] = assets[0]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8")
except IOError:
pass
for path in data["include_css"]:
if path.startswith('/assets/'):
path = path.replace('/assets/', 'assets/')
try:
with open(os.path.join(frappe.local.sites_path, path) ,"r") as f:
assets[1]["data"] = assets[1]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8")
except IOError:
pass
return {
"build_version": data["build_version"],
"boot": data["boot"],
"assets": assets
}
def get_cached_user_pass():
'''Get user and password if set.'''
user = pwd = None
tmp_id = frappe.form_dict.get('tmp_id')
if tmp_id:
user = frappe.safe_decode(frappe.cache().get(tmp_id + '_usr'))
pwd = frappe.safe_decode(frappe.cache().get(tmp_id + '_pwd'))
return (user, pwd)
def get_cached_user_pass():
"""Get user and password if set."""
user = pwd = None
tmp_id = frappe.form_dict.get("tmp_id")
if tmp_id:
user = frappe.safe_decode(frappe.cache().get(tmp_id + "_usr"))
pwd = frappe.safe_decode(frappe.cache().get(tmp_id + "_pwd"))
return (user, pwd)
def populate_payment_entries(self):
if self.bank_statement is None: return
file_url = self.bank_statement
if (len(self.new_transaction_items + self.reconciled_transaction_items)
> 0):
frappe.throw(
_("Transactions already retreived from the statement"))
date_format = frappe.get_value("Bank Statement Settings",
self.bank_settings, "date_format")
if (date_format is None):
date_format = '%Y-%m-%d'
if self.bank_settings:
mapped_items = frappe.get_doc("Bank Statement Settings",
self.bank_settings).mapped_items
statement_headers = self.get_statement_headers()
transactions = get_transaction_entries(file_url, statement_headers)
for entry in transactions:
date = entry[statement_headers["Date"]].strip()
#print("Processing entry DESC:{0}-W:{1}-D:{2}-DT:{3}".format(entry["Particulars"], entry["Withdrawals"], entry["Deposits"], entry["Date"]))
if (not date): continue
transaction_date = datetime.strptime(date, date_format).date()
if (self.from_date and transaction_date < datetime.strptime(
self.from_date, '%Y-%m-%d').date()):
continue
if (self.to_date and transaction_date > datetime.strptime(
self.to_date, '%Y-%m-%d').date()):
continue
bank_entry = self.append('new_transaction_items', {})
bank_entry.transaction_date = transaction_date
bank_entry.description = entry[statement_headers["Particulars"]]
mapped_item = next(
(entry for entry in mapped_items
if entry.mapping_type == "Transaction"
and frappe.safe_decode(entry.bank_data.lower()) in
frappe.safe_decode(bank_entry.description.lower())), None)
if (mapped_item is not None):
bank_entry.party_type = mapped_item.mapped_data_type
bank_entry.party = mapped_item.mapped_data
else:
bank_entry.party_type = "Supplier" if not entry[
statement_headers["Deposits"]].strip() else "Customer"
party_list = frappe.get_all(bank_entry.party_type,
fields=["name"])
parties = [party.name for party in party_list]
matches = difflib.get_close_matches(
frappe.safe_decode(bank_entry.description.lower()),
parties, 1, 0.4)
if len(matches) > 0: bank_entry.party = matches[0]
bank_entry.amount = -float(
entry[statement_headers["Withdrawals"]]) if not entry[
statement_headers["Deposits"]].strip() else float(
entry[statement_headers["Deposits"]])
self.map_unknown_transactions()
self.map_transactions_on_journal_entry()
def match_invoice_to_payment(self):
added_payments = []
for entry in self.new_transaction_items:
if (not entry.party or entry.party_type == "Account"): continue
entry.account = self.receivable_account if entry.party_type == "Customer" else self.payable_account
amount = abs(entry.amount)
payment, matching_invoices = None, []
for inv_entry in self.payment_invoice_items:
if (inv_entry.payment_description != frappe.safe_decode(
entry.description) or
inv_entry.transaction_date != entry.transaction_date):
continue
if (inv_entry.party != entry.party): continue
matching_invoices += [
inv_entry.invoice_type + "|" + inv_entry.invoice
]
payment = get_payments_matching_invoice(
inv_entry.invoice, entry.amount, entry.transaction_date)
doc = frappe.get_doc(inv_entry.invoice_type, inv_entry.invoice)
inv_entry.invoice_date = doc.posting_date
inv_entry.outstanding_amount = doc.outstanding_amount
inv_entry.allocated_amount = min(float(doc.outstanding_amount),
amount)
amount -= inv_entry.allocated_amount
if (amount < 0): break
amount = abs(entry.amount)
if (payment is None):
order_doctype = "Sales Order" if entry.party_type == "Customer" else "Purchase Order"
from erpbee.controllers.accounts_controller import get_advance_payment_entries
payment_entries = get_advance_payment_entries(
entry.party_type,
entry.party,
entry.account,
order_doctype,
against_all_orders=True)
payment_entries += self.get_matching_payments(
entry.party, amount, entry.transaction_date)
payment = next(
(payment
for payment in payment_entries if payment.amount == amount
and payment not in added_payments), None)
if (payment is None):
print("Failed to find payments for {0}:{1}".format(
entry.party, amount))
continue
added_payments += [payment]
entry.reference_type = payment.reference_type
entry.reference_name = payment.reference_name
entry.mode_of_payment = "Wire Transfer"
entry.outstanding_amount = min(amount, 0)
if (entry.payment_reference is None):
entry.payment_reference = frappe.safe_decode(entry.description)
entry.invoices = ",".join(matching_invoices)
def execute():
""" Create Contact for each User if not present """
frappe.reload_doc('contacts', 'doctype', 'contact')
users = frappe.get_all('User', filters={"name": ('not in', 'Administrator, Guest')}, fields=["*"])
for user in users:
if user.first_name:
user.first_name = re.sub("[<>]+", '', frappe.safe_decode(user.first_name))
if user.last_name:
user.last_name = re.sub("[<>]+", '', frappe.safe_decode(user.last_name))
create_contact(user, ignore_links=True, ignore_mandatory=True)
def set_subject(self):
"""Parse and decode `Subject` header."""
_subject = decode_header(self.mail.get("Subject", "No Subject"))
self.subject = _subject[0][0] or ""
if _subject[0][1]:
self.subject = safe_decode(self.subject, _subject[0][1])
else:
# assume that the encoding is utf-8
self.subject = safe_decode(self.subject)[:140]
if not self.subject:
self.subject = "No Subject"
def set_subject(self):
"""Parse and decode `Subject` header."""
_subject = decode_header(self.mail.get("Subject", "No Subject"))
self.subject = _subject[0][0] or ""
if _subject[0][1]:
self.subject = safe_decode(self.subject, _subject[0][1])
else:
# assume that the encoding is utf-8
self.subject = safe_decode(self.subject)[:140]
if not self.subject:
self.subject = "No Subject"
def make_tarfile(path, fname=None):
if not fname:
fname = "charts"
source_path = path
else:
source_path = os.path.join(path, fname + ".json").encode('utf-8')
target_path = os.path.join(path, fname + ".tar.gz").encode('utf-8')
source_path = frappe.safe_decode(source_path)
target_path = frappe.safe_decode(target_path)
with tarfile.open(target_path, "w:gz", encoding="utf-8") as tar:
tar.add(source_path, arcname=os.path.basename(source_path))
def pre_process(user):
name = frappe.safe_decode(user.name)
if "," in name:
firstname = name.split(',')[-1]
lastname = name.split(',')[0]
else:
firstname = name.split()[0]
lastname = name.split()[-1]
phone = frappe.safe_decode(
user.phone) if user.phone is not None else user.phone
condition = None
if user.email is not None and phone is not None:
condition = "(email_id = '{0}' OR phone = '{1}') AND (zendesk_sync_id != '{2}' OR zendesk_sync_id IS NULL)".format(
user.email, frappe.safe_encode(phone), user.id)
elif user.email is not None:
condition = "email_id = '{0}' AND (zendesk_sync_id != '{1}' OR zendesk_sync_id IS NULL)".format(
user.email, user.id)
elif phone is not None:
condition = "phone = '{0}' AND (zendesk_sync_id != '{1}' OR zendesk_sync_id IS NULL)".format(
frappe.safe_encode(phone), user.id)
if condition:
contacts = frappe.db.sql("""
SELECT
name
FROM
tabContact
WHERE
%s
""" % condition,
as_dict=True)
for contact in contacts:
try:
frappe.db.set_value("Contact",
frappe.safe_decode(contact.name),
"zendesk_sync_id", user.id)
except Exception as e:
frappe.log_error(e, user.name)
return {
'id': user.id,
'firstname': firstname,
'lastname': lastname,
'email': user.email,
'phone': phone
}
def load_assets(self):
import os
from frappe.modules import get_module_path, scrub
self.script = ""
page_name = scrub(self.name)
path = os.path.join(get_module_path(self.module), "page", page_name)
# script
fpath = os.path.join(path, page_name + ".js")
if os.path.exists(fpath):
with open(fpath, "r") as f:
self.script = render_include(f.read())
self.script += f"\n\n//# sourceURL={page_name}.js"
# css
fpath = os.path.join(path, page_name + ".css")
if os.path.exists(fpath):
with open(fpath, "r") as f:
self.style = safe_decode(f.read())
# html as js template
for fname in os.listdir(path):
if fname.endswith(".html"):
with open(os.path.join(path, fname), "r") as f:
template = f.read()
if "<!-- jinja -->" in template:
context = frappe._dict({})
try:
out = frappe.get_attr(
"{app}.{module}.page.{page}.{page}.get_context"
.format(app=frappe.local.module_app[scrub(
self.module)],
module=scrub(self.module),
page=page_name))(context)
if out:
context = out
except (AttributeError, ImportError):
pass
template = frappe.render_template(template, context)
self.script = html_to_js_template(fname,
template) + self.script
# flag for not caching this page
self._dynamic_page = True
if frappe.lang != "en":
from frappe.translate import get_lang_js
self.script += get_lang_js("page", self.name)
for path in get_code_files_via_hooks("page_js", self.name):
js = get_js(path)
if js:
self.script += "\n\n" + js
def generate_bootstrap_theme(self):
from subprocess import Popen, PIPE
file_name = frappe.scrub(self.name) + '_' + frappe.generate_hash(
'Website Theme', 8) + '.css'
output_path = join_path(frappe.utils.get_bench_path(), 'sites',
'assets', 'css', file_name)
content = self.theme_scss
content = content.replace('\n', '\\n')
command = ['node', 'generate_bootstrap_theme.js', output_path, content]
process = Popen(command,
cwd=frappe.get_app_path('frappe', '..'),
stdout=PIPE,
stderr=PIPE)
stderr = process.communicate()[1]
if stderr:
stderr = frappe.safe_decode(stderr)
stderr = stderr.replace('\n', '<br>')
frappe.throw(
'<div style="font-family: monospace;">{stderr}</div>'.format(
stderr=stderr))
else:
self.theme_url = '/assets/css/' + file_name
frappe.msgprint(_('Compiled Successfully'), alert=True)
def get_html_for_route(route): from frappe.website import render set_request(method="GET", path=route) response = render.render() html = frappe.safe_decode(response.get_data()) return html
def sync_user_settings():
'''Sync from cache to database (called asynchronously via the browser)'''
for key, data in iteritems(frappe.cache().hgetall('_user_settings')):
key = safe_decode(key)
doctype, user = key.split('::') # WTF?
frappe.db.sql('''insert into __UserSettings (user, doctype, data) values (%s, %s, %s)
on duplicate key update data=%s''', (user, doctype, data, data))
def test_homepage_section_custom_html(self):
frappe.get_doc({
'doctype':
'Homepage Section',
'name':
'Custom HTML Section',
'section_based_on':
'Custom HTML',
'section_html':
'<div class="custom-section">My custom html</div>',
}).insert()
set_request(method='GET', path='home')
response = render()
self.assertEquals(response.status_code, 200)
html = frappe.safe_decode(response.get_data())
soup = BeautifulSoup(html, 'html.parser')
sections = soup.find('main').find_all(class_='custom-section')
self.assertEqual(len(sections), 1)
homepage_section = sections[0]
self.assertEqual(homepage_section.text, 'My custom html')
# cleanup
frappe.db.rollback()
def populate_matching_invoices(self):
self.payment_invoice_items = []
self.map_unknown_transactions()
added_invoices = []
for entry in self.new_transaction_items:
if (not entry.party or entry.party_type == "Account"): continue
account = self.receivable_account if entry.party_type == "Customer" else self.payable_account
invoices = get_outstanding_invoices(entry.party_type, entry.party, account)
transaction_date = datetime.strptime(entry.transaction_date, "%Y-%m-%d").date()
outstanding_invoices = [invoice for invoice in invoices if invoice.posting_date <= transaction_date]
amount = abs(entry.amount)
matching_invoices = [invoice for invoice in outstanding_invoices if invoice.outstanding_amount == amount]
sorted(outstanding_invoices, key=lambda k: k['posting_date'])
for e in (matching_invoices + outstanding_invoices):
added = next((inv for inv in added_invoices if inv == e.get('voucher_no')), None)
if (added is not None): continue
ent = self.append('payment_invoice_items', {})
ent.transaction_date = entry.transaction_date
ent.payment_description = frappe.safe_decode(entry.description)
ent.party_type = entry.party_type
ent.party = entry.party
ent.invoice = e.get('voucher_no')
added_invoices += [ent.invoice]
ent.invoice_type = "Sales Invoice" if entry.party_type == "Customer" else "Purchase Invoice"
ent.invoice_date = e.get('posting_date')
ent.outstanding_amount = e.get('outstanding_amount')
ent.allocated_amount = min(float(e.get('outstanding_amount')), amount)
amount -= float(e.get('outstanding_amount'))
if (amount <= 5): break
self.match_invoice_to_payment()
self.populate_matching_vouchers()
self.map_transactions_on_journal_entry()
def validate_auth_via_api_keys(authorization_header):
"""
Authenticate request using API keys and set session user
Args:
authorization_header (list of str): The 'Authorization' header containing the prefix and token
"""
try:
auth_type, auth_token = authorization_header
if auth_type.lower() == 'basic':
api_key, api_secret = frappe.safe_decode(
base64.b64decode(auth_token)).split(":")
validate_api_key_secret(api_key, api_secret)
elif auth_type.lower() == 'token':
api_key, api_secret = auth_token.split(":")
validate_api_key_secret(api_key, api_secret)
except binascii.Error:
frappe.throw(
_("Failed to decode token, please provide a valid base64-encoded token."
), frappe.InvalidAuthorizationToken)
except (AttributeError, TypeError, ValueError):
frappe.throw(
_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."
), frappe.InvalidAuthorizationToken)
def map_transactions_on_journal_entry(self):
for entry in self.new_transaction_items:
vouchers = frappe.db.sql("""select name, posting_date from `tabJournal Entry`
where posting_date='{0}' and total_credit={1} and cheque_no='{2}' and docstatus != 2
""".format(entry.transaction_date, abs(entry.amount), frappe.safe_decode(entry.description)), as_dict=True)
if (len(vouchers) == 1):
entry.reference_name = vouchers[0].name
def log_touched_tables(self, query, values=None):
if values:
query = frappe.safe_decode(self._cursor.mogrify(query, values))
if query.strip().lower().split()[0] in ('insert', 'delete', 'update',
'alter'):
# single_word_regex is designed to match following patterns
# `tabXxx`, tabXxx and "tabXxx"
# multi_word_regex is designed to match following patterns
# `tabXxx Xxx` and "tabXxx Xxx"
# ([`"]?) Captures " or ` at the begining of the table name (if provided)
# \1 matches the first captured group (quote character) at the end of the table name
# multi word table name must have surrounding quotes.
# (tab([A-Z]\w+)( [A-Z]\w+)*) Captures table names that start with "tab"
# and are continued with multiple words that start with a captital letter
# e.g. 'tabXxx' or 'tabXxx Xxx' or 'tabXxx Xxx Xxx' and so on
single_word_regex = r'([`"]?)(tab([A-Z]\w+))\1'
multi_word_regex = r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1'
tables = []
for regex in (single_word_regex, multi_word_regex):
tables += [groups[1] for groups in re.findall(regex, query)]
if frappe.flags.touched_tables is None:
frappe.flags.touched_tables = set()
frappe.flags.touched_tables.update(tables)
def test_expose(self):
from frappe.utils.verified_command import verify_request
frappe.sendmail(recipients=['*****@*****.**'],
cc=['*****@*****.**'],
sender="*****@*****.**",
reference_doctype='User', reference_name="Administrator",
subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe", now=True)
email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1)
self.assertEqual(len(email_queue), 1)
queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient`
where status='Sent'""", as_dict=1)]
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
message = frappe.db.sql("""select message from `tabEmail Queue`
where status='Sent'""", as_dict=1)[0].message
self.assertTrue('<!--recipient-->' in message)
email_obj = email.message_from_string(frappe.safe_decode(frappe.flags.sent_mail))
for part in email_obj.walk():
content = part.get_payload(decode=True)
if content:
frappe.local.flags.signed_query_string = re.search('(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=\n)', content.decode()).group(0)
self.assertTrue(verify_request())
break
def populate_matching_invoices(self):
self.payment_invoice_items = []
self.map_unknown_transactions()
added_invoices = []
for entry in self.new_transaction_items:
if (not entry.party or entry.party_type == "Account"): continue
account = self.receivable_account if entry.party_type == "Customer" else self.payable_account
invoices = get_outstanding_invoices(entry.party_type, entry.party, account)
transaction_date = datetime.strptime(entry.transaction_date, "%Y-%m-%d").date()
outstanding_invoices = [invoice for invoice in invoices if invoice.posting_date <= transaction_date]
amount = abs(entry.amount)
matching_invoices = [invoice for invoice in outstanding_invoices if invoice.outstanding_amount == amount]
sorted(outstanding_invoices, key=lambda k: k['posting_date'])
for e in (matching_invoices + outstanding_invoices):
added = next((inv for inv in added_invoices if inv == e.get('voucher_no')), None)
if (added is not None): continue
ent = self.append('payment_invoice_items', {})
ent.transaction_date = entry.transaction_date
ent.payment_description = frappe.safe_decode(entry.description)
ent.party_type = entry.party_type
ent.party = entry.party
ent.invoice = e.get('voucher_no')
added_invoices += [ent.invoice]
ent.invoice_type = "Sales Invoice" if entry.party_type == "Customer" else "Purchase Invoice"
ent.invoice_date = e.get('posting_date')
ent.outstanding_amount = e.get('outstanding_amount')
ent.allocated_amount = min(float(e.get('outstanding_amount')), amount)
amount -= float(e.get('outstanding_amount'))
if (amount <= 5): break
self.match_invoice_to_payment()
self.populate_matching_vouchers()
self.map_transactions_on_journal_entry()
def __init__(self, content):
"""Parses headers, content, attachments from given raw message.
:param content: Raw message."""
self.raw = safe_encode(content) if six.PY2 else safe_decode(content)
self.mail = email.message_from_string(self.raw)
self.text_content = ''
self.html_content = ''
self.attachments = []
self.cid_map = {}
self.parse()
self.set_content_and_type()
self.set_subject()
self.set_from()
self.message_id = (self.mail.get('Message-ID') or "").strip(" <>")
if self.mail["Date"]:
try:
utc = email.utils.mktime_tz(
email.utils.parsedate_tz(self.mail["Date"]))
utc_dt = datetime.datetime.utcfromtimestamp(utc)
self.date = convert_utc_to_user_timezone(utc_dt).strftime(
'%Y-%m-%d %H:%M:%S')
except:
self.date = now()
else:
self.date = now()
if self.date > now():
self.date = now()
def to_cursor(self, row, sorting_fields):
# sorting_fields could be [custom_table.field_1],
# where only field_1 will be available on row
_json = frappe.as_json([
row.get(x.split('.')[1] if '.' in x else x) for x in sorting_fields
])
return frappe.safe_decode(base64.b64encode(_json.encode("utf-8")))
def map_transactions_on_journal_entry(self):
for entry in self.new_transaction_items:
vouchers = frappe.db.sql("""select name, posting_date from `tabJournal Entry`
where posting_date='{0}' and total_credit={1} and cheque_no='{2}' and docstatus != 2
""".format(entry.transaction_date, abs(entry.amount), frappe.safe_decode(entry.description)), as_dict=True)
if (len(vouchers) == 1):
entry.reference_name = vouchers[0].name
def get_web_image(file_url):
# download
file_url = frappe.utils.get_url(file_url)
r = requests.get(file_url, stream=True)
try:
r.raise_for_status()
except requests.exceptions.HTTPError as e:
if "404" in e.args[0]:
frappe.msgprint(_("File '{0}' not found").format(file_url))
else:
frappe.msgprint(
_("Unable to read file format for {0}").format(file_url))
raise
image = Image.open(StringIO(frappe.safe_decode(r.content)))
try:
filename, extn = file_url.rsplit("/", 1)[1].rsplit(".", 1)
except ValueError:
# the case when the file url doesn't have filename or extension
# but is fetched due to a query string. example: https://encrypted-tbn3.gstatic.com/images?q=something
filename = get_random_filename()
extn = None
extn = get_extension(filename, extn, r.content)
filename = "/files/" + strip(unquote(filename))
return image, filename, extn
def finalize_id_token(self, id_token, token, token_handler, request):
# Check whether frappe server URL is set
id_token_header = {"typ": "jwt", "alg": "HS256"}
user = frappe.get_doc(
"User",
frappe.session.user,
)
if request.nonce:
id_token["nonce"] = request.nonce
userinfo = get_userinfo(user)
if userinfo.get("iss"):
id_token["iss"] = userinfo.get("iss")
if "openid" in request.scopes:
id_token.update(userinfo)
id_token_encoded = jwt.encode(
payload=id_token,
key=request.client.client_secret,
algorithm="HS256",
headers=id_token_header,
)
return frappe.safe_decode(id_token_encoded)
def test_unsubscribe(self):
from frappe.email.queue import unsubscribe, send
unsubscribe(doctype="User", name="Administrator", email="*****@*****.**")
self.assertTrue(frappe.db.get_value("Email Unsubscribe",
{"reference_doctype": "User", "reference_name": "Administrator", "email": "*****@*****.**"}))
before = frappe.db.sql("""select count(name) from `tabEmail Queue` where status='Not Sent'""")[0][0]
send(recipients = ['*****@*****.**', '*****@*****.**'],
sender="*****@*****.**",
reference_doctype='User', reference_name= "Administrator",
subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe")
# this is sent async (?)
email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""",
as_dict=1)
self.assertEqual(len(email_queue), before + 1)
queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient`
where status='Not Sent'""", as_dict=1)]
self.assertFalse('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertEqual(len(queue_recipients), 1)
self.assertTrue('Unsubscribe' in frappe.safe_decode(frappe.flags.sent_mail))
def __init__(self, content):
"""Parses headers, content, attachments from given raw message.
:param content: Raw message."""
self.raw = safe_encode(content) if six.PY2 else safe_decode(content)
self.mail = email.message_from_string(self.raw)
self.text_content = ''
self.html_content = ''
self.attachments = []
self.cid_map = {}
self.parse()
self.set_content_and_type()
self.set_subject()
self.set_from()
self.message_id = (self.mail.get('Message-ID') or "").strip(" <>")
if self.mail["Date"]:
try:
utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))
utc_dt = datetime.datetime.utcfromtimestamp(utc)
self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S')
except:
self.date = now()
else:
self.date = now()
if self.date > now():
self.date = now()
def get_request_form_data():
if frappe.local.form_dict.data is None:
data = frappe.safe_decode(frappe.local.request.get_data())
else:
data = frappe.local.form_dict.data
return frappe.parse_json(data)
def test_homepage_section_custom_html(self):
frappe.get_doc({
"doctype":
"Homepage Section",
"name":
"Custom HTML Section",
"section_based_on":
"Custom HTML",
"section_html":
'<div class="custom-section">My custom html</div>',
}).insert()
set_request(method="GET", path="home")
response = get_response()
self.assertEqual(response.status_code, 200)
html = frappe.safe_decode(response.get_data())
soup = BeautifulSoup(html, "html.parser")
sections = soup.find("main").find_all(class_="custom-section")
self.assertEqual(len(sections), 1)
homepage_section = sections[0]
self.assertEqual(homepage_section.text, "My custom html")
# cleanup
frappe.db.rollback()
def validate_auth_via_api_keys(authorization_header):
"""
Authenticate request using API keys and set session user
Args:
authorization_header (list of str): The 'Authorization' header containing the prefix and token
"""
try:
auth_type, auth_token = authorization_header
authorization_source = frappe.get_request_header(
"Frappe-Authorization-Source")
if auth_type.lower() == "basic":
api_key, api_secret = frappe.safe_decode(
base64.b64decode(auth_token)).split(":")
validate_api_key_secret(api_key, api_secret, authorization_source)
elif auth_type.lower() == "token":
api_key, api_secret = auth_token.split(":")
validate_api_key_secret(api_key, api_secret, authorization_source)
except binascii.Error:
frappe.throw(
_("Failed to decode token, please provide a valid base64-encoded token."
),
frappe.InvalidAuthorizationToken,
)
except (AttributeError, TypeError, ValueError):
pass
def decode_email(self, email):
if not email: return
decoded = ""
for part, encoding in decode_header(frappe.as_unicode(email).replace("\""," ").replace("\'"," ")):
if encoding:
decoded += part.decode(encoding)
else:
decoded += safe_decode(part)
return decoded
def get_app_last_commit_ref(app):
try:
result = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD --short 7'.format(app),
shell=True)
result = safe_decode(result)
result = result.strip()
return result
except Exception:
return ''
def get_app_branch(app):
'''Returns branch of an app'''
try:
result = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app),
shell=True)
result = safe_decode(result)
result = result.strip()
return result
except Exception:
return ''
def load_assets(self):
from frappe.modules import get_module_path, scrub
import os
self.script = ''
page_name = scrub(self.name)
path = os.path.join(get_module_path(self.module), 'page', page_name)
# script
fpath = os.path.join(path, page_name + '.js')
if os.path.exists(fpath):
with open(fpath, 'r') as f:
self.script = render_include(f.read())
# css
fpath = os.path.join(path, page_name + '.css')
if os.path.exists(fpath):
with open(fpath, 'r') as f:
self.style = safe_decode(f.read())
# html as js template
for fname in os.listdir(path):
if fname.endswith(".html"):
with open(os.path.join(path, fname), 'r') as f:
template = f.read()
if "<!-- jinja -->" in template:
context = frappe._dict({})
try:
out = frappe.get_attr("{app}.{module}.page.{page}.{page}.get_context".format(
app = frappe.local.module_app[scrub(self.module)],
module = scrub(self.module),
page = page_name
))(context)
if out:
context = out
except (AttributeError, ImportError):
pass
template = frappe.render_template(template, context)
self.script = html_to_js_template(fname, template) + self.script
# flag for not caching this page
self._dynamic_page = True
if frappe.lang != 'en':
from frappe.translate import get_lang_js
self.script += get_lang_js("page", self.name)
for path in get_code_files_via_hooks("page_js", self.name):
js = get_js(path)
if js:
self.script += "\n\n" + js
def test_flush(self):
self.test_email_queue()
from frappe.email.queue import flush
flush(from_test=True)
email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1)
self.assertEqual(len(email_queue), 1)
queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient`
where status='Sent'""", as_dict=1)]
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertEqual(len(queue_recipients), 2)
self.assertTrue('Unsubscribe' in frappe.safe_decode(frappe.flags.sent_mail))
def test_page_load(self):
frappe.set_user('Guest')
set_request(method='POST', path='login')
response = render.render()
self.assertEquals(response.status_code, 200)
html = frappe.safe_decode(response.get_data())
self.assertTrue('/* login-css */' in html)
self.assertTrue('// login.js' in html)
self.assertTrue('<!-- login.html -->' in html)
frappe.set_user('Administrator')
def match_invoice_to_payment(self):
added_payments = []
for entry in self.new_transaction_items:
if (not entry.party or entry.party_type == "Account"): continue
entry.account = self.receivable_account if entry.party_type == "Customer" else self.payable_account
amount = abs(entry.amount)
payment, matching_invoices = None, []
for inv_entry in self.payment_invoice_items:
if (inv_entry.payment_description != frappe.safe_decode(entry.description) or inv_entry.transaction_date != entry.transaction_date): continue
if (inv_entry.party != entry.party): continue
matching_invoices += [inv_entry.invoice_type + "|" + inv_entry.invoice]
payment = get_payments_matching_invoice(inv_entry.invoice, entry.amount, entry.transaction_date)
doc = frappe.get_doc(inv_entry.invoice_type, inv_entry.invoice)
inv_entry.invoice_date = doc.posting_date
inv_entry.outstanding_amount = doc.outstanding_amount
inv_entry.allocated_amount = min(float(doc.outstanding_amount), amount)
amount -= inv_entry.allocated_amount
if (amount < 0): break
amount = abs(entry.amount)
if (payment is None):
order_doctype = "Sales Order" if entry.party_type=="Customer" else "Purchase Order"
from erpnext.controllers.accounts_controller import get_advance_payment_entries
payment_entries = get_advance_payment_entries(entry.party_type, entry.party, entry.account, order_doctype, against_all_orders=True)
payment_entries += self.get_matching_payments(entry.party, amount, entry.transaction_date)
payment = next((payment for payment in payment_entries if payment.amount == amount and payment not in added_payments), None)
if (payment is None):
print("Failed to find payments for {0}:{1}".format(entry.party, amount))
continue
added_payments += [payment]
entry.reference_type = payment.reference_type
entry.reference_name = payment.reference_name
entry.mode_of_payment = "Wire Transfer"
entry.outstanding_amount = min(amount, 0)
if (entry.payment_reference is None):
entry.payment_reference = frappe.safe_decode(entry.description)
entry.invoices = ",".join(matching_invoices)
def validate_auth_via_api_keys():
"""
authentication using api key and api secret
set user
"""
try:
authorization_header = frappe.get_request_header("Authorization", None).split(" ") if frappe.get_request_header("Authorization") else None
if authorization_header and authorization_header[0] == 'Basic':
token = frappe.safe_decode(base64.b64decode(authorization_header[1])).split(":")
validate_api_key_secret(token[0], token[1])
elif authorization_header and authorization_header[0] == 'token':
token = authorization_header[1].split(":")
validate_api_key_secret(token[0], token[1])
except Exception as e:
raise e
def read_csv_content(fcontent, ignore_encoding=False):
rows = []
if not isinstance(fcontent, text_type):
decoded = False
for encoding in ["utf-8", "windows-1250", "windows-1252"]:
try:
fcontent = text_type(fcontent, encoding)
decoded = True
break
except UnicodeDecodeError:
continue
if not decoded:
frappe.msgprint(_("Unknown file encoding. Tried utf-8, windows-1250, windows-1252."), raise_exception=True)
fcontent = fcontent.encode("utf-8")
content = [ ]
for line in fcontent.splitlines(True):
if six.PY2:
content.append(line)
else:
content.append(frappe.safe_decode(line))
try:
rows = []
for row in csv.reader(content):
r = []
for val in row:
# decode everything
val = val.strip()
if val=="":
# reason: in maraidb strict config, one cannot have blank strings for non string datatypes
r.append(None)
else:
r.append(val)
rows.append(r)
return rows
except Exception:
frappe.msgprint(_("Not a valid Comma Separated Value (CSV File)"))
raise
def create_payment_entries(self):
for payment_entry in self.new_transaction_items:
if (not payment_entry.party): continue
if (payment_entry.reference_name): continue
print("Creating payment entry for {0}".format(frappe.safe_decode(payment_entry.description)))
if (payment_entry.party_type == "Account"):
payment = self.create_journal_entry(payment_entry)
invoices = [payment.doctype + "|" + payment.name]
payment_entry.invoices = ",".join(invoices)
else:
payment = self.create_payment_entry(payment_entry)
invoices = [entry.reference_doctype + "|" + entry.reference_name for entry in payment.references if entry is not None]
payment_entry.invoices = ",".join(invoices)
payment_entry.mode_of_payment = payment.mode_of_payment
payment_entry.account = self.receivable_account if payment_entry.party_type == "Customer" else self.payable_account
payment_entry.reference_name = payment.name
payment_entry.reference_type = payment.doctype
frappe.msgprint(_("Successfully created payment entries"))
def send_sms(receiver_list, msg, sender_name = '', success_msg = True):
import json
if isinstance(receiver_list, string_types):
receiver_list = json.loads(receiver_list)
if not isinstance(receiver_list, list):
receiver_list = [receiver_list]
receiver_list = validate_receiver_nos(receiver_list)
arg = {
'receiver_list' : receiver_list,
'message' : frappe.safe_decode(msg).encode('utf-8'),
'success_msg' : success_msg
}
if frappe.db.get_value('SMS Settings', None, 'sms_gateway_url'):
send_via_gateway(arg)
else:
msgprint(_("Please Update SMS Settings"))
def verify_request():
"""Verify if the incoming signed request if it is correct."""
query_string = frappe.safe_decode(frappe.local.flags.signed_query_string or \
getattr(frappe.request, 'query_string', None))
valid = False
signature_string = '&_signature='
if signature_string in query_string:
params, signature = query_string.split(signature_string)
given_signature = hmac.new(params.encode('utf-8'))
given_signature.update(get_secret().encode())
valid = signature == given_signature.hexdigest()
if not valid:
frappe.respond_as_web_page(_("Invalid Link"),
_("This link is invalid or expired. Please make sure you have pasted correctly."))
return valid
def populate_payment_entries(self):
if self.bank_statement is None: return
filename = self.bank_statement.split("/")[-1]
if (len(self.new_transaction_items + self.reconciled_transaction_items) > 0):
frappe.throw("Transactions already retreived from the statement")
date_format = frappe.get_value("Bank Statement Settings", self.bank_settings, "date_format")
if (date_format is None):
date_format = '%Y-%m-%d'
if self.bank_settings:
mapped_items = frappe.get_doc("Bank Statement Settings", self.bank_settings).mapped_items
statement_headers = self.get_statement_headers()
transactions = get_transaction_entries(filename, statement_headers)
for entry in transactions:
date = entry[statement_headers["Date"]].strip()
#print("Processing entry DESC:{0}-W:{1}-D:{2}-DT:{3}".format(entry["Particulars"], entry["Withdrawals"], entry["Deposits"], entry["Date"]))
if (not date): continue
transaction_date = datetime.strptime(date, date_format).date()
if (self.from_date and transaction_date < datetime.strptime(self.from_date, '%Y-%m-%d').date()): continue
if (self.to_date and transaction_date > datetime.strptime(self.to_date, '%Y-%m-%d').date()): continue
bank_entry = self.append('new_transaction_items', {})
bank_entry.transaction_date = transaction_date
bank_entry.description = entry[statement_headers["Particulars"]]
mapped_item = next((entry for entry in mapped_items if entry.mapping_type == "Transaction" and frappe.safe_decode(entry.bank_data.lower()) in frappe.safe_decode(bank_entry.description.lower())), None)
if (mapped_item is not None):
bank_entry.party_type = mapped_item.mapped_data_type
bank_entry.party = mapped_item.mapped_data
else:
bank_entry.party_type = "Supplier" if not entry[statement_headers["Deposits"]].strip() else "Customer"
party_list = frappe.get_all(bank_entry.party_type, fields=["name"])
parties = [party.name for party in party_list]
matches = difflib.get_close_matches(frappe.safe_decode(bank_entry.description.lower()), parties, 1, 0.4)
if len(matches) > 0: bank_entry.party = matches[0]
bank_entry.amount = -float(entry[statement_headers["Withdrawals"]]) if not entry[statement_headers["Deposits"]].strip() else float(entry[statement_headers["Deposits"]])
self.map_unknown_transactions()
self.map_transactions_on_journal_entry()
def test_sendmail(self):
frappe.sendmail(sender="*****@*****.**", recipients="*****@*****.**",
content="test mail 001", subject="test-mail-001", delayed=False)
sent_mail = email.message_from_string(frappe.safe_decode(frappe.flags.sent_mail))
self.assertTrue("test-mail-001" in sent_mail.get("Subject"))
def get_feed(self):
return '{0}: {1}'.format(_(self.status), frappe.safe_decode(self.project_name))
def prepare_message(email, recipient, recipients_list):
message = email.message
if not message:
return ""
# Parse "Email Account" from "Email Sender"
email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender)
if frappe.conf.use_ssl and email_account.track_email_status:
# Using SSL => Publically available domain => Email Read Reciept Possible
message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode())
else:
# No SSL => No Email Read Reciept
message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode())
if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
email.unsubscribe_method, email.unsubscribe_params)
message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
if email.expose_recipients == "header":
pass
else:
if email.expose_recipients == "footer":
if isinstance(email.show_as_cc, string_types):
email.show_as_cc = email.show_as_cc.split(",")
email_sent_to = [r.recipient for r in recipients_list]
email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
if email_sent_cc:
email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
else:
email_sent_message = _("This email was sent to {0}").format(email_sent_to)
message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
message = message.replace("<!--recipient-->", recipient)
message = (message and message.encode('utf8')) or ''
message = safe_decode(message)
if not email.attachments:
return message
# On-demand attachments
from email.parser import Parser
msg_obj = Parser().parsestr(message)
attachments = json.loads(email.attachments)
for attachment in attachments:
if attachment.get('fcontent'): continue
fid = attachment.get("fid")
if fid:
fname, fcontent = get_file(fid)
attachment.update({
'fname': fname,
'fcontent': fcontent,
'parent': msg_obj
})
attachment.pop("fid", None)
add_attachment(**attachment)
elif attachment.get("print_format_attachment") == 1:
attachment.pop("print_format_attachment", None)
print_format_file = frappe.attach_print(**attachment)
print_format_file.update({"parent": msg_obj})
add_attachment(**print_format_file)
return msg_obj.as_string()
def validate_project_name(self):
if self.get("__islocal") and frappe.db.exists("Project", self.project_name):
frappe.throw(_("Project {0} already exists").format(frappe.safe_decode(self.project_name)))
def make_custom_fields(update=True):
invoice_item_fields = [
dict(fieldname='tax_rate', label='Tax Rate',
fieldtype='Float', insert_after='description',
print_hide=1, hidden=1, read_only=1),
dict(fieldname='tax_amount', label='Tax Amount',
fieldtype='Currency', insert_after='tax_rate',
print_hide=1, hidden=1, read_only=1, options="currency"),
dict(fieldname='total_amount', label='Total Amount',
fieldtype='Currency', insert_after='tax_amount',
print_hide=1, hidden=1, read_only=1, options="currency")
]
customer_po_fields = [
dict(fieldname='customer_po_details', label='Customer PO',
fieldtype='Section Break', insert_after='image'),
dict(fieldname='customer_po_no', label='Customer PO No',
fieldtype='Data', insert_after='customer_po_details',
fetch_from = 'sales_order.po_no',
print_hide=1, allow_on_submit=1, fetch_if_empty= 1, read_only=1, no_copy=1),
dict(fieldname='customer_po_clm_brk', label='',
fieldtype='Column Break', insert_after='customer_po_no',
print_hide=1, read_only=1),
dict(fieldname='customer_po_date', label='Customer PO Date',
fieldtype='Date', insert_after='customer_po_clm_brk',
fetch_from = 'sales_order.po_date',
print_hide=1, allow_on_submit=1, fetch_if_empty= 1, read_only=1, no_copy=1)
]
custom_fields = {
'Company': [
dict(fieldname='sb_e_invoicing', label='E-Invoicing',
fieldtype='Section Break', insert_after='date_of_establishment', print_hide=1),
dict(fieldname='fiscal_regime', label='Fiscal Regime',
fieldtype='Select', insert_after='sb_e_invoicing', print_hide=1,
options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), fiscal_regimes))),
dict(fieldname='fiscal_code', label='Fiscal Code', fieldtype='Data', insert_after='fiscal_regime', print_hide=1,
description=_("Applicable if the company is an Individual or a Proprietorship")),
dict(fieldname='vat_collectability', label='VAT Collectability',
fieldtype='Select', insert_after='fiscal_code', print_hide=1,
options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), vat_collectability_options))),
dict(fieldname='cb_e_invoicing1', fieldtype='Column Break', insert_after='vat_collectability', print_hide=1),
dict(fieldname='registrar_office_province', label='Province of the Registrar Office',
fieldtype='Data', insert_after='cb_e_invoicing1', print_hide=1, length=2),
dict(fieldname='registration_number', label='Registration Number',
fieldtype='Data', insert_after='registrar_office_province', print_hide=1, length=20),
dict(fieldname='share_capital_amount', label='Share Capital',
fieldtype='Currency', insert_after='registration_number', print_hide=1,
description=_('Applicable if the company is SpA, SApA or SRL')),
dict(fieldname='no_of_members', label='No of Members',
fieldtype='Select', insert_after='share_capital_amount', print_hide=1,
options="\nSU-Socio Unico\nSM-Piu Soci", description=_("Applicable if the company is a limited liability company")),
dict(fieldname='liquidation_state', label='Liquidation State',
fieldtype='Select', insert_after='no_of_members', print_hide=1,
options="\nLS-In Liquidazione\nLN-Non in Liquidazione")
],
'Sales Taxes and Charges': [
dict(fieldname='tax_exemption_reason', label='Tax Exemption Reason',
fieldtype='Select', insert_after='included_in_print_rate', print_hide=1,
depends_on='eval:doc.charge_type!="Actual" && doc.rate==0.0',
options="\n" + "\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), tax_exemption_reasons))),
dict(fieldname='tax_exemption_law', label='Tax Exempt Under',
fieldtype='Text', insert_after='tax_exemption_reason', print_hide=1,
depends_on='eval:doc.charge_type!="Actual" && doc.rate==0.0')
],
'Customer': [
dict(fieldname='fiscal_code', label='Fiscal Code', fieldtype='Data', insert_after='tax_id', print_hide=1),
dict(fieldname='recipient_code', label='Recipient Code',
fieldtype='Data', insert_after='fiscal_code', print_hide=1, default="0000000"),
dict(fieldname='pec', label='Recipient PEC',
fieldtype='Data', insert_after='fiscal_code', print_hide=1),
dict(fieldname='is_public_administration', label='Is Public Administration',
fieldtype='Check', insert_after='is_internal_customer', print_hide=1,
description=_("Set this if the customer is a Public Administration company."),
depends_on='eval:doc.customer_type=="Company"'),
dict(fieldname='first_name', label='First Name', fieldtype='Data',
insert_after='salutation', print_hide=1, depends_on='eval:doc.customer_type!="Company"'),
dict(fieldname='last_name', label='Last Name', fieldtype='Data',
insert_after='first_name', print_hide=1, depends_on='eval:doc.customer_type!="Company"')
],
'Mode of Payment': [
dict(fieldname='mode_of_payment_code', label='Code',
fieldtype='Select', insert_after='included_in_print_rate', print_hide=1,
options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), mode_of_payment_codes)))
],
'Payment Schedule': [
dict(fieldname='mode_of_payment_code', label='Code',
fieldtype='Select', insert_after='mode_of_payment', print_hide=1,
options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), mode_of_payment_codes)),
fetch_from="mode_of_payment.mode_of_payment_code", read_only=1),
dict(fieldname='bank_account', label='Bank Account',
fieldtype='Link', insert_after='mode_of_payment_code', print_hide=1,
options="Bank Account"),
dict(fieldname='bank_account_name', label='Bank Name',
fieldtype='Data', insert_after='bank_account', print_hide=1,
fetch_from="bank_account.bank", read_only=1),
dict(fieldname='bank_account_no', label='Bank Account No',
fieldtype='Data', insert_after='bank_account_name', print_hide=1,
fetch_from="bank_account.bank_account_no", read_only=1),
dict(fieldname='bank_account_iban', label='IBAN',
fieldtype='Data', insert_after='bank_account_name', print_hide=1,
fetch_from="bank_account.iban", read_only=1),
dict(fieldname='bank_account_swift_number', label='Swift Code (BIC)',
fieldtype='Data', insert_after='bank_account_iban', print_hide=1,
fetch_from="bank_account.swift_number", read_only=1),
],
"Sales Invoice": [
dict(fieldname='vat_collectability', label='VAT Collectability',
fieldtype='Select', insert_after='taxes_and_charges', print_hide=1,
options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), vat_collectability_options)),
fetch_from="company.vat_collectability"),
dict(fieldname='sb_e_invoicing_reference', label='E-Invoicing',
fieldtype='Section Break', insert_after='pos_total_qty', print_hide=1),
dict(fieldname='company_tax_id', label='Company Tax ID',
fieldtype='Data', insert_after='sb_e_invoicing_reference', print_hide=1, read_only=1,
fetch_from="company.tax_id"),
dict(fieldname='company_fiscal_code', label='Company Fiscal Code',
fieldtype='Data', insert_after='company_tax_id', print_hide=1, read_only=1,
fetch_from="company.fiscal_code"),
dict(fieldname='company_fiscal_regime', label='Company Fiscal Regime',
fieldtype='Data', insert_after='company_fiscal_code', print_hide=1, read_only=1,
fetch_from="company.fiscal_regime"),
dict(fieldname='cb_e_invoicing_reference', fieldtype='Column Break',
insert_after='company_fiscal_regime', print_hide=1),
dict(fieldname='customer_fiscal_code', label='Customer Fiscal Code',
fieldtype='Data', insert_after='cb_e_invoicing_reference', read_only=1,
fetch_from="customer.fiscal_code"),
],
'Purchase Invoice Item': invoice_item_fields,
'Sales Order Item': invoice_item_fields,
'Delivery Note Item': invoice_item_fields,
'Sales Invoice Item': invoice_item_fields + customer_po_fields,
'Quotation Item': invoice_item_fields,
'Purchase Order Item': invoice_item_fields,
'Purchase Receipt Item': invoice_item_fields,
'Supplier Quotation Item': invoice_item_fields,
'Address': [
dict(fieldname='country_code', label='Country Code',
fieldtype='Data', insert_after='country', print_hide=1, read_only=0,
fetch_from="country.code"),
dict(fieldname='state_code', label='State Code',
fieldtype='Data', insert_after='state', print_hide=1)
]
}
create_custom_fields(custom_fields, ignore_validate = frappe.flags.in_patch, update=update)
def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs): csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs) for row in csv_reader: yield [safe_decode(cell, 'utf-8') for cell in row]
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
skip_errors = True, data_import_doc=None, validate_template=False, user=None):
"""upload data"""
# for translations
if user:
frappe.cache().hdel("lang", user)
frappe.set_user_lang(user)
if data_import_doc and isinstance(data_import_doc, string_types):
data_import_doc = frappe.get_doc("Data Import", data_import_doc)
if data_import_doc and from_data_import == "Yes":
no_email = data_import_doc.no_email
ignore_encoding_errors = data_import_doc.ignore_encoding_errors
update_only = data_import_doc.only_update
submit_after_import = data_import_doc.submit_after_import
overwrite = data_import_doc.overwrite
skip_errors = data_import_doc.skip_errors
else:
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
frappe.flags.in_import = True
frappe.flags.mute_emails = no_email
def get_data_keys_definition():
return get_data_keys()
def bad_template():
frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator))
def check_data_length():
if not data:
frappe.throw(_("No data found in the file. Please reattach the new file with data."))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==get_data_keys_definition().data_separator:
return i+1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = list(filter(lambda x: x in ("", None), columns))
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"):
dt, parentfield = d, None
# xls format truncates the row, so it may not have more columns
if len(doctype_row) > i+2:
parentfield = doctype_row[i+2]
doctypes.append((dt, parentfield))
column_idx_to_fieldname[(dt, parentfield)] = {}
column_idx_to_fieldtype[(dt, parentfield)] = {}
if dt:
column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1]
column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1]
def get_doc(start_idx):
if doctypes:
doc = {}
attachments = []
last_error_row_idx = None
for idx in range(start_idx, len(rows)):
last_error_row_idx = idx # pylint: disable=W0612
if (not doc) or main_doc_empty(rows[idx]):
for dt, parentfield in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[(dt, parentfield)]:
try:
fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
if not fieldname or not rows[idx][column_idx]:
continue
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency", "Percent"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
if d[fieldname] and isinstance(d[fieldname], string_types):
d[fieldname] = getdate(parse_date(d[fieldname]))
elif fieldtype == "Datetime":
if d[fieldname]:
if " " in d[fieldname]:
_date, _time = d[fieldname].split()
else:
_date, _time = d[fieldname], '00:00:00'
_date = parse_date(d[fieldname])
d[fieldname] = get_datetime(_date + " " + _time)
else:
d[fieldname] = None
elif fieldtype in ("Image", "Attach Image", "Attach"):
# added file to attachments list
attachments.append(d[fieldname])
elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]:
# as fields can be saved in the number format(long type) in data import template
d[fieldname] = cstr(d[fieldname])
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite and doc.get("name"):
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = parentfield
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc, attachments, last_error_row_idx
else:
doc = frappe._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc, [], None
# used in testing whether a row is empty or parent row or child row
# checked only 3 first columns since first two columns can be blank for example the case of
# importing the item variant where item code and item name will be blank.
def main_doc_empty(row):
if row:
for i in range(3,0,-1):
if len(row) > i and row[i]:
return False
return True
def validate_naming(doc):
autoname = frappe.get_meta(doctype).autoname
if autoname:
if autoname[0:5] == 'field':
autoname = autoname[6:]
elif autoname == 'naming_series:':
autoname = 'naming_series'
else:
return True
if (autoname not in doc) or (not doc[autoname]):
from frappe.model.base_document import get_controller
if not hasattr(get_controller(doctype), "autoname"):
frappe.throw(_("{0} is a mandatory field".format(autoname)))
return True
users = frappe.db.sql_list("select name from tabUser")
def prepare_for_insert(doc):
# don't block data import if user is not set
# migrating from another system
if not doc.owner in users:
doc.owner = frappe.session.user
if not doc.modified_by in users:
doc.modified_by = frappe.session.user
def is_valid_url(url):
is_valid = False
if url.startswith("/files") or url.startswith("/private/files"):
url = get_url(url)
try:
r = requests.get(url)
is_valid = True if r.status_code == 200 else False
except Exception:
pass
return is_valid
def attach_file_to_doc(doctype, docname, file_url):
# check if attachment is already available
# check if the attachement link is relative or not
if not file_url:
return
if not is_valid_url(file_url):
return
files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and
attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format(
doctype=doctype,
docname=docname,
file_url=file_url
))
if files:
# file is already attached
return
save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
# header
filename, file_extension = ['','']
if not rows:
from frappe.utils.file_manager import get_file # get_file_doc
fname, fcontent = get_file(data_import_doc.import_file)
filename, file_extension = os.path.splitext(fname)
if file_extension == '.xlsx' and from_data_import == 'Yes':
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file)
elif file_extension == '.csv':
from frappe.utils.csvutils import read_csv_content
rows = read_csv_content(fcontent, ignore_encoding_errors)
else:
frappe.throw(_("Unsupported File Format"))
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
try:
doctype = get_header_row(get_data_keys_definition().main_table)[1]
columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:])
except:
frappe.throw(_("Cannot change header content"))
doctypes = []
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if skip_errors:
data_rows_with_error = header
if submit_after_import and not cint(frappe.db.get_value("DocType",
doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(get_data_keys_definition().parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not frappe.permissions.can_import(parenttype or doctype):
frappe.flags.mute_emails = False
return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
# Throw expception in case of the empty data file
check_data_length()
make_column_map()
total = len(data)
if validate_template:
if total:
data_import_doc.total_rows = total
return True
if overwrite==None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
parentfield = None
if parenttype:
parentfield = get_parent_field(doctype, parenttype)
if overwrite:
delete_child_rows(data, doctype)
import_log = []
def log(**kwargs):
if via_console:
print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
else:
import_log.append(kwargs)
def as_link(doctype, name):
if via_console:
return "{0}: {1}".format(doctype, name)
else:
return getlink(doctype, name)
# publish realtime task update
def publish_progress(achieved, reload=False):
if data_import_doc:
frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
error_flag = rollback_flag = False
batch_size = frappe.conf.data_import_batch_size or 1000
for batch_start in range(0, total, batch_size):
batch = data[batch_start:batch_start + batch_size]
for i, row in enumerate(batch):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
publish_progress(i)
try:
doc, attachments, last_error_row_idx = get_doc(row_idx)
validate_naming(doc)
if pre_process:
pre_process(doc)
original = None
if parentfield:
parent = frappe.get_doc(parenttype, doc["parent"])
doc = parent.append(parentfield, doc)
parent.save()
else:
if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
original_name = original.name
original.update(doc)
# preserve original name for case sensitivity
original.name = original_name
original.flags.ignore_links = ignore_links
original.save()
doc = original
else:
if not update_only:
doc = frappe.get_doc(doc)
prepare_for_insert(doc)
doc.flags.ignore_links = ignore_links
doc.insert()
if attachments:
# check file url and create a File document
for file_url in attachments:
attach_file_to_doc(doc.doctype, doc.name, file_url)
if submit_after_import:
doc.submit()
# log errors
if parentfield:
log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
"link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
elif submit_after_import:
log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"})
elif original:
log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
elif not update_only:
log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
else:
log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
"message": "Document updation ignored", "indicator": "orange"})
except Exception as e:
error_flag = True
# build error message
if frappe.local.message_log:
err_msg = "\n".join(['<p class="border-bottom small">{}</p>'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log])
else:
err_msg = '<p class="border-bottom small">{}</p>'.format(cstr(e))
error_trace = frappe.get_traceback()
if error_trace:
error_log_doc = frappe.log_error(error_trace)
error_link = get_url_to_form("Error Log", error_log_doc.name)
else:
error_link = None
log(**{
"row": row_idx + 1,
"title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""),
"message": err_msg,
"indicator": "red",
"link":error_link
})
# data with error to create a new file
# include the errored data in the last row as last_error_row_idx will not be updated for the last row
if skip_errors:
if last_error_row_idx == len(rows)-1:
last_error_row_idx = len(rows)
data_rows_with_error += rows[row_idx:last_error_row_idx]
else:
rollback_flag = True
finally:
frappe.local.message_log = []
start_row += batch_size
if rollback_flag:
frappe.db.rollback()
else:
frappe.db.commit()
frappe.flags.mute_emails = False
frappe.flags.in_import = False
log_message = {"messages": import_log, "error": error_flag}
if data_import_doc:
data_import_doc.log_details = json.dumps(log_message)
import_status = None
if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
import_status = "Partially Successful"
# write the file with the faulty row
from frappe.utils.file_manager import save_file
file_name = 'error_' + filename + file_extension
if file_extension == '.xlsx':
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
file_data = xlsx_file.getvalue()
else:
from frappe.utils.csvutils import to_csv
file_data = to_csv(data_rows_with_error)
error_data_file = save_file(file_name, file_data, "Data Import",
data_import_doc.name, "Home/Attachments")
data_import_doc.error_file = error_data_file.file_url
elif error_flag:
import_status = "Failed"
else:
import_status = "Successful"
data_import_doc.import_status = import_status
data_import_doc.save()
if data_import_doc.import_status in ["Successful", "Partially Successful"]:
data_import_doc.submit()
publish_progress(100, True)
else:
publish_progress(0, True)
frappe.db.commit()
else:
return log_message