def populate_matching_vouchers(self):
for entry in self.new_transaction_items:
if (not entry.party or entry.reference_name): continue
print("Finding matching voucher for {0}".format(dataent.safe_decode(entry.description)))
amount = abs(entry.amount)
invoices = []
vouchers = get_matching_journal_entries(self.from_date, self.to_date, entry.party, self.bank_account, amount)
if len(vouchers) == 0: continue
for voucher in vouchers:
added = next((entry.invoice for entry in self.payment_invoice_items if entry.invoice == voucher.voucher_no), None)
if (added):
print("Found voucher {0}".format(added))
continue
print("Adding voucher {0} {1} {2}".format(voucher.voucher_no, voucher.posting_date, voucher.debit))
ent = self.append('payment_invoice_items', {})
ent.invoice_date = voucher.posting_date
ent.invoice_type = "Journal Entry"
ent.invoice = voucher.voucher_no
ent.payment_description = dataent.safe_decode(entry.description)
ent.allocated_amount = max(voucher.debit, voucher.credit)
invoices += [ent.invoice_type + "|" + ent.invoice]
entry.reference_type = "Journal Entry"
entry.mode_of_payment = "Wire Transfer"
entry.reference_name = ent.invoice
#entry.account = entry.party
entry.invoices = ",".join(invoices)
break
def get_cached_user_pass():
'''Get user and password if set.'''
user = pwd = None
tmp_id = dataent.form_dict.get('tmp_id')
if tmp_id:
user = dataent.safe_decode(dataent.cache().get(tmp_id + '_usr'))
pwd = dataent.safe_decode(dataent.cache().get(tmp_id + '_pwd'))
return (user, pwd)
def set_subject(self):
"""Parse and decode `Subject` header."""
_subject = decode_header(self.mail.get("Subject", "No Subject"))
self.subject = _subject[0][0] or ""
if _subject[0][1]:
self.subject = safe_decode(self.subject, _subject[0][1])
else:
# assume that the encoding is utf-8
self.subject = safe_decode(self.subject)[:140]
if not self.subject:
self.subject = "No Subject"
def map_transactions_on_journal_entry(self):
for entry in self.new_transaction_items:
vouchers = dataent.db.sql("""select name, posting_date from `tabJournal Entry`
where posting_date='{0}' and total_credit={1} and cheque_no='{2}' and docstatus != 2
""".format(entry.transaction_date, abs(entry.amount), dataent.safe_decode(entry.description)), as_dict=True)
if (len(vouchers) == 1):
entry.reference_name = vouchers[0].name
def test_cc_footer(self):
#test if sending with cc's makes it into header
dataent.sendmail(recipients=['*****@*****.**'],
cc=['*****@*****.**'],
sender="*****@*****.**",
reference_doctype='User',
reference_name="Administrator",
subject='Testing Email Queue',
message='This is mail is queued!',
unsubscribe_message="Unsubscribe",
expose_recipients="footer",
now=True)
email_queue = dataent.db.sql(
"""select name from `tabEmail Queue` where status='Sent'""",
as_dict=1)
self.assertEqual(len(email_queue), 1)
queue_recipients = [
r.recipient for r in dataent.db.sql(
"""select recipient from `tabEmail Queue Recipient`
where status='Sent'""",
as_dict=1)
]
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue(
'This email was sent to [email protected] and copied to [email protected]'
in dataent.safe_decode(dataent.flags.sent_mail))
def populate_matching_invoices(self):
self.payment_invoice_items = []
self.map_unknown_transactions()
added_invoices = []
for entry in self.new_transaction_items:
if (not entry.party or entry.party_type == "Account"): continue
account = self.receivable_account if entry.party_type == "Customer" else self.payable_account
invoices = get_outstanding_invoices(entry.party_type, entry.party, account)
transaction_date = datetime.strptime(entry.transaction_date, "%Y-%m-%d").date()
outstanding_invoices = [invoice for invoice in invoices if invoice.posting_date <= transaction_date]
amount = abs(entry.amount)
matching_invoices = [invoice for invoice in outstanding_invoices if invoice.outstanding_amount == amount]
sorted(outstanding_invoices, key=lambda k: k['posting_date'])
for e in (matching_invoices + outstanding_invoices):
added = next((inv for inv in added_invoices if inv == e.get('voucher_no')), None)
if (added is not None): continue
ent = self.append('payment_invoice_items', {})
ent.transaction_date = entry.transaction_date
ent.payment_description = dataent.safe_decode(entry.description)
ent.party_type = entry.party_type
ent.party = entry.party
ent.invoice = e.get('voucher_no')
added_invoices += [ent.invoice]
ent.invoice_type = "Sales Invoice" if entry.party_type == "Customer" else "Purchase Invoice"
ent.invoice_date = e.get('posting_date')
ent.outstanding_amount = e.get('outstanding_amount')
ent.allocated_amount = min(float(e.get('outstanding_amount')), amount)
amount -= float(e.get('outstanding_amount'))
if (amount <= 5): break
self.match_invoice_to_payment()
self.populate_matching_vouchers()
self.map_transactions_on_journal_entry()
def execute():
""" Create Contact for each User if not present """
dataent.reload_doc('contacts', 'doctype', 'contact')
dataent.reload_doc('core', 'doctype', 'dynamic_link')
users = dataent.get_all(
'User',
filters={"name": ('not in', 'Administrator, Guest')},
fields=["*"])
for user in users:
if user.first_name:
user.first_name = re.sub("[<>]+", '',
dataent.safe_decode(user.first_name))
if user.last_name:
user.last_name = re.sub("[<>]+", '',
dataent.safe_decode(user.last_name))
create_contact(user, ignore_links=True, ignore_mandatory=True)
def sync_user_settings():
'''Sync from cache to database (called asynchronously via the browser)'''
for key, data in iteritems(dataent.cache().hgetall('_user_settings')):
key = safe_decode(key)
doctype, user = key.split('::') # WTF?
dataent.db.sql(
'''insert into __UserSettings (user, doctype, data) values (%s, %s, %s)
on duplicate key update data=%s''', (user, doctype, data, data))
def test_sendmail(self):
dataent.sendmail(sender="*****@*****.**",
recipients="*****@*****.**",
content="test mail 001",
subject="test-mail-001",
delayed=False)
sent_mail = email.message_from_string(
dataent.safe_decode(dataent.flags.sent_mail))
self.assertTrue("test-mail-001" in sent_mail.get("Subject"))
def get_app_last_commit_ref(app):
try:
result = subprocess.check_output(
'cd ../apps/{0} && git rev-parse HEAD --short 7'.format(app),
shell=True)
result = safe_decode(result)
result = result.strip()
return result
except Exception:
return ''
def decode_email(self, email):
if not email: return
decoded = ""
for part, encoding in decode_header(
dataent.as_unicode(email).replace("\"",
" ").replace("\'", " ")):
if encoding:
decoded += part.decode(encoding)
else:
decoded += safe_decode(part)
return decoded
def load_assets(self):
from dataent.modules import get_module_path, scrub
import os
self.script = ''
page_name = scrub(self.name)
path = os.path.join(get_module_path(self.module), 'page', page_name)
# script
fpath = os.path.join(path, page_name + '.js')
if os.path.exists(fpath):
with open(fpath, 'r') as f:
self.script = render_include(f.read())
# css
fpath = os.path.join(path, page_name + '.css')
if os.path.exists(fpath):
with open(fpath, 'r') as f:
self.style = safe_decode(f.read())
# html as js template
for fname in os.listdir(path):
if fname.endswith(".html"):
with open(os.path.join(path, fname), 'r') as f:
template = f.read()
if "<!-- jinja -->" in template:
context = dataent._dict({})
try:
out = dataent.get_attr("{app}.{module}.page.{page}.{page}.get_context".format(
app = dataent.local.module_app[scrub(self.module)],
module = scrub(self.module),
page = page_name
))(context)
if out:
context = out
except (AttributeError, ImportError):
pass
template = dataent.render_template(template, context)
self.script = html_to_js_template(fname, template) + self.script
# flag for not caching this page
self._dynamic_page = True
if dataent.lang != 'en':
from dataent.translate import get_lang_js
self.script += get_lang_js("page", self.name)
for path in get_code_files_via_hooks("page_js", self.name):
js = get_js(path)
if js:
self.script += "\n\n" + js
def get_app_branch(app):
'''Returns branch of an app'''
try:
result = subprocess.check_output(
'cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app),
shell=True)
result = safe_decode(result)
result = result.strip()
return result
except Exception:
return ''
def test_page_load(self):
dataent.set_user('Guest')
set_request(method='POST', path='login')
response = render.render()
self.assertEquals(response.status_code, 200)
html = dataent.safe_decode(response.get_data())
self.assertTrue('/* login-css */' in html)
self.assertTrue('// login.js' in html)
self.assertTrue('<!-- login.html -->' in html)
dataent.set_user('Administrator')
def match_invoice_to_payment(self):
added_payments = []
for entry in self.new_transaction_items:
if (not entry.party or entry.party_type == "Account"): continue
entry.account = self.receivable_account if entry.party_type == "Customer" else self.payable_account
amount = abs(entry.amount)
payment, matching_invoices = None, []
for inv_entry in self.payment_invoice_items:
if (inv_entry.payment_description != dataent.safe_decode(entry.description) or inv_entry.transaction_date != entry.transaction_date): continue
if (inv_entry.party != entry.party): continue
matching_invoices += [inv_entry.invoice_type + "|" + inv_entry.invoice]
payment = get_payments_matching_invoice(inv_entry.invoice, entry.amount, entry.transaction_date)
doc = dataent.get_doc(inv_entry.invoice_type, inv_entry.invoice)
inv_entry.invoice_date = doc.posting_date
inv_entry.outstanding_amount = doc.outstanding_amount
inv_entry.allocated_amount = min(float(doc.outstanding_amount), amount)
amount -= inv_entry.allocated_amount
if (amount < 0): break
amount = abs(entry.amount)
if (payment is None):
order_doctype = "Sales Order" if entry.party_type=="Customer" else "Purchase Order"
from epaas.controllers.accounts_controller import get_advance_payment_entries
payment_entries = get_advance_payment_entries(entry.party_type, entry.party, entry.account, order_doctype, against_all_orders=True)
payment_entries += self.get_matching_payments(entry.party, amount, entry.transaction_date)
payment = next((payment for payment in payment_entries if payment.amount == amount and payment not in added_payments), None)
if (payment is None):
print("Failed to find payments for {0}:{1}".format(entry.party, amount))
continue
added_payments += [payment]
entry.reference_type = payment.reference_type
entry.reference_name = payment.reference_name
entry.mode_of_payment = "Wire Transfer"
entry.outstanding_amount = min(amount, 0)
if (entry.payment_reference is None):
entry.payment_reference = dataent.safe_decode(entry.description)
entry.invoices = ",".join(matching_invoices)
def read_csv_content(fcontent, ignore_encoding=False):
rows = []
if not isinstance(fcontent, text_type):
decoded = False
for encoding in ["utf-8", "windows-1250", "windows-1252"]:
try:
fcontent = text_type(fcontent, encoding)
decoded = True
break
except UnicodeDecodeError:
continue
if not decoded:
dataent.msgprint(_("Unknown file encoding. Tried utf-8, windows-1250, windows-1252."), raise_exception=True)
fcontent = fcontent.encode("utf-8")
content = [ ]
for line in fcontent.splitlines(True):
if six.PY2:
content.append(line)
else:
content.append(dataent.safe_decode(line))
try:
rows = []
for row in csv.reader(content):
r = []
for val in row:
# decode everything
val = val.strip()
if val=="":
# reason: in maraidb strict config, one cannot have blank strings for non string datatypes
r.append(None)
else:
r.append(val)
rows.append(r)
return rows
except Exception:
dataent.msgprint(_("Not a valid Comma Separated Value (CSV File)"))
raise
def create_payment_entries(self):
for payment_entry in self.new_transaction_items:
if (not payment_entry.party): continue
if (payment_entry.reference_name): continue
print("Creating payment entry for {0}".format(dataent.safe_decode(payment_entry.description)))
if (payment_entry.party_type == "Account"):
payment = self.create_journal_entry(payment_entry)
invoices = [payment.doctype + "|" + payment.name]
payment_entry.invoices = ",".join(invoices)
else:
payment = self.create_payment_entry(payment_entry)
invoices = [entry.reference_doctype + "|" + entry.reference_name for entry in payment.references if entry is not None]
payment_entry.invoices = ",".join(invoices)
payment_entry.mode_of_payment = payment.mode_of_payment
payment_entry.account = self.receivable_account if payment_entry.party_type == "Customer" else self.payable_account
payment_entry.reference_name = payment.name
payment_entry.reference_type = payment.doctype
dataent.msgprint(_("Successfully created payment entries"))
def validate_auth_via_api_keys():
"""
authentication using api key and api secret
set user
"""
try:
authorization_header = dataent.get_request_header(
"Authorization", None).split(" ") if dataent.get_request_header(
"Authorization") else None
if authorization_header and authorization_header[0] == 'Basic':
token = dataent.safe_decode(
base64.b64decode(authorization_header[1])).split(":")
validate_api_key_secret(token[0], token[1])
elif authorization_header and authorization_header[0] == 'token':
token = authorization_header[1].split(":")
validate_api_key_secret(token[0], token[1])
except Exception as e:
raise e
def test_unsubscribe(self):
from dataent.email.queue import unsubscribe, send
unsubscribe(doctype="User",
name="Administrator",
email="*****@*****.**")
self.assertTrue(
dataent.db.get_value(
"Email Unsubscribe", {
"reference_doctype": "User",
"reference_name": "Administrator",
"email": "*****@*****.**"
}))
before = dataent.db.sql(
"""select count(name) from `tabEmail Queue` where status='Not Sent'"""
)[0][0]
send(recipients=['*****@*****.**', '*****@*****.**'],
sender="*****@*****.**",
reference_doctype='User',
reference_name="Administrator",
subject='Testing Email Queue',
message='This is mail is queued!',
unsubscribe_message="Unsubscribe")
# this is sent async (?)
email_queue = dataent.db.sql(
"""select name from `tabEmail Queue` where status='Not Sent'""",
as_dict=1)
self.assertEqual(len(email_queue), before + 1)
queue_recipients = [
r.recipient for r in dataent.db.sql(
"""select recipient from `tabEmail Queue Recipient`
where status='Not Sent'""",
as_dict=1)
]
self.assertFalse('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertEqual(len(queue_recipients), 1)
self.assertTrue(
'Unsubscribe' in dataent.safe_decode(dataent.flags.sent_mail))
def test_flush(self):
self.test_email_queue()
from dataent.email.queue import flush
flush(from_test=True)
email_queue = dataent.db.sql(
"""select name from `tabEmail Queue` where status='Sent'""",
as_dict=1)
self.assertEqual(len(email_queue), 1)
queue_recipients = [
r.recipient for r in dataent.db.sql(
"""select recipient from `tabEmail Queue Recipient`
where status='Sent'""",
as_dict=1)
]
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertEqual(len(queue_recipients), 2)
self.assertTrue(
'Unsubscribe' in dataent.safe_decode(dataent.flags.sent_mail))
def send_sms(receiver_list, msg, sender_name='', success_msg=True):
import json
if isinstance(receiver_list, string_types):
receiver_list = json.loads(receiver_list)
if not isinstance(receiver_list, list):
receiver_list = [receiver_list]
receiver_list = validate_receiver_nos(receiver_list)
arg = {
'receiver_list': receiver_list,
'message': dataent.safe_decode(msg).encode('utf-8'),
'success_msg': success_msg
}
if dataent.db.get_value('SMS Settings', None, 'sms_gateway_url'):
send_via_gateway(arg)
else:
msgprint(_("Please Update SMS Settings"))
def test_expose(self):
from dataent.utils.verified_command import verify_request
dataent.sendmail(recipients=['*****@*****.**'],
cc=['*****@*****.**'],
sender="*****@*****.**",
reference_doctype='User',
reference_name="Administrator",
subject='Testing Email Queue',
message='This is mail is queued!',
unsubscribe_message="Unsubscribe",
now=True)
email_queue = dataent.db.sql(
"""select name from `tabEmail Queue` where status='Sent'""",
as_dict=1)
self.assertEqual(len(email_queue), 1)
queue_recipients = [
r.recipient for r in dataent.db.sql(
"""select recipient from `tabEmail Queue Recipient`
where status='Sent'""",
as_dict=1)
]
self.assertTrue('*****@*****.**' in queue_recipients)
self.assertTrue('*****@*****.**' in queue_recipients)
message = dataent.db.sql("""select message from `tabEmail Queue`
where status='Sent'""",
as_dict=1)[0].message
self.assertTrue('<!--recipient-->' in message)
email_obj = email.message_from_string(
dataent.safe_decode(dataent.flags.sent_mail))
for part in email_obj.walk():
content = part.get_payload(decode=True)
if content:
dataent.local.flags.signed_query_string = re.search(
'(?<=/api/method/dataent.email.queue.unsubscribe\?).*(?=\n)',
content.decode()).group(0)
self.assertTrue(verify_request())
break
def verify_request():
"""Verify if the incoming signed request if it is correct."""
query_string = dataent.safe_decode(dataent.local.flags.signed_query_string or \
getattr(dataent.request, 'query_string', None))
valid = False
signature_string = '&_signature='
if signature_string in query_string:
params, signature = query_string.split(signature_string)
given_signature = hmac.new(params.encode('utf-8'))
given_signature.update(get_secret().encode())
valid = signature == given_signature.hexdigest()
if not valid:
dataent.respond_as_web_page(
_("Invalid Link"),
_("This link is invalid or expired. Please make sure you have pasted correctly."
))
return valid
def populate_payment_entries(self):
if self.bank_statement is None: return
filename = self.bank_statement.split("/")[-1]
if (len(self.new_transaction_items + self.reconciled_transaction_items) > 0):
dataent.throw(_("Transactions already retreived from the statement"))
date_format = dataent.get_value("Bank Statement Settings", self.bank_settings, "date_format")
if (date_format is None):
date_format = '%Y-%m-%d'
if self.bank_settings:
mapped_items = dataent.get_doc("Bank Statement Settings", self.bank_settings).mapped_items
statement_headers = self.get_statement_headers()
transactions = get_transaction_entries(filename, statement_headers)
for entry in transactions:
date = entry[statement_headers["Date"]].strip()
#print("Processing entry DESC:{0}-W:{1}-D:{2}-DT:{3}".format(entry["Particulars"], entry["Withdrawals"], entry["Deposits"], entry["Date"]))
if (not date): continue
transaction_date = datetime.strptime(date, date_format).date()
if (self.from_date and transaction_date < datetime.strptime(self.from_date, '%Y-%m-%d').date()): continue
if (self.to_date and transaction_date > datetime.strptime(self.to_date, '%Y-%m-%d').date()): continue
bank_entry = self.append('new_transaction_items', {})
bank_entry.transaction_date = transaction_date
bank_entry.description = entry[statement_headers["Particulars"]]
mapped_item = next((entry for entry in mapped_items if entry.mapping_type == "Transaction" and dataent.safe_decode(entry.bank_data.lower()) in dataent.safe_decode(bank_entry.description.lower())), None)
if (mapped_item is not None):
bank_entry.party_type = mapped_item.mapped_data_type
bank_entry.party = mapped_item.mapped_data
else:
bank_entry.party_type = "Supplier" if not entry[statement_headers["Deposits"]].strip() else "Customer"
party_list = dataent.get_all(bank_entry.party_type, fields=["name"])
parties = [party.name for party in party_list]
matches = difflib.get_close_matches(dataent.safe_decode(bank_entry.description.lower()), parties, 1, 0.4)
if len(matches) > 0: bank_entry.party = matches[0]
bank_entry.amount = -float(entry[statement_headers["Withdrawals"]]) if not entry[statement_headers["Deposits"]].strip() else float(entry[statement_headers["Deposits"]])
self.map_unknown_transactions()
self.map_transactions_on_journal_entry()
def upload(rows=None,
submit_after_import=None,
ignore_encoding_errors=False,
no_email=True,
overwrite=None,
update_only=None,
ignore_links=False,
pre_process=None,
via_console=False,
from_data_import="No",
skip_errors=True,
data_import_doc=None,
validate_template=False,
user=None):
"""upload data"""
# for translations
if user:
dataent.cache().hdel("lang", user)
dataent.set_user_lang(user)
if data_import_doc and isinstance(data_import_doc, string_types):
data_import_doc = dataent.get_doc("Data Import", data_import_doc)
if data_import_doc and from_data_import == "Yes":
no_email = data_import_doc.no_email
ignore_encoding_errors = data_import_doc.ignore_encoding_errors
update_only = data_import_doc.only_update
submit_after_import = data_import_doc.submit_after_import
overwrite = data_import_doc.overwrite
skip_errors = data_import_doc.skip_errors
else:
# extra input params
params = json.loads(dataent.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
dataent.flags.in_import = True
dataent.flags.mute_emails = no_email
def get_data_keys_definition():
return get_data_keys()
def bad_template():
dataent.throw(
_("Please do not change the rows above {0}").format(
get_data_keys_definition().data_separator))
def check_data_length():
if not data:
dataent.throw(
_("No data found in the file. Please reattach the new file with data."
))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0] == get_data_keys_definition().data_separator:
return i + 1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0] == key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = list(filter(lambda x: x in ("", None), columns))
if empty_cols:
if columns[-1 * len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1 * len(empty_cols)]
else:
dataent.msgprint(_(
"Please make sure that there are no empty columns in the file."
),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(
get_data_keys_definition().doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d and doctype_row[i] in (None, '', '~', '-',
_("DocType") + ":"):
dt, parentfield = d, None
# xls format truncates the row, so it may not have more columns
if len(doctype_row) > i + 2:
parentfield = doctype_row[i + 2]
doctypes.append((dt, parentfield))
column_idx_to_fieldname[(dt, parentfield)] = {}
column_idx_to_fieldtype[(dt, parentfield)] = {}
if dt:
column_idx_to_fieldname[(dt,
parentfield)][i +
1] = rows[row_idx +
2][i + 1]
column_idx_to_fieldtype[(dt,
parentfield)][i +
1] = rows[row_idx +
4][i + 1]
def get_doc(start_idx):
if doctypes:
doc = {}
attachments = []
last_error_row_idx = None
for idx in range(start_idx, len(rows)):
last_error_row_idx = idx # pylint: disable=W0612
if (not doc) or main_doc_empty(rows[idx]):
for dt, parentfield in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[(
dt, parentfield)]:
try:
fieldname = column_idx_to_fieldname[(
dt, parentfield)][column_idx]
fieldtype = column_idx_to_fieldtype[(
dt, parentfield)][column_idx]
if not fieldname or not rows[idx][column_idx]:
continue
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency",
"Percent"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
if d[fieldname] and isinstance(
d[fieldname], string_types):
d[fieldname] = getdate(
parse_date(d[fieldname]))
elif fieldtype == "Datetime":
if d[fieldname]:
if " " in d[fieldname]:
_date, _time = d[fieldname].split()
else:
_date, _time = d[
fieldname], '00:00:00'
_date = parse_date(d[fieldname])
d[fieldname] = get_datetime(_date +
" " +
_time)
else:
d[fieldname] = None
elif fieldtype in ("Image", "Attach Image",
"Attach"):
# added file to attachments list
attachments.append(d[fieldname])
elif fieldtype in ("Link", "Dynamic Link",
"Data") and d[fieldname]:
# as fields can be saved in the number format(long type) in data import template
d[fieldname] = cstr(d[fieldname])
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite and doc.get("name"):
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = parentfield
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc, attachments, last_error_row_idx
else:
doc = dataent._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc, [], None
# used in testing whether a row is empty or parent row or child row
# checked only 3 first columns since first two columns can be blank for example the case of
# importing the item variant where item code and item name will be blank.
def main_doc_empty(row):
if row:
for i in range(3, 0, -1):
if len(row) > i and row[i]:
return False
return True
def validate_naming(doc):
autoname = dataent.get_meta(doctype).autoname
if autoname:
if autoname[0:5] == 'field':
autoname = autoname[6:]
elif autoname == 'naming_series:':
autoname = 'naming_series'
else:
return True
if (autoname not in doc) or (not doc[autoname]):
from dataent.model.base_document import get_controller
if not hasattr(get_controller(doctype), "autoname"):
dataent.throw(
_("{0} is a mandatory field".format(autoname)))
return True
users = dataent.db.sql_list("select name from tabUser")
def prepare_for_insert(doc):
# don't block data import if user is not set
# migrating from another system
if not doc.owner in users:
doc.owner = dataent.session.user
if not doc.modified_by in users:
doc.modified_by = dataent.session.user
def is_valid_url(url):
is_valid = False
if url.startswith("/files") or url.startswith("/private/files"):
url = get_url(url)
try:
r = requests.get(url)
is_valid = True if r.status_code == 200 else False
except Exception:
pass
return is_valid
def attach_file_to_doc(doctype, docname, file_url):
# check if attachment is already available
# check if the attachement link is relative or not
if not file_url:
return
if not is_valid_url(file_url):
return
files = dataent.db.sql(
"""Select name from `tabFile` where attached_to_doctype='{doctype}' and
attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')"""
.format(doctype=doctype, docname=docname, file_url=file_url))
if files:
# file is already attached
return
save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
# header
filename, file_extension = ['', '']
if not rows:
from dataent.utils.file_manager import get_file # get_file_doc
fname, fcontent = get_file(data_import_doc.import_file)
filename, file_extension = os.path.splitext(fname)
if file_extension == '.xlsx' and from_data_import == 'Yes':
from dataent.utils.xlsxutils import read_xlsx_file_from_attached_file
rows = read_xlsx_file_from_attached_file(
file_id=data_import_doc.import_file)
elif file_extension == '.csv':
from dataent.utils.csvutils import read_csv_content
rows = read_csv_content(fcontent, ignore_encoding_errors)
else:
dataent.throw(_("Unsupported File Format"))
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
try:
doctype = get_header_row(get_data_keys_definition().main_table)[1]
columns = filter_empty_columns(
get_header_row(get_data_keys_definition().columns)[1:])
except:
dataent.throw(_("Cannot change header content"))
doctypes = []
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if skip_errors:
data_rows_with_error = header
if submit_after_import and not cint(
dataent.db.get_value("DocType", doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(get_data_keys_definition().parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not dataent.permissions.can_import(parenttype or doctype):
dataent.flags.mute_emails = False
return {
"messages": [_("Not allowed to Import") + ": " + _(doctype)],
"error": True
}
# Throw expception in case of the empty data file
check_data_length()
make_column_map()
total = len(data)
if validate_template:
if total:
data_import_doc.total_rows = total
return True
if overwrite == None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
parentfield = None
if parenttype:
parentfield = get_parent_field(doctype, parenttype)
if overwrite:
delete_child_rows(data, doctype)
import_log = []
def log(**kwargs):
if via_console:
print(
(kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
else:
import_log.append(kwargs)
def as_link(doctype, name):
if via_console:
return "{0}: {1}".format(doctype, name)
else:
return getlink(doctype, name)
# publish realtime task update
def publish_progress(achieved, reload=False):
if data_import_doc:
dataent.publish_realtime(
"data_import_progress", {
"progress": str(int(100.0 * achieved / total)),
"data_import": data_import_doc.name,
"reload": reload
},
user=dataent.session.user)
error_flag = rollback_flag = False
batch_size = dataent.conf.data_import_batch_size or 1000
for batch_start in range(0, total, batch_size):
batch = data[batch_start:batch_start + batch_size]
for i, row in enumerate(batch):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
publish_progress(i)
try:
doc, attachments, last_error_row_idx = get_doc(row_idx)
validate_naming(doc)
if pre_process:
pre_process(doc)
original = None
if parentfield:
parent = dataent.get_doc(parenttype, doc["parent"])
doc = parent.append(parentfield, doc)
parent.save()
else:
if overwrite and doc.get("name") and dataent.db.exists(
doctype, doc["name"]):
original = dataent.get_doc(doctype, doc["name"])
original_name = original.name
original.update(doc)
# preserve original name for case sensitivity
original.name = original_name
original.flags.ignore_links = ignore_links
original.save()
doc = original
else:
if not update_only:
doc = dataent.get_doc(doc)
prepare_for_insert(doc)
doc.flags.ignore_links = ignore_links
doc.insert()
if attachments:
# check file url and create a File document
for file_url in attachments:
attach_file_to_doc(doc.doctype, doc.name, file_url)
if submit_after_import:
doc.submit()
# log errors
if parentfield:
log(
**{
"row":
doc.idx,
"title":
'Inserted row for "%s"' %
(as_link(parenttype, doc.parent)),
"link":
get_absolute_url(parenttype, doc.parent),
"message":
'Document successfully saved',
"indicator":
"green"
})
elif submit_after_import:
log(
**{
"row":
row_idx + 1,
"title":
'Submitted row for "%s"' %
(as_link(doc.doctype, doc.name)),
"message":
"Document successfully submitted",
"link":
get_absolute_url(doc.doctype, doc.name),
"indicator":
"blue"
})
elif original:
log(
**{
"row":
row_idx + 1,
"title":
'Updated row for "%s"' %
(as_link(doc.doctype, doc.name)),
"message":
"Document successfully updated",
"link":
get_absolute_url(doc.doctype, doc.name),
"indicator":
"green"
})
elif not update_only:
log(
**{
"row":
row_idx + 1,
"title":
'Inserted row for "%s"' %
(as_link(doc.doctype, doc.name)),
"message":
"Document successfully saved",
"link":
get_absolute_url(doc.doctype, doc.name),
"indicator":
"green"
})
else:
log(
**{
"row": row_idx + 1,
"title": 'Ignored row for %s' % (row[1]),
"link": None,
"message": "Document updation ignored",
"indicator": "orange"
})
except Exception as e:
error_flag = True
# build error message
if dataent.local.message_log:
err_msg = "\n".join([
'<p class="border-bottom small">{}</p>'.format(
json.loads(msg).get('message'))
for msg in dataent.local.message_log
])
else:
err_msg = '<p class="border-bottom small">{}</p>'.format(
cstr(e))
error_trace = dataent.get_traceback()
if error_trace:
error_log_doc = dataent.log_error(error_trace)
error_link = get_absolute_url("Error Log",
error_log_doc.name)
else:
error_link = None
log(
**{
"row":
row_idx + 1,
"title":
'Error for row %s' %
(len(row) > 1 and dataent.safe_decode(row[1]) or ""),
"message":
err_msg,
"indicator":
"red",
"link":
error_link
})
# data with error to create a new file
# include the errored data in the last row as last_error_row_idx will not be updated for the last row
if skip_errors:
if last_error_row_idx == len(rows) - 1:
last_error_row_idx = len(rows)
data_rows_with_error += rows[row_idx:last_error_row_idx]
else:
rollback_flag = True
finally:
dataent.local.message_log = []
start_row += batch_size
if rollback_flag:
dataent.db.rollback()
else:
dataent.db.commit()
dataent.flags.mute_emails = False
dataent.flags.in_import = False
log_message = {"messages": import_log, "error": error_flag}
if data_import_doc:
data_import_doc.log_details = json.dumps(log_message)
import_status = None
if error_flag and data_import_doc.skip_errors and len(data) != len(
data_rows_with_error):
import_status = "Partially Successful"
# write the file with the faulty row
from dataent.utils.file_manager import save_file
file_name = 'error_' + filename + file_extension
if file_extension == '.xlsx':
from dataent.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(data_rows_with_error,
"Data Import Template")
file_data = xlsx_file.getvalue()
else:
from dataent.utils.csvutils import to_csv
file_data = to_csv(data_rows_with_error)
error_data_file = save_file(file_name, file_data, "Data Import",
data_import_doc.name,
"Home/Attachments")
data_import_doc.error_file = error_data_file.file_url
elif error_flag:
import_status = "Failed"
else:
import_status = "Successful"
data_import_doc.import_status = import_status
data_import_doc.save()
if data_import_doc.import_status in [
"Successful", "Partially Successful"
]:
data_import_doc.submit()
publish_progress(100, True)
else:
publish_progress(0, True)
dataent.db.commit()
else:
return log_message
def validate_project_name(self):
if self.get("__islocal") and dataent.db.exists("Project",
self.project_name):
dataent.throw(
_("Project {0} already exists").format(
dataent.safe_decode(self.project_name)))
def prepare_message(email, recipient, recipients_list):
message = email.message
if not message:
return ""
# Parse "Email Account" from "Email Sender"
email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender)
if dataent.conf.use_ssl and email_account.track_email_status:
# Using SSL => Publically available domain => Email Read Reciept Possible
message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/dataent.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(dataent.local.site, email.communication).encode()).decode())
else:
# No SSL => No Email Read Reciept
message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode())
if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
email.unsubscribe_method, email.unsubscribe_params)
message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
if email.expose_recipients == "header":
pass
else:
if email.expose_recipients == "footer":
if isinstance(email.show_as_cc, string_types):
email.show_as_cc = email.show_as_cc.split(",")
email_sent_to = [r.recipient for r in recipients_list]
email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
if email_sent_cc:
email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
else:
email_sent_message = _("This email was sent to {0}").format(email_sent_to)
message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
message = message.replace("<!--recipient-->", recipient)
message = (message and message.encode('utf8')) or ''
message = safe_decode(message)
if not email.attachments:
return message
# On-demand attachments
from email.parser import Parser
msg_obj = Parser().parsestr(message)
attachments = json.loads(email.attachments)
for attachment in attachments:
if attachment.get('fcontent'): continue
fid = attachment.get("fid")
if fid:
fname, fcontent = get_file(fid)
attachment.update({
'fname': fname,
'fcontent': fcontent,
'parent': msg_obj
})
attachment.pop("fid", None)
add_attachment(**attachment)
elif attachment.get("print_format_attachment") == 1:
attachment.pop("print_format_attachment", None)
print_format_file = dataent.attach_print(**attachment)
print_format_file.update({"parent": msg_obj})
add_attachment(**print_format_file)
return msg_obj.as_string()
def move_reconciled_entries(self):
idx = 0
while idx < len(self.new_transaction_items):
entry = self.new_transaction_items[idx]
try:
print("Checking transaction {0}: {2} in {1} entries".format(idx, len(self.new_transaction_items), dataent.safe_decode(entry.description)))
except UnicodeEncodeError:
pass
idx += 1
if entry.reference_name is None: continue
doc = dataent.get_doc(entry.reference_type, entry.reference_name)
if doc.docstatus == 1 and (entry.reference_type == "Journal Entry" or doc.unallocated_amount == 0):
self.remove(entry)
rc_entry = self.append('reconciled_transaction_items', {})
dentry = entry.as_dict()
dentry.pop('idx', None)
rc_entry.update(dentry)
idx -= 1
def get_feed(self):
return '{0}: {1}'.format(_(self.status),
dataent.safe_decode(self.project_name))
def make_custom_fields(update=True):
invoice_item_fields = [
dict(fieldname='tax_rate',
label='Tax Rate',
fieldtype='Float',
insert_after='description',
print_hide=1,
hidden=1,
read_only=1),
dict(fieldname='tax_amount',
label='Tax Amount',
fieldtype='Currency',
insert_after='tax_rate',
print_hide=1,
hidden=1,
read_only=1,
options="currency"),
dict(fieldname='total_amount',
label='Total Amount',
fieldtype='Currency',
insert_after='tax_amount',
print_hide=1,
hidden=1,
read_only=1,
options="currency")
]
customer_po_fields = [
dict(fieldname='customer_po_details',
label='Customer PO',
fieldtype='Section Break',
insert_after='image'),
dict(fieldname='customer_po_no',
label='Customer PO No',
fieldtype='Data',
insert_after='customer_po_details',
fetch_from='sales_order.po_no',
print_hide=1,
allow_on_submit=1,
fetch_if_empty=1,
read_only=1,
no_copy=1),
dict(fieldname='customer_po_clm_brk',
label='',
fieldtype='Column Break',
insert_after='customer_po_no',
print_hide=1,
read_only=1),
dict(fieldname='customer_po_date',
label='Customer PO Date',
fieldtype='Date',
insert_after='customer_po_clm_brk',
fetch_from='sales_order.po_date',
print_hide=1,
allow_on_submit=1,
fetch_if_empty=1,
read_only=1,
no_copy=1)
]
custom_fields = {
'Company': [
dict(fieldname='sb_e_invoicing',
label='E-Invoicing',
fieldtype='Section Break',
insert_after='date_of_establishment',
print_hide=1),
dict(fieldname='fiscal_regime',
label='Fiscal Regime',
fieldtype='Select',
insert_after='sb_e_invoicing',
print_hide=1,
options="\n".join(
map(lambda x: dataent.safe_decode(x, encoding='utf-8'),
fiscal_regimes))),
dict(
fieldname='fiscal_code',
label='Fiscal Code',
fieldtype='Data',
insert_after='fiscal_regime',
print_hide=1,
description=
_("Applicable if the company is an Individual or a Proprietorship"
)),
dict(fieldname='vat_collectability',
label='VAT Collectability',
fieldtype='Select',
insert_after='fiscal_code',
print_hide=1,
options="\n".join(
map(lambda x: dataent.safe_decode(x, encoding='utf-8'),
vat_collectability_options))),
dict(fieldname='cb_e_invoicing1',
fieldtype='Column Break',
insert_after='vat_collectability',
print_hide=1),
dict(fieldname='registrar_office_province',
label='Province of the Registrar Office',
fieldtype='Data',
insert_after='cb_e_invoicing1',
print_hide=1,
length=2),
dict(fieldname='registration_number',
label='Registration Number',
fieldtype='Data',
insert_after='registrar_office_province',
print_hide=1,
length=20),
dict(fieldname='share_capital_amount',
label='Share Capital',
fieldtype='Currency',
insert_after='registration_number',
print_hide=1,
description=_(
'Applicable if the company is SpA, SApA or SRL')),
dict(fieldname='no_of_members',
label='No of Members',
fieldtype='Select',
insert_after='share_capital_amount',
print_hide=1,
options="\nSU-Socio Unico\nSM-Piu Soci",
description=_(
"Applicable if the company is a limited liability company"
)),
dict(fieldname='liquidation_state',
label='Liquidation State',
fieldtype='Select',
insert_after='no_of_members',
print_hide=1,
options="\nLS-In Liquidazione\nLN-Non in Liquidazione")
],
'Sales Taxes and Charges': [
dict(fieldname='tax_exemption_reason',
label='Tax Exemption Reason',
fieldtype='Select',
insert_after='included_in_print_rate',
print_hide=1,
depends_on='eval:doc.charge_type!="Actual" && doc.rate==0.0',
options="\n" + "\n".join(
map(lambda x: dataent.safe_decode(x, encoding='utf-8'),
tax_exemption_reasons))),
dict(fieldname='tax_exemption_law',
label='Tax Exempt Under',
fieldtype='Text',
insert_after='tax_exemption_reason',
print_hide=1,
depends_on='eval:doc.charge_type!="Actual" && doc.rate==0.0')
],
'Customer': [
dict(fieldname='fiscal_code',
label='Fiscal Code',
fieldtype='Data',
insert_after='tax_id',
print_hide=1),
dict(fieldname='recipient_code',
label='Recipient Code',
fieldtype='Data',
insert_after='fiscal_code',
print_hide=1,
default="0000000"),
dict(fieldname='pec',
label='Recipient PEC',
fieldtype='Data',
insert_after='fiscal_code',
print_hide=1),
dict(
fieldname='is_public_administration',
label='Is Public Administration',
fieldtype='Check',
insert_after='is_internal_customer',
print_hide=1,
description=
_("Set this if the customer is a Public Administration company."
),
depends_on='eval:doc.customer_type=="Company"'),
dict(fieldname='first_name',
label='First Name',
fieldtype='Data',
insert_after='salutation',
print_hide=1,
depends_on='eval:doc.customer_type!="Company"'),
dict(fieldname='last_name',
label='Last Name',
fieldtype='Data',
insert_after='first_name',
print_hide=1,
depends_on='eval:doc.customer_type!="Company"')
],
'Mode of Payment': [
dict(fieldname='mode_of_payment_code',
label='Code',
fieldtype='Select',
insert_after='included_in_print_rate',
print_hide=1,
options="\n".join(
map(lambda x: dataent.safe_decode(x, encoding='utf-8'),
mode_of_payment_codes)))
],
'Payment Schedule': [
dict(fieldname='mode_of_payment_code',
label='Code',
fieldtype='Select',
insert_after='mode_of_payment',
print_hide=1,
options="\n".join(
map(lambda x: dataent.safe_decode(x, encoding='utf-8'),
mode_of_payment_codes)),
fetch_from="mode_of_payment.mode_of_payment_code",
read_only=1),
dict(fieldname='bank_account',
label='Bank Account',
fieldtype='Link',
insert_after='mode_of_payment_code',
print_hide=1,
options="Bank Account"),
dict(fieldname='bank_account_name',
label='Bank Name',
fieldtype='Data',
insert_after='bank_account',
print_hide=1,
fetch_from="bank_account.bank",
read_only=1),
dict(fieldname='bank_account_no',
label='Bank Account No',
fieldtype='Data',
insert_after='bank_account_name',
print_hide=1,
fetch_from="bank_account.bank_account_no",
read_only=1),
dict(fieldname='bank_account_iban',
label='IBAN',
fieldtype='Data',
insert_after='bank_account_name',
print_hide=1,
fetch_from="bank_account.iban",
read_only=1),
dict(fieldname='bank_account_swift_number',
label='Swift Code (BIC)',
fieldtype='Data',
insert_after='bank_account_iban',
print_hide=1,
fetch_from="bank_account.swift_number",
read_only=1),
],
"Sales Invoice": [
dict(fieldname='vat_collectability',
label='VAT Collectability',
fieldtype='Select',
insert_after='taxes_and_charges',
print_hide=1,
options="\n".join(
map(lambda x: dataent.safe_decode(x, encoding='utf-8'),
vat_collectability_options)),
fetch_from="company.vat_collectability"),
dict(fieldname='sb_e_invoicing_reference',
label='E-Invoicing',
fieldtype='Section Break',
insert_after='pos_total_qty',
print_hide=1),
dict(fieldname='company_tax_id',
label='Company Tax ID',
fieldtype='Data',
insert_after='sb_e_invoicing_reference',
print_hide=1,
read_only=1,
fetch_from="company.tax_id"),
dict(fieldname='company_fiscal_code',
label='Company Fiscal Code',
fieldtype='Data',
insert_after='company_tax_id',
print_hide=1,
read_only=1,
fetch_from="company.fiscal_code"),
dict(fieldname='company_fiscal_regime',
label='Company Fiscal Regime',
fieldtype='Data',
insert_after='company_fiscal_code',
print_hide=1,
read_only=1,
fetch_from="company.fiscal_regime"),
dict(fieldname='cb_e_invoicing_reference',
fieldtype='Column Break',
insert_after='company_fiscal_regime',
print_hide=1),
dict(fieldname='customer_fiscal_code',
label='Customer Fiscal Code',
fieldtype='Data',
insert_after='cb_e_invoicing_reference',
read_only=1,
fetch_from="customer.fiscal_code"),
],
'Purchase Invoice Item':
invoice_item_fields,
'Sales Order Item':
invoice_item_fields,
'Delivery Note Item':
invoice_item_fields,
'Sales Invoice Item':
invoice_item_fields + customer_po_fields,
'Quotation Item':
invoice_item_fields,
'Purchase Order Item':
invoice_item_fields,
'Purchase Receipt Item':
invoice_item_fields,
'Supplier Quotation Item':
invoice_item_fields,
'Address': [
dict(fieldname='country_code',
label='Country Code',
fieldtype='Data',
insert_after='country',
print_hide=1,
read_only=0,
fetch_from="country.code"),
dict(fieldname='state_code',
label='State Code',
fieldtype='Data',
insert_after='state',
print_hide=1)
]
}
create_custom_fields(custom_fields,
ignore_validate=dataent.flags.in_patch,
update=update)
