def execute(): """ Create Contact for each User if not present """ frappe.reload_doc('integrations', 'doctype', 'google_contacts') frappe.reload_doc('contacts', 'doctype', 'contact') frappe.reload_doc('core', 'doctype', 'dynamic_link') frappe.reload_doc('communication', 'doctype', 'call_log') contact_meta = frappe.get_meta("Contact") if contact_meta.has_field("phone_nos") and contact_meta.has_field( "email_ids"): frappe.reload_doc('contacts', 'doctype', 'contact_phone') frappe.reload_doc('contacts', 'doctype', 'contact_email') users = frappe.get_all( 'User', filters={"name": ('not in', 'Administrator, Guest')}, fields=["*"]) for user in users: if frappe.db.exists("Contact", {"email_id": user.email}) or frappe.db.exists( "Contact Email", {"email_id": user.email}): continue if user.first_name: user.first_name = re.sub("[<>]+", '', frappe.safe_decode(user.first_name)) if user.last_name: user.last_name = re.sub("[<>]+", '', frappe.safe_decode(user.last_name)) create_contact(user, ignore_links=True, ignore_mandatory=True)
def populate_matching_vouchers(self): for entry in self.new_transaction_items: if (not entry.party or entry.reference_name): continue print("Finding matching voucher for {0}".format( frappe.safe_decode(entry.description))) amount = abs(entry.amount) invoices = [] vouchers = get_matching_journal_entries(self.from_date, self.to_date, entry.party, self.bank_account, amount) if len(vouchers) == 0: continue for voucher in vouchers: added = next((entry.invoice for entry in self.payment_invoice_items if entry.invoice == voucher.voucher_no), None) if (added): print("Found voucher {0}".format(added)) continue print("Adding voucher {0} {1} {2}".format( voucher.voucher_no, voucher.posting_date, voucher.debit)) ent = self.append('payment_invoice_items', {}) ent.invoice_date = voucher.posting_date ent.invoice_type = "Journal Entry" ent.invoice = voucher.voucher_no ent.payment_description = frappe.safe_decode(entry.description) ent.allocated_amount = max(voucher.debit, voucher.credit) invoices += [ent.invoice_type + "|" + ent.invoice] entry.reference_type = "Journal Entry" entry.mode_of_payment = "Wire Transfer" entry.reference_name = ent.invoice #entry.account = entry.party entry.invoices = ",".join(invoices) break
def generate_sms_pin(): mobile = frappe.local.form_dict.mobile newPIN = cint(frappe.local.form_dict.newPIN or "0") # If mobile needs to automatically the received hash = frappe.local.form_dict.hash if not mobile: frappe.throw("No Mobile Number") # check cache for pin pin = frappe.safe_decode(frappe.cache().get("sms:" + mobile)) user = get_linked_user(mobile) if not pin and user: # check if available in db pin = frappe.db.get_value("User", user, "renovation_sms_pin") if not pin or newPIN: # generate a pin pin = frappe.safe_decode(str(get_pin())) frappe.cache().set("sms:" + mobile, pin) # save in User doc if mobile linked to any User if user: frappe.db.set_value("User", user, "renovation_sms_pin", pin, update_modified=False) msg = u"Your verification OTP is: " + pin if hash: msg = msg + u". " + hash sms = send_sms([mobile], msg, success_msg=False) status = "fail" if sms and isinstance(sms, list) and mobile in sms: status = "success" update_http_response({"status": status, "mobile": mobile})
def execute(): """Create Contact for each User if not present""" frappe.reload_doc("integrations", "doctype", "google_contacts") frappe.reload_doc("contacts", "doctype", "contact") frappe.reload_doc("core", "doctype", "dynamic_link") contact_meta = frappe.get_meta("Contact") if contact_meta.has_field("phone_nos") and contact_meta.has_field( "email_ids"): frappe.reload_doc("contacts", "doctype", "contact_phone") frappe.reload_doc("contacts", "doctype", "contact_email") users = frappe.get_all( "User", filters={"name": ("not in", "Administrator, Guest")}, fields=["*"]) for user in users: if frappe.db.exists("Contact", {"email_id": user.email}) or frappe.db.exists( "Contact Email", {"email_id": user.email}): continue if user.first_name: user.first_name = re.sub("[<>]+", "", frappe.safe_decode(user.first_name)) if user.last_name: user.last_name = re.sub("[<>]+", "", frappe.safe_decode(user.last_name)) create_contact(user, ignore_links=True, ignore_mandatory=True)
def get_desk_assets(build_version): """Get desk assets to be loaded for mobile app""" data = get_context({"for_mobile": True}) assets = [{"type": "js", "data": ""}, {"type": "css", "data": ""}] if build_version != data["build_version"]: # new build, send assets for path in data["include_js"]: # assets path shouldn't start with / # as it points to different location altogether if path.startswith('/assets/'): path = path.replace('/assets/', 'assets/') try: with open(os.path.join(frappe.local.sites_path, path) ,"r") as f: assets[0]["data"] = assets[0]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8") except IOError: pass for path in data["include_css"]: if path.startswith('/assets/'): path = path.replace('/assets/', 'assets/') try: with open(os.path.join(frappe.local.sites_path, path) ,"r") as f: assets[1]["data"] = assets[1]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8") except IOError: pass return { "build_version": data["build_version"], "boot": data["boot"], "assets": assets }
def migrate(): check_if_admin() can_migrate() status = frappe.safe_decode(frappe.cache().get(migration_status_key)) error = frappe.safe_decode(frappe.cache().get(migration_error_key)) if status == migration_status_migrating: return "Migration in progress" elif status == migration_status_done: frappe.cache().set(migration_status_key, migration_status_done + "_seen") return { "status": "SUCCESS: Last Migration was successfull, Please send a new request to restart migration", "error": error } elif status == migration_status_error: frappe.cache().set(migration_status_key, migration_status_error + "_seen") return { "status": "ERROR: Last Migration wasnt successfull, Please send a new request to retry", "error": error } frappe.enqueue("renovation_core.utils.site.start_migration") return { "status": "Started migration", "last_migration": { "status": status, "error": error } }
def test_category_link(self): # Make a temporary Blog Post (and a Blog Category) blog = make_test_blog() # Visit the blog post page set_request(path=blog.route) blog_page_response = render() blog_page_html = frappe.safe_decode(blog_page_response.get_data()) # On blog post page find link to the category page soup = BeautifulSoup(blog_page_html, "lxml") category_page_link = list( soup.find_all('a', href=re.compile(blog.blog_category)))[0] category_page_url = category_page_link["href"] # Visit the category page (by following the link found in above stage) set_request(path=category_page_url) category_page_response = render() category_page_html = frappe.safe_decode( category_page_response.get_data()) # Category page should contain the blog post title self.assertIn(blog.title, category_page_html) # Cleanup afterwords frappe.delete_doc("Blog Post", blog.name) frappe.delete_doc("Blog Category", blog.blog_category)
def test_cc_footer(self): frappe.conf.use_ssl = True # test if sending with cc's makes it into header frappe.sendmail(recipients=['*****@*****.**'], cc=['*****@*****.**'], sender="*****@*****.**", reference_doctype='User', reference_name="Administrator", subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe", expose_recipients="footer", now=True) email_queue = frappe.db.sql( """select name from `tabEmail Queue` where status='Sent'""", as_dict=1) self.assertEqual(len(email_queue), 1) queue_recipients = [ r.recipient for r in frappe.db.sql( """select recipient from `tabEmail Queue Recipient` where status='Sent'""", as_dict=1) ] self.assertTrue('*****@*****.**' in queue_recipients) self.assertTrue('*****@*****.**' in queue_recipients) self.assertTrue( 'This email was sent to [email protected] and copied to [email protected]' in frappe.safe_decode(frappe.flags.sent_mail)) # check for email tracker self.assertTrue( 'mark_email_as_seen' in frappe.safe_decode(frappe.flags.sent_mail)) frappe.conf.use_ssl = False
def populate_matching_vouchers(self): for entry in self.new_transaction_items: if (not entry.party or entry.reference_name): continue print("Finding matching voucher for {0}".format(frappe.safe_decode(entry.description))) amount = abs(entry.amount) invoices = [] vouchers = get_matching_journal_entries(self.from_date, self.to_date, entry.party, self.bank_account, amount) if len(vouchers) == 0: continue for voucher in vouchers: added = next((entry.invoice for entry in self.payment_invoice_items if entry.invoice == voucher.voucher_no), None) if (added): print("Found voucher {0}".format(added)) continue print("Adding voucher {0} {1} {2}".format(voucher.voucher_no, voucher.posting_date, voucher.debit)) ent = self.append('payment_invoice_items', {}) ent.invoice_date = voucher.posting_date ent.invoice_type = "Journal Entry" ent.invoice = voucher.voucher_no ent.payment_description = frappe.safe_decode(entry.description) ent.allocated_amount = max(voucher.debit, voucher.credit) invoices += [ent.invoice_type + "|" + ent.invoice] entry.reference_type = "Journal Entry" entry.mode_of_payment = "Wire Transfer" entry.reference_name = ent.invoice #entry.account = entry.party entry.invoices = ",".join(invoices) break
def get_cached_user_pass(): '''Get user and password if set.''' user = pwd = None tmp_id = frappe.form_dict.get('tmp_id') if tmp_id: user = frappe.safe_decode(frappe.cache().get(tmp_id + '_usr')) pwd = frappe.safe_decode(frappe.cache().get(tmp_id + '_pwd')) return (user, pwd)
def get_cached_user_pass(): """Get user and password if set.""" user = pwd = None tmp_id = frappe.form_dict.get("tmp_id") if tmp_id: user = frappe.safe_decode(frappe.cache().get(tmp_id + "_usr")) pwd = frappe.safe_decode(frappe.cache().get(tmp_id + "_pwd")) return (user, pwd)
def populate_payment_entries(self): if self.bank_statement is None: return file_url = self.bank_statement if (len(self.new_transaction_items + self.reconciled_transaction_items) > 0): frappe.throw( _("Transactions already retreived from the statement")) date_format = frappe.get_value("Bank Statement Settings", self.bank_settings, "date_format") if (date_format is None): date_format = '%Y-%m-%d' if self.bank_settings: mapped_items = frappe.get_doc("Bank Statement Settings", self.bank_settings).mapped_items statement_headers = self.get_statement_headers() transactions = get_transaction_entries(file_url, statement_headers) for entry in transactions: date = entry[statement_headers["Date"]].strip() #print("Processing entry DESC:{0}-W:{1}-D:{2}-DT:{3}".format(entry["Particulars"], entry["Withdrawals"], entry["Deposits"], entry["Date"])) if (not date): continue transaction_date = datetime.strptime(date, date_format).date() if (self.from_date and transaction_date < datetime.strptime( self.from_date, '%Y-%m-%d').date()): continue if (self.to_date and transaction_date > datetime.strptime( self.to_date, '%Y-%m-%d').date()): continue bank_entry = self.append('new_transaction_items', {}) bank_entry.transaction_date = transaction_date bank_entry.description = entry[statement_headers["Particulars"]] mapped_item = next( (entry for entry in mapped_items if entry.mapping_type == "Transaction" and frappe.safe_decode(entry.bank_data.lower()) in frappe.safe_decode(bank_entry.description.lower())), None) if (mapped_item is not None): bank_entry.party_type = mapped_item.mapped_data_type bank_entry.party = mapped_item.mapped_data else: bank_entry.party_type = "Supplier" if not entry[ statement_headers["Deposits"]].strip() else "Customer" party_list = frappe.get_all(bank_entry.party_type, fields=["name"]) parties = [party.name for party in party_list] matches = difflib.get_close_matches( frappe.safe_decode(bank_entry.description.lower()), parties, 1, 0.4) if len(matches) > 0: bank_entry.party = matches[0] bank_entry.amount = -float( entry[statement_headers["Withdrawals"]]) if not entry[ statement_headers["Deposits"]].strip() else float( entry[statement_headers["Deposits"]]) self.map_unknown_transactions() self.map_transactions_on_journal_entry()
def match_invoice_to_payment(self): added_payments = [] for entry in self.new_transaction_items: if (not entry.party or entry.party_type == "Account"): continue entry.account = self.receivable_account if entry.party_type == "Customer" else self.payable_account amount = abs(entry.amount) payment, matching_invoices = None, [] for inv_entry in self.payment_invoice_items: if (inv_entry.payment_description != frappe.safe_decode( entry.description) or inv_entry.transaction_date != entry.transaction_date): continue if (inv_entry.party != entry.party): continue matching_invoices += [ inv_entry.invoice_type + "|" + inv_entry.invoice ] payment = get_payments_matching_invoice( inv_entry.invoice, entry.amount, entry.transaction_date) doc = frappe.get_doc(inv_entry.invoice_type, inv_entry.invoice) inv_entry.invoice_date = doc.posting_date inv_entry.outstanding_amount = doc.outstanding_amount inv_entry.allocated_amount = min(float(doc.outstanding_amount), amount) amount -= inv_entry.allocated_amount if (amount < 0): break amount = abs(entry.amount) if (payment is None): order_doctype = "Sales Order" if entry.party_type == "Customer" else "Purchase Order" from erpbee.controllers.accounts_controller import get_advance_payment_entries payment_entries = get_advance_payment_entries( entry.party_type, entry.party, entry.account, order_doctype, against_all_orders=True) payment_entries += self.get_matching_payments( entry.party, amount, entry.transaction_date) payment = next( (payment for payment in payment_entries if payment.amount == amount and payment not in added_payments), None) if (payment is None): print("Failed to find payments for {0}:{1}".format( entry.party, amount)) continue added_payments += [payment] entry.reference_type = payment.reference_type entry.reference_name = payment.reference_name entry.mode_of_payment = "Wire Transfer" entry.outstanding_amount = min(amount, 0) if (entry.payment_reference is None): entry.payment_reference = frappe.safe_decode(entry.description) entry.invoices = ",".join(matching_invoices)
def execute(): """ Create Contact for each User if not present """ frappe.reload_doc('contacts', 'doctype', 'contact') users = frappe.get_all('User', filters={"name": ('not in', 'Administrator, Guest')}, fields=["*"]) for user in users: if user.first_name: user.first_name = re.sub("[<>]+", '', frappe.safe_decode(user.first_name)) if user.last_name: user.last_name = re.sub("[<>]+", '', frappe.safe_decode(user.last_name)) create_contact(user, ignore_links=True, ignore_mandatory=True)
def set_subject(self): """Parse and decode `Subject` header.""" _subject = decode_header(self.mail.get("Subject", "No Subject")) self.subject = _subject[0][0] or "" if _subject[0][1]: self.subject = safe_decode(self.subject, _subject[0][1]) else: # assume that the encoding is utf-8 self.subject = safe_decode(self.subject)[:140] if not self.subject: self.subject = "No Subject"
def make_tarfile(path, fname=None): if not fname: fname = "charts" source_path = path else: source_path = os.path.join(path, fname + ".json").encode('utf-8') target_path = os.path.join(path, fname + ".tar.gz").encode('utf-8') source_path = frappe.safe_decode(source_path) target_path = frappe.safe_decode(target_path) with tarfile.open(target_path, "w:gz", encoding="utf-8") as tar: tar.add(source_path, arcname=os.path.basename(source_path))
def pre_process(user): name = frappe.safe_decode(user.name) if "," in name: firstname = name.split(',')[-1] lastname = name.split(',')[0] else: firstname = name.split()[0] lastname = name.split()[-1] phone = frappe.safe_decode( user.phone) if user.phone is not None else user.phone condition = None if user.email is not None and phone is not None: condition = "(email_id = '{0}' OR phone = '{1}') AND (zendesk_sync_id != '{2}' OR zendesk_sync_id IS NULL)".format( user.email, frappe.safe_encode(phone), user.id) elif user.email is not None: condition = "email_id = '{0}' AND (zendesk_sync_id != '{1}' OR zendesk_sync_id IS NULL)".format( user.email, user.id) elif phone is not None: condition = "phone = '{0}' AND (zendesk_sync_id != '{1}' OR zendesk_sync_id IS NULL)".format( frappe.safe_encode(phone), user.id) if condition: contacts = frappe.db.sql(""" SELECT name FROM tabContact WHERE %s """ % condition, as_dict=True) for contact in contacts: try: frappe.db.set_value("Contact", frappe.safe_decode(contact.name), "zendesk_sync_id", user.id) except Exception as e: frappe.log_error(e, user.name) return { 'id': user.id, 'firstname': firstname, 'lastname': lastname, 'email': user.email, 'phone': phone }
def load_assets(self): import os from frappe.modules import get_module_path, scrub self.script = "" page_name = scrub(self.name) path = os.path.join(get_module_path(self.module), "page", page_name) # script fpath = os.path.join(path, page_name + ".js") if os.path.exists(fpath): with open(fpath, "r") as f: self.script = render_include(f.read()) self.script += f"\n\n//# sourceURL={page_name}.js" # css fpath = os.path.join(path, page_name + ".css") if os.path.exists(fpath): with open(fpath, "r") as f: self.style = safe_decode(f.read()) # html as js template for fname in os.listdir(path): if fname.endswith(".html"): with open(os.path.join(path, fname), "r") as f: template = f.read() if "<!-- jinja -->" in template: context = frappe._dict({}) try: out = frappe.get_attr( "{app}.{module}.page.{page}.{page}.get_context" .format(app=frappe.local.module_app[scrub( self.module)], module=scrub(self.module), page=page_name))(context) if out: context = out except (AttributeError, ImportError): pass template = frappe.render_template(template, context) self.script = html_to_js_template(fname, template) + self.script # flag for not caching this page self._dynamic_page = True if frappe.lang != "en": from frappe.translate import get_lang_js self.script += get_lang_js("page", self.name) for path in get_code_files_via_hooks("page_js", self.name): js = get_js(path) if js: self.script += "\n\n" + js
def generate_bootstrap_theme(self): from subprocess import Popen, PIPE file_name = frappe.scrub(self.name) + '_' + frappe.generate_hash( 'Website Theme', 8) + '.css' output_path = join_path(frappe.utils.get_bench_path(), 'sites', 'assets', 'css', file_name) content = self.theme_scss content = content.replace('\n', '\\n') command = ['node', 'generate_bootstrap_theme.js', output_path, content] process = Popen(command, cwd=frappe.get_app_path('frappe', '..'), stdout=PIPE, stderr=PIPE) stderr = process.communicate()[1] if stderr: stderr = frappe.safe_decode(stderr) stderr = stderr.replace('\n', '<br>') frappe.throw( '<div style="font-family: monospace;">{stderr}</div>'.format( stderr=stderr)) else: self.theme_url = '/assets/css/' + file_name frappe.msgprint(_('Compiled Successfully'), alert=True)
def get_html_for_route(route): from frappe.website import render set_request(method="GET", path=route) response = render.render() html = frappe.safe_decode(response.get_data()) return html
def sync_user_settings(): '''Sync from cache to database (called asynchronously via the browser)''' for key, data in iteritems(frappe.cache().hgetall('_user_settings')): key = safe_decode(key) doctype, user = key.split('::') # WTF? frappe.db.sql('''insert into __UserSettings (user, doctype, data) values (%s, %s, %s) on duplicate key update data=%s''', (user, doctype, data, data))
def test_homepage_section_custom_html(self): frappe.get_doc({ 'doctype': 'Homepage Section', 'name': 'Custom HTML Section', 'section_based_on': 'Custom HTML', 'section_html': '<div class="custom-section">My custom html</div>', }).insert() set_request(method='GET', path='home') response = render() self.assertEquals(response.status_code, 200) html = frappe.safe_decode(response.get_data()) soup = BeautifulSoup(html, 'html.parser') sections = soup.find('main').find_all(class_='custom-section') self.assertEqual(len(sections), 1) homepage_section = sections[0] self.assertEqual(homepage_section.text, 'My custom html') # cleanup frappe.db.rollback()
def populate_matching_invoices(self): self.payment_invoice_items = [] self.map_unknown_transactions() added_invoices = [] for entry in self.new_transaction_items: if (not entry.party or entry.party_type == "Account"): continue account = self.receivable_account if entry.party_type == "Customer" else self.payable_account invoices = get_outstanding_invoices(entry.party_type, entry.party, account) transaction_date = datetime.strptime(entry.transaction_date, "%Y-%m-%d").date() outstanding_invoices = [invoice for invoice in invoices if invoice.posting_date <= transaction_date] amount = abs(entry.amount) matching_invoices = [invoice for invoice in outstanding_invoices if invoice.outstanding_amount == amount] sorted(outstanding_invoices, key=lambda k: k['posting_date']) for e in (matching_invoices + outstanding_invoices): added = next((inv for inv in added_invoices if inv == e.get('voucher_no')), None) if (added is not None): continue ent = self.append('payment_invoice_items', {}) ent.transaction_date = entry.transaction_date ent.payment_description = frappe.safe_decode(entry.description) ent.party_type = entry.party_type ent.party = entry.party ent.invoice = e.get('voucher_no') added_invoices += [ent.invoice] ent.invoice_type = "Sales Invoice" if entry.party_type == "Customer" else "Purchase Invoice" ent.invoice_date = e.get('posting_date') ent.outstanding_amount = e.get('outstanding_amount') ent.allocated_amount = min(float(e.get('outstanding_amount')), amount) amount -= float(e.get('outstanding_amount')) if (amount <= 5): break self.match_invoice_to_payment() self.populate_matching_vouchers() self.map_transactions_on_journal_entry()
def validate_auth_via_api_keys(authorization_header): """ Authenticate request using API keys and set session user Args: authorization_header (list of str): The 'Authorization' header containing the prefix and token """ try: auth_type, auth_token = authorization_header if auth_type.lower() == 'basic': api_key, api_secret = frappe.safe_decode( base64.b64decode(auth_token)).split(":") validate_api_key_secret(api_key, api_secret) elif auth_type.lower() == 'token': api_key, api_secret = auth_token.split(":") validate_api_key_secret(api_key, api_secret) except binascii.Error: frappe.throw( _("Failed to decode token, please provide a valid base64-encoded token." ), frappe.InvalidAuthorizationToken) except (AttributeError, TypeError, ValueError): frappe.throw( _("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'." ), frappe.InvalidAuthorizationToken)
def map_transactions_on_journal_entry(self): for entry in self.new_transaction_items: vouchers = frappe.db.sql("""select name, posting_date from `tabJournal Entry` where posting_date='{0}' and total_credit={1} and cheque_no='{2}' and docstatus != 2 """.format(entry.transaction_date, abs(entry.amount), frappe.safe_decode(entry.description)), as_dict=True) if (len(vouchers) == 1): entry.reference_name = vouchers[0].name
def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) if query.strip().lower().split()[0] in ('insert', 'delete', 'update', 'alter'): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" # multi_word_regex is designed to match following patterns # `tabXxx Xxx` and "tabXxx Xxx" # ([`"]?) Captures " or ` at the begining of the table name (if provided) # \1 matches the first captured group (quote character) at the end of the table name # multi word table name must have surrounding quotes. # (tab([A-Z]\w+)( [A-Z]\w+)*) Captures table names that start with "tab" # and are continued with multiple words that start with a captital letter # e.g. 'tabXxx' or 'tabXxx Xxx' or 'tabXxx Xxx Xxx' and so on single_word_regex = r'([`"]?)(tab([A-Z]\w+))\1' multi_word_regex = r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1' tables = [] for regex in (single_word_regex, multi_word_regex): tables += [groups[1] for groups in re.findall(regex, query)] if frappe.flags.touched_tables is None: frappe.flags.touched_tables = set() frappe.flags.touched_tables.update(tables)
def test_expose(self): from frappe.utils.verified_command import verify_request frappe.sendmail(recipients=['*****@*****.**'], cc=['*****@*****.**'], sender="*****@*****.**", reference_doctype='User', reference_name="Administrator", subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe", now=True) email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1) self.assertEqual(len(email_queue), 1) queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` where status='Sent'""", as_dict=1)] self.assertTrue('*****@*****.**' in queue_recipients) self.assertTrue('*****@*****.**' in queue_recipients) message = frappe.db.sql("""select message from `tabEmail Queue` where status='Sent'""", as_dict=1)[0].message self.assertTrue('<!--recipient-->' in message) email_obj = email.message_from_string(frappe.safe_decode(frappe.flags.sent_mail)) for part in email_obj.walk(): content = part.get_payload(decode=True) if content: frappe.local.flags.signed_query_string = re.search('(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=\n)', content.decode()).group(0) self.assertTrue(verify_request()) break
def __init__(self, content): """Parses headers, content, attachments from given raw message. :param content: Raw message.""" self.raw = safe_encode(content) if six.PY2 else safe_decode(content) self.mail = email.message_from_string(self.raw) self.text_content = '' self.html_content = '' self.attachments = [] self.cid_map = {} self.parse() self.set_content_and_type() self.set_subject() self.set_from() self.message_id = (self.mail.get('Message-ID') or "").strip(" <>") if self.mail["Date"]: try: utc = email.utils.mktime_tz( email.utils.parsedate_tz(self.mail["Date"])) utc_dt = datetime.datetime.utcfromtimestamp(utc) self.date = convert_utc_to_user_timezone(utc_dt).strftime( '%Y-%m-%d %H:%M:%S') except: self.date = now() else: self.date = now() if self.date > now(): self.date = now()
def to_cursor(self, row, sorting_fields): # sorting_fields could be [custom_table.field_1], # where only field_1 will be available on row _json = frappe.as_json([ row.get(x.split('.')[1] if '.' in x else x) for x in sorting_fields ]) return frappe.safe_decode(base64.b64encode(_json.encode("utf-8")))
def get_web_image(file_url): # download file_url = frappe.utils.get_url(file_url) r = requests.get(file_url, stream=True) try: r.raise_for_status() except requests.exceptions.HTTPError as e: if "404" in e.args[0]: frappe.msgprint(_("File '{0}' not found").format(file_url)) else: frappe.msgprint( _("Unable to read file format for {0}").format(file_url)) raise image = Image.open(StringIO(frappe.safe_decode(r.content))) try: filename, extn = file_url.rsplit("/", 1)[1].rsplit(".", 1) except ValueError: # the case when the file url doesn't have filename or extension # but is fetched due to a query string. example: https://encrypted-tbn3.gstatic.com/images?q=something filename = get_random_filename() extn = None extn = get_extension(filename, extn, r.content) filename = "/files/" + strip(unquote(filename)) return image, filename, extn
def finalize_id_token(self, id_token, token, token_handler, request): # Check whether frappe server URL is set id_token_header = {"typ": "jwt", "alg": "HS256"} user = frappe.get_doc( "User", frappe.session.user, ) if request.nonce: id_token["nonce"] = request.nonce userinfo = get_userinfo(user) if userinfo.get("iss"): id_token["iss"] = userinfo.get("iss") if "openid" in request.scopes: id_token.update(userinfo) id_token_encoded = jwt.encode( payload=id_token, key=request.client.client_secret, algorithm="HS256", headers=id_token_header, ) return frappe.safe_decode(id_token_encoded)
def test_unsubscribe(self): from frappe.email.queue import unsubscribe, send unsubscribe(doctype="User", name="Administrator", email="*****@*****.**") self.assertTrue(frappe.db.get_value("Email Unsubscribe", {"reference_doctype": "User", "reference_name": "Administrator", "email": "*****@*****.**"})) before = frappe.db.sql("""select count(name) from `tabEmail Queue` where status='Not Sent'""")[0][0] send(recipients = ['*****@*****.**', '*****@*****.**'], sender="*****@*****.**", reference_doctype='User', reference_name= "Administrator", subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe") # this is sent async (?) email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""", as_dict=1) self.assertEqual(len(email_queue), before + 1) queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` where status='Not Sent'""", as_dict=1)] self.assertFalse('*****@*****.**' in queue_recipients) self.assertTrue('*****@*****.**' in queue_recipients) self.assertEqual(len(queue_recipients), 1) self.assertTrue('Unsubscribe' in frappe.safe_decode(frappe.flags.sent_mail))
def __init__(self, content): """Parses headers, content, attachments from given raw message. :param content: Raw message.""" self.raw = safe_encode(content) if six.PY2 else safe_decode(content) self.mail = email.message_from_string(self.raw) self.text_content = '' self.html_content = '' self.attachments = [] self.cid_map = {} self.parse() self.set_content_and_type() self.set_subject() self.set_from() self.message_id = (self.mail.get('Message-ID') or "").strip(" <>") if self.mail["Date"]: try: utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"])) utc_dt = datetime.datetime.utcfromtimestamp(utc) self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S') except: self.date = now() else: self.date = now() if self.date > now(): self.date = now()
def get_request_form_data(): if frappe.local.form_dict.data is None: data = frappe.safe_decode(frappe.local.request.get_data()) else: data = frappe.local.form_dict.data return frappe.parse_json(data)
def test_homepage_section_custom_html(self): frappe.get_doc({ "doctype": "Homepage Section", "name": "Custom HTML Section", "section_based_on": "Custom HTML", "section_html": '<div class="custom-section">My custom html</div>', }).insert() set_request(method="GET", path="home") response = get_response() self.assertEqual(response.status_code, 200) html = frappe.safe_decode(response.get_data()) soup = BeautifulSoup(html, "html.parser") sections = soup.find("main").find_all(class_="custom-section") self.assertEqual(len(sections), 1) homepage_section = sections[0] self.assertEqual(homepage_section.text, "My custom html") # cleanup frappe.db.rollback()
def validate_auth_via_api_keys(authorization_header): """ Authenticate request using API keys and set session user Args: authorization_header (list of str): The 'Authorization' header containing the prefix and token """ try: auth_type, auth_token = authorization_header authorization_source = frappe.get_request_header( "Frappe-Authorization-Source") if auth_type.lower() == "basic": api_key, api_secret = frappe.safe_decode( base64.b64decode(auth_token)).split(":") validate_api_key_secret(api_key, api_secret, authorization_source) elif auth_type.lower() == "token": api_key, api_secret = auth_token.split(":") validate_api_key_secret(api_key, api_secret, authorization_source) except binascii.Error: frappe.throw( _("Failed to decode token, please provide a valid base64-encoded token." ), frappe.InvalidAuthorizationToken, ) except (AttributeError, TypeError, ValueError): pass
def decode_email(self, email): if not email: return decoded = "" for part, encoding in decode_header(frappe.as_unicode(email).replace("\""," ").replace("\'"," ")): if encoding: decoded += part.decode(encoding) else: decoded += safe_decode(part) return decoded
def get_app_last_commit_ref(app): try: result = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD --short 7'.format(app), shell=True) result = safe_decode(result) result = result.strip() return result except Exception: return ''
def get_app_branch(app): '''Returns branch of an app''' try: result = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app), shell=True) result = safe_decode(result) result = result.strip() return result except Exception: return ''
def load_assets(self): from frappe.modules import get_module_path, scrub import os self.script = '' page_name = scrub(self.name) path = os.path.join(get_module_path(self.module), 'page', page_name) # script fpath = os.path.join(path, page_name + '.js') if os.path.exists(fpath): with open(fpath, 'r') as f: self.script = render_include(f.read()) # css fpath = os.path.join(path, page_name + '.css') if os.path.exists(fpath): with open(fpath, 'r') as f: self.style = safe_decode(f.read()) # html as js template for fname in os.listdir(path): if fname.endswith(".html"): with open(os.path.join(path, fname), 'r') as f: template = f.read() if "<!-- jinja -->" in template: context = frappe._dict({}) try: out = frappe.get_attr("{app}.{module}.page.{page}.{page}.get_context".format( app = frappe.local.module_app[scrub(self.module)], module = scrub(self.module), page = page_name ))(context) if out: context = out except (AttributeError, ImportError): pass template = frappe.render_template(template, context) self.script = html_to_js_template(fname, template) + self.script # flag for not caching this page self._dynamic_page = True if frappe.lang != 'en': from frappe.translate import get_lang_js self.script += get_lang_js("page", self.name) for path in get_code_files_via_hooks("page_js", self.name): js = get_js(path) if js: self.script += "\n\n" + js
def test_flush(self): self.test_email_queue() from frappe.email.queue import flush flush(from_test=True) email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1) self.assertEqual(len(email_queue), 1) queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` where status='Sent'""", as_dict=1)] self.assertTrue('*****@*****.**' in queue_recipients) self.assertTrue('*****@*****.**' in queue_recipients) self.assertEqual(len(queue_recipients), 2) self.assertTrue('Unsubscribe' in frappe.safe_decode(frappe.flags.sent_mail))
def test_page_load(self): frappe.set_user('Guest') set_request(method='POST', path='login') response = render.render() self.assertEquals(response.status_code, 200) html = frappe.safe_decode(response.get_data()) self.assertTrue('/* login-css */' in html) self.assertTrue('// login.js' in html) self.assertTrue('<!-- login.html -->' in html) frappe.set_user('Administrator')
def match_invoice_to_payment(self): added_payments = [] for entry in self.new_transaction_items: if (not entry.party or entry.party_type == "Account"): continue entry.account = self.receivable_account if entry.party_type == "Customer" else self.payable_account amount = abs(entry.amount) payment, matching_invoices = None, [] for inv_entry in self.payment_invoice_items: if (inv_entry.payment_description != frappe.safe_decode(entry.description) or inv_entry.transaction_date != entry.transaction_date): continue if (inv_entry.party != entry.party): continue matching_invoices += [inv_entry.invoice_type + "|" + inv_entry.invoice] payment = get_payments_matching_invoice(inv_entry.invoice, entry.amount, entry.transaction_date) doc = frappe.get_doc(inv_entry.invoice_type, inv_entry.invoice) inv_entry.invoice_date = doc.posting_date inv_entry.outstanding_amount = doc.outstanding_amount inv_entry.allocated_amount = min(float(doc.outstanding_amount), amount) amount -= inv_entry.allocated_amount if (amount < 0): break amount = abs(entry.amount) if (payment is None): order_doctype = "Sales Order" if entry.party_type=="Customer" else "Purchase Order" from erpnext.controllers.accounts_controller import get_advance_payment_entries payment_entries = get_advance_payment_entries(entry.party_type, entry.party, entry.account, order_doctype, against_all_orders=True) payment_entries += self.get_matching_payments(entry.party, amount, entry.transaction_date) payment = next((payment for payment in payment_entries if payment.amount == amount and payment not in added_payments), None) if (payment is None): print("Failed to find payments for {0}:{1}".format(entry.party, amount)) continue added_payments += [payment] entry.reference_type = payment.reference_type entry.reference_name = payment.reference_name entry.mode_of_payment = "Wire Transfer" entry.outstanding_amount = min(amount, 0) if (entry.payment_reference is None): entry.payment_reference = frappe.safe_decode(entry.description) entry.invoices = ",".join(matching_invoices)
def validate_auth_via_api_keys(): """ authentication using api key and api secret set user """ try: authorization_header = frappe.get_request_header("Authorization", None).split(" ") if frappe.get_request_header("Authorization") else None if authorization_header and authorization_header[0] == 'Basic': token = frappe.safe_decode(base64.b64decode(authorization_header[1])).split(":") validate_api_key_secret(token[0], token[1]) elif authorization_header and authorization_header[0] == 'token': token = authorization_header[1].split(":") validate_api_key_secret(token[0], token[1]) except Exception as e: raise e
def read_csv_content(fcontent, ignore_encoding=False): rows = [] if not isinstance(fcontent, text_type): decoded = False for encoding in ["utf-8", "windows-1250", "windows-1252"]: try: fcontent = text_type(fcontent, encoding) decoded = True break except UnicodeDecodeError: continue if not decoded: frappe.msgprint(_("Unknown file encoding. Tried utf-8, windows-1250, windows-1252."), raise_exception=True) fcontent = fcontent.encode("utf-8") content = [ ] for line in fcontent.splitlines(True): if six.PY2: content.append(line) else: content.append(frappe.safe_decode(line)) try: rows = [] for row in csv.reader(content): r = [] for val in row: # decode everything val = val.strip() if val=="": # reason: in maraidb strict config, one cannot have blank strings for non string datatypes r.append(None) else: r.append(val) rows.append(r) return rows except Exception: frappe.msgprint(_("Not a valid Comma Separated Value (CSV File)")) raise
def create_payment_entries(self): for payment_entry in self.new_transaction_items: if (not payment_entry.party): continue if (payment_entry.reference_name): continue print("Creating payment entry for {0}".format(frappe.safe_decode(payment_entry.description))) if (payment_entry.party_type == "Account"): payment = self.create_journal_entry(payment_entry) invoices = [payment.doctype + "|" + payment.name] payment_entry.invoices = ",".join(invoices) else: payment = self.create_payment_entry(payment_entry) invoices = [entry.reference_doctype + "|" + entry.reference_name for entry in payment.references if entry is not None] payment_entry.invoices = ",".join(invoices) payment_entry.mode_of_payment = payment.mode_of_payment payment_entry.account = self.receivable_account if payment_entry.party_type == "Customer" else self.payable_account payment_entry.reference_name = payment.name payment_entry.reference_type = payment.doctype frappe.msgprint(_("Successfully created payment entries"))
def send_sms(receiver_list, msg, sender_name = '', success_msg = True): import json if isinstance(receiver_list, string_types): receiver_list = json.loads(receiver_list) if not isinstance(receiver_list, list): receiver_list = [receiver_list] receiver_list = validate_receiver_nos(receiver_list) arg = { 'receiver_list' : receiver_list, 'message' : frappe.safe_decode(msg).encode('utf-8'), 'success_msg' : success_msg } if frappe.db.get_value('SMS Settings', None, 'sms_gateway_url'): send_via_gateway(arg) else: msgprint(_("Please Update SMS Settings"))
def verify_request(): """Verify if the incoming signed request if it is correct.""" query_string = frappe.safe_decode(frappe.local.flags.signed_query_string or \ getattr(frappe.request, 'query_string', None)) valid = False signature_string = '&_signature=' if signature_string in query_string: params, signature = query_string.split(signature_string) given_signature = hmac.new(params.encode('utf-8')) given_signature.update(get_secret().encode()) valid = signature == given_signature.hexdigest() if not valid: frappe.respond_as_web_page(_("Invalid Link"), _("This link is invalid or expired. Please make sure you have pasted correctly.")) return valid
def populate_payment_entries(self): if self.bank_statement is None: return filename = self.bank_statement.split("/")[-1] if (len(self.new_transaction_items + self.reconciled_transaction_items) > 0): frappe.throw("Transactions already retreived from the statement") date_format = frappe.get_value("Bank Statement Settings", self.bank_settings, "date_format") if (date_format is None): date_format = '%Y-%m-%d' if self.bank_settings: mapped_items = frappe.get_doc("Bank Statement Settings", self.bank_settings).mapped_items statement_headers = self.get_statement_headers() transactions = get_transaction_entries(filename, statement_headers) for entry in transactions: date = entry[statement_headers["Date"]].strip() #print("Processing entry DESC:{0}-W:{1}-D:{2}-DT:{3}".format(entry["Particulars"], entry["Withdrawals"], entry["Deposits"], entry["Date"])) if (not date): continue transaction_date = datetime.strptime(date, date_format).date() if (self.from_date and transaction_date < datetime.strptime(self.from_date, '%Y-%m-%d').date()): continue if (self.to_date and transaction_date > datetime.strptime(self.to_date, '%Y-%m-%d').date()): continue bank_entry = self.append('new_transaction_items', {}) bank_entry.transaction_date = transaction_date bank_entry.description = entry[statement_headers["Particulars"]] mapped_item = next((entry for entry in mapped_items if entry.mapping_type == "Transaction" and frappe.safe_decode(entry.bank_data.lower()) in frappe.safe_decode(bank_entry.description.lower())), None) if (mapped_item is not None): bank_entry.party_type = mapped_item.mapped_data_type bank_entry.party = mapped_item.mapped_data else: bank_entry.party_type = "Supplier" if not entry[statement_headers["Deposits"]].strip() else "Customer" party_list = frappe.get_all(bank_entry.party_type, fields=["name"]) parties = [party.name for party in party_list] matches = difflib.get_close_matches(frappe.safe_decode(bank_entry.description.lower()), parties, 1, 0.4) if len(matches) > 0: bank_entry.party = matches[0] bank_entry.amount = -float(entry[statement_headers["Withdrawals"]]) if not entry[statement_headers["Deposits"]].strip() else float(entry[statement_headers["Deposits"]]) self.map_unknown_transactions() self.map_transactions_on_journal_entry()
def test_sendmail(self): frappe.sendmail(sender="*****@*****.**", recipients="*****@*****.**", content="test mail 001", subject="test-mail-001", delayed=False) sent_mail = email.message_from_string(frappe.safe_decode(frappe.flags.sent_mail)) self.assertTrue("test-mail-001" in sent_mail.get("Subject"))
def get_feed(self): return '{0}: {1}'.format(_(self.status), frappe.safe_decode(self.project_name))
def prepare_message(email, recipient, recipients_list): message = email.message if not message: return "" # Parse "Email Account" from "Email Sender" email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender) if frappe.conf.use_ssl and email_account.track_email_status: # Using SSL => Publically available domain => Email Read Reciept Possible message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode()) else: # No SSL => No Email Read Reciept message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode()) if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient, email.unsubscribe_method, email.unsubscribe_params) message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode()) if email.expose_recipients == "header": pass else: if email.expose_recipients == "footer": if isinstance(email.show_as_cc, string_types): email.show_as_cc = email.show_as_cc.split(",") email_sent_to = [r.recipient for r in recipients_list] email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc]) email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc]) if email_sent_cc: email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc) else: email_sent_message = _("This email was sent to {0}").format(email_sent_to) message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode()) message = message.replace("<!--recipient-->", recipient) message = (message and message.encode('utf8')) or '' message = safe_decode(message) if not email.attachments: return message # On-demand attachments from email.parser import Parser msg_obj = Parser().parsestr(message) attachments = json.loads(email.attachments) for attachment in attachments: if attachment.get('fcontent'): continue fid = attachment.get("fid") if fid: fname, fcontent = get_file(fid) attachment.update({ 'fname': fname, 'fcontent': fcontent, 'parent': msg_obj }) attachment.pop("fid", None) add_attachment(**attachment) elif attachment.get("print_format_attachment") == 1: attachment.pop("print_format_attachment", None) print_format_file = frappe.attach_print(**attachment) print_format_file.update({"parent": msg_obj}) add_attachment(**print_format_file) return msg_obj.as_string()
def validate_project_name(self): if self.get("__islocal") and frappe.db.exists("Project", self.project_name): frappe.throw(_("Project {0} already exists").format(frappe.safe_decode(self.project_name)))
def make_custom_fields(update=True): invoice_item_fields = [ dict(fieldname='tax_rate', label='Tax Rate', fieldtype='Float', insert_after='description', print_hide=1, hidden=1, read_only=1), dict(fieldname='tax_amount', label='Tax Amount', fieldtype='Currency', insert_after='tax_rate', print_hide=1, hidden=1, read_only=1, options="currency"), dict(fieldname='total_amount', label='Total Amount', fieldtype='Currency', insert_after='tax_amount', print_hide=1, hidden=1, read_only=1, options="currency") ] customer_po_fields = [ dict(fieldname='customer_po_details', label='Customer PO', fieldtype='Section Break', insert_after='image'), dict(fieldname='customer_po_no', label='Customer PO No', fieldtype='Data', insert_after='customer_po_details', fetch_from = 'sales_order.po_no', print_hide=1, allow_on_submit=1, fetch_if_empty= 1, read_only=1, no_copy=1), dict(fieldname='customer_po_clm_brk', label='', fieldtype='Column Break', insert_after='customer_po_no', print_hide=1, read_only=1), dict(fieldname='customer_po_date', label='Customer PO Date', fieldtype='Date', insert_after='customer_po_clm_brk', fetch_from = 'sales_order.po_date', print_hide=1, allow_on_submit=1, fetch_if_empty= 1, read_only=1, no_copy=1) ] custom_fields = { 'Company': [ dict(fieldname='sb_e_invoicing', label='E-Invoicing', fieldtype='Section Break', insert_after='date_of_establishment', print_hide=1), dict(fieldname='fiscal_regime', label='Fiscal Regime', fieldtype='Select', insert_after='sb_e_invoicing', print_hide=1, options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), fiscal_regimes))), dict(fieldname='fiscal_code', label='Fiscal Code', fieldtype='Data', insert_after='fiscal_regime', print_hide=1, description=_("Applicable if the company is an Individual or a Proprietorship")), dict(fieldname='vat_collectability', label='VAT Collectability', fieldtype='Select', insert_after='fiscal_code', print_hide=1, options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), vat_collectability_options))), dict(fieldname='cb_e_invoicing1', fieldtype='Column Break', insert_after='vat_collectability', print_hide=1), dict(fieldname='registrar_office_province', label='Province of the Registrar Office', fieldtype='Data', insert_after='cb_e_invoicing1', print_hide=1, length=2), dict(fieldname='registration_number', label='Registration Number', fieldtype='Data', insert_after='registrar_office_province', print_hide=1, length=20), dict(fieldname='share_capital_amount', label='Share Capital', fieldtype='Currency', insert_after='registration_number', print_hide=1, description=_('Applicable if the company is SpA, SApA or SRL')), dict(fieldname='no_of_members', label='No of Members', fieldtype='Select', insert_after='share_capital_amount', print_hide=1, options="\nSU-Socio Unico\nSM-Piu Soci", description=_("Applicable if the company is a limited liability company")), dict(fieldname='liquidation_state', label='Liquidation State', fieldtype='Select', insert_after='no_of_members', print_hide=1, options="\nLS-In Liquidazione\nLN-Non in Liquidazione") ], 'Sales Taxes and Charges': [ dict(fieldname='tax_exemption_reason', label='Tax Exemption Reason', fieldtype='Select', insert_after='included_in_print_rate', print_hide=1, depends_on='eval:doc.charge_type!="Actual" && doc.rate==0.0', options="\n" + "\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), tax_exemption_reasons))), dict(fieldname='tax_exemption_law', label='Tax Exempt Under', fieldtype='Text', insert_after='tax_exemption_reason', print_hide=1, depends_on='eval:doc.charge_type!="Actual" && doc.rate==0.0') ], 'Customer': [ dict(fieldname='fiscal_code', label='Fiscal Code', fieldtype='Data', insert_after='tax_id', print_hide=1), dict(fieldname='recipient_code', label='Recipient Code', fieldtype='Data', insert_after='fiscal_code', print_hide=1, default="0000000"), dict(fieldname='pec', label='Recipient PEC', fieldtype='Data', insert_after='fiscal_code', print_hide=1), dict(fieldname='is_public_administration', label='Is Public Administration', fieldtype='Check', insert_after='is_internal_customer', print_hide=1, description=_("Set this if the customer is a Public Administration company."), depends_on='eval:doc.customer_type=="Company"'), dict(fieldname='first_name', label='First Name', fieldtype='Data', insert_after='salutation', print_hide=1, depends_on='eval:doc.customer_type!="Company"'), dict(fieldname='last_name', label='Last Name', fieldtype='Data', insert_after='first_name', print_hide=1, depends_on='eval:doc.customer_type!="Company"') ], 'Mode of Payment': [ dict(fieldname='mode_of_payment_code', label='Code', fieldtype='Select', insert_after='included_in_print_rate', print_hide=1, options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), mode_of_payment_codes))) ], 'Payment Schedule': [ dict(fieldname='mode_of_payment_code', label='Code', fieldtype='Select', insert_after='mode_of_payment', print_hide=1, options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), mode_of_payment_codes)), fetch_from="mode_of_payment.mode_of_payment_code", read_only=1), dict(fieldname='bank_account', label='Bank Account', fieldtype='Link', insert_after='mode_of_payment_code', print_hide=1, options="Bank Account"), dict(fieldname='bank_account_name', label='Bank Name', fieldtype='Data', insert_after='bank_account', print_hide=1, fetch_from="bank_account.bank", read_only=1), dict(fieldname='bank_account_no', label='Bank Account No', fieldtype='Data', insert_after='bank_account_name', print_hide=1, fetch_from="bank_account.bank_account_no", read_only=1), dict(fieldname='bank_account_iban', label='IBAN', fieldtype='Data', insert_after='bank_account_name', print_hide=1, fetch_from="bank_account.iban", read_only=1), dict(fieldname='bank_account_swift_number', label='Swift Code (BIC)', fieldtype='Data', insert_after='bank_account_iban', print_hide=1, fetch_from="bank_account.swift_number", read_only=1), ], "Sales Invoice": [ dict(fieldname='vat_collectability', label='VAT Collectability', fieldtype='Select', insert_after='taxes_and_charges', print_hide=1, options="\n".join(map(lambda x: frappe.safe_decode(x, encoding='utf-8'), vat_collectability_options)), fetch_from="company.vat_collectability"), dict(fieldname='sb_e_invoicing_reference', label='E-Invoicing', fieldtype='Section Break', insert_after='pos_total_qty', print_hide=1), dict(fieldname='company_tax_id', label='Company Tax ID', fieldtype='Data', insert_after='sb_e_invoicing_reference', print_hide=1, read_only=1, fetch_from="company.tax_id"), dict(fieldname='company_fiscal_code', label='Company Fiscal Code', fieldtype='Data', insert_after='company_tax_id', print_hide=1, read_only=1, fetch_from="company.fiscal_code"), dict(fieldname='company_fiscal_regime', label='Company Fiscal Regime', fieldtype='Data', insert_after='company_fiscal_code', print_hide=1, read_only=1, fetch_from="company.fiscal_regime"), dict(fieldname='cb_e_invoicing_reference', fieldtype='Column Break', insert_after='company_fiscal_regime', print_hide=1), dict(fieldname='customer_fiscal_code', label='Customer Fiscal Code', fieldtype='Data', insert_after='cb_e_invoicing_reference', read_only=1, fetch_from="customer.fiscal_code"), ], 'Purchase Invoice Item': invoice_item_fields, 'Sales Order Item': invoice_item_fields, 'Delivery Note Item': invoice_item_fields, 'Sales Invoice Item': invoice_item_fields + customer_po_fields, 'Quotation Item': invoice_item_fields, 'Purchase Order Item': invoice_item_fields, 'Purchase Receipt Item': invoice_item_fields, 'Supplier Quotation Item': invoice_item_fields, 'Address': [ dict(fieldname='country_code', label='Country Code', fieldtype='Data', insert_after='country', print_hide=1, read_only=0, fetch_from="country.code"), dict(fieldname='state_code', label='State Code', fieldtype='Data', insert_after='state', print_hide=1) ] } create_custom_fields(custom_fields, ignore_validate = frappe.flags.in_patch, update=update)
def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs): csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs) for row in csv_reader: yield [safe_decode(cell, 'utf-8') for cell in row]
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", skip_errors = True, data_import_doc=None, validate_template=False, user=None): """upload data""" # for translations if user: frappe.cache().hdel("lang", user) frappe.set_user_lang(user) if data_import_doc and isinstance(data_import_doc, string_types): data_import_doc = frappe.get_doc("Data Import", data_import_doc) if data_import_doc and from_data_import == "Yes": no_email = data_import_doc.no_email ignore_encoding_errors = data_import_doc.ignore_encoding_errors update_only = data_import_doc.only_update submit_after_import = data_import_doc.submit_after_import overwrite = data_import_doc.overwrite skip_errors = data_import_doc.skip_errors else: # extra input params params = json.loads(frappe.form_dict.get("params") or '{}') if params.get("submit_after_import"): submit_after_import = True if params.get("ignore_encoding_errors"): ignore_encoding_errors = True if not params.get("no_email"): no_email = False if params.get('update_only'): update_only = True if params.get('from_data_import'): from_data_import = params.get('from_data_import') if not params.get('skip_errors'): skip_errors = params.get('skip_errors') frappe.flags.in_import = True frappe.flags.mute_emails = no_email def get_data_keys_definition(): return get_data_keys() def bad_template(): frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator)) def check_data_length(): if not data: frappe.throw(_("No data found in the file. Please reattach the new file with data.")) def get_start_row(): for i, row in enumerate(rows): if row and row[0]==get_data_keys_definition().data_separator: return i+1 bad_template() def get_header_row(key): return get_header_row_and_idx(key)[0] def get_header_row_and_idx(key): for i, row in enumerate(header): if row and row[0]==key: return row, i return [], -1 def filter_empty_columns(columns): empty_cols = list(filter(lambda x: x in ("", None), columns)) if empty_cols: if columns[-1*len(empty_cols):] == empty_cols: # filter empty columns if they exist at the end columns = columns[:-1*len(empty_cols)] else: frappe.msgprint(_("Please make sure that there are no empty columns in the file."), raise_exception=1) return columns def make_column_map(): doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype) if row_idx == -1: # old style return dt = None for i, d in enumerate(doctype_row[1:]): if d not in ("~", "-"): if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"): dt, parentfield = d, None # xls format truncates the row, so it may not have more columns if len(doctype_row) > i+2: parentfield = doctype_row[i+2] doctypes.append((dt, parentfield)) column_idx_to_fieldname[(dt, parentfield)] = {} column_idx_to_fieldtype[(dt, parentfield)] = {} if dt: column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1] column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1] def get_doc(start_idx): if doctypes: doc = {} attachments = [] last_error_row_idx = None for idx in range(start_idx, len(rows)): last_error_row_idx = idx # pylint: disable=W0612 if (not doc) or main_doc_empty(rows[idx]): for dt, parentfield in doctypes: d = {} for column_idx in column_idx_to_fieldname[(dt, parentfield)]: try: fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx] fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx] if not fieldname or not rows[idx][column_idx]: continue d[fieldname] = rows[idx][column_idx] if fieldtype in ("Int", "Check"): d[fieldname] = cint(d[fieldname]) elif fieldtype in ("Float", "Currency", "Percent"): d[fieldname] = flt(d[fieldname]) elif fieldtype == "Date": if d[fieldname] and isinstance(d[fieldname], string_types): d[fieldname] = getdate(parse_date(d[fieldname])) elif fieldtype == "Datetime": if d[fieldname]: if " " in d[fieldname]: _date, _time = d[fieldname].split() else: _date, _time = d[fieldname], '00:00:00' _date = parse_date(d[fieldname]) d[fieldname] = get_datetime(_date + " " + _time) else: d[fieldname] = None elif fieldtype in ("Image", "Attach Image", "Attach"): # added file to attachments list attachments.append(d[fieldname]) elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]: # as fields can be saved in the number format(long type) in data import template d[fieldname] = cstr(d[fieldname]) except IndexError: pass # scrub quotes from name and modified if d.get("name") and d["name"].startswith('"'): d["name"] = d["name"][1:-1] if sum([0 if not val else 1 for val in d.values()]): d['doctype'] = dt if dt == doctype: doc.update(d) else: if not overwrite and doc.get("name"): d['parent'] = doc["name"] d['parenttype'] = doctype d['parentfield'] = parentfield doc.setdefault(d['parentfield'], []).append(d) else: break return doc, attachments, last_error_row_idx else: doc = frappe._dict(zip(columns, rows[start_idx][1:])) doc['doctype'] = doctype return doc, [], None # used in testing whether a row is empty or parent row or child row # checked only 3 first columns since first two columns can be blank for example the case of # importing the item variant where item code and item name will be blank. def main_doc_empty(row): if row: for i in range(3,0,-1): if len(row) > i and row[i]: return False return True def validate_naming(doc): autoname = frappe.get_meta(doctype).autoname if autoname: if autoname[0:5] == 'field': autoname = autoname[6:] elif autoname == 'naming_series:': autoname = 'naming_series' else: return True if (autoname not in doc) or (not doc[autoname]): from frappe.model.base_document import get_controller if not hasattr(get_controller(doctype), "autoname"): frappe.throw(_("{0} is a mandatory field".format(autoname))) return True users = frappe.db.sql_list("select name from tabUser") def prepare_for_insert(doc): # don't block data import if user is not set # migrating from another system if not doc.owner in users: doc.owner = frappe.session.user if not doc.modified_by in users: doc.modified_by = frappe.session.user def is_valid_url(url): is_valid = False if url.startswith("/files") or url.startswith("/private/files"): url = get_url(url) try: r = requests.get(url) is_valid = True if r.status_code == 200 else False except Exception: pass return is_valid def attach_file_to_doc(doctype, docname, file_url): # check if attachment is already available # check if the attachement link is relative or not if not file_url: return if not is_valid_url(file_url): return files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format( doctype=doctype, docname=docname, file_url=file_url )) if files: # file is already attached return save_url(file_url, None, doctype, docname, "Home/Attachments", 0) # header filename, file_extension = ['',''] if not rows: from frappe.utils.file_manager import get_file # get_file_doc fname, fcontent = get_file(data_import_doc.import_file) filename, file_extension = os.path.splitext(fname) if file_extension == '.xlsx' and from_data_import == 'Yes': from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file) elif file_extension == '.csv': from frappe.utils.csvutils import read_csv_content rows = read_csv_content(fcontent, ignore_encoding_errors) else: frappe.throw(_("Unsupported File Format")) start_row = get_start_row() header = rows[:start_row] data = rows[start_row:] try: doctype = get_header_row(get_data_keys_definition().main_table)[1] columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:]) except: frappe.throw(_("Cannot change header content")) doctypes = [] column_idx_to_fieldname = {} column_idx_to_fieldtype = {} if skip_errors: data_rows_with_error = header if submit_after_import and not cint(frappe.db.get_value("DocType", doctype, "is_submittable")): submit_after_import = False parenttype = get_header_row(get_data_keys_definition().parent_table) if len(parenttype) > 1: parenttype = parenttype[1] # check permissions if not frappe.permissions.can_import(parenttype or doctype): frappe.flags.mute_emails = False return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True} # Throw expception in case of the empty data file check_data_length() make_column_map() total = len(data) if validate_template: if total: data_import_doc.total_rows = total return True if overwrite==None: overwrite = params.get('overwrite') # delete child rows (if parenttype) parentfield = None if parenttype: parentfield = get_parent_field(doctype, parenttype) if overwrite: delete_child_rows(data, doctype) import_log = [] def log(**kwargs): if via_console: print((kwargs.get("title") + kwargs.get("message")).encode('utf-8')) else: import_log.append(kwargs) def as_link(doctype, name): if via_console: return "{0}: {1}".format(doctype, name) else: return getlink(doctype, name) # publish realtime task update def publish_progress(achieved, reload=False): if data_import_doc: frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)), "data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user) error_flag = rollback_flag = False batch_size = frappe.conf.data_import_batch_size or 1000 for batch_start in range(0, total, batch_size): batch = data[batch_start:batch_start + batch_size] for i, row in enumerate(batch): # bypass empty rows if main_doc_empty(row): continue row_idx = i + start_row doc = None publish_progress(i) try: doc, attachments, last_error_row_idx = get_doc(row_idx) validate_naming(doc) if pre_process: pre_process(doc) original = None if parentfield: parent = frappe.get_doc(parenttype, doc["parent"]) doc = parent.append(parentfield, doc) parent.save() else: if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]): original = frappe.get_doc(doctype, doc["name"]) original_name = original.name original.update(doc) # preserve original name for case sensitivity original.name = original_name original.flags.ignore_links = ignore_links original.save() doc = original else: if not update_only: doc = frappe.get_doc(doc) prepare_for_insert(doc) doc.flags.ignore_links = ignore_links doc.insert() if attachments: # check file url and create a File document for file_url in attachments: attach_file_to_doc(doc.doctype, doc.name, file_url) if submit_after_import: doc.submit() # log errors if parentfield: log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)), "link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"}) elif submit_after_import: log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)), "message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"}) elif original: log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)), "message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"}) elif not update_only: log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)), "message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"}) else: log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None, "message": "Document updation ignored", "indicator": "orange"}) except Exception as e: error_flag = True # build error message if frappe.local.message_log: err_msg = "\n".join(['<p class="border-bottom small">{}</p>'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log]) else: err_msg = '<p class="border-bottom small">{}</p>'.format(cstr(e)) error_trace = frappe.get_traceback() if error_trace: error_log_doc = frappe.log_error(error_trace) error_link = get_url_to_form("Error Log", error_log_doc.name) else: error_link = None log(**{ "row": row_idx + 1, "title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""), "message": err_msg, "indicator": "red", "link":error_link }) # data with error to create a new file # include the errored data in the last row as last_error_row_idx will not be updated for the last row if skip_errors: if last_error_row_idx == len(rows)-1: last_error_row_idx = len(rows) data_rows_with_error += rows[row_idx:last_error_row_idx] else: rollback_flag = True finally: frappe.local.message_log = [] start_row += batch_size if rollback_flag: frappe.db.rollback() else: frappe.db.commit() frappe.flags.mute_emails = False frappe.flags.in_import = False log_message = {"messages": import_log, "error": error_flag} if data_import_doc: data_import_doc.log_details = json.dumps(log_message) import_status = None if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error): import_status = "Partially Successful" # write the file with the faulty row from frappe.utils.file_manager import save_file file_name = 'error_' + filename + file_extension if file_extension == '.xlsx': from frappe.utils.xlsxutils import make_xlsx xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template") file_data = xlsx_file.getvalue() else: from frappe.utils.csvutils import to_csv file_data = to_csv(data_rows_with_error) error_data_file = save_file(file_name, file_data, "Data Import", data_import_doc.name, "Home/Attachments") data_import_doc.error_file = error_data_file.file_url elif error_flag: import_status = "Failed" else: import_status = "Successful" data_import_doc.import_status = import_status data_import_doc.save() if data_import_doc.import_status in ["Successful", "Partially Successful"]: data_import_doc.submit() publish_progress(100, True) else: publish_progress(0, True) frappe.db.commit() else: return log_message