def upload_file_to_dropbox(filename, folder, dropbox_client): """upload files with chunk of 15 mb to reduce session append calls""" if not os.path.exists(filename): return create_folder_if_not_exists(folder, dropbox_client) chunk_size = 15 * 1024 * 1024 file_size = os.path.getsize(encode(filename)) mode = (dropbox.files.WriteMode.overwrite) f = open(encode(filename), 'rb') path = "{0}/{1}".format(folder, os.path.basename(filename)) try: if file_size <= chunk_size: dropbox_client.files_upload(f.read(), path, mode) else: upload_session_start_result = dropbox_client.files_upload_session_start(f.read(chunk_size)) cursor = dropbox.files.UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=f.tell()) commit = dropbox.files.CommitInfo(path=path, mode=mode) while f.tell() < file_size: if ((file_size - f.tell()) <= chunk_size): dropbox_client.files_upload_session_finish(f.read(chunk_size), cursor, commit) else: dropbox_client.files_upload_session_append(f.read(chunk_size), cursor.session_id,cursor.offset) cursor.offset = f.tell() except dropbox.exceptions.ApiError as e: if isinstance(e.error, dropbox.files.UploadError): error = "File Path: {path}\n".format(path=path) error += dataent.get_traceback() dataent.log_error(error) else: raise
def make_error_snapshot(exception): if dataent.conf.disable_error_snapshot: return logger = dataent.logger(__name__, with_more_info=False) try: error_id = '{timestamp:s}-{ip:s}-{hash:s}'.format( timestamp=cstr(datetime.datetime.now()), ip=dataent.local.request_ip or '127.0.0.1', hash=dataent.generate_hash(length=3)) snapshot_folder = get_error_snapshot_path() dataent.create_folder(snapshot_folder) snapshot_file_path = os.path.join(snapshot_folder, "{0}.json".format(error_id)) snapshot = get_snapshot(exception) with open(encode(snapshot_file_path), 'wb') as error_file: error_file.write(encode(dataent.as_json(snapshot))) logger.error('New Exception collected with id: {}'.format(error_id)) except Exception as e: logger.error('Could not take error snapshot: {0}'.format(e), exc_info=True)
def encrypt(pwd): if len(pwd) > 100: # encrypting > 100 chars will lead to truncation dataent.throw(_('Password cannot be more than 100 characters long')) cipher_suite = Fernet(encode(get_encryption_key())) cipher_text = cstr(cipher_suite.encrypt(encode(pwd))) return cipher_text
def decrypt(pwd): try: cipher_suite = Fernet(encode(get_encryption_key())) plain_text = cstr(cipher_suite.decrypt(encode(pwd))) return plain_text except InvalidToken: # encryption_key in site_config is changed and not valid dataent.throw( _('Encryption key is invalid, Please check site_config.json'))
def execute(): """Fix the File records created via item.py even if the website_image file didn't exist""" for item in dataent.db.sql_list("""select name from `tabItem` where website_image is not null and website_image != '' and website_image like '/files/%' and exists ( select name from `tabFile` where attached_to_doctype='Item' and attached_to_name=`tabItem`.name and file_url=`tabItem`.website_image and (file_name is null or file_name = '') )"""): item = dataent.get_doc("Item", item) file = dataent.get_doc( "File", { "attached_to_doctype": "Item", "attached_to_name": item.name, "file_url": item.website_image }) try: file.validate_file() except IOError: print(encode(item.website_image), "does not exist") file.delete() item.db_set("website_image", None, update_modified=False)
def get_file(fname): """Returns [`file_name`, `content`] for given file name `fname`""" file_path = get_file_path(fname) # read the file if PY2: with open(encode(file_path)) as f: content = f.read() else: with io.open(encode(file_path), mode='rb') as f: content = f.read() try: # for plain text files content = content.decode() except UnicodeDecodeError: # for .png, .jpg, etc pass return [file_path.rsplit("/", 1)[-1], content]
def get_website_settings(): hooks = dataent.get_hooks() context = dataent._dict({ 'top_bar_items': get_items('top_bar_items'), 'footer_items': get_items('footer_items'), "post_login": [ {"label": _("My Account"), "url": "/me"}, # {"class": "divider"}, {"label": _("Logout"), "url": "/?cmd=web_logout"} ] }) settings = dataent.get_single("Website Settings") for k in ["banner_html", "brand_html", "copyright", "twitter_share_via", "facebook_share", "google_plus_one", "twitter_share", "linked_in_share", "disable_signup", "hide_footer_signup", "head_html", "title_prefix", "navbar_search"]: if hasattr(settings, k): context[k] = settings.get(k) if settings.address: context["footer_address"] = settings.address for k in ["facebook_share", "google_plus_one", "twitter_share", "linked_in_share", "disable_signup"]: context[k] = int(context.get(k) or 0) if dataent.request: context.url = quote(str(get_request_site_address(full_address=True)), safe="/:") context.encoded_title = quote(encode(context.title or ""), str("")) for update_website_context in hooks.update_website_context or []: dataent.get_attr(update_website_context)(context) context.web_include_js = hooks.web_include_js or [] context.web_include_css = hooks.web_include_css or [] via_hooks = dataent.get_hooks("website_context") for key in via_hooks: context[key] = via_hooks[key] if key not in ("top_bar_items", "footer_items", "post_login") \ and isinstance(context[key], (list, tuple)): context[key] = context[key][-1] add_website_theme(context) if not context.get("favicon"): context["favicon"] = "/assets/dataent/images/favicon.png" if settings.favicon and settings.favicon != "attach_files:": context["favicon"] = settings.favicon return context
def get_file_name(fname, optional_suffix): # convert to unicode fname = cstr(fname) n_records = dataent.db.sql("select name from `tabFile` where file_name=%s", fname) if len(n_records) > 0 or os.path.exists(encode(get_files_path(fname))): f = fname.rsplit('.', 1) if len(f) == 1: partial, extn = f[0], "" else: partial, extn = f[0], "." + f[1] return '{partial}{suffix}{extn}'.format(partial=partial, extn=extn, suffix=optional_suffix) return fname
def delete_file(path): """Delete file from `public folder`""" if path: if ".." in path.split("/"): dataent.msgprint( _("It is risky to delete this file: {0}. Please contact your System Manager." ).format(path)) parts = os.path.split(path.strip("/")) if parts[0] == "files": path = dataent.utils.get_site_path("public", "files", parts[-1]) else: path = dataent.utils.get_site_path("private", "files", parts[-1]) path = encode(path) if os.path.exists(path): os.remove(path)
def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not_upload, error_log): if not os.path.exists(path): return if is_fresh_upload(): response = get_uploaded_files_meta(dropbox_folder, dropbox_client) else: response = dataent._dict({"entries": []}) path = text_type(path) for f in dataent.get_all("File", filters={"is_folder": 0, "is_private": is_private, "uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']): if is_private: filename = f.file_url.replace('/private/files/', '') else: if not f.file_url: f.file_url = '/files/' + f.file_name; filename = f.file_url.replace('/files/', '') filepath = os.path.join(path, filename) if filename in ignore_list: continue found = False for file_metadata in response.entries: if (os.path.basename(filepath) == file_metadata.name and os.stat(encode(filepath)).st_size == int(file_metadata.size)): found = True update_file_dropbox_status(f.name) break if not found: try: upload_file_to_dropbox(filepath, dropbox_folder, dropbox_client) update_file_dropbox_status(f.name) except Exception: did_not_upload.append(filepath) error_log.append(dataent.get_traceback())
def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=False): '''Send Email Queue with given smtpserver''' email = dataent.db.sql('''select name, status, communication, message, sender, reference_doctype, reference_name, unsubscribe_param, unsubscribe_method, expose_recipients, show_as_cc, add_unsubscribe_link, attachments, retry from `tabEmail Queue` where name=%s for update''', email, as_dict=True)[0] recipients_list = dataent.db.sql('''select name, recipient, status from `tabEmail Queue Recipient` where parent=%s''',email.name,as_dict=1) if dataent.are_emails_muted(): dataent.msgprint(_("Emails are muted")) return if cint(dataent.defaults.get_defaults().get("hold_queue"))==1 : return if email.status not in ('Not Sent','Partially Sent') : # rollback to release lock and return dataent.db.rollback() return dataent.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""", (now_datetime(), email.name), auto_commit=auto_commit) if email.communication: dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) try: if not dataent.flags.in_test: if not smtpserver: smtpserver = SMTPServer() smtpserver.setup_email_account(email.reference_doctype, sender=email.sender) for recipient in recipients_list: if recipient.status != "Not Sent": continue message = prepare_message(email, recipient.recipient, recipients_list) if not dataent.flags.in_test: smtpserver.sess.sendmail(email.sender, recipient.recipient, encode(message)) recipient.status = "Sent" dataent.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""", (now_datetime(), recipient.name), auto_commit=auto_commit) #if all are sent set status if any("Sent" == s.status for s in recipients_list): dataent.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""", (now_datetime(), email.name), auto_commit=auto_commit) else: dataent.db.sql("""update `tabEmail Queue` set status='Error', error=%s where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit) if dataent.flags.in_test: dataent.flags.sent_mail = message return if email.communication: dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) except (smtplib.SMTPServerDisconnected, smtplib.SMTPConnectError, smtplib.SMTPHeloError, smtplib.SMTPAuthenticationError, JobTimeoutException): # bad connection/timeout, retry later if any("Sent" == s.status for s in recipients_list): dataent.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""", (now_datetime(), email.name), auto_commit=auto_commit) else: dataent.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""", (now_datetime(), email.name), auto_commit=auto_commit) if email.communication: dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) # no need to attempt further return except Exception as e: dataent.db.rollback() if email.retry < 3: dataent.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""", (now_datetime(), email.name), auto_commit=auto_commit) else: if any("Sent" == s.status for s in recipients_list): dataent.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""", (text_type(e), email.name), auto_commit=auto_commit) else: dataent.db.sql("""update `tabEmail Queue` set status='Error', error=%s where name=%s""", (text_type(e), email.name), auto_commit=auto_commit) if email.communication: dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) if now: print(dataent.get_traceback()) raise e else: # log to Error Log log('dataent.email.queue.flush', text_type(e))
def get_snapshot(exception, context=10): """ Return a dict describing a given traceback (based on cgitb.text) """ etype, evalue, etb = sys.exc_info() if isinstance(etype, six.class_types): etype = etype.__name__ # creates a snapshot dict with some basic information s = { 'pyver': 'Python {version:s}: {executable:s} (prefix: {prefix:s})'.format( version=sys.version.split()[0], executable=sys.executable, prefix=sys.prefix), 'timestamp': cstr(datetime.datetime.now()), 'traceback': traceback.format_exc(), 'frames': [], 'etype': cstr(etype), 'evalue': cstr(repr(evalue)), 'exception': {}, 'locals': {} } # start to process frames records = inspect.getinnerframes(etb, 5) for frame, file, lnum, func, lines, index in records: file = file and os.path.abspath(file) or '?' args, varargs, varkw, locals = inspect.getargvalues(frame) call = '' if func != '?': call = inspect.formatargvalues( args, varargs, varkw, locals, formatvalue=lambda value: '={}'.format(pydoc.text.repr(value))) # basic frame information f = { 'file': file, 'func': func, 'call': call, 'lines': {}, 'lnum': lnum } def reader(lnum=[lnum]): try: return linecache.getline(file, lnum[0]) finally: lnum[0] += 1 vars = cgitb.scanvars(reader, frame, locals) # if it is a view, replace with generated code # if file.endswith('html'): # lmin = lnum > context and (lnum - context) or 0 # lmax = lnum + context # lines = code.split("\n")[lmin:lmax] # index = min(context, lnum) - 1 if index is not None: i = lnum - index for line in lines: f['lines'][i] = line.rstrip() i += 1 # dump local variable (referenced in current line only) f['dump'] = {} for name, where, value in vars: if name in f['dump']: continue if value is not cgitb.__UNDEF__: if where == 'global': name = 'global {name:s}'.format(name=name) elif where != 'local': name = where + ' ' + name.split('.')[-1] f['dump'][name] = pydoc.text.repr(value) else: f['dump'][name] = 'undefined' s['frames'].append(f) # add exception type, value and attributes if isinstance(evalue, BaseException): for name in dir(evalue): # prevent py26 DeprecationWarning if (name != 'messages' or sys.version_info < (2.6)) and not name.startswith('__'): value = pydoc.text.repr(getattr(evalue, name)) # render multilingual string properly if type(value) == str and value.startswith(b"u'"): value = eval(value) s['exception'][name] = encode(value) # add all local values (of last frame) to the snapshot for name, value in locals.items(): if type(value) == str and value.startswith(b"u'"): value = eval(value) s['locals'][name] = pydoc.text.repr(value) return s
def writerow(self, row): if six.PY2: row = encode(row, self.encoding) self.writer.writerow(row)