def upload_file_to_dropbox(filename, folder, dropbox_client):
"""upload files with chunk of 15 mb to reduce session append calls"""
if not os.path.exists(filename):
return
create_folder_if_not_exists(folder, dropbox_client)
chunk_size = 15 * 1024 * 1024
file_size = os.path.getsize(encode(filename))
mode = (dropbox.files.WriteMode.overwrite)
f = open(encode(filename), 'rb')
path = "{0}/{1}".format(folder, os.path.basename(filename))
try:
if file_size <= chunk_size:
dropbox_client.files_upload(f.read(), path, mode)
else:
upload_session_start_result = dropbox_client.files_upload_session_start(f.read(chunk_size))
cursor = dropbox.files.UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=f.tell())
commit = dropbox.files.CommitInfo(path=path, mode=mode)
while f.tell() < file_size:
if ((file_size - f.tell()) <= chunk_size):
dropbox_client.files_upload_session_finish(f.read(chunk_size), cursor, commit)
else:
dropbox_client.files_upload_session_append(f.read(chunk_size), cursor.session_id,cursor.offset)
cursor.offset = f.tell()
except dropbox.exceptions.ApiError as e:
if isinstance(e.error, dropbox.files.UploadError):
error = "File Path: {path}\n".format(path=path)
error += dataent.get_traceback()
dataent.log_error(error)
else:
raise
def make_error_snapshot(exception):
if dataent.conf.disable_error_snapshot:
return
logger = dataent.logger(__name__, with_more_info=False)
try:
error_id = '{timestamp:s}-{ip:s}-{hash:s}'.format(
timestamp=cstr(datetime.datetime.now()),
ip=dataent.local.request_ip or '127.0.0.1',
hash=dataent.generate_hash(length=3))
snapshot_folder = get_error_snapshot_path()
dataent.create_folder(snapshot_folder)
snapshot_file_path = os.path.join(snapshot_folder,
"{0}.json".format(error_id))
snapshot = get_snapshot(exception)
with open(encode(snapshot_file_path), 'wb') as error_file:
error_file.write(encode(dataent.as_json(snapshot)))
logger.error('New Exception collected with id: {}'.format(error_id))
except Exception as e:
logger.error('Could not take error snapshot: {0}'.format(e),
exc_info=True)
def encrypt(pwd):
if len(pwd) > 100:
# encrypting > 100 chars will lead to truncation
dataent.throw(_('Password cannot be more than 100 characters long'))
cipher_suite = Fernet(encode(get_encryption_key()))
cipher_text = cstr(cipher_suite.encrypt(encode(pwd)))
return cipher_text
def decrypt(pwd):
try:
cipher_suite = Fernet(encode(get_encryption_key()))
plain_text = cstr(cipher_suite.decrypt(encode(pwd)))
return plain_text
except InvalidToken:
# encryption_key in site_config is changed and not valid
dataent.throw(
_('Encryption key is invalid, Please check site_config.json'))
def execute():
"""Fix the File records created via item.py even if the website_image file didn't exist"""
for item in dataent.db.sql_list("""select name from `tabItem`
where website_image is not null and website_image != ''
and website_image like '/files/%'
and exists (
select name from `tabFile`
where attached_to_doctype='Item'
and attached_to_name=`tabItem`.name
and file_url=`tabItem`.website_image
and (file_name is null or file_name = '')
)"""):
item = dataent.get_doc("Item", item)
file = dataent.get_doc(
"File", {
"attached_to_doctype": "Item",
"attached_to_name": item.name,
"file_url": item.website_image
})
try:
file.validate_file()
except IOError:
print(encode(item.website_image), "does not exist")
file.delete()
item.db_set("website_image", None, update_modified=False)
def get_file(fname):
"""Returns [`file_name`, `content`] for given file name `fname`"""
file_path = get_file_path(fname)
# read the file
if PY2:
with open(encode(file_path)) as f:
content = f.read()
else:
with io.open(encode(file_path), mode='rb') as f:
content = f.read()
try:
# for plain text files
content = content.decode()
except UnicodeDecodeError:
# for .png, .jpg, etc
pass
return [file_path.rsplit("/", 1)[-1], content]
def get_website_settings():
hooks = dataent.get_hooks()
context = dataent._dict({
'top_bar_items': get_items('top_bar_items'),
'footer_items': get_items('footer_items'),
"post_login": [
{"label": _("My Account"), "url": "/me"},
# {"class": "divider"},
{"label": _("Logout"), "url": "/?cmd=web_logout"}
]
})
settings = dataent.get_single("Website Settings")
for k in ["banner_html", "brand_html", "copyright", "twitter_share_via",
"facebook_share", "google_plus_one", "twitter_share", "linked_in_share",
"disable_signup", "hide_footer_signup", "head_html", "title_prefix",
"navbar_search"]:
if hasattr(settings, k):
context[k] = settings.get(k)
if settings.address:
context["footer_address"] = settings.address
for k in ["facebook_share", "google_plus_one", "twitter_share", "linked_in_share",
"disable_signup"]:
context[k] = int(context.get(k) or 0)
if dataent.request:
context.url = quote(str(get_request_site_address(full_address=True)), safe="/:")
context.encoded_title = quote(encode(context.title or ""), str(""))
for update_website_context in hooks.update_website_context or []:
dataent.get_attr(update_website_context)(context)
context.web_include_js = hooks.web_include_js or []
context.web_include_css = hooks.web_include_css or []
via_hooks = dataent.get_hooks("website_context")
for key in via_hooks:
context[key] = via_hooks[key]
if key not in ("top_bar_items", "footer_items", "post_login") \
and isinstance(context[key], (list, tuple)):
context[key] = context[key][-1]
add_website_theme(context)
if not context.get("favicon"):
context["favicon"] = "/assets/dataent/images/favicon.png"
if settings.favicon and settings.favicon != "attach_files:":
context["favicon"] = settings.favicon
return context
def get_file_name(fname, optional_suffix):
# convert to unicode
fname = cstr(fname)
n_records = dataent.db.sql("select name from `tabFile` where file_name=%s",
fname)
if len(n_records) > 0 or os.path.exists(encode(get_files_path(fname))):
f = fname.rsplit('.', 1)
if len(f) == 1:
partial, extn = f[0], ""
else:
partial, extn = f[0], "." + f[1]
return '{partial}{suffix}{extn}'.format(partial=partial,
extn=extn,
suffix=optional_suffix)
return fname
def delete_file(path):
"""Delete file from `public folder`"""
if path:
if ".." in path.split("/"):
dataent.msgprint(
_("It is risky to delete this file: {0}. Please contact your System Manager."
).format(path))
parts = os.path.split(path.strip("/"))
if parts[0] == "files":
path = dataent.utils.get_site_path("public", "files", parts[-1])
else:
path = dataent.utils.get_site_path("private", "files", parts[-1])
path = encode(path)
if os.path.exists(path):
os.remove(path)
def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not_upload, error_log):
if not os.path.exists(path):
return
if is_fresh_upload():
response = get_uploaded_files_meta(dropbox_folder, dropbox_client)
else:
response = dataent._dict({"entries": []})
path = text_type(path)
for f in dataent.get_all("File", filters={"is_folder": 0, "is_private": is_private,
"uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']):
if is_private:
filename = f.file_url.replace('/private/files/', '')
else:
if not f.file_url:
f.file_url = '/files/' + f.file_name;
filename = f.file_url.replace('/files/', '')
filepath = os.path.join(path, filename)
if filename in ignore_list:
continue
found = False
for file_metadata in response.entries:
if (os.path.basename(filepath) == file_metadata.name
and os.stat(encode(filepath)).st_size == int(file_metadata.size)):
found = True
update_file_dropbox_status(f.name)
break
if not found:
try:
upload_file_to_dropbox(filepath, dropbox_folder, dropbox_client)
update_file_dropbox_status(f.name)
except Exception:
did_not_upload.append(filepath)
error_log.append(dataent.get_traceback())
def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=False):
'''Send Email Queue with given smtpserver'''
email = dataent.db.sql('''select
name, status, communication, message, sender, reference_doctype,
reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
show_as_cc, add_unsubscribe_link, attachments, retry
from
`tabEmail Queue`
where
name=%s
for update''', email, as_dict=True)[0]
recipients_list = dataent.db.sql('''select name, recipient, status from
`tabEmail Queue Recipient` where parent=%s''',email.name,as_dict=1)
if dataent.are_emails_muted():
dataent.msgprint(_("Emails are muted"))
return
if cint(dataent.defaults.get_defaults().get("hold_queue"))==1 :
return
if email.status not in ('Not Sent','Partially Sent') :
# rollback to release lock and return
dataent.db.rollback()
return
dataent.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
try:
if not dataent.flags.in_test:
if not smtpserver: smtpserver = SMTPServer()
smtpserver.setup_email_account(email.reference_doctype, sender=email.sender)
for recipient in recipients_list:
if recipient.status != "Not Sent":
continue
message = prepare_message(email, recipient.recipient, recipients_list)
if not dataent.flags.in_test:
smtpserver.sess.sendmail(email.sender, recipient.recipient, encode(message))
recipient.status = "Sent"
dataent.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
(now_datetime(), recipient.name), auto_commit=auto_commit)
#if all are sent set status
if any("Sent" == s.status for s in recipients_list):
dataent.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
else:
dataent.db.sql("""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
if dataent.flags.in_test:
dataent.flags.sent_mail = message
return
if email.communication:
dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
except (smtplib.SMTPServerDisconnected,
smtplib.SMTPConnectError,
smtplib.SMTPHeloError,
smtplib.SMTPAuthenticationError,
JobTimeoutException):
# bad connection/timeout, retry later
if any("Sent" == s.status for s in recipients_list):
dataent.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
else:
dataent.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
if email.communication:
dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
# no need to attempt further
return
except Exception as e:
dataent.db.rollback()
if email.retry < 3:
dataent.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""",
(now_datetime(), email.name), auto_commit=auto_commit)
else:
if any("Sent" == s.status for s in recipients_list):
dataent.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
(text_type(e), email.name), auto_commit=auto_commit)
else:
dataent.db.sql("""update `tabEmail Queue` set status='Error', error=%s
where name=%s""", (text_type(e), email.name), auto_commit=auto_commit)
if email.communication:
dataent.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
if now:
print(dataent.get_traceback())
raise e
else:
# log to Error Log
log('dataent.email.queue.flush', text_type(e))
def get_snapshot(exception, context=10):
"""
Return a dict describing a given traceback (based on cgitb.text)
"""
etype, evalue, etb = sys.exc_info()
if isinstance(etype, six.class_types):
etype = etype.__name__
# creates a snapshot dict with some basic information
s = {
'pyver':
'Python {version:s}: {executable:s} (prefix: {prefix:s})'.format(
version=sys.version.split()[0],
executable=sys.executable,
prefix=sys.prefix),
'timestamp':
cstr(datetime.datetime.now()),
'traceback':
traceback.format_exc(),
'frames': [],
'etype':
cstr(etype),
'evalue':
cstr(repr(evalue)),
'exception': {},
'locals': {}
}
# start to process frames
records = inspect.getinnerframes(etb, 5)
for frame, file, lnum, func, lines, index in records:
file = file and os.path.abspath(file) or '?'
args, varargs, varkw, locals = inspect.getargvalues(frame)
call = ''
if func != '?':
call = inspect.formatargvalues(
args,
varargs,
varkw,
locals,
formatvalue=lambda value: '={}'.format(pydoc.text.repr(value)))
# basic frame information
f = {
'file': file,
'func': func,
'call': call,
'lines': {},
'lnum': lnum
}
def reader(lnum=[lnum]):
try:
return linecache.getline(file, lnum[0])
finally:
lnum[0] += 1
vars = cgitb.scanvars(reader, frame, locals)
# if it is a view, replace with generated code
# if file.endswith('html'):
# lmin = lnum > context and (lnum - context) or 0
# lmax = lnum + context
# lines = code.split("\n")[lmin:lmax]
# index = min(context, lnum) - 1
if index is not None:
i = lnum - index
for line in lines:
f['lines'][i] = line.rstrip()
i += 1
# dump local variable (referenced in current line only)
f['dump'] = {}
for name, where, value in vars:
if name in f['dump']:
continue
if value is not cgitb.__UNDEF__:
if where == 'global':
name = 'global {name:s}'.format(name=name)
elif where != 'local':
name = where + ' ' + name.split('.')[-1]
f['dump'][name] = pydoc.text.repr(value)
else:
f['dump'][name] = 'undefined'
s['frames'].append(f)
# add exception type, value and attributes
if isinstance(evalue, BaseException):
for name in dir(evalue):
# prevent py26 DeprecationWarning
if (name != 'messages' or sys.version_info <
(2.6)) and not name.startswith('__'):
value = pydoc.text.repr(getattr(evalue, name))
# render multilingual string properly
if type(value) == str and value.startswith(b"u'"):
value = eval(value)
s['exception'][name] = encode(value)
# add all local values (of last frame) to the snapshot
for name, value in locals.items():
if type(value) == str and value.startswith(b"u'"):
value = eval(value)
s['locals'][name] = pydoc.text.repr(value)
return s
def writerow(self, row): if six.PY2: row = encode(row, self.encoding) self.writer.writerow(row)
