def backup_to_dropbox():
from dropbox import client, session
from conf import dropbox_access_key, dropbox_secret_key
from webnotes.utils.backups import new_backup
if not webnotes.conn:
webnotes.connect()
sess = session.DropboxSession(dropbox_access_key, dropbox_secret_key, "app_folder")
sess.set_token(webnotes.conn.get_value("Backup Manager", None, "dropbox_access_key"),
webnotes.conn.get_value("Backup Manager", None, "dropbox_access_secret"))
dropbox_client = client.DropboxClient(sess)
# upload database
backup = new_backup()
filename = backup.backup_path_db
upload_file_to_dropbox(filename, "database", dropbox_client)
# upload files
response = dropbox_client.metadata("files")
# add missing files
for filename in os.listdir(os.path.join("public", "files")):
found = False
for file_metadata in response["contents"]:
if filename==os.path.basename(file_metadata["path"]):
if os.stat(os.path.join("public", "files", filename)).st_size==file_metadata["bytes"]:
found=True
if not found:
upload_file_to_dropbox(os.path.join("public", "files", filename), "files", dropbox_client)
def run():
webnotes.connect()
# Confirmation from user
confirm = ''
while not confirm:
confirm = raw_input(
"Are you sure you want to delete the data from the system (N/Y)?")
if confirm.lower() != 'y':
raise Exception
cleanup_type = ''
while cleanup_type not in ['1', '2']:
cleanup_type = raw_input("""\nWhat type of cleanup you want ot perform?
1. Only Transactions
2. Both Masters and Transactions
Please enter your choice (1/2):
""")
# delete
delete_transactions()
if cleanup_type == '1':
reset_transaction_series()
else:
delete_masters()
reset_all_series()
delete_main_masters()
reset_global_defaults()
print "System cleaned up succesfully"
webnotes.conn.close()
def patch(patch_module, site=None, force=False): import webnotes.modules.patch_handler webnotes.connect(site=site) webnotes.local.patch_log_list = [] webnotes.modules.patch_handler.run_single(patch_module, force=force) print "\n".join(webnotes.local.patch_log_list) webnotes.destroy()
def clear_web(site=None): import webnotes.webutils webnotes.connect(site=site) from website.doctype.website_sitemap_config.website_sitemap_config import build_website_sitemap_config build_website_sitemap_config() webnotes.webutils.clear_cache() webnotes.destroy()
def reset_perms(site=None):
webnotes.connect(site=site)
for d in webnotes.conn.sql_list("""select name from `tabDocType`
where ifnull(istable, 0)=0 and ifnull(custom, 0)=0"""):
webnotes.clear_cache(doctype=d)
webnotes.reset_perms(d)
webnotes.destroy()
def set_admin_password(admin_password, site=None):
import webnotes
webnotes.connect(site=site)
webnotes.conn.sql("""update __Auth set `password`=password(%s)
where user='******'""", (admin_password,))
webnotes.conn.commit()
webnotes.destroy()
def latest(site=None, verbose=True): import webnotes.modules.patch_handler import webnotes.model.sync import webnotes.plugins from website.doctype.website_sitemap_config.website_sitemap_config import build_website_sitemap_config webnotes.connect(site=site) try: # run patches webnotes.local.patch_log_list = [] webnotes.modules.patch_handler.run_all() if verbose: print "\n".join(webnotes.local.patch_log_list) # sync webnotes.model.sync.sync_all() # remove __init__.py from plugins webnotes.plugins.remove_init_files() # build website config if any changes in templates etc. build_website_sitemap_config() except webnotes.modules.patch_handler.PatchError, e: print "\n".join(webnotes.local.patch_log_list) raise
def make(): webnotes.connect() webnotes.mute_emails = True install() complete_setup() make_items() make_customers_suppliers_contacts()
def request(args):
import webnotes.handler
webnotes.connect()
webnotes.form_dict = webnotes._dict([a.split("=") for a in args.split("&")])
webnotes.handler.execute_cmd(webnotes.form_dict.cmd)
print webnotes.response
webnotes.destroy()
def reset_perms(site=None):
webnotes.connect(site=site)
for d in webnotes.conn.sql_list("""select name from `tabDocType`
where ifnull(istable, 0)=0 and ifnull(custom, 0)=0"""):
webnotes.clear_cache(doctype=d)
webnotes.reset_perms(d)
webnotes.destroy()
def main():
import argparse
parser = argparse.ArgumentParser(description='Run tests.')
parser.add_argument('-d', '--doctype', nargs=1, metavar = "DOCTYPE",
help="test for doctype")
parser.add_argument('-v', '--verbose', default=False, action="store_true")
parser.add_argument('-e', '--export', nargs=2, metavar="DOCTYPE DOCNAME")
parser.add_argument('-a', '--all', default=False, action="store_true")
parser.add_argument('-m', '--module', default=1, metavar="MODULE")
args = parser.parse_args()
if not webnotes.conn:
webnotes.connect()
webnotes.flags.print_messages = args.verbose
webnotes.flags.in_test = True
if args.doctype:
run_unittest(args.doctype[0], verbose=args.verbose)
elif args.all:
run_all_tests(args.verbose)
elif args.export:
export_doc(args.export[0], args.export[1])
elif args.module:
import importlib
test_suite = unittest.TestSuite()
module = importlib.import_module(args.module)
if hasattr(module, "test_dependencies"):
for doctype in module.test_dependencies:
make_test_records(doctype, verbose=args.verbose)
test_suite.addTest(unittest.TestLoader().loadTestsFromModule(sys.modules[args.module]))
unittest.TextTestRunner(verbosity=1+(args.verbose and 1 or 0)).run(test_suite)
def run():
webnotes.connect()
# Confirmation from user
confirm = ''
while not confirm:
confirm = raw_input("Are you sure you want to delete the data from the system (N/Y)?")
if confirm.lower() != 'y':
raise Exception
cleanup_type = ''
while cleanup_type not in ['1', '2']:
cleanup_type = raw_input("""\nWhat type of cleanup you want ot perform?
1. Only Transactions
2. Both Masters and Transactions
Please enter your choice (1/2):
""")
# delete
delete_transactions()
if cleanup_type == '1':
reset_transaction_series()
else:
delete_masters()
reset_all_series()
delete_main_masters()
reset_global_defaults()
print "System cleaned up succesfully"
webnotes.conn.close()
def make():
import os
import webnotes
import website.utils
import startup.event_handlers
if not webnotes.conn:
webnotes.connect()
home_page = website.utils.get_home_page()
fname = 'js/wn-web.js'
if os.path.basename(os.path.abspath('.')) != 'public':
fname = os.path.join('public', fname)
if hasattr(startup.event_handlers, 'get_web_script'):
with open(fname, 'w') as f:
script = 'window.home_page = "%s";\n' % home_page
script += startup.event_handlers.get_web_script()
f.write(script)
fname = 'css/wn-web.css'
if os.path.basename(os.path.abspath('.')) != 'public':
fname = os.path.join('public', fname)
# style - wn.css
if hasattr(startup.event_handlers, 'get_web_style'):
with open(fname, 'w') as f:
f.write(startup.event_handlers.get_web_style())
def main():
import argparse
parser = argparse.ArgumentParser(description='Run tests.')
parser.add_argument('-d', '--doctype', nargs=1, metavar = "DOCTYPE",
help="test for doctype")
parser.add_argument('-v', '--verbose', default=False, action="store_true")
parser.add_argument('-e', '--export', nargs=2, metavar="DOCTYPE DOCNAME")
parser.add_argument('-a', '--all', default=False, action="store_true")
parser.add_argument('-m', '--module', default=1, metavar="MODULE")
args = parser.parse_args()
if not webnotes.conn:
webnotes.connect()
webnotes.flags.print_messages = args.verbose
webnotes.flags.in_test = True
if args.doctype:
run_unittest(args.doctype[0], verbose=args.verbose)
elif args.all:
run_all_tests(args.verbose)
elif args.export:
export_doc(args.export[0], args.export[1])
elif args.module:
import importlib
test_suite = unittest.TestSuite()
module = importlib.import_module(args.module)
if hasattr(module, "test_dependencies"):
for doctype in module.test_dependencies:
make_test_records(doctype, verbose=args.verbose)
test_suite.addTest(unittest.TestLoader().loadTestsFromModule(sys.modules[args.module]))
unittest.TextTestRunner(verbosity=1+(args.verbose and 1 or 0)).run(test_suite)
def latest(site=None, verbose=True):
import webnotes.modules.patch_handler
import webnotes.model.sync
import webnotes.plugins
from website.doctype.website_sitemap_config.website_sitemap_config import build_website_sitemap_config
webnotes.connect(site=site)
try:
# run patches
webnotes.local.patch_log_list = []
webnotes.modules.patch_handler.run_all()
if verbose:
print "\n".join(webnotes.local.patch_log_list)
# sync
webnotes.model.sync.sync_all()
# remove __init__.py from plugins
webnotes.plugins.remove_init_files()
# build website config if any changes in templates etc.
build_website_sitemap_config()
except webnotes.modules.patch_handler.PatchError, e:
print "\n".join(webnotes.local.patch_log_list)
raise
def clear_web(site=None):
import webnotes.webutils
webnotes.connect(site=site)
from website.doctype.website_sitemap_config.website_sitemap_config import build_website_sitemap_config
build_website_sitemap_config()
webnotes.webutils.clear_cache()
webnotes.destroy()
def make():
import os
import webnotes
import website.utils
import startup.event_handlers
if not webnotes.conn:
webnotes.connect()
home_page = website.utils.get_home_page()
fname = 'js/wn-web.js'
if os.path.basename(os.path.abspath('.'))!='public':
fname = os.path.join('public', fname)
if hasattr(startup.event_handlers, 'get_web_script'):
with open(fname, 'w') as f:
script = 'window.home_page = "%s";\n' % home_page
script += startup.event_handlers.get_web_script()
f.write(script)
fname = 'css/wn-web.css'
if os.path.basename(os.path.abspath('.'))!='public':
fname = os.path.join('public', fname)
# style - wn.css
if hasattr(startup.event_handlers, 'get_web_style'):
with open(fname, 'w') as f:
f.write(startup.event_handlers.get_web_style())
def backup_to_dropbox():
from dropbox import client, session
from conf import dropbox_access_key, dropbox_secret_key
from webnotes.utils.backups import new_backup
if not webnotes.conn:
webnotes.connect()
sess = session.DropboxSession(dropbox_access_key, dropbox_secret_key, "app_folder")
sess.set_token(webnotes.conn.get_value("Backup Manager", None, "dropbox_access_key"),
webnotes.conn.get_value("Backup Manager", None, "dropbox_access_secret"))
dropbox_client = client.DropboxClient(sess)
# upload database
backup = new_backup()
filename = os.path.join(get_base_path(), "public", "backups",
os.path.basename(backup.backup_path_db))
upload_file_to_dropbox(filename, "database", dropbox_client)
response = dropbox_client.metadata("/files")
# upload files to files folder
path = os.path.join(get_base_path(), "public", "files")
for filename in os.listdir(path):
found = False
filepath = os.path.join(path, filename)
for file_metadata in response["contents"]:
if os.path.basename(filepath) == os.path.basename(file_metadata["path"]) and os.stat(filepath).st_size == int(file_metadata["bytes"]):
found = True
break
if not found:
upload_file_to_dropbox(filepath, "files", dropbox_client)
def patch(patch_module, site=None, force=False):
import webnotes.modules.patch_handler
webnotes.connect(site=site)
webnotes.local.patch_log_list = []
webnotes.modules.patch_handler.run_single(patch_module, force=force)
print "\n".join(webnotes.local.patch_log_list)
webnotes.destroy()
def get_html(page_name):
"""get page html"""
page_name = scrub_page_name(page_name)
html = ''
# load from cache, if auto cache clear is falsy
if not (hasattr(conf, 'auto_cache_clear') and conf.auto_cache_clear or 0):
if not page_name in no_cache:
html = webnotes.cache().get_value("page:" + page_name)
from_cache = True
if not html:
webnotes.connect()
html = load_into_cache(page_name)
from_cache = False
if not html:
html = get_html("404")
if page_name=="error":
html = html.replace("%(error)s", webnotes.getTraceback())
else:
comments = "\n\npage:"+page_name+\
"\nload status: " + (from_cache and "cache" or "fresh")
html += """\n<!-- %s -->""" % webnotes.utils.cstr(comments)
return html
def set_admin_password(admin_password, site=None):
import webnotes
webnotes.connect(site=site)
webnotes.conn.sql("""update __Auth set `password`=password(%s)
where user='******'""", (admin_password,))
webnotes.conn.commit()
webnotes.destroy()
def build_page(page_name):
if not webnotes.conn:
webnotes.connect()
sitemap = get_website_sitemap()
page_options = sitemap.get(page_name)
if not page_options:
if page_name=="index":
# page not found, try home page
home_page = get_home_page()
page_options = sitemap.get(home_page)
if not page_options:
raise PageNotFoundError
page_options["page_name"] = home_page
else:
raise PageNotFoundError
else:
page_options["page_name"] = page_name
basepath = webnotes.utils.get_base_path()
module = None
no_cache = False
if page_options.get("controller"):
module = webnotes.get_module(page_options["controller"])
no_cache = getattr(module, "no_cache", False)
# if generator, then load bean, pass arguments
if page_options.get("is_generator"):
if not module:
raise Exception("Generator controller not defined")
name = webnotes.conn.get_value(module.doctype, {
page_options.get("page_name_field", "page_name"): page_options["page_name"]})
obj = webnotes.get_obj(module.doctype, name, with_children=True)
if hasattr(obj, 'get_context'):
obj.get_context()
context = webnotes._dict(obj.doc.fields)
context["obj"] = obj
else:
# page
context = webnotes._dict({ 'name': page_name })
if module and hasattr(module, "get_context"):
context.update(module.get_context())
context.update(get_website_settings())
jenv = webnotes.get_jenv()
context["base_template"] = jenv.get_template(webnotes.get_config().get("base_template"))
template_name = page_options['template']
html = jenv.get_template(template_name).render(context)
if not no_cache:
webnotes.cache().set_value("page:" + page_name, html)
return html
def backup_to_gdrive():
from webnotes.utils.backups import new_backup
if not webnotes.conn:
webnotes.connect()
get_gdrive_flow()
credentials_json = webnotes.conn.get_value("Backup Manager", None,
"gdrive_credentials")
credentials = oauth2client.client.Credentials.new_from_json(
credentials_json)
http = httplib2.Http()
http = credentials.authorize(http)
drive_service = build('drive', 'v2', http=http)
# upload database
backup = new_backup()
path = os.path.join(get_base_path(), "public", "backups")
filename = os.path.join(path, os.path.basename(backup.backup_path_db))
# upload files to database folder
upload_files(
filename, 'application/x-gzip', drive_service,
webnotes.conn.get_value("Backup Manager", None, "database_folder_id"))
# upload files to files folder
did_not_upload = []
error_log = []
files_folder_id = webnotes.conn.get_value("Backup Manager", None,
"files_folder_id")
webnotes.conn.close()
path = os.path.join(get_base_path(), "public", "files")
for filename in os.listdir(path):
filename = cstr(filename)
found = False
filepath = os.path.join(path, filename)
ext = filename.split('.')[-1]
size = os.path.getsize(filepath)
if ext == 'gz' or ext == 'gzip':
mimetype = 'application/x-gzip'
else:
mimetype = mimetypes.types_map.get(
"." + ext) or "application/octet-stream"
#Compare Local File with Server File
children = drive_service.children().list(
folderId=files_folder_id).execute()
for child in children.get('items', []):
file = drive_service.files().get(fileId=child['id']).execute()
if filename == file['title'] and size == int(file['fileSize']):
found = True
break
if not found:
try:
upload_files(filepath, mimetype, drive_service,
files_folder_id)
except Exception, e:
did_not_upload.append(filename)
error_log.append(cstr(e))
def domain(host_url=None, site=None):
webnotes.connect(site=site)
if host_url:
webnotes.conn.set_value("Website Settings", None, "subdomain", host_url)
webnotes.conn.commit()
else:
print webnotes.conn.get_value("Website Settings", None, "subdomain")
webnotes.destroy()
def run_scheduler():
from webnotes.utils.file_lock import create_lock, delete_lock
import webnotes.utils.scheduler
if create_lock('scheduler'):
webnotes.connect()
print webnotes.utils.scheduler.execute()
delete_lock('scheduler')
webnotes.destroy()
def domain(host_url=None, site=None):
webnotes.connect(site=site)
if host_url:
webnotes.conn.set_value("Website Settings", None, "subdomain", host_url)
webnotes.conn.commit()
else:
print webnotes.conn.get_value("Website Settings", None, "subdomain")
webnotes.destroy()
def make(): webnotes.connect() webnotes.print_messages = True webnotes.mute_emails = True install() complete_setup() make_items() make_customers_suppliers_contacts()
def create_erpnext_folder(service):
if not webnotes.conn:
webnotes.connect()
erpnext = {
'title': 'erpnext',
'mimeType': 'application/vnd.google-apps.folder'
}
erpnext = service.files().insert(body=erpnext).execute()
return erpnext['id']
def make(reset=False): webnotes.connect() #webnotes.print_messages = True webnotes.mute_emails = True webnotes.rollback_on_exception = True if reset: setup() simulate()
def setup(): install() webnotes.connect(db_name=webnotes.conf.demo_db_name) complete_setup() make_customers_suppliers_contacts() make_items() make_price_lists() make_users_and_employees() make_bank_account()
def setup():
install()
webnotes.connect(db_name=webnotes.conf.demo_db_name)
complete_setup()
make_customers_suppliers_contacts()
make_items()
make_price_lists()
make_users_and_employees()
make_bank_account()
def backup(site=None, with_files=False, verbose=True, backup_path_db=None, backup_path_files=None): from webnotes.utils.backups import scheduled_backup webnotes.connect(site=site) print backup_path_db odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files) if verbose: from webnotes.utils import now print "backup taken -", odb.backup_path_db, "- on", now() return odb
def run_scheduler(site=None):
from webnotes.utils.file_lock import create_lock, delete_lock
import webnotes.utils.scheduler
webnotes.init(site=site)
if create_lock('scheduler'):
webnotes.connect(site=site)
print webnotes.utils.scheduler.execute()
delete_lock('scheduler')
webnotes.destroy()
def make_demo_app():
webnotes.mute_emails = 1
webnotes.connect()
utilities.demo.make_demo.make(reset=True, simulate=False)
# setup demo user etc so that the site it up faster, while the data loads
make_demo_user()
make_demo_login_page()
make_demo_on_login_script()
utilities.demo.make_demo.make(reset=False, simulate=True)
def run_scheduler(site=None):
from webnotes.utils.file_lock import create_lock, delete_lock
import webnotes.utils.scheduler
webnotes.init(site=site)
if create_lock('scheduler'):
webnotes.connect(site=site)
print webnotes.utils.scheduler.execute()
delete_lock('scheduler')
webnotes.destroy()
def make_demo_app():
webnotes.mute_emails = 1
webnotes.connect()
utilities.demo.make_demo.make(reset=True, simulate=False)
# setup demo user etc so that the site it up faster, while the data loads
make_demo_user()
make_demo_login_page()
make_demo_on_login_script()
utilities.demo.make_demo.make(reset=False, simulate=True)
def make(reset=False): webnotes.connect() webnotes.print_messages = True webnotes.mute_emails = True webnotes.rollback_on_exception = True if reset: setup() simulate()
def create_owrang_folder(service):
if not webnotes.conn:
webnotes.connect()
owrang = {
'title': 'owrang',
'mimeType': 'application/vnd.google-apps.folder'
}
owrang = service.files().insert(body=owrang).execute()
return owrang['id']
def get_home_page():
if not webnotes.conn:
webnotes.connect()
doc_name = webnotes.conn.get_value('Website Settings', None, 'home_page')
if doc_name:
page_name = webnotes.conn.get_value('Web Page', doc_name, 'page_name')
else:
page_name = 'login'
return page_name
def get_home_page():
if not webnotes.conn:
webnotes.connect()
doc_name = webnotes.conn.get_value('Website Settings', None, 'home_page')
if doc_name:
page_name = webnotes.conn.get_value('Web Page', doc_name, 'page_name')
else:
page_name = 'login'
return page_name
def build_page(page_name):
if not webnotes.conn:
webnotes.connect()
sitemap_options = webnotes.doc("Website Sitemap", page_name).fields
page_options = webnotes.doc(
"Website Sitemap Config",
sitemap_options.get("website_sitemap_config")).fields.update({
"page_name":
sitemap_options.page_name,
"docname":
sitemap_options.docname
})
if not page_options:
raise PageNotFoundError
else:
page_options["page_name"] = page_name
basepath = webnotes.utils.get_base_path()
no_cache = page_options.get("no_cache")
# if generator, then load bean, pass arguments
if page_options.get("page_or_generator") == "Generator":
doctype = page_options.get("ref_doctype")
obj = webnotes.get_obj(doctype,
page_options["docname"],
with_children=True)
if hasattr(obj, 'get_context'):
obj.get_context()
context = webnotes._dict(obj.doc.fields)
context["obj"] = obj
else:
# page
context = webnotes._dict({'name': page_name})
if page_options.get("controller"):
module = webnotes.get_module(page_options.get("controller"))
if module and hasattr(module, "get_context"):
context.update(module.get_context())
context.update(get_website_settings())
jenv = webnotes.get_jenv()
context["base_template"] = jenv.get_template(
webnotes.get_config().get("base_template"))
template_name = page_options['template_path']
html = jenv.get_template(template_name).render(context)
if not no_cache:
webnotes.cache().set_value("page:" + page_name, html)
return html
def make():
import os
import webnotes
# TODO: why is jinja2 imported?
from jinja2 import Template
import webnotes.cms
if not webnotes.conn:
webnotes.connect()
make_web_core()
def backup(site=None, with_files=False, verbose=True, backup_path_db=None, backup_path_files=None): from webnotes.utils.backups import scheduled_backup webnotes.connect(site=site) odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files) if verbose: from webnotes.utils import now print "database backup taken -", odb.backup_path_db, "- on", now() if with_files: print "files backup taken -", odb.backup_path_files, "- on", now() webnotes.destroy() return odb
def backup(site=None, with_files=False, verbose=True, backup_path_db=None, backup_path_files=None, delete_older_than=6): from webnotes.utils.backups import scheduled_backup webnotes.connect(site=site) odb = scheduled_backup(delete_older_than=delete_older_than, ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files) if verbose: from webnotes.utils import now print "database backup taken -", odb.backup_path_db, "- on", now() if with_files: print "files backup taken -", odb.backup_path_files, "- on", now() webnotes.destroy() return odb
def make(): import os import webnotes # TODO: why is jinja2 imported? from jinja2 import Template import webnotes.cms if not webnotes.conn: webnotes.connect() make_web_core()
def execute(site=None):
"""
execute jobs
this method triggers the other scheduler events
Database connection: Ideally it should be connected from outside, if there is
no connection, it will connect from defs.py
"""
from datetime import datetime
import webnotes.utils
format = '%Y-%m-%d %H:%M:%S'
if not webnotes.conn:
webnotes.connect(site=site)
out = []
nowtime = webnotes.utils.now_datetime()
last = webnotes.conn.get_global('scheduler_last_event')
# set scheduler last event
webnotes.conn.begin()
webnotes.conn.set_global('scheduler_last_event', nowtime.strftime(format))
webnotes.conn.commit()
if last:
last = datetime.strptime(last, format)
if nowtime.day != last.day:
# if first task of the day execute daily tasks
out.append(
nowtime.strftime("%Y-%m-%d %H:%M:%S") + ' - daily:' +
trigger('execute_daily'))
if nowtime.month != last.month:
out.append(
nowtime.strftime("%Y-%m-%d %H:%M:%S") + ' - monthly:' +
trigger('execute_monthly'))
if nowtime.weekday() == 0:
out.append(
nowtime.strftime("%Y-%m-%d %H:%M:%S") + ' - weekly:' +
trigger('execute_weekly'))
if nowtime.hour != last.hour:
out.append(
nowtime.strftime("%Y-%m-%d %H:%M:%S") + ' - hourly:' +
trigger('execute_hourly'))
out.append(
nowtime.strftime("%Y-%m-%d %H:%M:%S") + ' - all:' +
trigger('execute_all'))
return '\n'.join(out)
def backup_to_dropbox():
from dropbox import client, session
from conf import dropbox_access_key, dropbox_secret_key
from webnotes.utils.backups import new_backup
from webnotes.utils import get_files_path, get_backups_path
if not webnotes.conn:
webnotes.connect()
sess = session.DropboxSession(dropbox_access_key, dropbox_secret_key,
"app_folder")
sess.set_token(
webnotes.conn.get_value("Backup Manager", None, "dropbox_access_key"),
webnotes.conn.get_value("Backup Manager", None,
"dropbox_access_secret"))
dropbox_client = client.DropboxClient(sess)
# upload database
backup = new_backup()
filename = os.path.join(get_backups_path(),
os.path.basename(backup.backup_path_db))
upload_file_to_dropbox(filename, "/database", dropbox_client)
webnotes.conn.close()
response = dropbox_client.metadata("/files")
# upload files to files folder
did_not_upload = []
error_log = []
path = get_files_path()
for filename in os.listdir(path):
filename = cstr(filename)
if filename in ignore_list:
continue
found = False
filepath = os.path.join(path, filename)
for file_metadata in response["contents"]:
if os.path.basename(filepath) == os.path.basename(
file_metadata["path"]) and os.stat(
filepath).st_size == int(file_metadata["bytes"]):
found = True
break
if not found:
try:
upload_file_to_dropbox(filepath, "/files", dropbox_client)
except Exception:
did_not_upload.append(filename)
error_log.append(webnotes.getTraceback())
webnotes.connect()
return did_not_upload, list(set(error_log))
def backup_to_gdrive():
from webnotes.utils.backups import new_backup
if not webnotes.conn:
webnotes.connect()
get_gdrive_flow()
credentials_json = webnotes.conn.get_value("Backup Manager", None, "gdrive_credentials")
credentials = oauth2client.client.Credentials.new_from_json(credentials_json)
http = httplib2.Http()
http = credentials.authorize(http)
drive_service = build('drive', 'v2', http=http)
# upload database
backup = new_backup()
path = os.path.join(get_base_path(), "public", "backups")
filename = os.path.join(path, os.path.basename(backup.backup_path_db))
# upload files to database folder
upload_files(filename, 'application/x-gzip', drive_service,
webnotes.conn.get_value("Backup Manager", None, "database_folder_id"))
# upload files to files folder
did_not_upload = []
error_log = []
files_folder_id = webnotes.conn.get_value("Backup Manager", None, "files_folder_id")
webnotes.conn.close()
path = os.path.join(get_base_path(), "public", "files")
for filename in os.listdir(path):
found = False
filepath = os.path.join(path, filename)
ext = filename.split('.')[-1]
size = os.path.getsize(filepath)
if ext == 'gz' or ext == 'gzip':
mimetype = 'application/x-gzip'
else:
mimetype = mimetypes.types_map.get("." + ext) or "application/octet-stream"
#Compare Local File with Server File
param = {}
children = drive_service.children().list(folderId=files_folder_id, **param).execute()
for child in children.get('items', []):
file = drive_service.files().get(fileId=child['id']).execute()
if filename == file['title'] and size == int(file['fileSize']):
found = True
break
if not found:
try:
upload_files(filepath, mimetype, drive_service, files_folder_id)
except Exception, e:
did_not_upload.append(filename)
error_log.append(cstr(e))
def backup_to_dropbox():
from dropbox import client, session
from conf import dropbox_access_key, dropbox_secret_key
from webnotes.utils.backups import new_backup
if not webnotes.conn:
webnotes.connect()
sess = session.DropboxSession(dropbox_access_key, dropbox_secret_key, "app_folder")
sess.set_token(
webnotes.conn.get_value("Backup Manager", None, "dropbox_access_key"),
webnotes.conn.get_value("Backup Manager", None, "dropbox_access_secret"),
)
dropbox_client = client.DropboxClient(sess)
# upload database
backup = new_backup()
filename = os.path.join(get_base_path(), "public", "backups", os.path.basename(backup.backup_path_db))
upload_file_to_dropbox(filename, "/database", dropbox_client)
webnotes.conn.close()
response = dropbox_client.metadata("/files")
# upload files to files folder
did_not_upload = []
error_log = []
path = os.path.join(get_base_path(), "public", "files")
for filename in os.listdir(path):
filename = cstr(filename)
if filename in ignore_list:
continue
found = False
filepath = os.path.join(path, filename)
for file_metadata in response["contents"]:
if os.path.basename(filepath) == os.path.basename(file_metadata["path"]) and os.stat(
filepath
).st_size == int(file_metadata["bytes"]):
found = True
break
if not found:
try:
upload_file_to_dropbox(filepath, "/files", dropbox_client)
except Exception:
did_not_upload.append(filename)
error_log.append(webnotes.getTraceback())
webnotes.connect()
return did_not_upload, list(set(error_log))
def build(page_name):
if not webnotes.conn:
webnotes.connect()
build_method = (build_json if is_ajax() else build_page)
try:
return build_method(page_name)
except webnotes.DoesNotExistError:
hooks = webnotes.get_hooks()
if hooks.website_catch_all:
return build_method(hooks.website_catch_all[0])
else:
return build_method("404")
def build_page(page_name):
if not webnotes.conn:
webnotes.connect()
sitemap_options = webnotes.doc("Website Sitemap", page_name).fields
page_options = webnotes.doc("Website Sitemap Config",
sitemap_options.get("website_sitemap_config")).fields.update({
"page_name":sitemap_options.page_name,
"docname":sitemap_options.docname
})
if not page_options:
raise PageNotFoundError
else:
page_options["page_name"] = page_name
basepath = webnotes.utils.get_base_path()
no_cache = page_options.get("no_cache")
# if generator, then load bean, pass arguments
if page_options.get("page_or_generator")=="Generator":
doctype = page_options.get("ref_doctype")
obj = webnotes.get_obj(doctype, page_options["docname"], with_children=True)
if hasattr(obj, 'get_context'):
obj.get_context()
context = webnotes._dict(obj.doc.fields)
context["obj"] = obj
else:
# page
context = webnotes._dict({ 'name': page_name })
if page_options.get("controller"):
module = webnotes.get_module(page_options.get("controller"))
if module and hasattr(module, "get_context"):
context.update(module.get_context())
context.update(get_website_settings())
jenv = webnotes.get_jenv()
context["base_template"] = jenv.get_template(webnotes.get_config().get("base_template"))
template_name = page_options['template_path']
html = jenv.get_template(template_name).render(context)
if not no_cache:
webnotes.cache().set_value("page:" + page_name, html)
return html
def make_test_records(doctype, verbose=0):
webnotes.mute_emails = True
if not webnotes.conn:
webnotes.connect()
for options in get_dependencies(doctype):
if options.startswith("link:"):
options = options[5:]
if options == "[Select]":
continue
if options not in webnotes.test_objects:
webnotes.test_objects[options] = []
make_test_records(options, verbose)
make_test_records_for_doctype(options, verbose)
def build_message_files():
"""build from doctypes, pages, database and framework"""
if not webnotes.conn:
webnotes.connect()
build_for_pages('lib/core')
build_for_pages('app')
build_from_doctype_code('lib/core')
build_from_doctype_code('app')
# doctype
build_from_database()
build_for_framework('lib/webnotes', 'py', with_doctype_names=True)
build_for_framework('lib/public/js/wn', 'js')
build_for_framework('app/public/js', 'js', with_doctype_names=True)
def upload_files(name, mimetype, service, folder_id):
if not webnotes.conn:
webnotes.connect()
file_name = os.path.basename(name)
media_body = MediaFileUpload(name, mimetype=mimetype, resumable=True)
body = {
'title': file_name,
'description': 'Backup File',
'mimetype': mimetype,
'parents': [{
'kind': 'drive#filelink',
'id': folder_id
}]
}
request = service.files().insert(body=body, media_body=media_body)
response = None
while response is None:
status, response = request.next_chunk()
def latest(site=None, verbose=True):
import webnotes.modules.patch_handler
import webnotes.model.sync
webnotes.connect(site=site)
try:
# run patches
webnotes.local.patch_log_list = []
webnotes.modules.patch_handler.run_all()
if verbose:
print "\n".join(webnotes.local.patch_log_list)
# sync
webnotes.model.sync.sync_all()
except webnotes.modules.patch_handler.PatchError, e:
print "\n".join(webnotes.local.patch_log_list)
raise e
def make(reset=False, simulate=True):
#webnotes.flags.print_messages = True
webnotes.flags.mute_emails = True
webnotes.flags.rollback_on_exception = True
if not webnotes.conf.demo_db_name:
raise Exception("conf.py does not have demo_db_name")
if reset:
setup()
else:
if webnotes.conn:
webnotes.conn.close()
webnotes.connect(db_name=webnotes.conf.demo_db_name)
if simulate:
_simulate()
def log(method):
"""log error in patch_log"""
import webnotes
if not (webnotes.conn and webnotes.conn._conn):
webnotes.connect()
webnotes.conn.rollback()
traceback = webnotes.getTraceback()
import webnotes.utils
webnotes.conn.begin()
d = webnotes.doc("Scheduler Log")
d.method = method
d.error = traceback
d.save()
webnotes.conn.commit()
return traceback
def take_backups_dropbox():
did_not_upload, error_log = [], []
try:
from setup.doctype.backup_manager.backup_dropbox import backup_to_dropbox
did_not_upload, error_log = backup_to_dropbox()
if did_not_upload: raise Exception
send_email(True, "Dropbox")
except Exception:
file_and_error = [
" - ".join(f) for f in zip(did_not_upload, error_log)
]
error_message = ("\n".join(file_and_error) + "\n" +
webnotes.getTraceback())
webnotes.errprint(error_message)
if not webnotes.conn:
webnotes.connect()
send_email(False, "Dropbox", error_message)
def make():
from webnotes.webutils import get_home_page
from webnotes.utils import get_path
if not webnotes.conn:
webnotes.connect()
home_page = get_home_page()
if not os.path.exists(get_path("public", "js")):
os.makedirs(get_path("public", "js"))
fname = os.path.join(get_path("public", "js", "wn-web.js"))
with open(fname, 'w') as f:
f.write(get_web_script())
if not os.path.exists(get_path("public", "css")):
os.makedirs(get_path("public", "css"))
fname = os.path.join(get_path("public", "css", "wn-web.css"))
with open(fname, 'w') as f:
f.write(get_web_style())