def do_bulkloader_passthru_argv():
"""
Execute bulkloader script with appropriate parameters. For more
details, please invoke 'python manage.py bulkloader --help'.
"""
progname = sys.argv[0]
models = get_all_models_as_dict()
args = []
for arg in sys.argv[1:]:
args.append(arg)
if '--help' in args:
print_status(bulkloader.__doc__ % {'arg0': "manage.py bulkloader"})
sys.stdout.flush()
sys.stderr.flush()
sys.exit(0)
sys.exit(bulkloader.main(args))
def do_bulkloader_passthru_argv():
"""
Execute bulkloader script with appropriate parameters. For more
details, please invoke 'python manage.py bulkloader --help'.
"""
progname = sys.argv[0]
models = get_all_models_as_dict()
args = []
for arg in sys.argv[1:]:
args.append(arg)
if '--help' in args:
print_status(bulkloader.__doc__ % {'arg0': "manage.py bulkloader"})
sys.stdout.flush()
sys.stderr.flush()
sys.exit(0)
sys.exit(bulkloader.main(args))
def do_appcfg_passthru_argv():
"""
Execute appcfg.py with specified parameters. For more details,
please invoke 'python manage.py appcfg --help'.
"""
from google.appengine.tools import appcfg
progname = sys.argv[0]
if len(sys.argv) < 3:
sys.stderr.write('action required.\n')
sys.exit(1)
if 'update' in sys.argv:
do_preparse_apps()
models = get_all_models_as_dict()
args = []
for arg in sys.argv[2:]:
args.append(arg)
if arg == "request_logs":
args.append(os.getcwdu())
if "--help" in args or "help" in args or "request_logs" in args:
args = [progname] + args
else:
args = [progname] + args + [os.getcwdu()]
logging.basicConfig(format=('%(asctime)s %(levelname)s %(filename)s:'
'%(lineno)s %(message)s '))
try:
app = appcfg.AppCfgApp(args, parser_class=HookedOptionParser)
result = app.Run()
if result:
sys.exit(result)
except KeyboardInterrupt:
print_status('Interrupted.')
sys.exit(1)
from kay.conf import settings
if settings.PROFILE and 'update' in sys.argv:
print_status(
'--------------------------\n'
'WARNING: PROFILER ENABLED!\n'
'--------------------------'
)
def dump_or_restore_all(help, data_set_name, app_id, url, directory, batch_size, op):
if help:
print_status('help for %s' % op)
sys.exit(0)
if not data_set_name:
data_set_name = datetime.datetime.now().strftime("%Y%m%d.%H%M%S")
if not app_id:
app_id = get_appid()
if not url:
url = "https://%s.appspot.com/remote_api" % app_id
if not directory:
directory = '_backup'
target_dir = os.path.join(kay.PROJECT_DIR, directory, data_set_name)
if not os.path.isdir(target_dir):
if op == DUMP:
makedirs(target_dir)
print_status('Directory "%s" created.' % target_dir)
else:
print_status('Directory "%s" is missing, exiting...' % target_dir)
sys.exit(1)
current_time = datetime.datetime.now().strftime("%Y%m%d.%H%M%S")
models = get_all_models_as_dict(only_polymodel_base=True)
results = {}
if op == RESTORE:
base_args = ["bulkloader", "--restore"]
else:
base_args = ["bulkloader", "--dump"]
base_args.append("--application=%s" % app_id)
if "localhost" in url:
bulkloader.RequestManager.AuthFunction = dummy_auth_func
else:
bulkloader.RequestManager.AuthFunction = real_auth_func
for key, model in models.iteritems():
kind = model.kind()
db_filename = os.path.join(target_dir, "bulkloader-%s-%s.progress" %
(kind, current_time))
log_file = os.path.join(target_dir, "bulkloader-%s-%s.log" %
(kind, current_time))
result_db_filename = os.path.join(target_dir, "bulkloader-%s-%s.result" %
(kind, current_time))
args = copy.copy(base_args)
args.append("--filename=%s" % os.path.join(target_dir, "%s.dat" % kind))
args.append("--kind=%s" % kind)
args.append("--db_filename=%s" % db_filename)
args.append("--log_file=%s" % log_file)
if batch_size:
args.append("--batch_size=%s" % int(batch_size))
if op == DUMP:
args.append("--result_db_filename=%s" % result_db_filename)
args.append("--url=%s" % url)
try:
from werkzeug.utils import import_string
backup_mod = import_string(directory)
if op == RESTORE:
args.extend(backup_mod.restore_options[kind])
else:
args.extend(backup_mod.dump_options[kind])
except:
pass
try:
results[key] = bulkloader.main(args)
except bulkloader.FileNotFoundError, e:
print_status("File not found, skipped: %s" % e)
results[key] = -1
continue
logging.getLogger('google.appengine.tools.bulkloader').handlers = []
def dump_or_restore_all(help, data_set_name, app_id, url, directory, op):
if help:
print_status('help for %s' % op)
sys.exit(0)
if not data_set_name:
data_set_name = datetime.datetime.now().strftime("%Y%m%d.%H%M%S")
if not app_id:
app_id = get_appid()
if not url:
url = "https://%s.appspot.com/remote_api" % app_id
if not directory:
directory = '_backup'
target_dir = os.path.join(kay.PROJECT_DIR, directory, data_set_name)
if not os.path.isdir(target_dir):
if op == DUMP:
makedirs(target_dir)
print_status('Directory "%s" created.' % target_dir)
else:
print_status('Directory "%s" is missing, exiting...' % target_dir)
sys.exit(1)
current_time = datetime.datetime.now().strftime("%Y%m%d.%H%M%S")
models = get_all_models_as_dict(only_polymodel_base=True)
results = {}
if op == RESTORE:
base_args = ["bulkloader", "--restore"]
else:
base_args = ["bulkloader", "--dump"]
if "localhost" in url:
base_args.append("--app_id=%s" % app_id)
bulkloader.RequestManager.AuthFunction = dummy_auth_func
else:
bulkloader.RequestManager.AuthFunction = real_auth_func
for key, model in models.iteritems():
kind = model.kind()
db_filename = os.path.join(
target_dir, "bulkloader-%s-%s.progress" % (kind, current_time))
log_file = os.path.join(target_dir,
"bulkloader-%s-%s.log" % (kind, current_time))
result_db_filename = os.path.join(
target_dir, "bulkloader-%s-%s.result" % (kind, current_time))
args = copy.copy(base_args)
args.append("--filename=%s" %
os.path.join(target_dir, "%s.dat" % kind))
args.append("--kind=%s" % kind)
args.append("--db_filename=%s" % db_filename)
args.append("--log_file=%s" % log_file)
if op == DUMP:
args.append("--result_db_filename=%s" % result_db_filename)
args.append("--url=%s" % url)
try:
from werkzeug.utils import import_string
backup_mod = import_string(directory)
if op == RESTORE:
args.extend(backup_mod.restore_options[kind])
else:
args.extend(backup_mod.dump_options[kind])
except:
pass
try:
results[key] = bulkloader.main(args)
except bulkloader.FileNotFoundError, e:
print_status("File not found, skipped: %s" % e)
results[key] = -1
continue
logging.getLogger('google.appengine.tools.bulkloader').handlers = []