def copy_in_data(self):
"""Copy over sqlite3 database, then run syncdb"""
if settings.DATABASES['default'][
'ENGINE'] != 'django.db.backends.sqlite3':
sys.stdout.write("Nothing to do for non-sqlite3 database.\n")
return
elif not os.path.exists(settings.DATABASES['default']['NAME']):
sys.stdout.write("KA Lite not yet installed; no data to copy.\n")
return
else:
# Copy over data for sqlite
sys.stdout.write(
"* Copying over database to the server update location\n")
shutil.copy(
settings.DATABASES['default']['NAME'],
os.path.realpath(self.working_dir +
"/kalite/database/data.sqlite"))
# Run the syncdb
sys.stdout.write("* Syncing database...")
sys.stdout.flush()
call_outside_command_with_output("setup",
interactive=False,
manage_py_dir=os.path.join(
self.working_dir, "kalite"))
sys.stdout.write("\n")
def start_server(self, port=None):
"""
Start the server, for real (not to test) (cron and web server)
Assumes the web server is shut down.
"""
self._print_message("Starting the server")
# Start the server to validate
manage_py_dir = os.path.join(self.dest_dir, 'kalite')
# shift to an existing directory first to remove the reference to a deleted directory
os.chdir(tempfile.gettempdir())
# now go back to the working directory
os.chdir(manage_py_dir)
stdout, stderr, exit_code, proc = call_outside_command_with_output(
'kaserve',
wait=False,
manage_py_dir=manage_py_dir,
output_to_stdin=True,
)
self._wait_for_server_to_be_up()
self._print_message("Server should be accessible @ port %s.\n" %
(port or settings.USER_FACING_PORT()))
def call_command(self,
commandname,
*args,
**kwargs):
'''
Run a command in the context of this distributed server, customized to
its own settings.
commandname -- the management command to run
output_to_stdout -- True to output the command's stdout to the console
instead of capturing to a variable
output_to_stderr -- True to output the command's stderr to the console
instead of capturing to a variable
'''
output_to_stdout = kwargs.pop('output_to_stdout', True)
output_to_stderr = kwargs.pop('output_to_stderr', True)
if self.running_process:
raise Exception('Command {} already started.'.format(commandname))
_, _err, _ret, self.running_process = call_outside_command_with_output(
commandname,
output_to_stdout=output_to_stdout,
output_to_stderr=output_to_stderr,
settings=self.settings_name,
manage_py_dir=self.distributed_dir.as_posix(),
wait=False,
*args,
**kwargs
)
return self
def start_server(self, port=None):
"""
Start the server, for real (not to test) (cron and web server)
Assumes the web server is shut down.
"""
self._print_message("Starting the server")
# Start the server to validate
target_dir = getattr(self, "dest_dir", os.path.join(settings.PROJECT_PATH, '..'))
manage_py_dir = os.path.join(target_dir, 'kalite')
# shift to an existing directory first to remove the reference to a deleted directory
os.chdir(tempfile.gettempdir())
# now go back to the working directory
os.chdir(manage_py_dir)
stdout, stderr, exit_code, proc = call_outside_command_with_output(
'kaserve',
wait=False,
manage_py_dir=manage_py_dir,
output_to_stdout=True,
production=True,
)
self._wait_for_server_to_be_up()
self._print_message("Server should be accessible @ port %s.\n" % (port or settings.USER_FACING_PORT()))
def call_command(self, commandname, *args, **kwargs):
'''
Run a command in the context of this distributed server, customized to
its own settings.
commandname -- the management command to run
output_to_stdout -- True to output the command's stdout to the console
instead of capturing to a variable
output_to_stderr -- True to output the command's stderr to the console
instead of capturing to a variable
'''
output_to_stdout = kwargs.pop('output_to_stdout', True)
output_to_stderr = kwargs.pop('output_to_stderr', True)
if self.running_process:
raise Exception('Command {} already started.'.format(commandname))
_, _err, _ret, self.running_process = call_outside_command_with_output(
commandname,
output_to_stdout=output_to_stdout,
output_to_stderr=output_to_stderr,
settings=self.settings_name,
manage_py_dir=self.distributed_dir.as_posix(),
wait=False,
*args,
**kwargs)
return self
def test_server_weak(self):
sys.stdout.write("* Testing the new server (simple)\n")
out = call_outside_command_with_output("update",
"test",
manage_py_dir=os.path.join(
self.working_dir, "kalite"))
if "Success!" not in out[0]:
raise CommandError(out[1] if out[1] else out[0])
def copy_in_data(self):
"""Copy over sqlite3 database, then run syncdb"""
if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
sys.stdout.write("Nothing to do for non-sqlite3 database.\n")
return
elif not os.path.exists(settings.DATABASES['default']['NAME']):
sys.stdout.write("KA Lite not yet installed; no data to copy.\n")
return
else:
# Copy over data for sqlite
sys.stdout.write("* Copying over database to the server update location\n")
shutil.copy(settings.DATABASES['default']['NAME'], os.path.realpath(self.working_dir + "/kalite/database/data.sqlite"))
# Run the syncdb
sys.stdout.write("* Syncing database...")
sys.stdout.flush()
call_outside_command_with_output("setup", interactive=False, manage_py_dir=os.path.join(self.working_dir, "kalite"))
sys.stdout.write("\n")
def update_via_git(self, branch=None, *args, **kwargs):
"""
Full update via git
"""
self.stages = [
"clean_pyc",
"git",
"download",
"syncdb",
]
# step 1: clean_pyc (has to be first)
call_command("clean_pyc", path=os.path.join(settings.PROJECT_PATH, ".."))
self.start(notes="Clean up pyc files")
# Step 2: update via git
self.next_stage(notes="Updating via git%s" % (" to branch %s" % branch if branch else ""))
repo = git.Repo()
try:
if not branch:
# old behavior--assume you're pulling to remote
self.stdout.write(repo.git.pull() + "\n")
elif "/" not in branch:
self.stdout.write(repo.git.fetch() + "\n") # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout(branch) + "\n")
else:
self.stdout.write(repo.git.fetch("--all", "-p"), "\n") # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout("-t", branch) + "\n")
except git.errors.GitCommandError as gce:
if not (branch and "There is no tracking information for the current branch" in gce.stderr):
# pull failed because the branch is local only. this is OK when you specify a branch (switch), but not when you don't (has to be a remote pull)
self.stderr.write("Error running %s\n" % gce.command)
self.stderr.write("%s\n" % gce.stderr)
exit(1)
# step 3: get other remote resources
self.next_stage("Download remote resources")
get_dubbed_video_map(force=True) # force a remote download
# step 4: syncdb / migrate, via setup
# NOTE: this MUST be done via an external process,
# to guarantee all the new code is begin used.
self.next_stage("Update the database [please wait; no interactive output]")
# should be interactive=False, but this method is a total hack
(out, err, rc) = call_outside_command_with_output("setup", noinput=True, manage_py_dir=settings.PROJECT_PATH)
sys.stderr.write(out)
if rc:
sys.stderr.write(err)
# Done!
self.complete()
def update_via_git(self, branch=None, *args, **kwargs):
"""
Full update via git
"""
self.stages = [
"clean_pyc",
"git",
"download",
"syncdb",
]
# TODO: use the git package to detect if we're in a git repo
if not os.path.exists(os.path.join(settings.PROJECT_PATH, "..",
".git")):
raise CommandError(
_("You have not installed KA Lite through Git. Please use the other update methods instead, e.g. 'internet' or 'localzip'"
))
# step 1: clean_pyc (has to be first)
call_command("clean_pyc",
path=os.path.join(settings.PROJECT_PATH, ".."))
self.start(notes="Clean up pyc files")
# Step 2: update via git
self.next_stage(notes="Updating via git%s" %
(" to branch %s" % branch if branch else ""))
repo = git.Repo()
try:
if not branch:
# old behavior--assume you're pulling to remote
self.stdout.write(repo.git.pull() + "\n")
elif "/" not in branch:
self.stdout.write(
repo.git.fetch() + "\n"
) # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout(branch) + "\n")
else:
self.stdout.write(
repo.git.fetch("--all", "-p"), "\n"
) # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout("-t", branch) + "\n")
except git.errors.GitCommandError as gce:
if not (branch and
"There is no tracking information for the current branch"
in gce.stderr):
# pull failed because the branch is local only. this is OK when you specify a branch (switch), but not when you don't (has to be a remote pull)
self.stderr.write("Error running %s\n" % gce.command)
self.stderr.write("%s\n" % gce.stderr)
exit(1)
# step 3: get other remote resources
self.next_stage("Download remote resources")
get_dubbed_video_map(force=True) # force a remote download
# step 4: syncdb / migrate, via setup
# NOTE: this MUST be done via an external process,
# to guarantee all the new code is begin used.
self.next_stage(
"Update the database [please wait; no interactive output]")
# should be interactive=False, but this method is a total hack
(out, err, rc) = call_outside_command_with_output(
"setup", noinput=True, manage_py_dir=settings.PROJECT_PATH)
sys.stderr.write(out)
if rc:
sys.stderr.write(err)
# Done!
self.complete()
def update_via_git(self, *args, **kwargs):
"""
Full update via git
"""
self.stages = [
"clean_pyc",
"gitpull",
"download",
"syncdb",
"stop_server",
"start_server",
]
remote_name = settings.GIT_UPDATE_REMOTE_NAME
remote_branch = settings.GIT_UPDATE_BRANCH
remote_url = settings.GIT_UPDATE_REPO_URL
# step 0: check that we are in a git repo first!
try:
repo = git.Repo(os.path.dirname(__file__))
except git.exc.InvalidGitRepositoryError:
raise CommandError(
_("You have not installed KA Lite through Git. Please use the other update methods instead, e.g. 'internet' or 'localzip'"
))
# step 1: clean_pyc (has to be first)
call_command("clean_pyc",
path=os.path.join(settings.PROJECT_PATH, ".."))
self.start(notes="Clean up pyc files")
# Step 2: update via git
self.next_stage(notes="Updating via git branch %s in remote %s" %
(remote_branch, remote_name))
# Step 2a: add the remote first
try:
remote = repo.create_remote(remote_name, remote_url)
except git.exc.GitCommandError: # remote already exists
repo.git.remote(
'set-url', remote_name,
remote_url) # update it in case we change the remote url
remote = repo.remote(remote_name)
# Step 2b: fetch the update remote
try:
remote.fetch()
except AssertionError as e: # sometimes we get this, but the fetch is successful anyway, so continue on!
pass
# Step 2c: stash any changes we have
now = datetime.datetime.now().isoformat()
repo.git.stash("save", "Stash from update initiated at %s" % now)
# Step 2c: checkout the remote branch
remote_name_branch = '%s/%s' % (remote_name, remote_branch)
repo.git.checkout(remote_name_branch)
# step 3: get other remote resources
self.next_stage("Download remote resources")
get_dubbed_video_map(force=True) # force a remote download
# step 4: syncdb / migrate, via setup
# NOTE: this MUST be done via an external process,
# to guarantee all the new code is begin used.
self.next_stage(
"Update the database [please wait; no interactive output]")
# should be interactive=False, but this method is a total hack
(out, err, rc, p) = call_outside_command_with_output(
"setup", noinput=True, manage_py_dir=settings.PROJECT_PATH)
sys.stderr.write(out)
if rc:
sys.stderr.write(err)
# step 5: stop the server
self.stop_server()
# step 6: start the server
self.start_server()
# Done!
self.complete()
def test_server_weak(self):
sys.stdout.write("* Testing the new server (simple)\n")
out = call_outside_command_with_output("update", "test", manage_py_dir=os.path.join(self.working_dir, "kalite"))
if "Success!" not in out[0]:
raise CommandError(out[1] if out[1] else out[0])
def update_via_git(self, *args, **kwargs):
"""
Full update via git
"""
self.stages = [
"clean_pyc",
"gitpull",
"download",
"syncdb",
"stop_server",
"start_server",
]
remote_name = settings.GIT_UPDATE_REMOTE_NAME
remote_branch = settings.GIT_UPDATE_BRANCH
remote_url = settings.GIT_UPDATE_REPO_URL
# step 0: check that we are in a git repo first!
try:
repo = git.Repo(os.path.dirname(__file__))
except git.exc.InvalidGitRepositoryError:
raise CommandError(_("You have not installed KA Lite through Git. Please use the other update methods instead, e.g. 'internet' or 'localzip'"))
# step 1: clean_pyc (has to be first)
call_command("clean_pyc", path=os.path.join(settings.PROJECT_PATH, ".."))
self.start(notes="Clean up pyc files")
# Step 2: update via git
self.next_stage(notes="Updating via git branch %s in remote %s" % (remote_branch, remote_name))
# Step 2a: add the remote first
try:
remote = repo.create_remote(remote_name, remote_url)
except git.exc.GitCommandError: # remote already exists
repo.git.remote('set-url', remote_name, remote_url) # update it in case we change the remote url
remote = repo.remote(remote_name)
# Step 2b: fetch the update remote
try:
remote.fetch()
except AssertionError as e: # sometimes we get this, but the fetch is successful anyway, so continue on!
pass
# Step 2c: stash any changes we have
now = datetime.datetime.now().isoformat()
repo.git.stash("save", "Stash from update initiated at %s" % now)
# Step 2c: checkout the remote branch
remote_name_branch = '%s/%s' % (remote_name, remote_branch)
repo.git.checkout(remote_name_branch)
# step 3: get other remote resources
self.next_stage("Download remote resources")
get_dubbed_video_map(force=True) # force a remote download
# step 4: syncdb / migrate, via setup
# NOTE: this MUST be done via an external process,
# to guarantee all the new code is begin used.
self.next_stage("Update the database [please wait; no interactive output]")
# should be interactive=False, but this method is a total hack
(out, err, rc, p) = call_outside_command_with_output("setup", noinput=True, manage_py_dir=settings.PROJECT_PATH)
sys.stderr.write(out)
if rc:
sys.stderr.write(err)
# step 5: stop the server
self.stop_server()
# step 6: start the server
self.start_server()
# Done!
self.complete()
