def handle(self, *args, **options):
if not settings.CENTRAL_SERVER:
raise CommandError("Disabled for distributed servers, until we can figure out what to do with ")
options['platform'] = options['platform'].lower() # normalize
if options['platform'] not in ["all", "linux", "macos", "darwin", "windows"]:
raise CommandError("Unrecognized platform: %s; will include ALL files." % options['platform'])
# Step 0: refresh all resources
get_dubbed_video_map(force=True) # force a remote download
# Step 1: recursively add all static files
kalite_base = os.path.realpath(settings.PROJECT_PATH + "/../")
files_dict = recursively_add_files(dirpath=kalite_base, **options)
# Step 2: Add a local_settings.py file.
# For distributed servers, this is a copy of the local local_settings.py,
# with a few properties (specified as command-line options) overridden
ls_file = create_local_settings_file(location=os.path.realpath(kalite_base+"/kalite/local_settings.py"), server_type=options['server_type'], locale=options['locale'], central_server=options["central_server"])
files_dict[ls_file] = { "dest_path": "kalite/local_settings.py" }
# Step 3: select output file.
if not options['file']:
options['file'] = create_default_archive_filename(options)
# Step 4: package into a zip file
ensure_dir(os.path.realpath(os.path.dirname(options["file"]))) # allows relative paths to be passed.===
system_specific_zipping(
files_dict = dict([(v["dest_path"], src_path) for src_path, v in files_dict.iteritems()]),
zip_file = options["file"],
compression=ZIP_DEFLATED if options['compress'] else ZIP_STORED,
callback=_default_callback_zip if options["verbosity"] else None,
)
def update_via_git(self, branch=None, *args, **kwargs):
"""
Full update via git
"""
self.stages = [
"clean_pyc",
"git",
"download",
"syncdb",
]
# step 1: clean_pyc (has to be first)
call_command("clean_pyc", path=os.path.join(settings.PROJECT_PATH, ".."))
self.start(notes="Clean up pyc files")
# Step 2: update via git
self.next_stage(notes="Updating via git%s" % (" to branch %s" % branch if branch else ""))
repo = git.Repo()
try:
if not branch:
# old behavior--assume you're pulling to remote
self.stdout.write(repo.git.pull() + "\n")
elif "/" not in branch:
self.stdout.write(repo.git.fetch() + "\n") # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout(branch) + "\n")
else:
self.stdout.write(repo.git.fetch("--all", "-p"), "\n") # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout("-t", branch) + "\n")
except git.errors.GitCommandError as gce:
if not (branch and "There is no tracking information for the current branch" in gce.stderr):
# pull failed because the branch is local only. this is OK when you specify a branch (switch), but not when you don't (has to be a remote pull)
self.stderr.write("Error running %s\n" % gce.command)
self.stderr.write("%s\n" % gce.stderr)
exit(1)
# step 3: get other remote resources
self.next_stage("Download remote resources")
get_dubbed_video_map(force=True) # force a remote download
# step 4: syncdb / migrate, via setup
# NOTE: this MUST be done via an external process,
# to guarantee all the new code is begin used.
self.next_stage("Update the database [please wait; no interactive output]")
# should be interactive=False, but this method is a total hack
(out, err, rc) = call_outside_command_with_output("setup", noinput=True, manage_py_dir=settings.PROJECT_PATH)
sys.stderr.write(out)
if rc:
sys.stderr.write(err)
# Done!
self.complete()
def handle(self, *args, **options):
if not settings.CENTRAL_SERVER:
raise CommandError(
"Disabled for distributed servers, until we can figure out what to do with "
)
options['platform'] = options['platform'].lower() # normalize
if options['platform'] not in [
"all", "linux", "macos", "darwin", "windows"
]:
raise CommandError(
"Unrecognized platform: %s; will include ALL files." %
options['platform'])
# Step 0: refresh all resources
get_dubbed_video_map(force=True) # force a remote download
# Step 1: recursively add all static files
kalite_base = os.path.realpath(settings.PROJECT_PATH + "/../")
files_dict = recursively_add_files(dirpath=kalite_base, **options)
# Step 2: Add a local_settings.py file.
# For distributed servers, this is a copy of the local local_settings.py,
# with a few properties (specified as command-line options) overridden
ls_file = create_local_settings_file(
location=os.path.realpath(kalite_base +
"/kalite/local_settings.py"),
server_type=options['server_type'],
locale=options['locale'],
central_server=options["central_server"])
files_dict[ls_file] = {"dest_path": "kalite/local_settings.py"}
# Step 3: select output file.
if not options['file']:
options['file'] = create_default_archive_filename(options)
# Step 4: package into a zip file
ensure_dir(os.path.realpath(os.path.dirname(
options["file"]))) # allows relative paths to be passed.===
system_specific_zipping(
files_dict=dict([(v["dest_path"], src_path)
for src_path, v in files_dict.iteritems()]),
zip_file=options["file"],
compression=ZIP_DEFLATED if options['compress'] else ZIP_STORED,
callback=_default_callback_zip if options["verbosity"] else None,
)
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the distributed server.")
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
#
ensure_dir(settings.CONTENT_ROOT)
# Get list of videos
lang_code = lcode_to_ietf(options["lang_code"])
video_map = get_dubbed_video_map(lang_code) or {}
video_ids = options["video_ids"].split(",") if options["video_ids"] else None
video_ids = video_ids or ([vid["id"] for vid in get_topic_videos(topic_id=options["topic_id"])] if options["topic_id"] else None)
video_ids = video_ids or video_map.keys()
# Download the videos
for video_id in video_ids:
if video_id in video_map:
youtube_id = video_map[video_id]
elif video_id in video_map.values():
# Perhaps they sent in a youtube ID? We can handle that!
youtube_id = video_id
else:
logging.error("No mapping for video_id=%s; skipping" % video_id)
continue
try:
scrape_video(youtube_id=youtube_id, format=options["format"], force=options["force"])
#scrape_thumbnail(youtube_id=youtube_id)
logging.info("Access video %s at %s" % (youtube_id, get_node_cache("Video")[video_id][0]["path"]))
except Exception as e:
logging.error("Failed to download video %s: %s" % (youtube_id, e))
logging.info("Process complete.")
def update_via_git(self, branch=None, *args, **kwargs):
"""
Full update via git
"""
self.stages = [
"clean_pyc",
"git",
"download",
"syncdb",
]
# step 1: clean_pyc (has to be first)
call_command("clean_pyc",
path=os.path.join(settings.PROJECT_PATH, ".."))
self.start(notes="Clean up pyc files")
# Step 2: update via git
self.next_stage(notes="Updating via git%s" %
(" to branch %s" % branch if branch else ""))
repo = git.Repo()
try:
if not branch:
# old behavior--assume you're pulling to remote
self.stdout.write(repo.git.pull() + "\n")
elif "/" not in branch:
self.stdout.write(
repo.git.fetch() + "\n"
) # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout(branch) + "\n")
else:
self.stdout.write(
repo.git.fetch("--all", "-p"), "\n"
) # update all branches across all repos, to make sure all branches exist
self.stdout.write(repo.git.checkout("-t", branch) + "\n")
except git.errors.GitCommandError as gce:
if not (branch and
"There is no tracking information for the current branch"
in gce.stderr):
# pull failed because the branch is local only. this is OK when you specify a branch (switch), but not when you don't (has to be a remote pull)
self.stderr.write("Error running %s\n" % gce.command)
self.stderr.write("%s\n" % gce.stderr)
exit(1)
# step 3: get other remote resources
self.next_stage("Download remote resources")
get_dubbed_video_map(force=True) # force a remote download
# step 4: syncdb / migrate, via setup
# NOTE: this MUST be done via an external process,
# to guarantee all the new code is begin used.
self.next_stage(
"Update the database [please wait; no interactive output]")
# should be interactive=False, but this method is a total hack
(out, err, rc) = call_outside_command_with_output(
"setup", noinput=True, manage_py_dir=settings.PROJECT_PATH)
sys.stderr.write(out)
if rc:
sys.stderr.write(err)
# Done!
self.complete()
