def handle(self, *args, **options): if not settings.CENTRAL_SERVER: raise CommandError("Disabled for distributed servers, until we can figure out what to do with ") options['platform'] = options['platform'].lower() # normalize if options['platform'] not in ["all", "linux", "macos", "darwin", "windows"]: raise CommandError("Unrecognized platform: %s; will include ALL files." % options['platform']) # Step 0: refresh all resources get_dubbed_video_map(force=True) # force a remote download # Step 1: recursively add all static files kalite_base = os.path.realpath(settings.PROJECT_PATH + "/../") files_dict = recursively_add_files(dirpath=kalite_base, **options) # Step 2: Add a local_settings.py file. # For distributed servers, this is a copy of the local local_settings.py, # with a few properties (specified as command-line options) overridden ls_file = create_local_settings_file(location=os.path.realpath(kalite_base+"/kalite/local_settings.py"), server_type=options['server_type'], locale=options['locale'], central_server=options["central_server"]) files_dict[ls_file] = { "dest_path": "kalite/local_settings.py" } # Step 3: select output file. if not options['file']: options['file'] = create_default_archive_filename(options) # Step 4: package into a zip file ensure_dir(os.path.realpath(os.path.dirname(options["file"]))) # allows relative paths to be passed.=== system_specific_zipping( files_dict = dict([(v["dest_path"], src_path) for src_path, v in files_dict.iteritems()]), zip_file = options["file"], compression=ZIP_DEFLATED if options['compress'] else ZIP_STORED, callback=_default_callback_zip if options["verbosity"] else None, )
def update_via_git(self, branch=None, *args, **kwargs): """ Full update via git """ self.stages = [ "clean_pyc", "git", "download", "syncdb", ] # step 1: clean_pyc (has to be first) call_command("clean_pyc", path=os.path.join(settings.PROJECT_PATH, "..")) self.start(notes="Clean up pyc files") # Step 2: update via git self.next_stage(notes="Updating via git%s" % (" to branch %s" % branch if branch else "")) repo = git.Repo() try: if not branch: # old behavior--assume you're pulling to remote self.stdout.write(repo.git.pull() + "\n") elif "/" not in branch: self.stdout.write(repo.git.fetch() + "\n") # update all branches across all repos, to make sure all branches exist self.stdout.write(repo.git.checkout(branch) + "\n") else: self.stdout.write(repo.git.fetch("--all", "-p"), "\n") # update all branches across all repos, to make sure all branches exist self.stdout.write(repo.git.checkout("-t", branch) + "\n") except git.errors.GitCommandError as gce: if not (branch and "There is no tracking information for the current branch" in gce.stderr): # pull failed because the branch is local only. this is OK when you specify a branch (switch), but not when you don't (has to be a remote pull) self.stderr.write("Error running %s\n" % gce.command) self.stderr.write("%s\n" % gce.stderr) exit(1) # step 3: get other remote resources self.next_stage("Download remote resources") get_dubbed_video_map(force=True) # force a remote download # step 4: syncdb / migrate, via setup # NOTE: this MUST be done via an external process, # to guarantee all the new code is begin used. self.next_stage("Update the database [please wait; no interactive output]") # should be interactive=False, but this method is a total hack (out, err, rc) = call_outside_command_with_output("setup", noinput=True, manage_py_dir=settings.PROJECT_PATH) sys.stderr.write(out) if rc: sys.stderr.write(err) # Done! self.complete()
def handle(self, *args, **options): if not settings.CENTRAL_SERVER: raise CommandError( "Disabled for distributed servers, until we can figure out what to do with " ) options['platform'] = options['platform'].lower() # normalize if options['platform'] not in [ "all", "linux", "macos", "darwin", "windows" ]: raise CommandError( "Unrecognized platform: %s; will include ALL files." % options['platform']) # Step 0: refresh all resources get_dubbed_video_map(force=True) # force a remote download # Step 1: recursively add all static files kalite_base = os.path.realpath(settings.PROJECT_PATH + "/../") files_dict = recursively_add_files(dirpath=kalite_base, **options) # Step 2: Add a local_settings.py file. # For distributed servers, this is a copy of the local local_settings.py, # with a few properties (specified as command-line options) overridden ls_file = create_local_settings_file( location=os.path.realpath(kalite_base + "/kalite/local_settings.py"), server_type=options['server_type'], locale=options['locale'], central_server=options["central_server"]) files_dict[ls_file] = {"dest_path": "kalite/local_settings.py"} # Step 3: select output file. if not options['file']: options['file'] = create_default_archive_filename(options) # Step 4: package into a zip file ensure_dir(os.path.realpath(os.path.dirname( options["file"]))) # allows relative paths to be passed.=== system_specific_zipping( files_dict=dict([(v["dest_path"], src_path) for src_path, v in files_dict.iteritems()]), zip_file=options["file"], compression=ZIP_DEFLATED if options['compress'] else ZIP_STORED, callback=_default_callback_zip if options["verbosity"] else None, )
def handle(self, *args, **options): if settings.CENTRAL_SERVER: raise CommandError("This must only be run on the distributed server.") if not options["lang_code"]: raise CommandError("You must specify a language code.") # ensure_dir(settings.CONTENT_ROOT) # Get list of videos lang_code = lcode_to_ietf(options["lang_code"]) video_map = get_dubbed_video_map(lang_code) or {} video_ids = options["video_ids"].split(",") if options["video_ids"] else None video_ids = video_ids or ([vid["id"] for vid in get_topic_videos(topic_id=options["topic_id"])] if options["topic_id"] else None) video_ids = video_ids or video_map.keys() # Download the videos for video_id in video_ids: if video_id in video_map: youtube_id = video_map[video_id] elif video_id in video_map.values(): # Perhaps they sent in a youtube ID? We can handle that! youtube_id = video_id else: logging.error("No mapping for video_id=%s; skipping" % video_id) continue try: scrape_video(youtube_id=youtube_id, format=options["format"], force=options["force"]) #scrape_thumbnail(youtube_id=youtube_id) logging.info("Access video %s at %s" % (youtube_id, get_node_cache("Video")[video_id][0]["path"])) except Exception as e: logging.error("Failed to download video %s: %s" % (youtube_id, e)) logging.info("Process complete.")
def update_via_git(self, branch=None, *args, **kwargs): """ Full update via git """ self.stages = [ "clean_pyc", "git", "download", "syncdb", ] # step 1: clean_pyc (has to be first) call_command("clean_pyc", path=os.path.join(settings.PROJECT_PATH, "..")) self.start(notes="Clean up pyc files") # Step 2: update via git self.next_stage(notes="Updating via git%s" % (" to branch %s" % branch if branch else "")) repo = git.Repo() try: if not branch: # old behavior--assume you're pulling to remote self.stdout.write(repo.git.pull() + "\n") elif "/" not in branch: self.stdout.write( repo.git.fetch() + "\n" ) # update all branches across all repos, to make sure all branches exist self.stdout.write(repo.git.checkout(branch) + "\n") else: self.stdout.write( repo.git.fetch("--all", "-p"), "\n" ) # update all branches across all repos, to make sure all branches exist self.stdout.write(repo.git.checkout("-t", branch) + "\n") except git.errors.GitCommandError as gce: if not (branch and "There is no tracking information for the current branch" in gce.stderr): # pull failed because the branch is local only. this is OK when you specify a branch (switch), but not when you don't (has to be a remote pull) self.stderr.write("Error running %s\n" % gce.command) self.stderr.write("%s\n" % gce.stderr) exit(1) # step 3: get other remote resources self.next_stage("Download remote resources") get_dubbed_video_map(force=True) # force a remote download # step 4: syncdb / migrate, via setup # NOTE: this MUST be done via an external process, # to guarantee all the new code is begin used. self.next_stage( "Update the database [please wait; no interactive output]") # should be interactive=False, but this method is a total hack (out, err, rc) = call_outside_command_with_output( "setup", noinput=True, manage_py_dir=settings.PROJECT_PATH) sys.stderr.write(out) if rc: sys.stderr.write(err) # Done! self.complete()