def update_json(youtube_id, lang_code, downloaded, api_response, time_of_attempt):
"""Update language_srt_map to reflect download status"""
# Open JSON file
filepath = get_lang_map_filepath(lang_code)
try:
language_srt_map = json.loads(open(filepath).read())
except Exception as e:
logging.error("Something went wrong while trying to open the json file (%s): %s" % (filepath, e))
return False
# create updated entry
entry = language_srt_map[youtube_id]
entry["downloaded"] = downloaded
entry["api_response"] = api_response
entry["last_attempt"] = time_of_attempt
if api_response == "success":
entry["last_success"] = time_of_attempt
# update full-size JSON with new information
language_srt_map[youtube_id].update(entry)
# write it to file
logging.info("File updated.")
json_file = open(filepath, "wb")
json_file.write(json.dumps(language_srt_map))
json_file.close()
return True
def generate_zipped_srts(lang_codes_to_update, download_path=DOWNLOAD_PATH):
# Create media directory if it doesn't yet exist
ensure_dir(settings.MEDIA_ROOT)
zip_path = settings.MEDIA_ROOT + "subtitles/"
ensure_dir(zip_path)
lang_codes_to_update = lang_codes_to_update or os.listdir(download_path)
for lang_code in lang_codes_to_update:
srt_dir = os.path.join(download_path, lang_code, "subtitles")
zip_file = os.path.join(zip_path, "%s_subtitles.zip" % lang_code)
# Remove any old version (as we may not re-create)
if os.path.exists(zip_file):
os.remove(zip_file)
if not os.path.exists(srt_dir):
logging.warn("No srt directory for %s; skipping." % lang_code)
continue
srts = glob.glob(os.path.join(srt_dir, "*.srt"))
if len(srts) == 0:
logging.warn("No srts for %s; skipping." % lang_code)
continue
logging.info("Zipping up a new pack for language code: %s" % lang_code)
zf = zipfile.ZipFile(zip_file, 'w')
for f in srts:
zf.write(f, arcname=os.path.basename(f))
zf.close()
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError(
"Run this command on the distributed server only.")
# Load videos
video_sizes = softload_json(REMOTE_VIDEO_SIZE_FILEPATH,
logger=logging.debug)
# Query current files
all_video_filepaths = glob.glob(
os.path.join(settings.CONTENT_ROOT, "*.mp4"))
logging.info("Querying sizes for %d video(s)." %
len(all_video_filepaths))
# Get all current sizes
for video_filepath in all_video_filepaths:
youtube_id = os.path.splitext(os.path.basename(video_filepath))[0]
# Set to max, so that local compressed videos will not affect things.
video_sizes[youtube_id] = max(video_sizes.get(youtube_id, 0),
os.path.getsize(video_filepath))
# Sort results
video_sizes = OrderedDict([(key, video_sizes[key])
for key in sorted(video_sizes.keys())])
logging.info("Saving results to disk.")
ensure_dir(os.path.dirname(REMOTE_VIDEO_SIZE_FILEPATH))
with open(REMOTE_VIDEO_SIZE_FILEPATH, "w") as fp:
json.dump(video_sizes, fp, indent=2)
def handle(self, *args, **options):
# Check that we can run
if not settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the central server.")
supported_langs = get_supported_languages()
if not options["lang_codes"]:
lang_codes = supported_langs
else:
requested_codes = set(options["lang_codes"].split(","))
lang_codes = [lcode_to_ietf(lc) for lc in requested_codes if lc in supported_langs]
unsupported_codes = requested_codes - set(lang_codes)
if unsupported_codes:
raise CommandError("Requested unsupported languages: %s" % sorted(list(unsupported_codes)))
# Scrub options
for key in options:
# If no_update is set, then disable all update options.
if key.startswith("update_"):
options[key] = options[key] and not options["no_update"]
if version_diff(options["version"], "0.10.3") < 0:
raise CommandError("This command cannot be used for versions before 0.10.3")
if options['low_mem']:
logging.info('Making the GC more aggressive...')
gc.set_threshold(36, 2, 2)
# For dealing with central server changes across versions
upgrade_old_schema()
# Now, we're going to build the language packs, collecting metadata long the way.
package_metadata = update_language_packs(lang_codes, options)
def handle(self, *args, **options):
# Get the CSV data, either from a recent cache_file
# or from the internet
cache_dir = settings.MEDIA_ROOT
cache_file = os.path.join(cache_dir, "dubbed_videos.csv")
if os.path.exists(cache_file) and datediff(datetime.datetime.now(), datetime.datetime.fromtimestamp(os.path.getctime(cache_file)), units="days") <= 14.0:
# Use cached data to generate the video map
csv_data = open(cache_file, "r").read()
(video_map, _) = generate_dubbed_video_mappings(csv_data=csv_data)
else:
# Use cached data to generate the video map
(video_map, csv_data) = generate_dubbed_video_mappings()
try:
ensure_dir(cache_dir)
with open(cache_file, "w") as fp:
fp.write(csv_data)
except Exception as e:
logging.error("Failed to make a local cache of the CSV data: %s" % e)
# Now we've built the map. Save it.
out_file = DUBBED_VIDEOS_MAPPING_FILE
ensure_dir(os.path.dirname(out_file))
logging.info("Saving data to %s" % out_file)
with open(out_file, "w") as fp:
json.dump(video_map, fp)
logging.info("Done.")
def invalidate_inmemory_caches():
for module in (i18n, topic_tools):
for cache_var in getattr(module, "CACHE_VARS", []):
logging.debug("Emptying cache %s.%s" % (module.__name__, cache_var))
setattr(module, cache_var, None)
logging.info("Great success emptying the in-memory cache.")
def handle(self, *args, **options):
if not options["lang_code"]:
raise CommandError("You must specify a language code.")
lang_code = lcode_to_ietf(options["lang_code"])
if lang_code not in AVAILABLE_EXERCISE_LANGUAGE_CODES:
logging.info("No exercises available for language %s" % lang_code)
else:
# Get list of exercises
exercise_ids = options["exercise_ids"].split(
",") if options["exercise_ids"] else None
exercise_ids = exercise_ids or ([
ex["id"]
for ex in get_topic_exercises(topic_id=options["topic_id"])
] if options["topic_id"] else None)
exercise_ids = exercise_ids or get_node_cache("Exercise").keys()
# Download the exercises
for exercise_id in exercise_ids:
scrape_exercise(exercise_id=exercise_id,
lang_code=lang_code,
force=options["force"])
logging.info("Process complete.")
def generate_fake_coachreport_logs():
teacher_password = make_password('hellothere')
t,_ = FacilityUser.objects.get_or_create(
facility=Facility.objects.all()[0],
username=random.choice(firstnames),
defaults={
'password' : teacher_password,
'is_teacher' : True,
}
)
# TODO: create flags later
num_logs = 20
logs = []
for _ in xrange(num_logs):
date_logged_in = datetime.datetime.now() - datetime.timedelta(days=random.randint(1,10))
date_viewed_coachreport = date_logged_in + datetime.timedelta(minutes=random.randint(0, 30))
date_logged_out = date_viewed_coachreport + datetime.timedelta(minutes=random.randint(0, 30))
login_log = UserLog.objects.create(
user=t,
activity_type=UserLog.get_activity_int("login"),
start_datetime=date_logged_in,
last_active_datetime=date_viewed_coachreport,
end_datetime=date_logged_out,
)
logging.info("created login log for teacher %s" % t.username)
coachreport_log = UserLog.objects.create(
user=t,
activity_type=UserLog.get_activity_int("coachreport"),
start_datetime=date_viewed_coachreport,
last_active_datetime=date_viewed_coachreport,
end_datetime=date_viewed_coachreport,
)
logs.append((login_log, coachreport_log))
logging.info("created coachreport log for teacher %s" % t.username)
return logs
def update_metadata(package_metadata, version=VERSION):
"""
We've zipped the packages, and now have unzipped & zipped sizes.
Update this info in the local metadata (but not inside the zip)
"""
master_filepath = get_language_pack_availability_filepath(version=version)
master_metadata = softload_json(master_filepath, logger=logging.warn, errmsg="Error opening master language pack metadata")
for lc, updated_meta in package_metadata.iteritems():
lang_code_ietf = lcode_to_ietf(lc)
# Gather existing metadata
metadata_filepath = get_language_pack_metadata_filepath(lang_code_ietf, version=version)
stored_meta = softload_json(metadata_filepath, logger=logging.warn, errmsg="Error opening %s language pack metadata" % lc)
stored_meta.update(updated_meta)
# Write locally (this is used on download by distributed server to update it's database)
with open(metadata_filepath, 'w') as output:
json.dump(stored_meta, output)
# Update master (this is used for central server to handle API requests for data)
master_metadata[lang_code_ietf] = stored_meta
# Save updated master
ensure_dir(os.path.dirname(master_filepath))
with open(master_filepath, 'w') as output:
json.dump(master_metadata, output)
logging.info("Local record of translations updated")
def update_json(youtube_id, lang_code, downloaded, api_response,
time_of_attempt):
"""Update language_srt_map to reflect download status"""
# Open JSON file
filepath = get_lang_map_filepath(lang_code)
try:
language_srt_map = json.loads(open(filepath).read())
except Exception as e:
logging.error(
"Something went wrong while trying to open the json file (%s): %s"
% (filepath, e))
return False
# create updated entry
entry = language_srt_map[youtube_id]
entry["downloaded"] = downloaded
entry["api_response"] = api_response
entry["last_attempt"] = time_of_attempt
if api_response == "success":
entry["last_success"] = time_of_attempt
# update full-size JSON with new information
language_srt_map[youtube_id].update(entry)
# write it to file
logging.info("File updated.")
json_file = open(filepath, "wb")
json_file.write(json.dumps(language_srt_map))
json_file.close()
return True
def handle(self, *args, **options):
# Get the CSV data, either from a recent cache_file
# or from the internet
cache_dir = settings.MEDIA_ROOT
cache_file = os.path.join(cache_dir, "dubbed_videos.csv")
if os.path.exists(cache_file) and datediff(
datetime.datetime.now(),
datetime.datetime.fromtimestamp(os.path.getctime(cache_file)),
units="days") <= 14.0:
# Use cached data to generate the video map
csv_data = open(cache_file, "r").read()
(video_map, _) = generate_dubbed_video_mappings(csv_data=csv_data)
else:
# Use cached data to generate the video map
(video_map, csv_data) = generate_dubbed_video_mappings()
try:
ensure_dir(cache_dir)
with open(cache_file, "w") as fp:
fp.write(csv_data)
except Exception as e:
logging.error(
"Failed to make a local cache of the CSV data: %s" % e)
# Now we've built the map. Save it.
out_file = DUBBED_VIDEOS_MAPPING_FILE
ensure_dir(os.path.dirname(out_file))
logging.info("Saving data to %s" % out_file)
with open(out_file, "w") as fp:
json.dump(video_map, fp)
logging.info("Done.")
def zip_language_packs(lang_codes=None):
"""Zip up and expose all language packs"""
lang_codes = lang_codes or listdir(LOCALE_ROOT)
logging.info("Zipping up %d language pack(s)" % len(lang_codes))
ensure_dir(settings.LANGUAGE_PACK_ROOT)
for lang in lang_codes:
lang_locale_path = os.path.join(LOCALE_ROOT, lang)
if not os.path.exists(lang_locale_path):
logging.warn("Unexpectedly skipping missing directory: %s" % lang)
elif not os.path.isdir(lang_locale_path):
logging.error("Skipping language where a file exists: %s" % lang)
# Create a zipfile for this language
zip_path = os.path.join(settings.LANGUAGE_PACK_ROOT, version.VERSION)
ensure_dir(zip_path)
z = zipfile.ZipFile(os.path.join(zip_path, "%s.zip" % convert_language_code_format(lang)), 'w')
# Get every single file in the directory and zip it up
for metadata_file in glob.glob('%s/*.json' % lang_locale_path):
z.write(os.path.join(lang_locale_path, metadata_file), arcname=os.path.basename(metadata_file))
for mo_file in glob.glob('%s/LC_MESSAGES/*.mo' % lang_locale_path):
z.write(os.path.join(lang_locale_path, mo_file), arcname=os.path.join("LC_MESSAGES", os.path.basename(mo_file)))
for srt_file in glob.glob('%s/subtitles/*.srt' % lang_locale_path):
z.write(os.path.join(lang_locale_path, srt_file), arcname=os.path.join("subtitles", os.path.basename(srt_file)))
z.close()
logging.info("Done.")
def handle(self, *args, **options):
if len(args)==1 and args[0]== "test":
# Callback for "weak" test--checks at least that the django project compiles (local_settings is OK)
sys.stdout.write("Success!\n")
exit(0)
try:
if options.get("branch", None):
# Specified a repo
self.update_via_git(**options)
elif options.get("zip_file", None):
# Specified a file
if not os.path.exists(options.get("zip_file")):
raise CommandError("Specified zip file does not exist: %s" % options.get("zip_file"))
self.update_via_zip(**options)
elif options.get("url", None):
self.update_via_zip(**options)
elif os.path.exists(settings.PROJECT_PATH + "/../.git"):
# If we detect a git repo, try git
if len(args) == 1 and not options["branch"]:
options["branch"] = args[0]
elif len(args) != 0:
raise CommandError("Specified too many command-line arguments")
self.update_via_git(**options)
elif len(args) > 1:
raise CommandError("Too many command-line arguments.")
elif len(args) == 1:
# Specify zip via first command-line arg
if options['zip_file'] is not None:
raise CommandError("Cannot specify a zipfile as unnamed and named command-line arguments at the same time.")
options['zip_file'] = args[0]
self.update_via_zip(**options)
else:
# No params, no git repo: try to get a file online.
zip_file = tempfile.mkstemp()[1]
for url in ["http://%s/api/download/kalite/latest/%s/%s/" % (settings.CENTRAL_SERVER_HOST, platform.system().lower(), "en")]:
logging.info("Downloading repo snapshot from %s to %s" % (url, zip_file))
try:
urllib.urlretrieve(url, zip_file)
sys.stdout.write("success @ %s\n" % url)
break;
except Exception as e:
logging.debug("Failed to get zipfile from %s: %s" % (url, e))
continue
options["zip_file"] = zip_file
self.update_via_zip(**options)
except Exception as e:
if self.started() and not not self.ended():
self.cancel(stage_status="error", notes=unicode(e))
raise
assert self.ended(), "Subroutines should complete() if they start()!"
def download_latest_translations(project_id=settings.CROWDIN_PROJECT_ID, project_key=settings.CROWDIN_PROJECT_KEY, language_code="all"):
"""Download latest translations from CrowdIn to corresponding locale directory."""
## Build latest package
build_translations()
## Get zip file of translations
logging.info("Attempting to download a zip archive of current translations")
request_url = "http://api.crowdin.net/api/project/%s/download/%s.zip?key=%s" % (project_id, language_code, project_key)
r = requests.get(request_url)
try:
r.raise_for_status()
except Exception as e:
if r.status_code == 401:
raise CommandError("Error: 401 Unauthorized while trying to access the CrowdIn API. Be sure to set CROWDIN_PROJECT_ID and CROWDIN_PROJECT_KEY in local_settings.py.")
else:
raise CommandError("Error: %s - couldn't connect to CrowdIn API - cannot continue without that zip file!" % e)
else:
logging.info("Successfully downloaded zip archive")
## Unpack into temp dir
z = zipfile.ZipFile(StringIO.StringIO(r.content))
tmp_dir_path = os.path.join(LOCALE_ROOT, "tmp")
z.extractall(tmp_dir_path)
## Copy over new translations
extract_new_po(tmp_dir_path, language_codes=[language_code] if language_code != "all" else None)
# Clean up tracks
if os.path.exists(tmp_dir_path):
shutil.rmtree(tmp_dir_path)
def move_exercises(lang_code):
lang_pack_location = os.path.join(LOCALE_ROOT, lang_code)
src_exercise_dir = os.path.join(lang_pack_location, "exercises")
dest_exercise_dir = get_localized_exercise_dirpath(lang_code,
is_central_server=False)
if not os.path.exists(src_exercise_dir):
logging.warn("Could not find downloaded exercises; skipping: %s" %
src_exercise_dir)
else:
# Move over one at a time, to combine with any other resources that were there before.
ensure_dir(dest_exercise_dir)
all_exercise_files = glob.glob(os.path.join(src_exercise_dir,
"*.html"))
logging.info("Moving %d downloaded exercises to %s" %
(len(all_exercise_files), dest_exercise_dir))
for exercise_file in all_exercise_files:
shutil.move(
exercise_file,
os.path.join(dest_exercise_dir,
os.path.basename(exercise_file)))
logging.debug("Removing emtpy directory")
try:
shutil.rmtree(src_exercise_dir)
except Exception as e:
logging.error("Error removing dubbed video directory (%s): %s" %
(src_exercise_dir, e))
def handle(self, *args, **options):
logging.basicConfig(stream=sys.stdout,
datefmt="%Y-%m-%d %H:%M:%S",
format="[%(asctime)-15s] %(message)s")
try:
time_wait = getattr(
settings, "CRONSERVER_FREQUENCY",
60) if not args or not args[0].strip() else float(args[0])
except:
raise CommandError("Invalid wait time: %s is not a number." %
args[0])
try:
sys.stdout.write(
"Starting cronserver. Jobs will run every %d seconds.\n" %
time_wait)
#sys.stdout.write("Quit the server with CONTROL-C.\n")
# Run server until killed
while True:
thread = CronThread(gc=options.get("gc", False),
mp=options.get("prof", False))
thread.start()
sleep(time_wait)
except KeyboardInterrupt:
logger.info("Exiting...\n")
sys.exit()
def generate_fake_coachreport_logs(password="******"):
t,_ = FacilityUser.objects.get_or_create(
facility=Facility.objects.all()[0],
username=random.choice(firstnames)
)
t.set_password(password)
# TODO: create flags later
num_logs = 20
logs = []
for _ in xrange(num_logs):
date_logged_in = datetime.datetime.now() - datetime.timedelta(days=random.randint(1,10))
date_viewed_coachreport = date_logged_in + datetime.timedelta(minutes=random.randint(0, 30))
date_logged_out = date_viewed_coachreport + datetime.timedelta(minutes=random.randint(0, 30))
login_log = UserLog.objects.create(
user=t,
activity_type=UserLog.get_activity_int("login"),
start_datetime=date_logged_in,
last_active_datetime=date_viewed_coachreport,
end_datetime=date_logged_out,
)
logging.info("created login log for teacher %s" % t.username)
coachreport_log = UserLog.objects.create(
user=t,
activity_type=UserLog.get_activity_int("coachreport"),
start_datetime=date_viewed_coachreport,
last_active_datetime=date_viewed_coachreport,
end_datetime=date_viewed_coachreport,
)
logs.append((login_log, coachreport_log))
logging.info("created coachreport log for teacher %s" % t.username)
return logs
def generate_test_files():
"""Insert asterisks as translations in po files"""
# Open them up and insert asterisks for all empty msgstrs
logging.info("Generating test po files")
en_po_dir = os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/")
for po_file in glob.glob(os.path.join(en_po_dir, "*.po")):
msgid_pattern = re.compile(r'msgid \"(.*)\"\nmsgstr', re.S | re.M)
content = open(os.path.join(en_po_dir, po_file), 'r').read()
results = content.split("\n\n")
with open(os.path.join(en_po_dir, "tmp.po"), 'w') as temp_file:
# We know the first block is static, so just dump that.
temp_file.write(results[0])
# Now work through actual translations
for result in results[1:]:
try:
msgid = re.findall(msgid_pattern, result)[0]
temp_file.write("\n\n")
temp_file.write(result.replace("msgstr \"\"", "msgstr \"***%s***\"" % msgid))
except Exception as e:
logging.error("Failed to insert test string: %s\n\n%s\n\n" % (e, result))
# Once done replacing, rename temp file to overwrite original
os.rename(os.path.join(en_po_dir, "tmp.po"), os.path.join(en_po_dir, po_file))
(out, err, rc) = compile_po_files("en")
if err:
logging.debug("Error executing compilemessages: %s" % err)
def handle(self, *args, **options):
if not settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the central server.")
# Set up the refresh date
if not options["date_since_attempt"]:
date_since_attempt = datetime.datetime.now() - datetime.timedelta(
days=options["days_since_attempt"])
options["date_since_attempt"] = date_since_attempt.strftime(
"%m/%d/%Y")
converted_date = convert_date_input(options.get("date_since_attempt"))
updated_mappings = create_all_mappings(
force=options.get("force"),
frequency_to_save=5,
response_to_check=options.get("response_code"),
date_to_check=converted_date)
logging.info(
"Executed successfully. Updating language => subtitle mapping to record any changes!"
)
if updated_mappings:
language_srt_map = update_language_srt_map()
print_language_availability_table(language_srt_map)
logging.info("Process complete.")
def verify_inner_zip(self, zip_file):
"""
Extract contents of outer zip, verify the inner zip
"""
zip = ZipFile(zip_file, "r")
nfiles = len(zip.namelist())
for fi, afile in enumerate(zip.namelist()):
zip.extract(afile, path=self.working_dir)
self.signature_file = os.path.join(self.working_dir,
Command.signature_filename)
self.inner_zip_file = os.path.join(self.working_dir,
Command.inner_zip_filename)
central_server = Device.get_central_server()
lines = open(self.signature_file, "r").read().split("\n")
chunk_size = int(lines.pop(0))
if not central_server:
logging.warn(
"No central server device object found; trusting zip file because you asked me to..."
)
elif central_server.key.verify_large_file(self.inner_zip_file,
signature=lines,
chunk_size=chunk_size):
logging.info("Verified file!")
else:
raise Exception("Failed to verify inner zip file.")
return self.inner_zip_file
def move_srts(lang_code):
"""
Srts live in the locale directory, but that's not exposed at any URL. So instead,
we have to move the srts out to /static/subtitles/[lang_code]/
"""
lang_code_ietf = lcode_to_ietf(lang_code)
lang_code_django = lcode_to_django_dir(lang_code)
subtitles_static_dir = os.path.join(settings.STATIC_ROOT, "subtitles")
src_dir = os.path.join(LOCALE_ROOT, lang_code_django, "subtitles")
dest_dir = get_srt_path(lang_code_django)
ensure_dir(dest_dir)
lang_subtitles = glob.glob(os.path.join(src_dir, "*.srt"))
logging.info("Moving %d subtitles from %s to %s" % (len(lang_subtitles), src_dir, dest_dir))
for fil in lang_subtitles:
srt_dest_path = os.path.join(dest_dir, os.path.basename(fil))
if os.path.exists(srt_dest_path):
os.remove(srt_dest_path)
shutil.move(fil, srt_dest_path)
if os.listdir(src_dir):
logging.warn("%s is not empty; will not remove. Please check that all subtitles were moved." % src_dir)
else:
logging.info("Removing empty source directory (%s)." % src_dir)
shutil.rmtree(src_dir)
def generate_zipped_srts(lang_codes_to_update, download_path):
# Create media directory if it doesn't yet exist
ensure_dir(settings.MEDIA_ROOT)
zip_path = settings.MEDIA_ROOT + "subtitles/"
ensure_dir(zip_path)
lang_codes_to_update = lang_codes_to_update or os.listdir(download_path)
for lang_code in lang_codes_to_update:
srt_dir = os.path.join(download_path, lang_code, "subtitles")
zip_file = os.path.join(zip_path, "%s_subtitles.zip" % lang_code)
# Remove any old version (as we may not re-create)
if os.path.exists(zip_file):
os.remove(zip_file)
if not os.path.exists(srt_dir):
logging.warn("No srt directory for %s; skipping." % lang_code)
continue
srts = glob.glob(os.path.join(srt_dir, "*.srt"))
if len(srts) == 0:
logging.warn("No srts for %s; skipping." % lang_code)
continue
logging.info("Zipping up a new pack for language code: %s" % lang_code)
zf = zipfile.ZipFile(zip_file, 'w')
for f in srts:
zf.write(f, arcname=os.path.basename(f))
zf.close()
def handle_po_compile_errors(lang_codes=None, out=None, err=None, rc=None):
"""
Return list of languages to not rezip due to errors in compile process.
Then email admins errors.
"""
broken_codes = re.findall(r'(?<=ka-lite/locale/)\w+(?=/LC_MESSAGES)', err) or []
if lang_codes:
# Only show the errors relevant to the list of language codes passed in.
lang_codes = set([lcode_to_django_dir(lc) for lc in lang_codes])
broken_codes = list(set(broken_codes).intersection(lang_codes))
if broken_codes:
logging.warning("Found %d errors while compiling in codes %s. Mailing admins report now." % (len(broken_codes), ', '.join(broken_codes)))
subject = "Error while compiling po files"
commands = "\n".join(["python manage.py compilemessages -l %s" % lc for lc in broken_codes])
message = """The following codes had errors when compiling their po files: %s.
Please rerun the following commands to see specific line numbers
that need to be corrected on CrowdIn, before we can update the language packs.
%s""" % (
', '.join([lcode_to_ietf(lc) for lc in broken_codes]),
commands,
)
if not settings.DEBUG:
mail_admins(subject=subject, message=message)
logging.info("Report sent.")
else:
logging.info("DEBUG is True so not sending email, but would have sent the following: SUBJECT: %s; MESSAGE: %s" % (subject, message))
return broken_codes
def invalidate_inmemory_caches():
for module in (i18n, topic_tools):
for cache_var in getattr(module, "CACHE_VARS", []):
logging.debug("Emptying cache %s.%s" %
(module.__name__, cache_var))
setattr(module, cache_var, None)
logging.info("Great success emptying the in-memory cache.")
def get_language_pack(lang_code, software_version, callback):
"""Download language pack for specified language"""
lang_code = lcode_to_ietf(lang_code)
logging.info("Retrieving language pack: %s" % lang_code)
request_url = get_language_pack_url(lang_code, software_version)
path, response = download_file(request_url, callback=callback_percent_proxy(callback))
return path
def preload_global_data():
if not settings.CENTRAL_SERVER:
logging.info("Preloading topic data.")
from main.topic_tools import get_topic_tree
from updates import stamp_availability_on_topic
stamp_availability_on_topic(get_topic_tree(),
force=True,
stamp_urls=True)
def update_templates():
"""Update template po files"""
logging.info("Posting template po files to static/pot/")
## post them to exposed URL
static_path = os.path.join(settings.STATIC_ROOT, "pot/")
ensure_dir(static_path)
shutil.copy(os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/django.po"), os.path.join(static_path, "kalite.pot"))
shutil.copy(os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/djangojs.po"), os.path.join(static_path, "kalitejs.pot"))
def generate_fake_facility_users(nusers=20,
facilities=None,
facility_groups=None,
password="******"):
"""Add the given fake facility users to each of the given fake facilities.
If no facilities are given, they are created."""
if not facility_groups:
(facility_groups,
facilities) = generate_fake_facility_groups(facilities=facilities)
facility_users = []
cur_usernum = 0
users_per_group = nusers / len(facility_groups)
for facility in facilities:
for facility_group in facility_groups:
for i in range(0, users_per_group):
user_data = {
"first_name": random.choice(firstnames),
"last_name": random.choice(lastnames),
}
user_data["username"] = username_from_name(
user_data["first_name"], user_data["last_name"])
try:
facility_user = FacilityUser.objects.get(
facility=facility, username=user_data["username"])
facility_user.group = facility_group
facility_user.save()
logging.info("Retrieved facility user '%s/%s'" %
(facility.name, user_data["username"]))
except FacilityUser.DoesNotExist as e:
notes = json.dumps(sample_user_settings())
facility_user = FacilityUser(
facility=facility,
username=user_data["username"],
first_name=user_data["first_name"],
last_name=user_data["last_name"],
notes=notes,
group=facility_group,
)
facility_user.set_password(
password) # set same password for every user
facility_user.full_clean()
facility_user.save()
logging.info("Created facility user '%s/%s'" %
(facility.name, user_data["username"]))
facility_users.append(facility_user)
cur_usernum += 1 # this is messy and could be done more intelligently;
# could also randomize to add more users, as this function
# seems to be generic, but really is not.
return (facility_users, facility_groups, facilities)
def run_makemessages():
"""Run makemessages command for english po files"""
logging.info("Executing makemessages command")
# Generate english po file
ignore_pattern = ['python-packages/*'] + ['kalite/%s/*' % dirname for dirname in ['central', 'contact', 'faq', 'registration', 'tests', 'stats']]
call_command('makemessages', locale='en', ignore_patterns=ignore_pattern, no_obsolete=True)
# Generate english po file for javascript
ignore_pattern = ['kalite/static/admin/js/*', 'python-packages/*', 'kalite/static/js/i18n/*', 'kalite/static/js/khan-exercises/*']
call_command('makemessages', domain='djangojs', locale='en', ignore_patterns=ignore_pattern, no_obsolete=True)
def get_language_pack(lang_code, software_version, callback):
"""Download language pack for specified language"""
lang_code = lcode_to_ietf(lang_code)
logging.info("Retrieving language pack: %s" % lang_code)
request_url = get_language_pack_url(lang_code, software_version)
path, response = download_file(request_url,
callback=callback_percent_proxy(callback))
return path
def update_srts(days, lang_code):
"""
Run the commands to update subtitles that haven't been updated in the number of days provided.
Default is to update all srt files that haven't been requested in 30 days
"""
date = '{0.month}/{0.day}/{0.year}'.format(datetime.date.today()-datetime.timedelta(int(days)))
logging.info("Updating subtitles that haven't been refreshed since %s" % date)
call_command("generate_subtitle_map", date_since_attempt=date)
call_command("cache_subtitles", date_since_attempt=date, lang_code=lang_code)
def update_language_list(sub_counts, data_path):
"""Update hardcoded language codes if any supported subtitle languages aren't there."""
for data in sub_counts.values():
lang_code = data.get("code")
if lang_code not in LANGUAGE_LIST:
logging.info("Adding %s to language code list" % lang_code)
LANGUAGE_LIST.append(lang_code)
with open(os.path.join(data_path, "listedlanguages.json"), 'wb') as fp:
json.dump(LANGUAGE_LIST, fp)
def handle(self, *args, **options):
if len(args) == 1 and args[0] == "test":
# Callback for "weak" test--checks at least that the django project compiles (local_settings is OK)
sys.stdout.write("Success!\n")
exit(0)
if options.get("repo", None):
# Specified a repo
self.update_via_git(**options)
elif options.get("zip_file", None):
# Specified a file
if not os.path.exists(options.get("zip_file")):
raise CommandError("Specified zip file does not exist: %s" %
options.get("zip_file"))
self.update_via_zip(**options)
elif os.path.exists(settings.PROJECT_PATH + "/../.git"):
# Without params, if we detect a git repo, try git
self.update_via_git(**options)
elif len(args) > 1:
raise CommandError("Too many command-line arguments.")
elif len(args) == 1:
# Specify zip via first command-line arg
if options['zip_file'] is not None:
raise CommandError(
"Cannot specify a zipfile as unnamed and named command-line arguments at the same time."
)
options['zip_file'] = args[0]
self.update_via_zip(**options)
else:
# No params, no git repo: try to get a file online.
zip_file = tempfile.mkstemp()[1]
for url in [
"https://github.com/learningequality/ka-lite/archive/master.zip",
"http://%s/download/kalite/%s/%s/" %
(settings.CENTRAL_SERVER_HOST, platform.system().lower(),
"all")
]:
logging.info("Downloading repo snapshot from %s to %s" %
(url, zip_file))
try:
urllib.urlretrieve(url, zip_file)
sys.stdout.write("success @ %s\n" % url)
break
except Exception as e:
logging.debug("Failed to get zipfile from %s: %s" %
(url, e))
continue
self.update_via_zip(zip_file=zip_file, **options)
self.stdout.write("Update is complete!\n")
def print_language_availability_table(language_srt_map):
logging.info("=============================================")
logging.info("=\tLanguage\t=\tNum Videos\t=")
for lang_code in sorted(language_srt_map.keys()):
logging.info("=\t%-8s\t=\t%4d srts\t=" % (lang_code, len(language_srt_map[lang_code])))
logging.info("=============================================")
n_srts = sum([len(dict) for dict in language_srt_map.values()])
logging.info("Great success! Subtitles support found for %d languages, %d total dubbings!" % (len(language_srt_map), n_srts))
def print_language_availability_table(language_srt_map):
logging.info("=============================================")
logging.info("=\tLanguage\t=\tNum Videos\t=")
for lang_code in sorted(language_srt_map.keys()):
logging.info("=\t%-8s\t=\t%4d srts\t=" % (lang_code, len(language_srt_map[lang_code])))
logging.info("=============================================")
n_srts = sum([len(dict) for dict in language_srt_map.values()])
logging.info("Great success! Subtitles support found for %d languages, %d total dubbings!" % (len(language_srt_map), n_srts))
def unpack_language(code, zip_file):
"""Unpack zipped language pack into locale directory"""
logging.info("Unpacking new translations")
ensure_dir(os.path.join(LOCALE_ROOT, code, "LC_MESSAGES"))
## Unpack into temp dir
z = zipfile.ZipFile(StringIO(zip_file))
z.extractall(os.path.join(LOCALE_ROOT, code))
def update_language_list(sub_counts, data_path):
"""Update hardcoded language codes if any supported subtitle languages aren't there."""
for data in sub_counts.values():
lang_code = data.get("code")
if lang_code not in LANGUAGE_LIST:
logging.info("Adding %s to language code list" % lang_code)
LANGUAGE_LIST.append(lang_code)
with open(os.path.join(data_path, "listedlanguages.json"), 'wb') as fp:
json.dump(LANGUAGE_LIST, fp)
def run_makemessages():
"""Run makemessages command for english po files"""
logging.info("Executing makemessages command")
# Generate english po file
ignore_pattern = ['python-packages/*']
management.call_command('makemessages', locale='en', ignore_patterns=ignore_pattern, no_obsolete=True)
# Generate english po file for javascript
ignore_pattern = ['kalite/static/admin/js/*', 'python-packages/*', 'kalite/static/js/i18n/*']
management.call_command('makemessages', domain='djangojs', locale='en', ignore_patterns=ignore_pattern, no_obsolete=True)
def update_templates():
"""Update template po files"""
pot_path = os.path.join(settings.DATA_PATH_SECURE, "i18n", "pot")
logging.info("Copying english po files to %s" % pot_path)
# post them to exposed URL
ensure_dir(pot_path)
shutil.copy(os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/django.po"), os.path.join(pot_path, "kalite.pot"))
shutil.copy(os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/djangojs.po"), os.path.join(pot_path, "kalitejs.pot"))
def handle(self, *args, **options):
if settings.CENTRAL_SERVER:
raise CommandError(
"This must only be run on distributed servers server.")
lang_code = lcode_to_ietf(options["lang_code"])
software_version = options["software_version"]
logging.info(
"Downloading language pack for lang_code=%s, software_version=%s" %
(lang_code, software_version))
# Download the language pack
try:
if options['file']:
self.start(
_("Using local language pack '%(filepath)s'") %
{"filepath": options['file']})
zip_filepath = options['file']
else:
self.start(
_("Downloading language pack '%(lang_code)s'") %
{"lang_code": lang_code})
zip_filepath = get_language_pack(lang_code,
software_version,
callback=self.cb)
# Unpack into locale directory
self.next_stage(
_("Unpacking language pack '%(lang_code)s'") %
{"lang_code": lang_code})
unpack_language(lang_code, zip_filepath=zip_filepath)
#
self.next_stage(
_("Creating static files for language pack '%(lang_code)s'") %
{"lang_code": lang_code})
update_jsi18n_file(lang_code)
self.next_stage(
_("Moving files to their appropriate local disk locations."))
move_dubbed_video_map(lang_code)
move_exercises(lang_code)
move_srts(lang_code)
move_video_sizes_file(lang_code)
self.next_stage(_("Invalidate caches"))
caching.invalidate_all_caches()
self.complete(
_("Finished processing language pack %(lang_code)s") %
{"lang_code": lang_code})
except Exception as e:
self.cancel(stage_status="error",
notes=_("Error: %(error_msg)s") %
{"error_msg": unicode(e)})
raise
def build_translations(project_id=settings.CROWDIN_PROJECT_ID, project_key=settings.CROWDIN_PROJECT_KEY):
"""Build latest translations into zip archive on CrowdIn."""
logging.info("Requesting that CrowdIn build a fresh zip of our translations")
request_url = "http://api.crowdin.net/api/project/%s/export?key=%s" % (project_id, project_key)
resp = requests.get(request_url)
try:
resp.raise_for_status()
except Exception as e:
logging.error(e)
def cancel_progress(self, notes=None):
"""
Stamps end time.
"""
logging.info("Cancelling process %s" % (self.process_name))
self.end_time = datetime.datetime.now()
self.completed=False
self.notes = notes
self.save()
def unpack_language(lang_code, zip_filepath=None, zip_fp=None, zip_data=None):
"""Unpack zipped language pack into locale directory"""
lang_code = lcode_to_django_dir(lang_code)
logging.info("Unpacking new translations")
ensure_dir(os.path.join(LOCALE_ROOT, lang_code, "LC_MESSAGES"))
## Unpack into temp dir
z = zipfile.ZipFile(zip_fp or (StringIO(zip_data) if zip_data else open(zip_filepath, "rb")))
z.extractall(os.path.join(LOCALE_ROOT, lang_code))
def build_translations(project_id=settings.CROWDIN_PROJECT_ID, project_key=settings.CROWDIN_PROJECT_KEY):
"""Build latest translations into zip archive on CrowdIn"""
logging.info("Requesting that CrowdIn build a fresh zip of our translations")
request_url = "http://api.crowdin.net/api/project/%s/export?key=%s" % (project_id, project_key)
r = requests.get(request_url)
try:
r.raise_for_status()
except Exception as e:
logging.error(e)
def cancel_progress(self, notes=None):
"""
Stamps end time.
"""
logging.info("Cancelling process %s" % (self.process_name))
self.end_time = datetime.datetime.now()
self.completed = False
self.notes = notes
self.save()
def handle(self, *args, **options):
try:
converted_date = convert_date_input(options.get("date_since_attempt"))
# create_all_mappings(force=options.get("force"), frequency_to_save=5, response_to_check=options.get("response_code"), date_to_check=converted_date)
logging.info("Executed successfully. Updating language => subtitle mapping to record any changes!")
language_srt_map = update_language_srt_map()
print_language_availability_table(language_srt_map)
logging.info("Process complete.")
except Exception as e:
raise CommandError(str(e))
def handle(self, *args, **options):
try:
converted_date = convert_date_input(options.get("date_since_attempt"))
# create_all_mappings(force=options.get("force"), frequency_to_save=5, response_to_check=options.get("response_code"), date_to_check=converted_date)
logging.info("Executed successfully. Updating language => subtitle mapping to record any changes!")
language_srt_map = update_language_srt_map()
print_language_availability_table(language_srt_map)
logging.info("Process complete.")
except Exception as e:
raise CommandError(str(e))
def cancel_current_stage(self, notes=None):
"""
Delete the current stage--it's reported progress, and contribution to the total # of stages
"""
logging.info("Cancelling stage %s of process %s" % (self.stage_name, self.process_name))
self.stage_percent = 0.
self.update_total_stages(self.total_stages - 1)
self.stage_name = None
self.notes = notes
self.save()
def handle(self, *args, **options):
if not settings.CENTRAL_SERVER:
raise CommandError("This must only be run on the central server.")
converted_date = convert_date_input(options.get("date_since_attempt"))
create_all_mappings(force=options.get("force"), frequency_to_save=5, response_to_check=options.get("response_code"), date_to_check=converted_date)
logging.info("Executed successfully. Updating language => subtitle mapping to record any changes!")
language_srt_map = update_language_srt_map()
print_language_availability_table(language_srt_map)
logging.info("Process complete.")
def cancel_current_stage(self, notes=None):
"""
Delete the current stage--it's reported progress, and contribution to the total # of stages
"""
logging.info("Cancelling stage %s of process %s" %
(self.stage_name, self.process_name))
self.stage_percent = 0.
self.update_total_stages(self.total_stages - 1)
self.stage_name = None
self.notes = notes
self.save()
def update_templates():
"""Update template po files"""
logging.info("Posting template po files to static/pot/")
## post them to exposed URL
static_path = os.path.join(settings.STATIC_ROOT, "pot/")
ensure_dir(static_path)
shutil.copy(
os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/django.po"),
os.path.join(static_path, "kalite.pot"))
shutil.copy(
os.path.join(settings.LOCALE_PATHS[0], "en/LC_MESSAGES/djangojs.po"),
os.path.join(static_path, "kalitejs.pot"))
def run(self):
jobs = Job.objects.due()
prof_string = "" if not self.do_profile else "[%8.2f MB] " % memory_profiler.memory_usage()[0]
if jobs:
logger.info("%sRunning %d due jobs... (%s)" % (prof_string, jobs.count(), ", ".join(['"%s"' % job.name for job in jobs])))
call_command('cron')
else:
logger.debug("%sNo jobs due to run." % prof_string)
if self.do_gc:
gc.collect()
def unpack_language(lang_code, zip_filepath=None, zip_fp=None, zip_data=None):
"""Unpack zipped language pack into locale directory"""
lang_code = lcode_to_django_dir(lang_code)
logging.info("Unpacking new translations")
ensure_dir(os.path.join(LOCALE_ROOT, lang_code, "LC_MESSAGES"))
## Unpack into temp dir
z = zipfile.ZipFile(
zip_fp
or (StringIO(zip_data) if zip_data else open(zip_filepath, "rb")))
z.extractall(os.path.join(LOCALE_ROOT, lang_code))
def scrub_locale_paths():
for locale_root in settings.LOCALE_PATHS:
if not os.path.exists(locale_root):
continue
for lang in os.listdir(locale_root):
# Skips if not a directory
if not os.path.isdir(os.path.join(locale_root, lang)):
continue
# If it isn't crowdin/django format, keeeeeeellllllll
if lang != lcode_to_django_dir(lang):
logging.info("Deleting %s directory because it does not fit our language code format standards" % lang)
shutil.rmtree(os.path.join(locale_root, lang))
def get_language_pack(code, software_version):
"""Download language pack for specified language"""
logging.info("Retrieving language pack: %s" % code)
request_url = _language_pack_url(code, software_version)
r = requests.get(request_url)
try:
r.raise_for_status()
except Exception as e:
raise CommandError(e)
return r.content
def update_srts(days, lang_codes):
"""
Run the commands to update subtitles that haven't been updated in the number of days provided.
Default is to update all srt files that haven't been requested in 30 days
"""
date = '{0.month}/{0.day}/{0.year}'.format(datetime.date.today() - datetime.timedelta(int(days)))
logging.info("Updating subtitles that haven't been refreshed since %s" % date)
call_command("generate_subtitle_map", date_since_attempt=date)
if lang_codes:
for lang_code in lang_codes:
call_command("cache_subtitles", date_since_attempt=date, lang_code=lang_code)
else:
call_command("cache_subtitles", date_since_attempt=date)
def get_language_pack(lang_code, software_version):
"""Download language pack for specified language"""
lang_code = lcode_to_ietf(lang_code)
logging.info("Retrieving language pack: %s" % lang_code)
request_url = get_language_pack_url(lang_code, software_version)
r = requests.get(request_url)
try:
r.raise_for_status()
except Exception as e:
raise CommandError(e)
return r.content
def write_new_json(subtitle_counts, data_path):
"""Write JSON to file in static/data/subtitles/"""
filename = "subtitle_counts.json"
filepath = data_path + filename
try:
current_counts = json.loads(open(filepath).read())
except Exception as e:
logging.error("Subtitle counts file appears to be corrupted (%s). Starting from scratch." % e)
current_counts = {}
current_counts.update(subtitle_counts)
logging.info("Writing fresh srt counts to %s" % filepath)
with open(filepath, 'wb') as fp:
json.dump(current_counts, fp)
def download_latest_translations(project_id=settings.CROWDIN_PROJECT_ID,
project_key=settings.CROWDIN_PROJECT_KEY,
lang_code="all",
zip_file=None,
combine_with_po_file=None,
rebuild=True):
"""
Download latest translations from CrowdIn to corresponding locale
directory. If zip_file is given, use that as the zip file
instead of going through CrowdIn.
"""
lang_code = lcode_to_ietf(lang_code)
# Get zip file of translations
if zip_file and os.path.exists(zip_file):
logging.info("Using local zip file at %s" % zip_file)
z = zipfile.ZipFile(zip_file)
# use the name of the zip file to infer the language code, if needed
lang_code = lang_code or os.path.splitext(os.path.basename(zip_file))[0]
else:
# Tell CrowdIn to Build latest package
if rebuild:
build_translations()
logging.info("Attempting to download a zip archive of current translations")
request_url = "http://api.crowdin.net/api/project/%s/download/%s.zip?key=%s" % (project_id, lang_code, project_key)
try:
resp = requests.get(request_url)
resp.raise_for_status()
except Exception as e:
if resp.status_code == 404:
logging.info("No translations found for language %s" % lang_code)
return None # no translations
elif resp.status_code == 401:
raise CommandError("401 Unauthorized while trying to access the CrowdIn API. Be sure to set CROWDIN_PROJECT_ID and CROWDIN_PROJECT_KEY in local_settings.py.")
else:
raise CommandError("%s - couldn't connect to CrowdIn API - cannot continue without downloading %s!" % (e, request_url))
else:
logging.info("Successfully downloaded zip archive")
# Unpack into temp dir
z = zipfile.ZipFile(StringIO.StringIO(resp.content))
if zip_file:
with open(zip_file, "wb") as fp: # save the zip file
fp.write(resp.content)
tmp_dir_path = tempfile.mkdtemp()
z.extractall(tmp_dir_path)
# Copy over new translations
po_file = extract_new_po(tmp_dir_path, combine_with_po_file=combine_with_po_file, lang=lang_code)
# Clean up tracks
if os.path.exists(tmp_dir_path):
shutil.rmtree(tmp_dir_path)
return po_file
