def _scan_list(marshall, scan_id, filenames): # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == scan_id: mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save()
def _scan_list(marshall, scan_id, filenames): # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = f.read() results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == scan_id: mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save()
def _scan_list(marshall, scan_id, filenames): """ Given a list of filenames (file paths), of templates and/or python files, scan them for translatable strings and create corresponding MasterTranslation objects. """ # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn("Empty translation discovered: '{}', '{}', '{}', '{}'".format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE ) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == unicode(scan_id): mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = { group } mt.last_updated_by_scan_uuid = scan_id mt.save() # Update the ScanMarshall object with the reduced number of `files_left_to_process`. # Do this with several retries, so that if the transction collides with another task (which is # quite likely) this whole task doesn't fail and retry (which would be fine but inefficient). for retry in xrange(3): try: with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save() return except TransactionFailedError: msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, " msg += ("retrying..." if retry < 2 else "giving up, task will error and retry.") logger.info(msg) if retry < 2: # Back off by random number of ms. This helps prevent 2 colliding tasks from # repeatedly re-colliding. time.sleep(random.randint(0, 1000) / 1000.0) # Tried 3 times, give up, let the task fail and retry raise
def _scan_list(marshall, scan_id, filenames): """ Given a list of filenames (file paths), of templates and/or python files, scan them for translatable strings and create corresponding MasterTranslation objects. """ # FIXME: Need to clean up the translations which aren't in use anymore for filename in filenames: # Redeploying to a new version can cause this if not os.path.exists(filename): continue with open(filename) as f: content = unicode(f.read(), settings.DEFAULT_CHARSET) results = parse_file(content, os.path.splitext(filename)[-1]) for text, plural, hint, group in results: if not text: logger.warn( "Empty translation discovered: '{}', '{}', '{}', '{}'". format(text, plural, hint, group)) continue with transaction.atomic(xg=True): key = MasterTranslation.generate_key(text, hint, settings.LANGUAGE_CODE) try: mt = MasterTranslation.objects.get(pk=key) except MasterTranslation.DoesNotExist: mt = MasterTranslation( pk=key, text=text, hint=hint, language_code=settings.LANGUAGE_CODE) # By the very act of getting here, this is true mt.used_in_code_or_templates = True # If we last updated during this scan, then append, otherwise replace if mt.last_updated_by_scan_uuid == unicode(scan_id): mt.used_by_groups_in_code_or_templates.add(group) else: mt.used_by_groups_in_code_or_templates = {group} mt.last_updated_by_scan_uuid = scan_id mt.save() # Update the ScanMarshall object with the reduced number of `files_left_to_process`. # Do this with several retries, so that if the transction collides with another task (which is # quite likely) this whole task doesn't fail and retry (which would be fine but inefficient). for retry in xrange(3): try: with transaction.atomic(): marshall.refresh_from_db() marshall.files_left_to_process -= len(filenames) marshall.save() return except TransactionFailedError: msg = "Transaction failed trying to decrement 'files_left_to_process' on ScanMarshall, " msg += ("retrying..." if retry < 2 else "giving up, task will error and retry.") logger.info(msg) if retry < 2: # Back off by random number of ms. This helps prevent 2 colliding tasks from # repeatedly re-colliding. time.sleep(random.randint(0, 1000) / 1000.0) # Tried 3 times, give up, let the task fail and retry raise