def add_user_to_registered_group(sender, **kwargs):
"""
Add any user created on the system to the `registered` group.
This signal must be called by the post_save signal from the User class.
This signal also creates a public profile for the user if it does not exist.
"""
if 'created' in kwargs and kwargs['created'] is True:
user = kwargs['instance']
# Add user to registered group
group, created = Group.objects.get_or_create(name='registered')
user.groups.add(group)
sid = transaction.savepoint()
# Create Public Profile
try:
profile, created = Profile.objects.get_or_create(user=user)
profile.save()
transaction.savepoint_commit(sid)
except:
logger.debug("User profile not created.")
transaction.savepoint_rollback(sid)
transaction.commit()
def post_resource_save_handler(sender, instance, created, user, **kwargs):
if settings.ENABLE_NOTICES:
resource = instance
project = resource.project
users = [
watch.user
for watch in notification.ObservedItem.objects.filter(
content_type__model="project", object_id=project.id, signal="project_changed"
).select_related("user")
]
for user in users:
try:
notification.ObservedItem.objects.get_for(resource.project, user, "project_changed")
if created:
for signal in resource_signals:
try:
notification.ObservedItem.objects.get_for(resource, user, signal)
except notification.ObservedItem.DoesNotExist:
notification.observe(resource, user, signal, signal)
nt = "project_resource_added"
else:
nt = "project_resource_changed"
project = resource.project
_notify_resourcewatchers(project, resource, nt)
except notification.ObservedItem.DoesNotExist, e:
logger.debug("Watches: %s" % unicode(e))
def search(request):
query_string = prepare_solr_query_string(request.GET.get('q', ""))
search_terms = query_string.split()
index_query = SearchQuerySet().models(Project)
spelling_suggestion = None
if not FULLTEXT:
try:
results = index_query.auto_query(query_string)
count = results.count()
except TypeError:
count = 0
else:
try:
qfilter = fulltext_project_search_filter(query_string)
results = index_query.filter(qfilter)
spelling_suggestion = results.spelling_suggestion(query_string)
count = results.count()
except TypeError:
results = []
count = 0
logger.debug("Searched for %s. Found %s results." % (query_string, count))
return render_to_response("search.html", {
'query': query_string,
'terms': search_terms,
'results': results,
'spelling_suggestion': spelling_suggestion
},
context_instance=RequestContext(request))
def search(request):
query_string = prepare_solr_query_string(request.GET.get('q', ""))
search_terms = query_string.split()
index_query = SearchQuerySet().models(Project)
spelling_suggestion = None
if not FULLTEXT:
try:
results = index_query.auto_query(query_string)
count = results.count()
except TypeError:
count = 0
else:
try:
qfilter = fulltext_project_search_filter(query_string)
results = index_query.filter(qfilter)
spelling_suggestion = results.spelling_suggestion(query_string)
count = results.count()
except TypeError:
results = []
count = 0
logger.debug("Searched for %s. Found %s results." % (query_string, count))
return render_to_response("search.html",
{'query': query_string,
'terms': search_terms,
'results': results,
'spelling_suggestion': spelling_suggestion},
context_instance = RequestContext(request))
def post_release_save_handler(sender, instance, created, user, **kwargs):
if settings.ENABLE_NOTICES:
release = instance
project = release.project
users = [
watch.user for watch in notification.ObservedItem.objects.filter(
content_type__model='project',
object_id=project.id,
signal="project_changed").select_related('user')
]
for user in users:
try:
notification.ObservedItem.objects.get_for(
release.project, user, "project_changed")
if created:
for signal in release_signals:
try:
notification.ObservedItem.objects.get_for(
release, user, signal)
except notification.ObservedItem.DoesNotExist:
notification.observe(release, user, signal, signal)
nt = "project_release_added"
else:
nt = "project_release_changed"
project = release.project
_notify_releasewatchers(project, release, nt)
except notification.ObservedItem.DoesNotExist, e:
logger.debug("Watches: %s" % unicode(e))
def add_user_to_registered_group(sender, **kwargs):
"""
Add any user created on the system to the `registered` group.
This signal must be called by the post_save signal from the User class.
This signal also creates a public profile for the user if it does not exist.
"""
if 'created' in kwargs and kwargs['created'] is True:
user = kwargs['instance']
# Add user to registered group
group, created = Group.objects.get_or_create(name='registered')
user.groups.add(group)
sid = transaction.savepoint()
# Create Public Profile
try:
profile, created = Profile.objects.get_or_create(user=user)
profile.save()
transaction.savepoint_commit(sid)
except:
logger.debug("User profile not created.")
transaction.savepoint_rollback(sid)
transaction.commit()
def check_and_notify_resource_full_reviewed(**kwargs):
"""
Handler to notify maintainers about 100% reviewed translations.
"""
rlstats = kwargs.pop('sender')
if (settings.ENABLE_NOTICES
and rlstats.resource.source_language != rlstats.language):
logger.debug("resource: Checking if resource translation is fully "
"reviewed: %s (%s)" %
(rlstats.resource, rlstats.language.code))
if rlstats.reviewed_perc == 100:
logger.debug("resource: Resource translation is fully reviewed.")
# Notification
context = {
'project': rlstats.resource.project,
'resource': rlstats.resource,
'language': rlstats.language,
}
nt = "project_resource_full_reviewed"
notification.send(rlstats.resource.project.maintainers.all(), nt,
context)
def msgfmt_check(po_contents, ispot=False, with_exceptions=True):
"""
Run a `msgfmt -c` on the file contents.
Raise a FileCheckError in case the stderror has errors/warnings or
the command execution returns Error.
"""
try:
if ispot:
command = "msgfmt -o /dev/null --check-format --check-domain -"
else:
command = "msgfmt -o /dev/null -c -"
status, stdout, stderr = run_command(
command, _input=po_contents, with_extended_output=True, with_exceptions=with_exceptions
)
# Not sure why msgfmt sends its output to stderr instead of stdout
# if 'warning:' in stderr or 'too many errors, aborting' in stderr:
if "too many errors, aborting" in stderr:
raise CommandError(command, status, stderr)
except CommandError:
logger.debug("pofile: The 'msgfmt -c' check failed.")
raise FileCheckError, ugettext(
"Your file failed a correctness check "
"(msgfmt -c). Please run this command on "
"your system to see the errors."
)
def priority_creation(**kwargs):
"""Create the default priority on Resource creation."""
if 'created' in kwargs and kwargs['created'] is True:
resource = kwargs['instance']
if resource:
ResourcePriority.objects.create(resource=resource)
logger.debug("Resource %s: New ResourcePriority created." %
(resource.name))
def handle_exception_mailing(request, exception):
"""Handle an exception if in production mode."""
exc_info = sys.exc_info()
subject, message = exception_email(request, exc_info)
if not settings.DEBUG:
logger.debug('Sending handled exception to admins.')
mail_admins(('%s - %s') % (subject, exception.message), message,
fail_silently=True)
def priority_creation(**kwargs):
"""Create the default priority on Resource creation."""
if 'created' in kwargs and kwargs['created'] is True:
resource = kwargs['instance']
if resource:
ResourcePriority.objects.create(resource=resource)
logger.debug("Resource %s: New ResourcePriority created." % (
resource.name))
def _get_stats(self, request, pslug, rslug, lang_code):
try:
resource = Resource.objects.get(project__slug=pslug, slug=rslug)
except Resource.DoesNotExist, e:
logger.debug("Resource %s.%s requested, but it does not exist" %
(pslug, rslug),
exc_info=True)
return rc.NOT_FOUND
def _get_stats(self, request, pslug, rslug, lang_code):
try:
resource = Resource.objects.get(project__slug=pslug, slug=rslug)
except Resource.DoesNotExist, e:
logger.debug(
"Resource %s.%s requested, but it does not exist" % (pslug, rslug),
exc_info=True
)
return rc.NOT_FOUND
def handle_exception_mailing(request, exception):
"""Handle an exception if in production mode."""
exc_info = sys.exc_info()
subject, message = exception_email(request, exc_info)
if not settings.DEBUG:
logger.debug('Sending handled exception to admins.')
mail_admins(('%s - %s') % (subject, exception.message),
message,
fail_silently=True)
def delete(self, *args, **kwargs):
"""
Delete file from filesystem even if object has not been saved yet.
"""
try:
os.remove(self.get_storage_path())
except OSError, e:
if self.id:
logger.debug("Error deleting StorageFile: %s" % str(e))
def combine_strings(source_entities, language):
source_entities = list(source_entities)
result = old_get_strings(source_entities, language)
for entity in source_entities:
if not result.get(entity, None):
trans = handler._get_translation(entity, source_language, 5)
if trans:
logger.debug(trans.string)
result[entity] = trans.string
return result
def _notify_resourcewatchers(project, resource, signal):
"""
Notify watchers of a resource add/change
"""
context = {"project": project, "resource": resource}
logger.debug("addon-watches: Sending notification for '%s'" % resource)
if signal == "project_resource_added":
observed_instance = project
else:
observed_instance = resource
txnotification.send_observation_notices_for(observed_instance, signal=signal, extra_context=context)
def combine_strings(source_entities, language):
source_entities = list(source_entities)
result = old_get_strings(source_entities, language)
for entity in source_entities:
if not result.get(entity, None):
trans = handler._get_translation(entity, source_language,
5)
if trans:
logger.debug(trans.string)
result[entity] = trans.string
return result
def _notify_translationwatchers(resource, language):
"""
Notify the watchers for a specific TranslationWatch
"""
context = {"project": resource.project, "resource": resource, "language": language}
twatch = TranslationWatch.objects.get_or_create(resource=resource, language=language)[0]
logger.debug("addon-watches: Sending notification for '%s'" % twatch)
txnotification.send_observation_notices_for(
twatch, signal="project_resource_translation_changed", extra_context=context
)
def test_pseudo_file_api_calls(self):
"""Test Pseudo translation requests through the API."""
source_language = self.project.resources.all()[0].source_language
for i18n_type, v in FORMATS.items():
resource_slug = 'resource_%s' % i18n_type.lower()
resource_url = reverse('apiv2_resources',
kwargs={'project_slug': self.project.slug})
# Creating resource using the API
f = open(v['file'])
res = self.client['maintainer'].post(
resource_url,
data={
'slug': resource_slug,
'name': resource_slug,
'i18n_type': i18n_type,
'attachment': f
},
)
f.close()
logger.debug('-----------------------')
logger.debug(i18n_type)
logger.debug(settings.I18N_METHODS[i18n_type]['mimetype'])
logger.debug(res.content)
logger.debug('-----------------------')
# Pseudo file API URL
url = reverse('apiv2_pseudo_content',
args=[self.project.slug, resource_slug])
for pseudo_type in settings.PSEUDO_TYPES:
# Get resource file using a specific pseudo type
resp = self.client['registered'].get(
url, data={'pseudo_type': pseudo_type})
# Get response and check encoding
resp_content = eval(resp.content)['content']
if type(resp_content) != unicode:
try:
resp_content = resp_content.decode('utf-8')
except UnicodeDecodeError:
resp_content = resp_content.decode('iso-8859-1')
#FIXME: We have a bug related to spaces being escaped in
# .properties files. This can be dropped after fixing it.
if i18n_type == 'PROPERTIES' and \
pseudo_type in ['PLANGUAGE', 'UNICODE']:
resp_content = resp_content.replace('\\ ', ' ')
# Assert expected value in the generated file
for message in v['pseudo_messages'][pseudo_type]:
self.assertTrue(message in resp_content)
def delete(self, request, uuid=None, api_version=1):
"""
Deletes file by storage UUID
"""
if request.user.is_anonymous():
return rc.FORBIDDEN
try:
StorageFile.objects.get(uuid=uuid, user=request.user).delete()
except StorageFile.DoesNotExist:
return rc.NOT_FOUND
logger.debug("Deleted file %s" % uuid)
return rc.DELETED
def update_template_cache(template_name, fragment_names, key_vars, context):
"""Update the template cache with the new data.
The caches will be invalidated in the order given.
"""
logger.debug("Invalidating %s in %s" % (fragment_names, template_name))
t = get_template(template_name)
nodes = t.nodelist.get_nodes_by_type(CacheNode)
for f_name in fragment_names:
for node in nodes:
if f_name == node.fragment_name:
set_fragment_content(node, key_vars, context)
break
def update_template_cache(template_name, fragment_names, key_vars, context):
"""Update the template cache with the new data.
The caches will be invalidated in the order given.
"""
logger.debug("Invalidating %s in %s" % (fragment_names, template_name))
t = get_template(template_name)
nodes = t.nodelist.get_nodes_by_type(CacheNode)
for f_name in fragment_names:
for node in nodes:
if f_name == node.fragment_name:
set_fragment_content(node, key_vars, context)
break
def test_pseudo_file_api_calls(self):
"""Test Pseudo translation requests through the API."""
source_language = self.project.resources.all()[0].source_language
for i18n_type, v in FORMATS.items():
resource_slug = 'resource_%s' % i18n_type.lower()
resource_url = reverse('apiv2_resources', kwargs={
'project_slug': self.project.slug})
# Creating resource using the API
f = open(v['file'])
res = self.client['maintainer'].post(
resource_url,
data={
'slug': resource_slug,
'name': resource_slug,
'i18n_type': i18n_type,
'attachment': f},
)
f.close()
logger.debug('-----------------------')
logger.debug(i18n_type)
logger.debug(settings.I18N_METHODS[i18n_type]['mimetype'])
logger.debug(res.content)
logger.debug('-----------------------')
# Pseudo file API URL
url = reverse('apiv2_pseudo_content', args=[self.project.slug,
resource_slug])
for pseudo_type in settings.PSEUDO_TYPES:
# Get resource file using a specific pseudo type
resp = self.client['registered'].get(url, data={'pseudo_type':pseudo_type})
# Get response and check encoding
resp_content = eval(resp.content)['content']
if type(resp_content) != unicode:
try:
resp_content = resp_content.decode('utf-8')
except UnicodeDecodeError:
resp_content = resp_content.decode('iso-8859-1')
#FIXME: We have a bug related to spaces being escaped in
# .properties files. This can be dropped after fixing it.
if i18n_type == 'PROPERTIES' and \
pseudo_type in ['PLANGUAGE', 'UNICODE']:
resp_content = resp_content.replace('\\ ', ' ')
# Assert expected value in the generated file
for message in v['pseudo_messages'][pseudo_type]:
self.assertTrue(message in resp_content)
def check_and_notify_string_freeze_breakage(sender, **kwargs):
"""
Handler to notify people about string freeze breakage of releases.
This happens whenever a resource source file changes in the string freeze
period.
"""
resource = kwargs.pop('resource')
language = kwargs.pop('language')
# Check it only for source languages
if kwargs.pop('is_source'):
logger.debug("release: Checking string freeze breakage.")
# FIXME: Get timestamp from RLStats last_update field, but it depends
# on some changes on formats/core.py. At this point the RLStats object
# wasn't created yet.
timestamp = datetime.datetime.now()
project = resource.project.outsource or resource.project
releases = Release.objects.filter(resources=resource,
project=project,
stringfreeze_date__lte=timestamp,
develfreeze_date__gte=timestamp)
for release in releases:
logger.debug("release: Sending notifications about string "
"freeze breakage for '%s'" % release)
project = release.project.outsource or release.project
# User list with project maintainers and team coordinators of the
# given release PLUS maintainers of the project that the RLStats
# object belongs to PLUS
users = User.objects.filter(
Q(projects_maintaining=resource.project)
| Q(projects_maintaining=project)
| Q(team_coordinators__project=project)).distinct()
# Notification
context = {
'project': release.project,
'release': release,
'resource': resource
}
if release.project != project:
context.update({'parent_project': project})
nt = "project_release_stringfreeze_breakage"
#TODO: Add support for actionlog without a user author.
#action_logging(None, [project, release], nt, context=context)
if settings.ENABLE_NOTICES:
notification.send(users, nt, context)
class StatsHandler(BaseHandler):
allowed_methods = ('GET', )
def read(self,
request,
project_slug,
resource_slug,
lang_code=None,
api_version=1):
"""
This is an API handler to display translation statistics for individual
resources.
"""
if api_version != 2:
return BAD_REQUEST('Wrong API version called.')
return self._get_stats(request, project_slug, resource_slug, lang_code)
def _get_stats(self, request, pslug, rslug, lang_code):
try:
resource = Resource.objects.get(project__slug=pslug, slug=rslug)
except Resource.DoesNotExist, e:
logger.debug("Resource %s.%s requested, but it does not exist" %
(pslug, rslug),
exc_info=True)
return rc.NOT_FOUND
language = None
if lang_code is not None:
try:
language = Language.objects.by_code_or_alias(lang_code)
except Language.DoesNotExist, e:
logger.debug(
"Language %s was requested, but it does not exist." %
lang_code,
exc_info=True)
return BAD_REQUEST("Unknown language code %s" % lang_code)
def _notify_resourcewatchers(project, resource, signal):
"""
Notify watchers of a resource add/change
"""
context = {
'project': project,
'resource': resource,
}
logger.debug("addon-watches: Sending notification for '%s'" % resource)
if signal == 'project_resource_added':
observed_instance = project
else:
observed_instance = resource
txnotification.send_observation_notices_for(observed_instance,
signal=signal,
extra_context=context)
def create(self):
"""
Creates a new translation from file.
Returns:
A dict with information for the translation.
Raises:
BadRequestError: There was a problem with the request.
NoContentError: There was no file in the request.
"""
if not self.request.FILES:
raise NoContentError("No file has been uploaded.")
submitted_file = self.request.FILES.values()[0]
name = str(submitted_file.name)
size = submitted_file.size
try:
file_ = tempfile.NamedTemporaryFile(
mode='wb',
suffix=name[name.rfind('.'):],
delete=False
)
for chunk in submitted_file.chunks():
file_.write(chunk)
file_.close()
parser = registry.appropriate_handler(
self.resource,
language=self.language,
filename=name
)
parser.bind_file(file_.name)
if parser is None:
raise BadRequestError("Unknown file type")
if size == 0:
raise BadRequestError("Empty file")
try:
parser.is_content_valid()
logger.debug("Uploaded file %s" % file_.name)
except (FileCheckError, ParseError), e:
raise BadRequestError("Error uploading file: %s" % e)
except Exception, e:
logger.error(unicode(e), exc_info=True)
raise BadRequestError("A strange error happened.")
def _notify_all_on_source_change(resource, context):
"""
Send notifications to everyone involved with a resource.
Args:
resource: The updated resource.
"""
signal_name = 'project_resource_translation_changed'
msg = "addon-watches: Sending notification for '%s'"
TWatch = get_model('watches', 'TranslationWatch')
for l in resource.available_languages:
twatch = TWatch.objects.get_or_create(resource=resource, language=l)[0]
logger.debug(msg % twatch)
txnotification.send_observation_notices_for(twatch,
signal=signal_name,
extra_context=context)
def _notify_all_on_source_change(resource, context):
"""
Send notifications to everyone involved with a resource.
Args:
resource: The updated resource.
"""
signal_name = 'project_resource_translation_changed'
msg = "addon-watches: Sending notification for '%s'"
TWatch = get_model('watches', 'TranslationWatch')
for l in resource.available_languages:
twatch = TWatch.objects.get_or_create(resource=resource, language=l)[0]
logger.debug(msg % twatch)
txnotification.send_observation_notices_for(
twatch, signal=signal_name, extra_context=context
)
def get_from_app(request, project_slug, txapp_slug):
"""Get a template string from a tx app."""
txapp = get_object_or_404(TxApp, slug=txapp_slug)
project = get_object_or_404(Project, slug=project_slug)
root_path = _root_namespace(request.path, txapp)
requested_path = _remove_namespace_from_path(root_path, request.path)
if not txapp.access_is_allowed(request.user, project, requested_path):
return HttpResponseForbidden()
logger.debug("Path requested from tx app %s is %s" %
(txapp_slug, requested_path))
url = '/'.join([txapp.url, project_slug, requested_path])
try:
res = _forward_to_app(url, request.method, dict(request.POST.items()))
except RemoteTxAppError, e:
return error_contacting_app(request, url, txapp, e)
def get_from_app(request, project_slug, txapp_slug):
"""Get a template string from a tx app."""
txapp = get_object_or_404(TxApp, slug=txapp_slug)
project = get_object_or_404(Project, slug=project_slug)
root_path = _root_namespace(request.path, txapp)
requested_path = _remove_namespace_from_path(root_path, request.path)
if not txapp.access_is_allowed(request.user, project, requested_path):
return HttpResponseForbidden()
logger.debug(
"Path requested from tx app %s is %s" % (txapp_slug, requested_path)
)
url = '/'.join([txapp.url, project_slug, requested_path])
try:
res = _forward_to_app(url, request.method, dict(request.POST.items()))
except RemoteTxAppError, e:
return error_contacting_app(request, url, txapp, e)
def _notify_translationwatchers(resource, language):
"""
Notify the watchers for a specific TranslationWatch
"""
context = {
'project': resource.project,
'resource': resource,
'language': language,
}
twatch = TranslationWatch.objects.get_or_create(resource=resource,
language=language)[0]
logger.debug("addon-watches: Sending notification for '%s'" % twatch)
txnotification.send_observation_notices_for(
twatch,
signal='project_resource_translation_changed',
extra_context=context)
def test_pseudo_file_api_calls(self):
"""Test Pseudo translation requests through the API."""
source_language = self.project.resources.all()[0].source_language
for i18n_type, v in FORMATS.items():
resource_slug = "resource_%s" % i18n_type.lower()
resource_url = reverse("apiv2_resources", kwargs={"project_slug": self.project.slug})
# Creating resource using the API
f = open(v["file"])
res = self.client["maintainer"].post(
resource_url,
data={"slug": resource_slug, "name": resource_slug, "i18n_type": i18n_type, "attachment": f},
)
f.close()
logger.debug("-----------------------")
logger.debug(i18n_type)
logger.debug(settings.I18N_METHODS[i18n_type]["mimetype"])
logger.debug(res.content)
logger.debug("-----------------------")
# Pseudo file API URL
url = reverse("apiv2_pseudo_content", args=[self.project.slug, resource_slug])
for pseudo_type in settings.PSEUDO_TYPES:
# Get resource file using a specific pseudo type
resp = self.client["registered"].get(url, data={"pseudo_type": pseudo_type})
# Get response and check encoding
resp_content = eval(resp.content)["content"]
if type(resp_content) != unicode:
try:
resp_content = resp_content.decode("utf-8")
except UnicodeDecodeError:
resp_content = resp_content.decode("iso-8859-1")
# FIXME: We have a bug related to spaces being escaped in
# .properties files. This can be dropped after fixing it.
if i18n_type == "PROPERTIES" and pseudo_type in ["PLANGUAGE", "UNICODE"]:
resp_content = resp_content.replace("\\ ", " ")
# Assert expected value in the generated file
for message in v["pseudo_messages"][pseudo_type]:
self.assertTrue(message in resp_content)
def check_and_notify_string_freeze_breakage(sender, **kwargs):
"""
Handler to notify people about string freeze breakage of releases.
This happens whenever a resource source file changes in the string freeze
period.
"""
resource = kwargs.pop('resource')
language = kwargs.pop('language')
# Check it only for source languages
if kwargs.pop('is_source'):
logger.debug("release: Checking string freeze breakage.")
# FIXME: Get timestamp from RLStats last_update field, but it depends
# on some changes on formats/core.py. At this point the RLStats object
# wasn't created yet.
timestamp = datetime.datetime.now()
project = resource.project.outsource or resource.project
releases = Release.objects.filter(resources=resource, project=project,
stringfreeze_date__lte=timestamp, develfreeze_date__gte=timestamp)
for release in releases:
logger.debug("release: Sending notifications about string "
"freeze breakage for '%s'" % release)
project = release.project.outsource or release.project
# User list with project maintainers and team coordinators of the
# given release PLUS maintainers of the project that the RLStats
# object belongs to PLUS
users = User.objects.filter(
Q(projects_maintaining=resource.project) |
Q(projects_maintaining=project) |
Q(team_coordinators__project=project)).distinct()
# Notification
context = {'project': release.project, 'release': release,
'resource': resource}
if release.project != project:
context.update({'parent_project': project})
nt = "project_release_stringfreeze_breakage"
#TODO: Add support for actionlog without a user author.
#action_logging(None, [project, release], nt, context=context)
if settings.ENABLE_NOTICES:
notification.send(users, nt, context)
def create(self):
"""
Creates a new translation from file.
Returns:
A dict with information for the translation.
Raises:
BadRequestError: There was a problem with the request.
NoContentError: There was no file in the request.
"""
if not self.request.FILES:
raise NoContentError("No file has been uploaded.")
submitted_file = self.request.FILES.values()[0]
name = str(submitted_file.name)
size = submitted_file.size
try:
file_ = tempfile.NamedTemporaryFile(mode='wb',
suffix=name[name.rfind('.'):],
delete=False)
for chunk in submitted_file.chunks():
file_.write(chunk)
file_.close()
parser = registry.appropriate_handler(self.resource,
language=self.language,
filename=name)
parser.bind_file(file_.name)
if parser is None:
raise BadRequestError("Unknown file type")
if size == 0:
raise BadRequestError("Empty file")
try:
parser.is_content_valid()
logger.debug("Uploaded file %s" % file_.name)
except (FileCheckError, ParseError), e:
raise BadRequestError("Error uploading file: %s" % e)
except Exception, e:
logger.error(unicode(e), exc_info=True)
raise BadRequestError("A strange error happened.")
def invalidate_cache(sender, instance, created=True, **kwargs):
"""
Invalidate caching on places related to the lock icon in the stats table
row.
"""
if created:
logger.debug("lock-addon: Invalidating cache: %s" % instance)
invalidate_template_cache('resource_details_lang',
instance.rlstats.resource.project.slug,
instance.rlstats.resource.slug,
instance.rlstats.language.code)
invalidate_template_cache('resource_details',
instance.rlstats.resource.project.slug,
instance.rlstats.resource.slug)
invalidate_template_cache("team_details",
instance.rlstats.resource.project.slug,
instance.rlstats.language.code,
instance.rlstats.resource.id)
def expiration_notification(sender, **kwargs):
"""
FIXME: Migrate it (txcron) to work with the String Level.
"""
logger.debug("lock-addon: Sending expiration notifications...")
if not settings.ENABLE_NOTICES:
logger.debug("lock-addon: ENABLE_NOTICES is not enabled")
return
current_site = Site.objects.get_current()
locks = Lock.objects.expiring()
nt = 'project_resource_language_lock_expiring'
for lock in locks:
context = {
'resource': lock.rlstats.resource,
'language': lock.rlstats.language,
'project': lock.rlstats.resource.project,
'user': lock.owner,
'expires': lock.expires,
'current_site': current_site
}
logger.debug("lock-addon: Sending notification about lock: %s" % lock)
notification.send_now([
lock.owner,
], nt, context)
lock.notified = True
lock.save()
def invalidate_cache(sender, instance, created=True, **kwargs):
"""
Invalidate caching on places related to the lock icon in the stats table
row.
"""
if created:
logger.debug("lock-addon: Invalidating cache: %s" % instance)
invalidate_template_cache('resource_details_lang',
instance.rlstats.resource.project.slug,
instance.rlstats.resource.slug,
instance.rlstats.language.code)
invalidate_template_cache('resource_details',
instance.rlstats.resource.project.slug,
instance.rlstats.resource.slug)
team = Team.objects.get_or_none(instance.rlstats.resource.project,
instance.rlstats.language.code)
if team:
invalidate_template_cache('team_details', team.id,
instance.rlstats.resource.id)
def file_check(self):
"""
Check whether the uploaded file if valid.
Run a number of checks on the StorageFile contents, including specific
checks, depending on the i18n format of the file.
"""
# Get appropriate parser
parser = self.find_parser()
if not parser:
logger.debug("file: File is not supported.")
raise FileCheckError(ugettext("File you are trying to upload does not "
"seem to belong to a known i18n format."))
# Empty file check
if self.size == 0:
logger.debug("file: Empty file!")
raise FileCheckError(ugettext("You have uploaded an empty file!"))
# Specific check for the i18n format
parser.contents_check(self.get_storage_path())
def read(self, request, uuid=None, api_version=1):
"""
Returns list of StorageFile objects
[
{
"total_strings": 1102,
"uuid": "71f4964c-817b-4778-b3e0-693375cb1355",
"language": {
"code": "et"
},
"created": "2010-05-13 07:22:36",
"size": 187619,
"mime_type": "application/x-gettext",
"name": "kmess.master.et.po"
},
...
]
"""
if request.user.is_anonymous():
return rc.FORBIDDEN
retval = StorageFile.objects.filter(user=request.user, bound=False)
logger.debug("Returned list of users uploaded files: %s" % retval)
return retval
def check_and_notify_resource_full_reviewed(**kwargs):
"""
Handler to notify maintainers about 100% reviewed translations.
"""
rlstats = kwargs.pop('sender')
if (settings.ENABLE_NOTICES and
rlstats.resource.source_language != rlstats.language):
logger.debug("resource: Checking if resource translation is fully "
"reviewed: %s (%s)" % (rlstats.resource, rlstats.language.code))
if rlstats.reviewed_perc == 100:
logger.debug("resource: Resource translation is fully reviewed.")
# Notification
context = {
'project': rlstats.resource.project,
'resource': rlstats.resource,
'language': rlstats.language,}
nt = "project_resource_full_reviewed"
notification.send(rlstats.resource.project.maintainers.all(),
nt, context)
def search(request):
query_string = prepare_solr_query_string(request.GET.get('q', ""))
search_terms = query_string.split()
index_query = SearchQuerySet().models(Project)
spelling_suggestion = None
#FIXME: Workaround for https://github.com/toastdriven/django-haystack/issues/364
# Only the else part should be necessary.
if settings.HAYSTACK_SEARCH_ENGINE == 'simple':
results = index_query.auto_query(query_string)
else:
try:
qfilter = fulltext_fuzzy_match_filter(query_string)
results = index_query.filter(qfilter)
spelling_suggestion = results.spelling_suggestion(query_string)
except TypeError:
results = []
logger.debug("Searched for %s. Found %s results." % (query_string, len(results)))
return render_to_response("search.html",
{'query': query_string,
'terms': search_terms,
'results': results,
'spelling_suggestion': spelling_suggestion},
context_instance = RequestContext(request))
def lotte_init_handler(sender, request, resources=None, language=None, **kwargs):
user = request.user
logger.debug("lock-addon: Started editing in Lotte")
for resource in resources:
try:
lock = Lock.objects.create_update(resource, language, user)
logger.debug("lock-addon: Lock acquired/extended: '%s'" % lock)
except LockError, e:
logger.debug("lock-addon: %s" % e.message)
messages.error(request, _("This translation is " "locked by someone else."))
def visit_url(sender, **kwargs):
"""Visit the URL for the project.
Send the slug of the project, the slug of the resource and the language
of the translation as identifiers. Send the translation percentage
as information.
Args:
sender: The rlstats object itself.
Returns:
True of False for success (or not).
"""
# TODO make it a celery task
# TODO increase the timeout in celery
stats = sender
resource = stats.resource
project = resource.project
language = stats.language
if 'post_function' in kwargs:
post_function = kwargs['post_function']
else:
post_function = requests.post
hooks = WebHook.objects.filter(project=project)
if not hooks:
logger.debug("Project %s has no web hooks" % project.slug)
return
event_info = {
'project': project.slug,
'resource': resource.slug,
'language': language.code,
'percent': stats.translated_perc,
}
logger.debug("POST data for %s: %s" %
(stats.resource.project.slug, event_info))
for hook in hooks:
res = post_function(hook.url,
data=event_info,
allow_redirects=False,
timeout=2.0)
if res.ok:
logger.debug("POST for project %s successful." % project)
else:
msg = "Error visiting webhook %s: HTTP code is %s" % (
hook, res.status_code)
logger.error(msg)
def lotte_done_handler(sender, request, resources=None, language=None,
**kwargs):
user = request.user
logger.debug("lock-addon: Finished editing in Lotte")
for resource in resources:
lock = Lock.objects.get_valid(resource, language)
if lock:
try:
lock.delete_by_user(user)
logger.debug("lock-addon: Lock deleted: '%s'" % lock)
except LockError, e:
logger.debug("lock-addon: User '%s' sent translations to a "
"resource/language locked by someone else: %s" %
(user, e.message))
def visit_url(sender, **kwargs):
"""Visit the URL for the project.
Send the slug of the project, the slug of the resource and the language
of the translation as identifiers. Send the translation percentage
as information.
Args:
sender: The rlstats object itself.
Returns:
True of False for success (or not).
"""
# TODO make it a celery task
# TODO increase the timeout in celery
stats = sender
resource = stats.resource
project = resource.project
language = stats.language
if 'post_function' in kwargs:
post_function = kwargs['post_function']
else:
post_function = requests.post
hooks = WebHook.objects.filter(project=project)
if not hooks:
logger.debug("Project %s has no web hooks" % project.slug)
return
event_info = {
'project': project.slug,
'resource': resource.slug,
'language': language.code,
'percent': stats.translated_perc,
}
logger.debug(
"POST data for %s: %s" % (stats.resource.project.slug, event_info)
)
for hook in hooks:
res = post_function(hook.url,
data=event_info, allow_redirects=False, timeout=2.0)
if res.ok:
logger.debug("POST for project %s successful." % project)
else:
msg = "Error visiting webhook %s: HTTP code is %s" % (
hook, res.status_code)
logger.error(msg)
def lotte_init_handler(sender,
request,
resources=None,
language=None,
**kwargs):
user = request.user
logger.debug("lock-addon: Started editing in Lotte")
for resource in resources:
try:
lock = Lock.objects.create_update(resource, language, user)
logger.debug("lock-addon: Lock acquired/extended: '%s'" % lock)
except LockError, e:
logger.debug("lock-addon: %s" % e.message)
messages.error(request,
_("This translation is "
"locked by someone else."))
def lotte_done_handler(sender,
request,
resources=None,
language=None,
**kwargs):
user = request.user
logger.debug("lock-addon: Finished editing in Lotte")
for resource in resources:
lock = Lock.objects.get_valid(resource, language)
if lock:
try:
lock.delete_by_user(user)
logger.debug("lock-addon: Lock deleted: '%s'" % lock)
except LockError, e:
logger.debug("lock-addon: User '%s' sent translations to a "
"resource/language locked by someone else: %s" %
(user, e.message))
def expiration_notification(sender, **kwargs):
"""
FIXME: Migrate it (txcron) to work with the String Level.
"""
logger.debug("lock-addon: Sending expiration notifications...")
if not settings.ENABLE_NOTICES:
logger.debug("lock-addon: ENABLE_NOTICES is not enabled")
return
current_site = Site.objects.get_current()
locks = Lock.objects.expiring()
nt = 'project_resource_language_lock_expiring'
for lock in locks:
context = { 'resource': lock.rlstats.resource,
'language': lock.rlstats.language,
'project' : lock.rlstats.resource.project,
'user': lock.owner,
'expires': lock.expires,
'current_site' : current_site }
logger.debug("lock-addon: Sending notification about lock: %s" % lock)
notification.send_now([lock.owner,], nt, context)
lock.notified = True
lock.save()
def push_translation(request, project_slug, lang_code, *args, **kwargs):
"""
Client pushes an id and a translation string.
Id is considered to be of the source translation string and the string is
in the target_lang.
FIXME: Document in detail the form of the 'strings' POST variable.
"""
logger.debug("POST data when saving translation: %s" % request.POST)
# Permissions handling
# Project should always be available
project = get_object_or_404(Project, slug=project_slug)
team = Team.objects.get_or_none(project, lang_code)
check = ProjectPermission(request.user)
if not check.submit_translations(team or project) and not\
check.maintain(project):
return permission_denied(request)
if not request.POST:
return HttpResponseBadRequest()
data = simplejson.loads(request.raw_post_data)
strings = data["strings"]
try:
target_language = Language.objects.by_code_or_alias(lang_code)
except Language.DoesNotExist:
raise Http404
# This dictionary will hold the results of the save operation and will map
# status code for each translation pushed, to indicate the result on each
# translation push separately.
push_response_dict = {}
# Form the strings dictionary, get as Json object
# The fields are the following:
# id-> source_entity id
# translations-> translation strings (includes all plurals)
# context-> source_entity context
# occurrence-> occurrence (not yet well supported)
# Iterate through all the row data that have been sent.
for row in strings:
source_id = int(row['id'])
try:
source_string = Translation.objects.select_related(depth=1).get(
id=source_id)
except Translation.DoesNotExist:
# TODO: Log or inform here
push_response_dict[source_id] = {
'status': 400,
'message': _("Source string cannot be identified in the DB")
}
# If the source_string cannot be identified in the DB then go to next
# translation pair.
continue
if not source_string.resource.accept_translations:
push_response_dict[source_id] = {
'status':
400,
'message':
_("The resource of this source string is not "
"accepting translations.")
}
# If the translated source string is pluralized check that all the
# source language supported rules have been filled in, else return error
# and donot save the translations.
if source_string.source_entity.pluralized:
error_flag = False
for rule in target_language.get_pluralrules():
if rule in row[
'translations'] and row['translations'][rule] != "":
continue
else:
error_flag = True
if error_flag:
error_flag = False
# Check also if all of them are "". If yes, delete all the plurals!
for rule in target_language.get_pluralrules():
if rule in row['translations'] and row['translations'][
rule] == "":
continue
else:
error_flag = True
if error_flag:
push_response_dict[source_id] = {
'status':
400,
'message':
(_("Cannot save unless plural translations are either "
"completely specified or entirely empty!"))
}
# Skip the save as we hit on an error.
continue
try:
msgs = _save_translation(source_string, row['translations'],
target_language, request.user)
if not msgs:
push_response_dict[source_id] = {'status': 200}
else:
push_response_dict[source_id] = {
'status': 200,
'message': msgs[-1]
}
except LotteBadRequestError, e:
push_response_dict[source_id] = {
'status': 400,
'message': e.message
}
except Exception, e:
logger.error("Unexpected exception raised: %s" % e.message,
exc_info=True)
push_response_dict[source_id] = {
'status': 400,
'message': e.message
}
def db_cleanup(sender, **kwargs):
logger.debug("lock-addon: Looking for expired locks")
locks = Lock.objects.expired()
for lock in locks:
logger.debug("lock-addon: Deleting lock: %s" % lock)
lock.delete()
# Write the contents to the pipe
if _input:
if isinstance(_input, basestring):
proc.stdin.write(_input)
elif isinstance(_input, (file, UploadedFile)):
_input.seek(0)
for content in _input:
proc.stdin.write(content)
# Wait for the process to return
stdout_value, stderr_value = proc.communicate()
status = proc.returncode
# Strip off trailing whitespace by default
if not with_raw_output:
stdout_value = stdout_value.rstrip()
stderr_value = stderr_value.rstrip()
if with_exceptions and status != 0:
logger.debug('stdout: %s' % stdout_value)
logger.error(stderr_value)
raise CommandError(command, status, stderr_value, stdout_value)
# Allow access to the command's status code
if with_extended_output:
return (status, stdout_value, stderr_value)
else:
return stdout_value
def run_command(command, *args, **kw):
"""
Handle shell command execution.
Consume and return the returned information (stdout).
``command``
The command argument list to execute
``cwd``
Use cwd as the working dir.
``with_extended_output``
Whether to return a (status, stdout, stderr) tuple.
``with_exceptions``
Whether to raise an exception when command returns a non-zero status.
``with_raw_output``
Whether to avoid stripping off trailing whitespace.
``convert_args``
Converts python arguments to command line arguments.
``env``
A dictionary mapping environment variables to be used with the command.
Returns
str(output) # extended_output = False (Default)
tuple(int(status), str(stdout), str(stderr)) # extended_output = True
"""
_input= kw.pop('_input', None)
cwd = kw.pop('cwd', os.getcwd())
with_extended_output = kw.pop('with_extended_output', False)
with_exceptions = kw.pop('with_exceptions', True)
with_raw_output = kw.pop('with_raw_output', False)
env = kw.pop('env', None)
# if command is a string split to a list
if isinstance(command, basestring):
command = command.split()
# if more kwargs are given, convert them to command line args
if kw:
kwarglist = python_to_args(**kw)
else:
kwarglist = []
command += kwarglist + list(args)
# If stdin is a string, create a pipe so we can write the contents
if _input:
stdin = subprocess.PIPE
else:
stdin = None
logger.debug("Running low-level command '%s'" % ' '.join(command))
logger.debug(" CWD: '%s'" % cwd)
# Start the process
try:
proc = subprocess.Popen(command, cwd=cwd, stdin=stdin,
stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env,)
except OSError, e:
status = "Command '%s' not found." % command[0]
logger.debug(status)
raise CommandError(command, status, '', '')
def decorator(func):
msg = "%s skipped. Please implement it in your project path." % func_name
if settings.TX_ROOT != settings.PROJECT_PATH:
logger.debug(msg)
return unittest.skipUnless(settings.TX_ROOT == settings.PROJECT_PATH,
msg)
def push_translation(request, project_slug, lang_code, *args, **kwargs):
"""
Client pushes an id and a translation string.
Id is considered to be of the source translation string and the string is
in the target_lang.
FIXME: Document in detail the form of the 'strings' POST variable.
"""
logger.debug("POST data when saving translation: %s" % request.POST)
# Permissions handling
# Project should always be available
project = get_object_or_404(Project, slug=project_slug)
team = Team.objects.get_or_none(project, lang_code)
check = ProjectPermission(request.user)
if not check.submit_translations(team or project) and not\
check.maintain(project):
return permission_denied(request)
if not request.POST:
return HttpResponseBadRequest()
data = simplejson.loads(request.raw_post_data)
strings = data["strings"]
try:
target_language = Language.objects.by_code_or_alias(lang_code)
except Language.DoesNotExist:
raise Http404
# This dictionary will hold the results of the save operation and will map
# status code for each translation pushed, to indicate the result on each
# translation push separately.
push_response_dict = {}
# Form the strings dictionary, get as Json object
# The fields are the following:
# id-> source_entity id
# translations-> translation strings (includes all plurals)
# context-> source_entity context
# occurrence-> occurrence (not yet well supported)
# Iterate through all the row data that have been sent.
for row in strings:
source_id = int(row['id'])
try:
source_string = Translation.objects.select_related(depth=1).get(
id=source_id
)
except Translation.DoesNotExist:
# TODO: Log or inform here
push_response_dict[source_id] = { 'status':400,
'message':_("Source string cannot be identified in the DB")}
# If the source_string cannot be identified in the DB then go to next
# translation pair.
continue
if not source_string.resource.accept_translations:
push_response_dict[source_id] = { 'status':400,
'message':_("The resource of this source string is not "
"accepting translations.") }
# If the translated source string is pluralized check that all the
# source language supported rules have been filled in, else return error
# and donot save the translations.
if source_string.source_entity.pluralized:
error_flag = False
for rule in target_language.get_pluralrules():
if rule in row['translations'] and row['translations'][rule] != "":
continue
else:
error_flag = True
if error_flag:
error_flag = False
# Check also if all of them are "". If yes, delete all the plurals!
for rule in target_language.get_pluralrules():
if rule in row['translations'] and row['translations'][rule] == "":
continue
else:
error_flag = True
if error_flag:
push_response_dict[source_id] = { 'status':400,
'message':(_("Cannot save unless plural translations are either "
"completely specified or entirely empty!"))}
# Skip the save as we hit on an error.
continue
try:
msgs = _save_translation(
source_string, row['translations'],
target_language, request.user
)
if not msgs:
push_response_dict[source_id] = {'status': 200}
else:
push_response_dict[source_id] = {
'status': 200, 'message': msgs[-1]
}
except LotteBadRequestError, e:
push_response_dict[source_id] = {
'status': 400, 'message': e.message
}
except Exception, e:
logger.error(
"Unexpected exception raised: %s" % e.message, exc_info=True
)
push_response_dict[source_id] = {
'status': 400, 'message': e.message
}
language=target_language, rule=rule, string=target_string,
resource=resource
)
_add_copyright(source_string, target_language, user)
invalidate_stats_cache(resource, target_language, user=user)
else:
# In cases of pluralized translations, sometimes only one
# translation will exist and the rest plural forms will be
# empty. If the user wants to delete all of them, we need
# to let by the ones that don't already have a translation.
if not source_string.source_entity.pluralized:
raise LotteBadRequestError(
_("The translation string is empty")
)
except LotteBadRequestError, e:
logger.debug("%s" % e, exc_info=True)
raise
# catch-all. if we don't save we _MUST_ inform the user
except Exception, e:
msg = _(
"Error occurred while trying to save translation: %s" % unicode(e)
)
logger.error(msg, exc_info=True)
raise LotteBadRequestError(msg)
return warnings
def _add_copyright(source_string, target_language, user):
from transifex.addons.copyright.handlers import lotte_copyrights
lotte_save_translation.connect(lotte_copyrights)
lotte_save_translation.send(
language=target_language,
rule=rule,
string=target_string,
resource=resource)
_add_copyright(source_string, target_language, user)
invalidate_stats_cache(resource, target_language, user=user)
else:
# In cases of pluralized translations, sometimes only one
# translation will exist and the rest plural forms will be
# empty. If the user wants to delete all of them, we need
# to let by the ones that don't already have a translation.
if not source_string.source_entity.pluralized:
raise LotteBadRequestError(
_("The translation string is empty"))
except LotteBadRequestError, e:
logger.debug("%s" % e, exc_info=True)
raise
# catch-all. if we don't save we _MUST_ inform the user
except Exception, e:
msg = _("Error occurred while trying to save translation: %s" %
unicode(e))
logger.error(msg, exc_info=True)
raise LotteBadRequestError(msg)
return warnings
def _add_copyright(source_string, target_language, user):
from transifex.addons.copyright.handlers import lotte_copyrights
lotte_save_translation.connect(lotte_copyrights)
lotte_save_translation.send(None,
resource=source_string.resource,
def notify_string_freeze(sender=None, instance=None, **kwargs):
"""
Handler to notify people about string freeze of releases.
"""
now = datetime.datetime.now()
# 48hs before
logger.debug("release: Sending notifications 48hs before entering the "
"String Freeze period.")
timestamp = now + datetime.timedelta(hours=48)
releases = Release.objects.filter(stringfreeze_date__lte=timestamp,
notifications__before_stringfreeze=False)
for release in releases:
logger.debug("release: Sending notifications for '%s'." % release)
project = release.project.outsource or release.project
# List with project maintainers of the given release PLUS maintainers
# of projects that outsource theirs team to the project of the release
resource_ids = release.resources.all().values('id').query
users = User.objects.filter(
(Q(projects_maintaining__resources__in=resource_ids)
& Q(projects_maintaining__outsource=project))
| Q(projects_maintaining=project)).distinct()
# Notification
context = {'project': release.project, 'release': release}
if release.project != project:
context.update({'parent_project': project})
nt = "project_release_before_stringfreeze"
#TODO: Add support for actionlog without a user author.
#action_logging(None, [project, release], nt, context=context)
if settings.ENABLE_NOTICES:
notification.send(users, nt, context)
release.notifications.before_stringfreeze = True
release.notifications.save()
# Exactly on time
logger.debug("release: Sending notifications about being in String "
"Freeze period.")
releases = Release.objects.filter(stringfreeze_date__lte=now,
notifications__in_stringfreeze=False)
for release in releases:
logger.debug("release: Sending notifications for '%s'." % release)
project = release.project.outsource or release.project
# List with project maintainers of the given release PLUS maintainers
# of projects that outsource theirs team to the project of the release
# PLUS team coordinators and team members
resource_ids = release.resources.all().values('id').query
users = User.objects.filter(
(Q(projects_maintaining__resources__in=resource_ids)
& Q(projects_maintaining__outsource=project))
| Q(projects_maintaining=project)
| Q(team_coordinators__project=project)
| Q(team_members__project=project)).distinct()
# Notification
context = {'project': release.project, 'release': release}
if release.project != project:
context.update({'parent_project': project})
nt = "project_release_in_stringfreeze"
#TODO: Add support for actionlog without a user author.
#action_logging(None, [project, release], nt, context=context)
if settings.ENABLE_NOTICES:
notification.send(users, nt, context)
release.notifications.in_stringfreeze = True
release.notifications.save()
"""
Return actionlog entries for the given project plus the actionlogs of
the hub projects, in case it's a hub.
"""
ids = [self.id]
if self.is_hub:
ids += self.outsourcing.all().values_list('id', flat=True)
return LogEntry.objects.filter(
content_type=ContentType.objects.get_for_model(Project),
object_id__in=ids)
try:
tagging.register(Project, tag_descriptor_attr='tagsobj')
except tagging.AlreadyRegistered, e:
logger.debug('Tagging: %s' % str(e))
log_model(Project)
class HubRequest(models.Model):
"""
Model to handle outsource requests of project to project hubs. A project
only can have one request for a project hub and it can only be associated
to a project hub once at a time.
"""
project_hub = models.ForeignKey(
Project,
verbose_name=_('Project Hub'),
blank=False,
null=False,