def youtube_sync(video, language):
"""
Used on debug page for video.
Simplified version of what's found in
``ThirdPartyAccount.mirror_on_third_party``. It doesn't bother checking if
we should be syncing this or not. Only does the new Youtube/Amara
integration syncing.
"""
version = language.get_tip()
always_push_account = ThirdPartyAccount.objects.always_push_account()
for vurl in video.videourl_set.all():
vt = video_type_registrar.video_type_for_url(vurl.url)
try:
vt.update_subtitles(version, always_push_account)
Meter('youtube.push.success').inc()
except:
Meter('youtube.push.fail').inc()
logger.error('Always pushing to youtoube has failed.',
extra={
'video': video.video_id,
'vurl': vurl.pk
})
finally:
Meter('youtube.push.request').inc()
def team_member_new(member_pk):
if getattr(settings, "MESSAGES_DISABLED", False):
return
from messages.models import Message
from teams.models import TeamMember
member = TeamMember.objects.get(pk=member_pk)
if not _team_sends_notification(member.team,
'block_team_member_new_message'):
return False
from videos.models import Action
from teams.models import TeamMember
# the feed item should appear on the timeline for all team members
# as a team might have thousands of members, this one item has
# to show up on all of them
Action.create_new_member_handler(member)
# notify admins and owners through messages
notifiable = TeamMember.objects.filter(
team=member.team,
role__in=[TeamMember.ROLE_ADMIN,
TeamMember.ROLE_OWNER]).exclude(pk=member.pk)
for m in notifiable:
context = {
"new_member": member.user,
"team": member.team,
"user": m.user,
"role": member.role,
"url_base": get_url_base(),
}
body = render_to_string("messages/team-new-member.txt", context)
subject = ugettext("%s team has a new member" % (member.team))
if m.user.notify_by_message:
msg = Message()
msg.subject = subject
msg.content = body
msg.user = m.user
msg.object = m.team
msg.save()
template_name = "messages/email/team-new-member.html"
Meter('templated-emails-sent-by-type.teams.new-member').inc()
send_templated_email(m.user, subject, template_name, context)
# now send welcome mail to the new member
template_name = "messages/team-welcome.txt"
context = {
"team": member.team,
"url_base": get_url_base(),
"role": member.role,
"user": member.user,
}
body = render_to_string(template_name, context)
msg = Message()
msg.subject = ugettext("You've joined the %s team!" % (member.team))
msg.content = body
msg.user = member.user
msg.object = member.team
msg.save()
template_name = "messages/email/team-welcome.html"
Meter('templated-emails-sent-by-type.teams.welcome').inc()
send_templated_email(msg.user, msg.subject, template_name, context)
def _sync_versions(language_pk=None):
"""Sync a single language worth of SubtitleVersions."""
from utils.metrics import Meter
meter = Meter('data-model-refactor.version-syncs')
sl = get_unsynced_subtitle_version_language()
if not sl:
return False
if sl.can_writelock(TERN_REQUEST):
sl.writelock(TERN_REQUEST)
else:
# If we picked a writelocked language, bail for now, but come back to it
# later.
log('SubtitleLanguage', 'ERROR_WRITELOCKED', sl.pk, None)
Meter('data-model-refactor.version-errors.writelocked').inc()
return True
try:
# First update any versions that have been synced but have changed since.
versions = sl.subtitleversion_set.filter(
needs_sync=True, new_subtitle_version__isnull=False)
for version in versions.order_by('version_no'):
_update_subtitle_version(version)
if not dry:
meter.inc()
# Then sync any new versions.
versions = sl.subtitleversion_set.filter(needs_sync=True,
new_subtitle_version=None)
# This is ugly, but we (may) need to do something special on the last
# version we sync.
new_versions = list(versions.order_by('version_no'))
for version in new_versions[:-1]:
_create_subtitle_version(version, False)
if not dry:
meter.inc()
for version in new_versions[-1:]:
_create_subtitle_version(version, True)
if not dry:
meter.inc()
except:
Meter('data-model-refactor.version-errors.other').inc()
raise
finally:
sl.release_writelock()
if not dry:
if random.random() < 0.01:
report_metrics()
return True
def team_member_leave(team_pk, user_pk):
if getattr(settings, "MESSAGES_DISABLED", False):
return
from messages.models import Message
from teams.models import TeamMember, Team
user = User.objects.get(pk=user_pk)
team = Team.objects.get(pk=team_pk)
if not _team_sends_notification(team,'block_team_member_leave_message'):
return False
from videos.models import Action
# the feed item should appear on the timeline for all team members
# as a team might have thousands of members, this one item has
# to show up on all of them
Action.create_member_left_handler(team, user)
# notify admins and owners through messages
notifiable = TeamMember.objects.filter( team=team,
role__in=[TeamMember.ROLE_ADMIN, TeamMember.ROLE_OWNER])
subject = fmt(
ugettext(u"%(user)s has left the %(team)s team"),
user=user, team=team)
for m in notifiable:
context = {
"parting_member": user,
"team":team,
"user":m.user,
"url_base":get_url_base(),
}
body = render_to_string("messages/team-member-left.txt",context)
if m.user.notify_by_message:
msg = Message()
msg.subject = subject
msg.content = body
msg.user = m.user
msg.object = team
msg.save()
Meter('templated-emails-sent-by-type.teams.someone-left').inc()
send_templated_email(m.user, subject, "messages/email/team-member-left.html", context)
context = {
"team":team,
"user":user,
"url_base":get_url_base(),
}
subject = fmt(ugettext("You've left the %(team)s team!"), team=team)
if user.notify_by_message:
template_name = "messages/team-member-you-have-left.txt"
msg = Message()
msg.subject = subject
msg.content = render_to_string(template_name,context)
msg.user = user
msg.object = team
msg.save()
template_name = "messages/email/team-member-you-have-left.html"
Meter('templated-emails-sent-by-type.teams.you-left').inc()
send_templated_email(user, subject, template_name, context)
def _sync_language(language_pk=None):
"""Try to sync one SubtitleLanguage.
Returns True if a language was synced (or skipped, but we should try it
again later), False if there were no more left.
"""
from utils.metrics import Meter
sl = (get_specific_language(language_pk)
if language_pk else get_unsynced_subtitle_language())
if not sl:
return False
if sl.can_writelock(TERN_REQUEST):
sl.writelock(TERN_REQUEST)
else:
# If we picked a writelocked language, bail for now, but come back to it
# later.
log('SubtitleLanguage', 'ERROR_WRITELOCKED', sl.pk, None)
Meter('data-model-refactor.language-errors.writelocked').inc()
return True
try:
if sl.language == '':
fix_blank_original(sl.video)
# For now, we'll actually bail on this language and come back to it
# later. Hopefully it will have been fixed by the above call, but
# there's a chance that it's is_original=False and so is still borked.
log('SubtitleLanguage', 'ERROR_EMPTY_LANGUAGE', sl.pk, None)
Meter('data-model-refactor.language-errors.empty-language').inc()
return True
if sl.new_subtitle_language:
_update_subtitle_language(sl)
else:
_create_subtitle_language(sl)
except:
Meter('data-model-refactor.language-errors.other').inc()
raise
finally:
sl.release_writelock()
if not dry:
Meter('data-model-refactor.language-syncs').inc()
if random.random() < 0.01:
report_metrics()
return True
def test(self, model: nn.Module) -> Dict[str, Union[int, Meter]]:
model.eval()
loss_meter = Meter()
acc_meter = Meter()
num_all_samples = 0
with torch.no_grad():
for batch_idx, (X, y) in enumerate(self.dataset_loader):
X, y = X.to(self.device), y.to(self.device)
pred = model(X)
loss = self.criterion(pred, y)
correct = self.count_correct(pred, y)
#
num_samples = y.size(0)
loss_meter.update(loss.item(), n=num_samples)
acc_meter.update(correct.item() / num_samples, n=num_samples)
num_all_samples += num_samples
return {
'loss_meter': loss_meter,
'acc_meter': acc_meter,
'num_samples': num_all_samples
}
def send_http_request(self, url, basic_auth_username, basic_auth_password):
h = Http(disable_ssl_certificate_validation=True)
if basic_auth_username and basic_auth_password:
h.add_credentials(basic_auth_username, basic_auth_password)
project = self.video.get_team_video().project.slug if self.video else None
data = {
'event': self.event_name,
'api_url': self.api_url,
}
if self.team:
data['team'] = self.team.slug
if self.partner:
data['partner'] = self.partner.slug
if project:
data['project'] = project
if self.video:
data['video_id'] = self.video_id
if self.application_pk:
data['application_id'] = self.application_pk
if self.language:
data.update({
"language_code": self.language_code,
"language_id": self.language.pk,
})
data_sent = data
data = urlencode(data)
url = "%s?%s" % (url , data)
try:
resp, content = h.request(url, method="POST", body=data, headers={
'referer': '%s://%s' % (DEFAULT_PROTOCOL, Site.objects.get_current().domain)
})
success = 200 <= resp.status < 400
if success is False:
logger.error("Failed to notify team %s " % (self.team),
extra={
'team': self.team or self.partner,
'url': url,
'response': resp,
'content': content,
'data_sent':data_sent,
})
Meter('http-callback-notification-error').inc()
else:
Meter('http-callback-notification-success').inc()
return success, content
except:
logger.exception("Failed to send http notification ")
return None, None
def _get_entry(self, video_id):
Meter('youtube.api_request').inc()
try:
return yt_service.GetYouTubeVideoEntry(video_id=str(video_id))
except RequestError, e:
err = e[0].get('body', 'Undefined error')
raise VideoTypeError('Youtube error: %s' % err)
def video_url_create(request):
output = {}
form = CreateVideoUrlForm(request.user, request.POST)
if form.is_valid():
obj = form.save()
video = form.cleaned_data['video']
users = video.notification_list(request.user)
for user in users:
subject = u'New video URL added by %(username)s to "%(video_title)s" on universalsubtitles.org'
subject = subject % {'url': obj.url, 'username': obj.added_by, 'video_title': video}
context = {
'video': video,
'video_url': obj,
'user': user,
'domain': Site.objects.get_current().domain,
'hash': user.hash_for_video(video.video_id)
}
Meter('templated-emails-sent-by-type.videos.video-url-added').inc()
send_templated_email(user, subject,
'videos/email_video_url_add.html',
context, fail_silently=not settings.DEBUG)
else:
output['errors'] = form.get_errors()
return HttpResponse(json.dumps(output))
def application_sent(application_pk):
if getattr(settings, "MESSAGES_DISABLED", False):
return
from messages.models import Message
from teams.models import Application, TeamMember
application = Application.objects.get(pk=application_pk)
notifiable = TeamMember.objects.filter(
team=application.team,
role__in=[TeamMember.ROLE_ADMIN, TeamMember.ROLE_OWNER])
for m in notifiable:
template_name = "messages/application-sent.txt"
context = {
"applicant": application.user,
"url_base": get_url_base(),
"team": application.team,
"note": application.note,
"user": m.user,
}
body = render_to_string(template_name, context)
subject = ugettext(u'%(user)s is applying for team %(team)s') % dict(
user=application.user, team=application.team.name)
if m.user.notify_by_message:
msg = Message()
msg.subject = subject
msg.content = body
msg.user = m.user
msg.object = application.team
msg.author = application.user
msg.save()
Meter('templated-emails-sent-by-type.teams.application-sent').inc()
send_templated_email(m.user, subject,
"messages/email/application-sent-email.html",
context)
return True
def team_application_denied(application_pk):
if getattr(settings, "MESSAGES_DISABLED", False):
return
from messages.models import Message
from teams.models import Application
application = Application.objects.get(pk=application_pk)
if not _team_sends_notification(application.team,'block_application_denided_message'):
return False
template_name = "messages/email/team-application-denied.html"
context = {
"team": application.team,
"user": application.user,
"url_base": get_url_base(),
"note": application.note,
}
subject = fmt(
ugettext(u'Your application to join the %(team)s '
u'team has been declined'),
team=application.team.name)
if application.user.notify_by_message:
msg = Message()
msg.subject = subject
msg.content = render_to_string("messages/team-application-denied.txt", context)
msg.user = application.user
msg.object = application.team
msg.save()
Meter('templated-emails-sent-by-type.teams.application-declined').inc()
send_templated_email(application.user, subject, template_name, context)
def send_new_translation_notification(translation_version):
domain = Site.objects.get_current().domain
video = translation_version.language.video
language = translation_version.language
for user in video.notification_list(translation_version.user):
context = {
'version':
translation_version,
'domain':
domain,
'video_url':
'%s://%s%s' % (DEFAULT_PROTOCOL, domain, video.get_absolute_url()),
'user':
user,
'language':
language,
'video':
video,
'hash':
user.hash_for_video(video.video_id),
"STATIC_URL":
settings.STATIC_URL,
}
subject = 'New %s translation by %s of "%s"' % \
(language.language_display(), translation_version.user.__unicode__(), video.__unicode__())
Meter('templated-emails-sent-by-type.videos.new-translation-started'
).inc()
send_templated_email(user,
subject,
'videos/email_start_notification.html',
context,
fail_silently=not settings.DEBUG)
return True
def send_new_message_notification(message_id):
from messages.models import Message
try:
message = Message.objects.get(pk=message_id)
except Message.DoesNotExist:
msg = '**send_new_message_notification**. Message does not exist. ID: %s' % message_id
client.create_from_text(msg, logger='celery')
return
user = message.user
if not user.email or not user.is_active or not user.notify_by_email:
return
if message.author:
subject = _(u"New message from %(author)s on Amara: %(subject)s")
else:
subject = _("New message on Amara: %(subject)s")
subject = fmt(subject, author=message.author, subject=message.subject)
context = {
"message": message,
"domain": Site.objects.get_current().domain,
"STATIC_URL": settings.STATIC_URL,
}
Meter('templated-emails-sent-by-type.message-received').inc()
send_templated_email(user, subject, "messages/email/message_received.html", context)
def send_change_title_email(video_id, user_id, old_title, new_title):
from videos.models import Video
domain = Site.objects.get_current().domain
try:
video = Video.objects.get(id=video_id)
user = user_id and User.objects.get(id=user_id)
except ObjectDoesNotExist:
return
users = video.notification_list(user)
for obj in users:
subject = u'Video\'s title changed on Amara'
context = {
'user': obj,
'domain': domain,
'video': video,
'editor': user,
'old_title': old_title,
'hash': obj.hash_for_video(video.video_id),
'new_title': new_title,
"STATIC_URL": settings.STATIC_URL,
}
Meter('templated-emails-sent-by-type.videos.title-changed').inc()
send_templated_email(obj,
subject,
'videos/email_title_changed.html',
context,
fail_silently=not settings.DEBUG)
def _notify_teams_of_new_videos(team_qs):
from messages.tasks import _team_sends_notification
from teams.models import TeamVideo
domain = Site.objects.get_current().domain
for team in team_qs:
if not _team_sends_notification(team, 'block_new_video_message'):
continue
team_videos = TeamVideo.objects.filter(team=team, created__gt=team.last_notification_time)
team.last_notification_time = datetime.now()
team.save()
members = team.users.filter( notify_by_email=True, is_active=True) \
.distinct()
subject = fmt(_(u'New %(team)s videos ready for subtitling!'),
team=team)
for user in members:
if not user.email:
continue
context = {
'domain': domain,
'user': user,
'team': team,
'team_videos': team_videos,
"STATIC_URL": settings.STATIC_URL,
}
Meter('templated-emails-sent-by-type.team.new-videos-ready').inc()
send_templated_email(user, subject,
'teams/email_new_videos.html',
context, fail_silently=not settings.DEBUG)
def send_confirmation(self, user):
assert user.email
from messages.models import Message
self.filter(user=user).delete()
salt = sha_constructor(str(random()) +
settings.SECRET_KEY).hexdigest()[:5]
confirmation_key = sha_constructor(
salt + user.email.encode('utf-8')).hexdigest()
try:
current_site = Site.objects.get_current()
except Site.DoesNotExist:
return
path = reverse("auth:confirm_email", args=[confirmation_key])
activate_url = u"http://%s%s" % (unicode(current_site.domain), path)
context = {
"user": user,
"activate_url": activate_url,
"current_site": current_site,
"confirmation_key": confirmation_key,
}
subject = u'Please confirm your email address for %s' % current_site.name
Meter('templated-emails-sent-by-type.email-address-confirmation').inc()
send_templated_email_async(user, subject,
"messages/email/email-confirmation.html",
context)
return self.create(user=user,
sent=datetime.now(),
confirmation_key=confirmation_key)
def _create_subtitle_language(sl):
"""Sync the given subtitle language, creating a new one."""
from apps.subtitles.models import VALID_LANGUAGE_CODES
from apps.videos.models import SubtitleLanguage, Video
from utils.metrics import Meter
try:
duplicates = (SubtitleLanguage.objects.filter(
video=sl.video, language=sl.language).exclude(pk=sl.pk).exists())
except Video.DoesNotExist:
log('SubtitleLanguage', 'ERROR_MISSING_VIDEO', sl.pk, None)
return
if duplicates:
log('SubtitleLanguage', 'ERROR_DUPLICATE_LANGUAGE', sl.pk, None)
log('SubtitleLanguage', 'duplicate_version_counts', sl.pk, None, [
l.subtitleversion_set.count()
for l in sl.video.subtitlelanguage_set.filter(language=sl.language)
])
log('SubtitleLanguage', 'duplicate_subtitle_counts', sl.pk, None, [[
len(v.subtitles())
for v in l.subtitleversion_set.order_by('version_no')
] for l in sl.video.subtitlelanguage_set.filter(language=sl.language)])
Meter('data-model-refactor.language-errors.duplicate-language').inc()
_handle_duplicate_languages(sl)
return
if sl.language not in VALID_LANGUAGE_CODES:
if sl.language == 'no':
log('SubtitleLanguage', 'FIXED_LANGUAGE_CODE', sl.pk, None,
sl.language)
sl.language = 'nb'
sl.save()
elif sl.language == 'iw':
log('SubtitleLanguage', 'FIXED_LANGUAGE_CODE', sl.pk, None,
sl.language)
sl.language = 'he'
sl.save()
else:
log('SubtitleLanguage', 'ERROR_INVALID_LANGUAGE_CODE', sl.pk, None,
sl.language)
Meter('data-model-refactor.language-errors.invalid-language-code'
).inc()
return
nsl = _add_sl(sl)
log('SubtitleLanguage', 'create', sl.pk, nsl.pk)
def send_reject_notification(task_pk, sent_back):
raise NotImplementedError()
from teams.models import Task
from videos.models import Action
from messages.models import Message
try:
task = Task.objects.select_related("team_video__video", "team_video",
"assignee",
"subtitle_version").get(pk=task_pk)
except Task.DoesNotExist:
return False
version = task.get_subtitle_version()
subject = ugettext(u"Your subtitles were not accepted")
if task.new_review_base_version:
user = task.new_review_base_version.author
else:
user = version.author
task_language = get_language_label(task.language)
reviewer = task.assignee
video = task.team_video.video
subs_url = "%s%s" % (get_url_base(),
reverse("videos:translation_history",
kwargs={
'video_id': video.video_id,
'lang': task.language,
'lang_id': version.subtitle_language.pk,
}))
reviewer_message_url = "%s%s?user=%s" % (
get_url_base(), reverse("messages:new"), reviewer.username)
context = {
"team": task.team,
"title": version.subtitle_language.get_title(),
"user": user,
"task_language": task_language,
"url_base": get_url_base(),
"task": task,
"reviewer": reviewer,
"note": task.body,
"sent_back": sent_back,
"subs_url": subs_url,
"reviewer_message_url": reviewer_message_url,
}
msg = None
if user.notify_by_message:
template_name = "messages/team-task-rejected.txt"
msg = Message()
msg.subject = subject
msg.content = render_to_string(template_name, context)
msg.user = user
msg.object = task.team
msg.save()
template_name = "messages/email/team-task-rejected.html"
Meter('templated-emails-sent-by-type.teams.task-rejected').inc()
email_res = send_templated_email(user, subject, template_name, context)
Action.create_rejected_video_handler(version, reviewer)
return msg, email_res
def solve_epochs_delta(
self,
round_i,
model: Module,
global_state: STATE_TYPE,
num_epochs,
hide_output: bool = False
) -> Tuple[Dict[str, Union[int, Meter]], Dict[str, torch.Tensor]]:
loss_meter = Meter()
dice_meter = Meter('dice_coeff')
num_all_samples = 0
optimizer = self.create_optimizer(model)
optimizer.step_lr_scheduler(round_i=round_i)
model.train()
with tqdm.trange(num_epochs, disable=hide_output) as t:
for epoch in t:
t.set_description(
f'Client: {self.id}, Round: {round_i}, Epoch :{epoch}')
for batch_idx, (X, y) in enumerate(self.dataset_loader):
# from IPython import embed
X, y = X.to(self.device), y.to(self.device)
optimizer.zero_grad()
pred = model(X)
loss = self.criterion(pred, y)
#
activated = torch.sigmoid(pred)
dice_coeff = self.compute_dice_coefficient(activated, y)
#
loss.backward()
# torch.nn.utils.clip_grad_norm(self.model.parameters(), 60)
optimizer.step()
num_samples = y.size(0)
num_all_samples += num_samples
loss_meter.update(loss.item(), n=num_samples)
dice_meter.update(dice_coeff.item(), n=num_samples)
if (batch_idx % 10 == 0):
# 纯数值, 这里使用平均的损失
t.set_postfix(mean_loss=loss.item())
# 返回参数
state_dict = model.state_dict()
result = {
'loss_meter': loss_meter,
'dice_coeff_meter': dice_meter,
'num_samples': num_all_samples,
'lr': optimizer.get_current_lr()
}
# 计算差值 latest - init
for k, v in state_dict.items():
v.sub_(global_state[k])
# 输出相关的参数
return result, state_dict
def log_session(self, request, log):
dialog_log = WidgetDialogLog(browser_id=request.browser_id, log=log)
dialog_log.save()
Meter('templated-emails-sent-by-type.subtitle-save-failure').inc()
send_templated_email(settings.WIDGET_LOG_EMAIL,
'Subtitle save failure',
'widget/session_log_email.txt',
{'log_pk': dialog_log.pk},
fail_silently=False)
return {'response': 'ok'}
def send_email(self, email_to):
Meter('templated-emails-sent-by-type.teams.team-video-activity').inc()
send_templated_email(email_to,
_("New activity on your team video"),
"teams/emails/new-activity.html",
{
"video": self.video,
"event-name": self.event_name,
"team": self.team,
"laguage":self.language,
}
)
def jsonp(request, method_name, null=False):
Meter('widget-jsonp-calls').inc()
callback = request.GET.get('callback', 'callback')
args = {'request': request}
for k, v in request.GET.items():
if k != 'callback':
args[k.encode('ascii')] = json.loads(v)
rpc_module = null_rpc_views if null else rpc_views
func = getattr(rpc_module, method_name)
result = func(**args)
return HttpResponse("{0}({1});".format(callback, json.dumps(result)),
"text/javascript")
def process_request(self, request):
request.init_time = time.time()
Meter('requests.started').inc()
# Tracking the section of the site isn't trivial, because sometimes we
# have the language prefix, like "/en/foo".
paths = request.path.lstrip('/').split('/')[:2]
p1 = paths[0] if len(paths) >= 1 else None
p2 = paths[1] if len(paths) >= 2 else None
if p1 in SECTIONS:
request._metrics_section = SECTIONS[paths[0]]
elif p2 in SECTIONS:
request._metrics_section = SECTIONS[paths[1]]
else:
request._metrics_section = None
if request._metrics_section:
label = 'site-sections.%s-response-time' % request._metrics_section
Meter(label).inc()
return None
def email_confirmed(user_pk):
from messages.models import Message
user = User.objects.get(pk=user_pk)
subject = "Welcome aboard!"
context = {"user": user}
if user.notify_by_message:
body = render_to_string("messages/email-confirmed.txt", context)
message = Message(user=user, subject=subject, content=body)
message.save()
template_name = "messages/email/email-confirmed.html"
Meter('templated-emails-sent-by-type.email-confirmed').inc()
send_templated_email(user, subject, template_name, context)
return True
def process_response(self, request, response):
if hasattr(request, "init_time"):
delta = time.time() - request.init_time
ms = delta * 1000
Meter('requests.completed').inc()
try:
response_type = response.status_code / 100 * 100
except:
response_type = 'unknown'
Meter('requests.response-types.%s' % response_type).inc()
response.set_cookie('response_time', str(ms))
ManualTimer('response-time').record(ms)
if request._metrics_section:
label = 'site-sections.%s-response-time' % request._metrics_section
ManualTimer(label).record(ms)
return response
def request(self, *args, **kwargs):
"""
Override the very low-level request method to catch possible
too_many_recent_calls errors.
"""
Meter('youtube.api_request').inc()
try:
return super(YouTubeApiBridge, self).request(*args, **kwargs)
except gdata.client.RequestError, e:
if 'too_many_recent_calls' in str(e):
raise TooManyRecentCallsException(e.headers, e.reason,
e.status, e.body)
else:
raise e
def team_invitation_sent(invite_pk):
from messages.models import Message
from teams.models import Invite, Setting, TeamMember
invite = Invite.objects.get(pk=invite_pk)
if not _team_sends_notification(invite.team,
'block_invitation_sent_message'):
return False
# does this team have a custom message for this?
team_default_message = None
messages = Setting.objects.messages().filter(team=invite.team)
if messages.exists():
data = {}
for m in messages:
data[m.get_key_display()] = m.data
mapping = {
TeamMember.ROLE_ADMIN: data['messages_admin'],
TeamMember.ROLE_MANAGER: data['messages_manager'],
TeamMember.ROLE_CONTRIBUTOR: data['messages_invite'],
}
team_default_message = mapping.get(invite.role, None)
context = {
'invite': invite,
'role': invite.role,
"user": invite.user,
"inviter": invite.author,
"team": invite.team,
"invite_pk": invite_pk,
'note': invite.note,
'custom_message': team_default_message,
'url_base': get_url_base(),
}
title = ugettext(u"You've been invited to team %s on Amara" %
invite.team.name)
if invite.user.notify_by_message:
body = render_to_string("messages/team-you-have-been-invited.txt",
context)
msg = Message()
msg.subject = title
msg.user = invite.user
msg.object = invite
msg.author = invite.author
msg.content = body
msg.save()
template_name = 'messages/email/team-you-have-been-invited.html'
Meter('templated-emails-sent-by-type.teams.invitation').inc()
return send_templated_email(invite.user, title, template_name, context)
def solve_epochs_with_global(self, round_i, model: Module, global_model: Module, num_epochs, hide_output: bool = False) -> Tuple[Dict[str, Union[int, Meter]], Dict[str, torch.Tensor]]:
loss_meter = Meter()
acc_meter = Meter()
num_all_samples = 0
optimizer = self.create_optimizer(model)
# TODO 直接引用上一次的 global 模型, 避免复制, optimizer 的 step 中也是不记录梯度的
optimizer.set_old_weights(old_weights=[p for p in global_model.parameters()])
model.train()
with tqdm.trange(num_epochs, disable=hide_output) as t:
for epoch in t:
t.set_description(f'Client: {self.id}, Round: {round_i}, Epoch :{epoch}')
for batch_idx, (X, y) in enumerate(self.dataset_loader):
# from IPython import embed
X, y = X.to(self.device), y.to(self.device)
optimizer.zero_grad()
pred = model(X)
loss = self.criterion(pred, y)
loss.backward()
# torch.nn.utils.clip_grad_norm(self.model.parameters(), 60)
optimizer.step()
correct_sum = self.count_correct(pred, y)
num_samples = y.size(0)
num_all_samples += num_samples
loss_meter.update(loss.item(), n=num_samples)
acc_meter.update(correct_sum.item() / num_samples, n=num_samples)
if (batch_idx % 10 == 0):
# 纯数值, 这里使用平均的损失
t.set_postfix(mean_loss=loss.item())
# 返回参数
result = {
'loss_meter': loss_meter,
'acc_meter': acc_meter,
'num_samples': num_all_samples
}
state_dict = model.state_dict()
# 输出相关的参数
return result, state_dict
def videos_imported_message(user_pk, imported_videos):
from messages.models import Message
user = User.objects.get(pk=user_pk)
subject = _(u"Your videos were imported!")
url = "%s%s" % (get_url_base(), reverse("profiles:my_videos"))
context = {
"user": user,
"imported_videos": imported_videos,
"my_videos_url": url
}
if user.notify_by_message:
body = render_to_string("messages/videos-imported.txt", context)
message = Message(user=user, subject=subject, content=body)
message.save()
template_name = "messages/email/videos-imported.html"
Meter('templated-emails-sent-by-type.videos-imported').inc()
send_templated_email(user, subject, template_name, context)
def _make_update_request(self, uri, entry):
Meter('youtube.api_request').inc()
headers = {
'Content-Type': 'application/atom+xml',
'Authorization': 'Bearer %s' % self.access_token,
'GData-Version': '2',
'X-GData-Key': 'key=%s' % YOUTUBE_API_SECRET
}
status_code = 0
retry_count = 0
while True:
r = self._do_update_request(uri, data=entry, headers=headers)
# if not 400 or 403, assume success (i.e. 200, 201, etc.)
if r.status_code != 400 and r.status_code != 403:
break
if r.status_code == 403 and 'too_many_recent_calls' in r.content:
#raise TooManyRecentCallsException(r.headers, r.raw)
extra = r.headers
extra['raw'] = r.raw
logger.error('Youtube too many recent calls', extra=extra)
if r.status_code == 400:
extra = {'raw': r.raw, 'content': r.content}
logger.error('Youtube API request failed', extra=extra)
retry_count += 1
if retry_count > 60: # retry for a max of ~ 10 min
logger.error('Retries exhausted for Youtube API request',
extra={
'content': r.content,
'status': r.status_code,
'headers': r.headers,
'uri': uri
})
break
time.sleep(10)
status_code = r.status_code
return status_code
