def youtube_sync(video, language): """ Used on debug page for video. Simplified version of what's found in ``ThirdPartyAccount.mirror_on_third_party``. It doesn't bother checking if we should be syncing this or not. Only does the new Youtube/Amara integration syncing. """ version = language.get_tip() always_push_account = ThirdPartyAccount.objects.always_push_account() for vurl in video.videourl_set.all(): vt = video_type_registrar.video_type_for_url(vurl.url) try: vt.update_subtitles(version, always_push_account) Meter('youtube.push.success').inc() except: Meter('youtube.push.fail').inc() logger.error('Always pushing to youtoube has failed.', extra={ 'video': video.video_id, 'vurl': vurl.pk }) finally: Meter('youtube.push.request').inc()
def team_member_new(member_pk): if getattr(settings, "MESSAGES_DISABLED", False): return from messages.models import Message from teams.models import TeamMember member = TeamMember.objects.get(pk=member_pk) if not _team_sends_notification(member.team, 'block_team_member_new_message'): return False from videos.models import Action from teams.models import TeamMember # the feed item should appear on the timeline for all team members # as a team might have thousands of members, this one item has # to show up on all of them Action.create_new_member_handler(member) # notify admins and owners through messages notifiable = TeamMember.objects.filter( team=member.team, role__in=[TeamMember.ROLE_ADMIN, TeamMember.ROLE_OWNER]).exclude(pk=member.pk) for m in notifiable: context = { "new_member": member.user, "team": member.team, "user": m.user, "role": member.role, "url_base": get_url_base(), } body = render_to_string("messages/team-new-member.txt", context) subject = ugettext("%s team has a new member" % (member.team)) if m.user.notify_by_message: msg = Message() msg.subject = subject msg.content = body msg.user = m.user msg.object = m.team msg.save() template_name = "messages/email/team-new-member.html" Meter('templated-emails-sent-by-type.teams.new-member').inc() send_templated_email(m.user, subject, template_name, context) # now send welcome mail to the new member template_name = "messages/team-welcome.txt" context = { "team": member.team, "url_base": get_url_base(), "role": member.role, "user": member.user, } body = render_to_string(template_name, context) msg = Message() msg.subject = ugettext("You've joined the %s team!" % (member.team)) msg.content = body msg.user = member.user msg.object = member.team msg.save() template_name = "messages/email/team-welcome.html" Meter('templated-emails-sent-by-type.teams.welcome').inc() send_templated_email(msg.user, msg.subject, template_name, context)
def _sync_versions(language_pk=None): """Sync a single language worth of SubtitleVersions.""" from utils.metrics import Meter meter = Meter('data-model-refactor.version-syncs') sl = get_unsynced_subtitle_version_language() if not sl: return False if sl.can_writelock(TERN_REQUEST): sl.writelock(TERN_REQUEST) else: # If we picked a writelocked language, bail for now, but come back to it # later. log('SubtitleLanguage', 'ERROR_WRITELOCKED', sl.pk, None) Meter('data-model-refactor.version-errors.writelocked').inc() return True try: # First update any versions that have been synced but have changed since. versions = sl.subtitleversion_set.filter( needs_sync=True, new_subtitle_version__isnull=False) for version in versions.order_by('version_no'): _update_subtitle_version(version) if not dry: meter.inc() # Then sync any new versions. versions = sl.subtitleversion_set.filter(needs_sync=True, new_subtitle_version=None) # This is ugly, but we (may) need to do something special on the last # version we sync. new_versions = list(versions.order_by('version_no')) for version in new_versions[:-1]: _create_subtitle_version(version, False) if not dry: meter.inc() for version in new_versions[-1:]: _create_subtitle_version(version, True) if not dry: meter.inc() except: Meter('data-model-refactor.version-errors.other').inc() raise finally: sl.release_writelock() if not dry: if random.random() < 0.01: report_metrics() return True
def team_member_leave(team_pk, user_pk): if getattr(settings, "MESSAGES_DISABLED", False): return from messages.models import Message from teams.models import TeamMember, Team user = User.objects.get(pk=user_pk) team = Team.objects.get(pk=team_pk) if not _team_sends_notification(team,'block_team_member_leave_message'): return False from videos.models import Action # the feed item should appear on the timeline for all team members # as a team might have thousands of members, this one item has # to show up on all of them Action.create_member_left_handler(team, user) # notify admins and owners through messages notifiable = TeamMember.objects.filter( team=team, role__in=[TeamMember.ROLE_ADMIN, TeamMember.ROLE_OWNER]) subject = fmt( ugettext(u"%(user)s has left the %(team)s team"), user=user, team=team) for m in notifiable: context = { "parting_member": user, "team":team, "user":m.user, "url_base":get_url_base(), } body = render_to_string("messages/team-member-left.txt",context) if m.user.notify_by_message: msg = Message() msg.subject = subject msg.content = body msg.user = m.user msg.object = team msg.save() Meter('templated-emails-sent-by-type.teams.someone-left').inc() send_templated_email(m.user, subject, "messages/email/team-member-left.html", context) context = { "team":team, "user":user, "url_base":get_url_base(), } subject = fmt(ugettext("You've left the %(team)s team!"), team=team) if user.notify_by_message: template_name = "messages/team-member-you-have-left.txt" msg = Message() msg.subject = subject msg.content = render_to_string(template_name,context) msg.user = user msg.object = team msg.save() template_name = "messages/email/team-member-you-have-left.html" Meter('templated-emails-sent-by-type.teams.you-left').inc() send_templated_email(user, subject, template_name, context)
def _sync_language(language_pk=None): """Try to sync one SubtitleLanguage. Returns True if a language was synced (or skipped, but we should try it again later), False if there were no more left. """ from utils.metrics import Meter sl = (get_specific_language(language_pk) if language_pk else get_unsynced_subtitle_language()) if not sl: return False if sl.can_writelock(TERN_REQUEST): sl.writelock(TERN_REQUEST) else: # If we picked a writelocked language, bail for now, but come back to it # later. log('SubtitleLanguage', 'ERROR_WRITELOCKED', sl.pk, None) Meter('data-model-refactor.language-errors.writelocked').inc() return True try: if sl.language == '': fix_blank_original(sl.video) # For now, we'll actually bail on this language and come back to it # later. Hopefully it will have been fixed by the above call, but # there's a chance that it's is_original=False and so is still borked. log('SubtitleLanguage', 'ERROR_EMPTY_LANGUAGE', sl.pk, None) Meter('data-model-refactor.language-errors.empty-language').inc() return True if sl.new_subtitle_language: _update_subtitle_language(sl) else: _create_subtitle_language(sl) except: Meter('data-model-refactor.language-errors.other').inc() raise finally: sl.release_writelock() if not dry: Meter('data-model-refactor.language-syncs').inc() if random.random() < 0.01: report_metrics() return True
def test(self, model: nn.Module) -> Dict[str, Union[int, Meter]]: model.eval() loss_meter = Meter() acc_meter = Meter() num_all_samples = 0 with torch.no_grad(): for batch_idx, (X, y) in enumerate(self.dataset_loader): X, y = X.to(self.device), y.to(self.device) pred = model(X) loss = self.criterion(pred, y) correct = self.count_correct(pred, y) # num_samples = y.size(0) loss_meter.update(loss.item(), n=num_samples) acc_meter.update(correct.item() / num_samples, n=num_samples) num_all_samples += num_samples return { 'loss_meter': loss_meter, 'acc_meter': acc_meter, 'num_samples': num_all_samples }
def send_http_request(self, url, basic_auth_username, basic_auth_password): h = Http(disable_ssl_certificate_validation=True) if basic_auth_username and basic_auth_password: h.add_credentials(basic_auth_username, basic_auth_password) project = self.video.get_team_video().project.slug if self.video else None data = { 'event': self.event_name, 'api_url': self.api_url, } if self.team: data['team'] = self.team.slug if self.partner: data['partner'] = self.partner.slug if project: data['project'] = project if self.video: data['video_id'] = self.video_id if self.application_pk: data['application_id'] = self.application_pk if self.language: data.update({ "language_code": self.language_code, "language_id": self.language.pk, }) data_sent = data data = urlencode(data) url = "%s?%s" % (url , data) try: resp, content = h.request(url, method="POST", body=data, headers={ 'referer': '%s://%s' % (DEFAULT_PROTOCOL, Site.objects.get_current().domain) }) success = 200 <= resp.status < 400 if success is False: logger.error("Failed to notify team %s " % (self.team), extra={ 'team': self.team or self.partner, 'url': url, 'response': resp, 'content': content, 'data_sent':data_sent, }) Meter('http-callback-notification-error').inc() else: Meter('http-callback-notification-success').inc() return success, content except: logger.exception("Failed to send http notification ") return None, None
def _get_entry(self, video_id): Meter('youtube.api_request').inc() try: return yt_service.GetYouTubeVideoEntry(video_id=str(video_id)) except RequestError, e: err = e[0].get('body', 'Undefined error') raise VideoTypeError('Youtube error: %s' % err)
def video_url_create(request): output = {} form = CreateVideoUrlForm(request.user, request.POST) if form.is_valid(): obj = form.save() video = form.cleaned_data['video'] users = video.notification_list(request.user) for user in users: subject = u'New video URL added by %(username)s to "%(video_title)s" on universalsubtitles.org' subject = subject % {'url': obj.url, 'username': obj.added_by, 'video_title': video} context = { 'video': video, 'video_url': obj, 'user': user, 'domain': Site.objects.get_current().domain, 'hash': user.hash_for_video(video.video_id) } Meter('templated-emails-sent-by-type.videos.video-url-added').inc() send_templated_email(user, subject, 'videos/email_video_url_add.html', context, fail_silently=not settings.DEBUG) else: output['errors'] = form.get_errors() return HttpResponse(json.dumps(output))
def application_sent(application_pk): if getattr(settings, "MESSAGES_DISABLED", False): return from messages.models import Message from teams.models import Application, TeamMember application = Application.objects.get(pk=application_pk) notifiable = TeamMember.objects.filter( team=application.team, role__in=[TeamMember.ROLE_ADMIN, TeamMember.ROLE_OWNER]) for m in notifiable: template_name = "messages/application-sent.txt" context = { "applicant": application.user, "url_base": get_url_base(), "team": application.team, "note": application.note, "user": m.user, } body = render_to_string(template_name, context) subject = ugettext(u'%(user)s is applying for team %(team)s') % dict( user=application.user, team=application.team.name) if m.user.notify_by_message: msg = Message() msg.subject = subject msg.content = body msg.user = m.user msg.object = application.team msg.author = application.user msg.save() Meter('templated-emails-sent-by-type.teams.application-sent').inc() send_templated_email(m.user, subject, "messages/email/application-sent-email.html", context) return True
def team_application_denied(application_pk): if getattr(settings, "MESSAGES_DISABLED", False): return from messages.models import Message from teams.models import Application application = Application.objects.get(pk=application_pk) if not _team_sends_notification(application.team,'block_application_denided_message'): return False template_name = "messages/email/team-application-denied.html" context = { "team": application.team, "user": application.user, "url_base": get_url_base(), "note": application.note, } subject = fmt( ugettext(u'Your application to join the %(team)s ' u'team has been declined'), team=application.team.name) if application.user.notify_by_message: msg = Message() msg.subject = subject msg.content = render_to_string("messages/team-application-denied.txt", context) msg.user = application.user msg.object = application.team msg.save() Meter('templated-emails-sent-by-type.teams.application-declined').inc() send_templated_email(application.user, subject, template_name, context)
def send_new_translation_notification(translation_version): domain = Site.objects.get_current().domain video = translation_version.language.video language = translation_version.language for user in video.notification_list(translation_version.user): context = { 'version': translation_version, 'domain': domain, 'video_url': '%s://%s%s' % (DEFAULT_PROTOCOL, domain, video.get_absolute_url()), 'user': user, 'language': language, 'video': video, 'hash': user.hash_for_video(video.video_id), "STATIC_URL": settings.STATIC_URL, } subject = 'New %s translation by %s of "%s"' % \ (language.language_display(), translation_version.user.__unicode__(), video.__unicode__()) Meter('templated-emails-sent-by-type.videos.new-translation-started' ).inc() send_templated_email(user, subject, 'videos/email_start_notification.html', context, fail_silently=not settings.DEBUG) return True
def send_new_message_notification(message_id): from messages.models import Message try: message = Message.objects.get(pk=message_id) except Message.DoesNotExist: msg = '**send_new_message_notification**. Message does not exist. ID: %s' % message_id client.create_from_text(msg, logger='celery') return user = message.user if not user.email or not user.is_active or not user.notify_by_email: return if message.author: subject = _(u"New message from %(author)s on Amara: %(subject)s") else: subject = _("New message on Amara: %(subject)s") subject = fmt(subject, author=message.author, subject=message.subject) context = { "message": message, "domain": Site.objects.get_current().domain, "STATIC_URL": settings.STATIC_URL, } Meter('templated-emails-sent-by-type.message-received').inc() send_templated_email(user, subject, "messages/email/message_received.html", context)
def send_change_title_email(video_id, user_id, old_title, new_title): from videos.models import Video domain = Site.objects.get_current().domain try: video = Video.objects.get(id=video_id) user = user_id and User.objects.get(id=user_id) except ObjectDoesNotExist: return users = video.notification_list(user) for obj in users: subject = u'Video\'s title changed on Amara' context = { 'user': obj, 'domain': domain, 'video': video, 'editor': user, 'old_title': old_title, 'hash': obj.hash_for_video(video.video_id), 'new_title': new_title, "STATIC_URL": settings.STATIC_URL, } Meter('templated-emails-sent-by-type.videos.title-changed').inc() send_templated_email(obj, subject, 'videos/email_title_changed.html', context, fail_silently=not settings.DEBUG)
def _notify_teams_of_new_videos(team_qs): from messages.tasks import _team_sends_notification from teams.models import TeamVideo domain = Site.objects.get_current().domain for team in team_qs: if not _team_sends_notification(team, 'block_new_video_message'): continue team_videos = TeamVideo.objects.filter(team=team, created__gt=team.last_notification_time) team.last_notification_time = datetime.now() team.save() members = team.users.filter( notify_by_email=True, is_active=True) \ .distinct() subject = fmt(_(u'New %(team)s videos ready for subtitling!'), team=team) for user in members: if not user.email: continue context = { 'domain': domain, 'user': user, 'team': team, 'team_videos': team_videos, "STATIC_URL": settings.STATIC_URL, } Meter('templated-emails-sent-by-type.team.new-videos-ready').inc() send_templated_email(user, subject, 'teams/email_new_videos.html', context, fail_silently=not settings.DEBUG)
def send_confirmation(self, user): assert user.email from messages.models import Message self.filter(user=user).delete() salt = sha_constructor(str(random()) + settings.SECRET_KEY).hexdigest()[:5] confirmation_key = sha_constructor( salt + user.email.encode('utf-8')).hexdigest() try: current_site = Site.objects.get_current() except Site.DoesNotExist: return path = reverse("auth:confirm_email", args=[confirmation_key]) activate_url = u"http://%s%s" % (unicode(current_site.domain), path) context = { "user": user, "activate_url": activate_url, "current_site": current_site, "confirmation_key": confirmation_key, } subject = u'Please confirm your email address for %s' % current_site.name Meter('templated-emails-sent-by-type.email-address-confirmation').inc() send_templated_email_async(user, subject, "messages/email/email-confirmation.html", context) return self.create(user=user, sent=datetime.now(), confirmation_key=confirmation_key)
def _create_subtitle_language(sl): """Sync the given subtitle language, creating a new one.""" from apps.subtitles.models import VALID_LANGUAGE_CODES from apps.videos.models import SubtitleLanguage, Video from utils.metrics import Meter try: duplicates = (SubtitleLanguage.objects.filter( video=sl.video, language=sl.language).exclude(pk=sl.pk).exists()) except Video.DoesNotExist: log('SubtitleLanguage', 'ERROR_MISSING_VIDEO', sl.pk, None) return if duplicates: log('SubtitleLanguage', 'ERROR_DUPLICATE_LANGUAGE', sl.pk, None) log('SubtitleLanguage', 'duplicate_version_counts', sl.pk, None, [ l.subtitleversion_set.count() for l in sl.video.subtitlelanguage_set.filter(language=sl.language) ]) log('SubtitleLanguage', 'duplicate_subtitle_counts', sl.pk, None, [[ len(v.subtitles()) for v in l.subtitleversion_set.order_by('version_no') ] for l in sl.video.subtitlelanguage_set.filter(language=sl.language)]) Meter('data-model-refactor.language-errors.duplicate-language').inc() _handle_duplicate_languages(sl) return if sl.language not in VALID_LANGUAGE_CODES: if sl.language == 'no': log('SubtitleLanguage', 'FIXED_LANGUAGE_CODE', sl.pk, None, sl.language) sl.language = 'nb' sl.save() elif sl.language == 'iw': log('SubtitleLanguage', 'FIXED_LANGUAGE_CODE', sl.pk, None, sl.language) sl.language = 'he' sl.save() else: log('SubtitleLanguage', 'ERROR_INVALID_LANGUAGE_CODE', sl.pk, None, sl.language) Meter('data-model-refactor.language-errors.invalid-language-code' ).inc() return nsl = _add_sl(sl) log('SubtitleLanguage', 'create', sl.pk, nsl.pk)
def send_reject_notification(task_pk, sent_back): raise NotImplementedError() from teams.models import Task from videos.models import Action from messages.models import Message try: task = Task.objects.select_related("team_video__video", "team_video", "assignee", "subtitle_version").get(pk=task_pk) except Task.DoesNotExist: return False version = task.get_subtitle_version() subject = ugettext(u"Your subtitles were not accepted") if task.new_review_base_version: user = task.new_review_base_version.author else: user = version.author task_language = get_language_label(task.language) reviewer = task.assignee video = task.team_video.video subs_url = "%s%s" % (get_url_base(), reverse("videos:translation_history", kwargs={ 'video_id': video.video_id, 'lang': task.language, 'lang_id': version.subtitle_language.pk, })) reviewer_message_url = "%s%s?user=%s" % ( get_url_base(), reverse("messages:new"), reviewer.username) context = { "team": task.team, "title": version.subtitle_language.get_title(), "user": user, "task_language": task_language, "url_base": get_url_base(), "task": task, "reviewer": reviewer, "note": task.body, "sent_back": sent_back, "subs_url": subs_url, "reviewer_message_url": reviewer_message_url, } msg = None if user.notify_by_message: template_name = "messages/team-task-rejected.txt" msg = Message() msg.subject = subject msg.content = render_to_string(template_name, context) msg.user = user msg.object = task.team msg.save() template_name = "messages/email/team-task-rejected.html" Meter('templated-emails-sent-by-type.teams.task-rejected').inc() email_res = send_templated_email(user, subject, template_name, context) Action.create_rejected_video_handler(version, reviewer) return msg, email_res
def solve_epochs_delta( self, round_i, model: Module, global_state: STATE_TYPE, num_epochs, hide_output: bool = False ) -> Tuple[Dict[str, Union[int, Meter]], Dict[str, torch.Tensor]]: loss_meter = Meter() dice_meter = Meter('dice_coeff') num_all_samples = 0 optimizer = self.create_optimizer(model) optimizer.step_lr_scheduler(round_i=round_i) model.train() with tqdm.trange(num_epochs, disable=hide_output) as t: for epoch in t: t.set_description( f'Client: {self.id}, Round: {round_i}, Epoch :{epoch}') for batch_idx, (X, y) in enumerate(self.dataset_loader): # from IPython import embed X, y = X.to(self.device), y.to(self.device) optimizer.zero_grad() pred = model(X) loss = self.criterion(pred, y) # activated = torch.sigmoid(pred) dice_coeff = self.compute_dice_coefficient(activated, y) # loss.backward() # torch.nn.utils.clip_grad_norm(self.model.parameters(), 60) optimizer.step() num_samples = y.size(0) num_all_samples += num_samples loss_meter.update(loss.item(), n=num_samples) dice_meter.update(dice_coeff.item(), n=num_samples) if (batch_idx % 10 == 0): # 纯数值, 这里使用平均的损失 t.set_postfix(mean_loss=loss.item()) # 返回参数 state_dict = model.state_dict() result = { 'loss_meter': loss_meter, 'dice_coeff_meter': dice_meter, 'num_samples': num_all_samples, 'lr': optimizer.get_current_lr() } # 计算差值 latest - init for k, v in state_dict.items(): v.sub_(global_state[k]) # 输出相关的参数 return result, state_dict
def log_session(self, request, log): dialog_log = WidgetDialogLog(browser_id=request.browser_id, log=log) dialog_log.save() Meter('templated-emails-sent-by-type.subtitle-save-failure').inc() send_templated_email(settings.WIDGET_LOG_EMAIL, 'Subtitle save failure', 'widget/session_log_email.txt', {'log_pk': dialog_log.pk}, fail_silently=False) return {'response': 'ok'}
def send_email(self, email_to): Meter('templated-emails-sent-by-type.teams.team-video-activity').inc() send_templated_email(email_to, _("New activity on your team video"), "teams/emails/new-activity.html", { "video": self.video, "event-name": self.event_name, "team": self.team, "laguage":self.language, } )
def jsonp(request, method_name, null=False): Meter('widget-jsonp-calls').inc() callback = request.GET.get('callback', 'callback') args = {'request': request} for k, v in request.GET.items(): if k != 'callback': args[k.encode('ascii')] = json.loads(v) rpc_module = null_rpc_views if null else rpc_views func = getattr(rpc_module, method_name) result = func(**args) return HttpResponse("{0}({1});".format(callback, json.dumps(result)), "text/javascript")
def process_request(self, request): request.init_time = time.time() Meter('requests.started').inc() # Tracking the section of the site isn't trivial, because sometimes we # have the language prefix, like "/en/foo". paths = request.path.lstrip('/').split('/')[:2] p1 = paths[0] if len(paths) >= 1 else None p2 = paths[1] if len(paths) >= 2 else None if p1 in SECTIONS: request._metrics_section = SECTIONS[paths[0]] elif p2 in SECTIONS: request._metrics_section = SECTIONS[paths[1]] else: request._metrics_section = None if request._metrics_section: label = 'site-sections.%s-response-time' % request._metrics_section Meter(label).inc() return None
def email_confirmed(user_pk): from messages.models import Message user = User.objects.get(pk=user_pk) subject = "Welcome aboard!" context = {"user": user} if user.notify_by_message: body = render_to_string("messages/email-confirmed.txt", context) message = Message(user=user, subject=subject, content=body) message.save() template_name = "messages/email/email-confirmed.html" Meter('templated-emails-sent-by-type.email-confirmed').inc() send_templated_email(user, subject, template_name, context) return True
def process_response(self, request, response): if hasattr(request, "init_time"): delta = time.time() - request.init_time ms = delta * 1000 Meter('requests.completed').inc() try: response_type = response.status_code / 100 * 100 except: response_type = 'unknown' Meter('requests.response-types.%s' % response_type).inc() response.set_cookie('response_time', str(ms)) ManualTimer('response-time').record(ms) if request._metrics_section: label = 'site-sections.%s-response-time' % request._metrics_section ManualTimer(label).record(ms) return response
def request(self, *args, **kwargs): """ Override the very low-level request method to catch possible too_many_recent_calls errors. """ Meter('youtube.api_request').inc() try: return super(YouTubeApiBridge, self).request(*args, **kwargs) except gdata.client.RequestError, e: if 'too_many_recent_calls' in str(e): raise TooManyRecentCallsException(e.headers, e.reason, e.status, e.body) else: raise e
def team_invitation_sent(invite_pk): from messages.models import Message from teams.models import Invite, Setting, TeamMember invite = Invite.objects.get(pk=invite_pk) if not _team_sends_notification(invite.team, 'block_invitation_sent_message'): return False # does this team have a custom message for this? team_default_message = None messages = Setting.objects.messages().filter(team=invite.team) if messages.exists(): data = {} for m in messages: data[m.get_key_display()] = m.data mapping = { TeamMember.ROLE_ADMIN: data['messages_admin'], TeamMember.ROLE_MANAGER: data['messages_manager'], TeamMember.ROLE_CONTRIBUTOR: data['messages_invite'], } team_default_message = mapping.get(invite.role, None) context = { 'invite': invite, 'role': invite.role, "user": invite.user, "inviter": invite.author, "team": invite.team, "invite_pk": invite_pk, 'note': invite.note, 'custom_message': team_default_message, 'url_base': get_url_base(), } title = ugettext(u"You've been invited to team %s on Amara" % invite.team.name) if invite.user.notify_by_message: body = render_to_string("messages/team-you-have-been-invited.txt", context) msg = Message() msg.subject = title msg.user = invite.user msg.object = invite msg.author = invite.author msg.content = body msg.save() template_name = 'messages/email/team-you-have-been-invited.html' Meter('templated-emails-sent-by-type.teams.invitation').inc() return send_templated_email(invite.user, title, template_name, context)
def solve_epochs_with_global(self, round_i, model: Module, global_model: Module, num_epochs, hide_output: bool = False) -> Tuple[Dict[str, Union[int, Meter]], Dict[str, torch.Tensor]]: loss_meter = Meter() acc_meter = Meter() num_all_samples = 0 optimizer = self.create_optimizer(model) # TODO 直接引用上一次的 global 模型, 避免复制, optimizer 的 step 中也是不记录梯度的 optimizer.set_old_weights(old_weights=[p for p in global_model.parameters()]) model.train() with tqdm.trange(num_epochs, disable=hide_output) as t: for epoch in t: t.set_description(f'Client: {self.id}, Round: {round_i}, Epoch :{epoch}') for batch_idx, (X, y) in enumerate(self.dataset_loader): # from IPython import embed X, y = X.to(self.device), y.to(self.device) optimizer.zero_grad() pred = model(X) loss = self.criterion(pred, y) loss.backward() # torch.nn.utils.clip_grad_norm(self.model.parameters(), 60) optimizer.step() correct_sum = self.count_correct(pred, y) num_samples = y.size(0) num_all_samples += num_samples loss_meter.update(loss.item(), n=num_samples) acc_meter.update(correct_sum.item() / num_samples, n=num_samples) if (batch_idx % 10 == 0): # 纯数值, 这里使用平均的损失 t.set_postfix(mean_loss=loss.item()) # 返回参数 result = { 'loss_meter': loss_meter, 'acc_meter': acc_meter, 'num_samples': num_all_samples } state_dict = model.state_dict() # 输出相关的参数 return result, state_dict
def videos_imported_message(user_pk, imported_videos): from messages.models import Message user = User.objects.get(pk=user_pk) subject = _(u"Your videos were imported!") url = "%s%s" % (get_url_base(), reverse("profiles:my_videos")) context = { "user": user, "imported_videos": imported_videos, "my_videos_url": url } if user.notify_by_message: body = render_to_string("messages/videos-imported.txt", context) message = Message(user=user, subject=subject, content=body) message.save() template_name = "messages/email/videos-imported.html" Meter('templated-emails-sent-by-type.videos-imported').inc() send_templated_email(user, subject, template_name, context)
def _make_update_request(self, uri, entry): Meter('youtube.api_request').inc() headers = { 'Content-Type': 'application/atom+xml', 'Authorization': 'Bearer %s' % self.access_token, 'GData-Version': '2', 'X-GData-Key': 'key=%s' % YOUTUBE_API_SECRET } status_code = 0 retry_count = 0 while True: r = self._do_update_request(uri, data=entry, headers=headers) # if not 400 or 403, assume success (i.e. 200, 201, etc.) if r.status_code != 400 and r.status_code != 403: break if r.status_code == 403 and 'too_many_recent_calls' in r.content: #raise TooManyRecentCallsException(r.headers, r.raw) extra = r.headers extra['raw'] = r.raw logger.error('Youtube too many recent calls', extra=extra) if r.status_code == 400: extra = {'raw': r.raw, 'content': r.content} logger.error('Youtube API request failed', extra=extra) retry_count += 1 if retry_count > 60: # retry for a max of ~ 10 min logger.error('Retries exhausted for Youtube API request', extra={ 'content': r.content, 'status': r.status_code, 'headers': r.headers, 'uri': uri }) break time.sleep(10) status_code = r.status_code return status_code