def handle(self, **options): database = options.get('database') cursor = connections[database].cursor() if options.get('force'): cohorts = list(Cohort.objects.all()) else: cohorts = list( Cohort.objects.filter( Q(allele_freq_modified=None) | Q(modified__gt=F('allele_freq_modified')))) log.debug('Computing for {0} cohorts'.format(len(cohorts))) with transaction.commit_manually(database): try: cursor.execute('TRUNCATE "cohort_variant" RESTART IDENTITY') transaction.commit() except DatabaseError: transaction.rollback() log.exception('Could not truncate CohortVariant table') for cohort in cohorts: log.debug('"{0}" ({1} samples)...'.format(cohort, cohort.count)) t0 = time.time() with transaction.commit_manually(database): try: cohort.compute_allele_frequencies(database) log.debug('done in {0}s\n'.format(int(time.time() - t0))) transaction.commit() except DatabaseError, e: transaction.rollback() log.exception(e)
def handle(self, **options): database = options.get('database') cursor = connections[database].cursor() if options.get('force'): cohorts = list(Cohort.objects.all()) else: cohorts = list(Cohort.objects.filter( Q(allele_freq_modified=None) | Q(modified__gt=F('allele_freq_modified')))) log.debug('Computing for {0} cohorts'.format(len(cohorts))) with transaction.commit_manually(database): try: cursor.execute('TRUNCATE "cohort_variant" RESTART IDENTITY') transaction.commit() except DatabaseError: transaction.rollback() log.exception('Could not truncate CohortVariant table') for cohort in cohorts: log.debug('"{0}" ({1} samples)...'.format(cohort, cohort.count)) t0 = time.time() with transaction.commit_manually(database): try: cohort.compute_allele_frequencies(database) log.debug('done in {0}s\n'.format(int(time.time() - t0))) transaction.commit() except DatabaseError, e: transaction.rollback() log.exception(e)
def commit_all_or_rollback(): transaction.commit_manually() try: yield transaction.commit() except: transaction.rollback() raise
def get_object(self): '''Returns a jsPhyloSVG PhyloXML string.''' # open a transaction so that taxa may be pruned during export # and then rolled back with transaction.commit_manually(): # if there is a pruning filter, delete all children of matching taxa pruning_filter = self.pruning_filter if pruning_filter: for taxon in Taxon.objects.filter(**pruning_filter): for child in taxon.get_children(): child.delete() # get template, context, and render the template template_path = 'phylogeny/exporters/%s/%s.%s' template = get_template(template_path % (self.format_name, 'phylogeny', self.extension,)) context = Context({ 'taxa_categories': TaxaCategory.objects.all, 'colors_app_installed': ('colors' in settings.INSTALLED_APPS), 'object': self.taxon, 'clade_template_path': template_path % (self.format_name, 'clade', self.extension,) }) rendered_template = template.render(context) # rollback the transaction # (don't actually allow any taxon deletions to stand) transaction.rollback() return rendered_template
def post(self, request, *args, **kwargs): cart_data = self.parse_request() session.cart.set_data(self.request, cart_data) customer = session.auth.get_data(self.request) if customer is None: password = cart_data.get('password', None) if password is not None: # if not logged in and password is provided - create a customer # if no password provided - create customerless order with transaction.commit_manually(): try: customer = Customer.objects.create(password=password, **cart_data['order']) transaction.commit() except IntegrityError: transaction.rollback() return http.HttpResponseForbidden(_("Cannot create customer: duplicate phone")) session.auth.set_data(self.request, customer) order = Order.objects.create(customer, cart_data) if order is None: return http.HttpResponseForbidden(_("Cannot make empty order")) session.cart.clear_cart_data(self.request) self.send_order_notification(order) return http.HttpResponse(order.get_absolute_url())
def load_1000g(database, **kwargs): if not db.utils.table_exists('1000g', schema='raw'): return cursor = connections[database].cursor() cursor.execute(db.utils.sequence_reset_sql(ThousandG, database)) with transaction.commit_manually(database): try: # Get all missing records.. cursor.execute(''' INSERT INTO "1000g" ( "variant_id", "an", "ac", "af", "aa", "amr_af", "asn_af", "afr_af", "eur_af" ) ( SELECT variant.id, r.an, r.ac, r.af, r.aa, r.amr_af, r.asn_af, r.afr_af, r.eur_af FROM "variant" INNER JOIN "raw"."1000g" r ON ("variant".md5 = r."md5") LEFT OUTER JOIN "1000g" ON ("1000g"."variant_id" = "variant"."id") WHERE "1000g"."id" IS NULL ) ''') transaction.commit() except Exception as e: transaction.rollback() log.exception(e)
def test_manually_managed_mistake(self): """ If you forget, you'll get bad errors. """ with self.assertRaises(transaction.TransactionManagementError): with transaction.commit_manually(): Reporter.objects.create(first_name="Scott", last_name="Browning")
def post(self, request, *args, **kwargs): if not request.user.is_staff: return HttpResponseForbidden('Forbidden') context = self._get_initial_data(**kwargs) with transaction.commit_manually(): try: # TODO check if the user is authorized to delete the lessons # (e.g. if they have course permissions) cells = request.POST.getlist('cells[]') for cell in cells: l = get_object_or_404( Lesson, pk=cell, ) logger.info(u'%s deleted %s (lesson id=%d, course id=%d)' % (request.user, l, l.pk, l.group.course.pk)) l.delete() transaction.commit() return HttpResponse('ok') except KeyError: transaction.rollback() return HttpResponseBadRequest('key error') except IndexError: transaction.rollback() return HttpResponseBadRequest('index error') except ValueError: transaction.rollback() return HttpResponseBadRequest('value error') except IntegrityError: transaction.rollback() return HttpResponseBadRequest('db integrity error') except: transaction.rollback() raise
def deserialize(cls, fixtures, initial_obj=None, walking_classes=None, using='default', natural_keys=True, exclude_contents=None, deserialize_options=None, request=None, pretreatment_fixtures=False, pretreatment_fixtures_sorted_function=None): with transaction.commit_manually(): try: if pretreatment_fixtures: fixtures = cls.pretreatment_fixtures(initial_obj, fixtures, walking_classes, deserialize_options, pretreatment_fixtures_sorted_function) contents = cls._deserialize(fixtures, initial_obj=initial_obj, walking_classes=walking_classes, using=using, natural_keys=natural_keys, exclude_contents=exclude_contents, deserialize_options=deserialize_options, request=request) transaction.commit() return contents except Exception as e: if settings.DEBUG: import traceback logger.error(traceback.format_exc()) transaction.rollback() raise e
def serialize(cls, initial_obj, walking_classes=None, walking_always=False, natural_keys=True, indent=None, serialize_options=None, can_get_objs_from_several_path=False, request=None): serialize_options = serialize_options or {} walking_classes = walking_classes or [] object_list = [] if natural_keys: serialize_options['use_natural_primary_keys'] = True serialize_options['use_natural_foreign_keys'] = True contents_to_serialize = [] with transaction.commit_manually(): try: cls.objects_to_serialize(initial_obj, initial_obj, object_list, walking_classes=walking_classes, walking_always=walking_always, natural_keys=natural_keys, several_path=can_get_objs_from_several_path, request=request) for content in object_list: meta_walking_class = cls.get_meta_walking_class(content, walking_classes) content_to_serialize = meta_walking_class.pre_serialize(initial_obj, content, request, serialize_options) if content_to_serialize and not content_to_serialize in contents_to_serialize: contents_to_serialize.append(content_to_serialize) fixtures = serializers.serialize(cls.format, contents_to_serialize, indent=indent, **serialize_options) finally: transaction.rollback() return fixtures
def handle(self, path, **options): database = options.get('database') transripts = options.get('transripts') stdout = options.get('stdout') with open(path) as fin: stream = EffectStream(fin, skip_existing=False) if stdout: while True: line = stream.readline() if line == '': break log.debug(line) else: cursor = connections[database].cursor() with transaction.commit_manually(database): try: cursor.execute('TRUNCATE {0}'.format( VariantEffect._meta.db_table)) if transripts: cursor.execute('TRUNCATE {0} CASCADE'.format( Transcript._meta.db_table)) columns = stream.output_columns db_table = VariantEffect._meta.db_table pgcopy_batch(stream, db_table, columns, cursor, database) transaction.commit(database) except Exception as e: transaction.rollback(database) log.exception(e) raise
def load_answers(filepath): questions = list(Question.objects.all()) with transaction.commit_manually(): with open(filepath) as fh: reader = csv.reader(fh) itr = iter(reader) top = itr.next() data_rows = list(itr) print "Respondents..." respondents = [Respondent.objects.create(row_num=i+1) for i in range(len(data_rows))] print "Answers..." for question in questions: q_cols = [c for c in top if c.startswith("q%04i" % question.index)] for c, q_col in enumerate(q_cols): if q_col in COLUMN_BLACKLIST: continue for respondent, row in zip(respondents, data_rows): val = dict(zip(top, row))[q_col] val = clean_answer(q_col, val) if val: Answer.objects.create( respondent=respondent, question=question, answer=val, subquestion=c, ) transaction.commit()
def persist(request): global mc v_str = mc.get('djangotracer') if v_str == None: return HttpResponse('empty') v = v_str.split('||') with transaction.commit_manually(): try: for zv in v: if zv == '': continue zvv = json.loads(zv) timestamp_format = "%Y-%m-%d %H:%M:%S.%f" reqts = dt.strptime(zvv['reqts'],timestamp_format) rests = dt.strptime(zvv['rests'],timestamp_format) elapsed = (rests - reqts).microseconds td = models.TraceData( path=zvv['path'], reqts=reqts, rests=rests, elapsed=elapsed, ) td.save() except Exception, e: transaction.rollback() return HttpResponse('error: "%s" on zv=%s'%(e,zv)) else:
def log_ore_discovery(request): username = request.POST.get('username') type = request.POST.get('type') x = int(request.POST.get('x')) y = int(request.POST.get('y')) z = int(request.POST.get('z')) with transaction.commit_manually(): try: player = MinecraftPlayer.factory(username=username) material_type = MaterialType.factory(type=type) ore_event = OreDiscoveryEvent(server=request.server, material_type=material_type, player=player, x=x, y=y, z=z) OreDiscoveryCount.increment(request.server, material_type, player) ore_event.save() except: transaction.rollback() rollbar.report_exc_info() else: transaction.commit() return HttpResponse()
def wrapper(request, *args, **kwargs): data = request.GET.dict() data['remote_ip'] = request.META.get('REMOTE_ADDR') gen_exception = False log_enabled = kw.get('log', False) name = '%s.%s' % (fun.__module__.replace('cm.views.', ''), fun.__name__) if log_enabled: log.debug(0, '=' * 100) log.debug(0, 'Function: %s' % name) log.debug(0, 'Args:\n%s' % json.dumps(data, indent=4)) with transaction.commit_manually(): try: # Execute function resp = fun(**data) transaction.commit() except CMException, e: transaction.rollback() log.exception(0, 'CMException %s' % e) resp = e.response except Exception, e: transaction.rollback() gen_exception = True resp = response('cm_error', str(e))
def form_valid(self, form): with transaction.commit_manually(): try: super(ItemCreate, self).form_valid(form) except ValidationError as e: transaction.rollback() errors = ErrorList() for error in e.messages: errors.append(error) errors = form._errors.setdefault(NON_FIELD_ERRORS, errors) return super(ItemCreate, self).form_invalid(form) context = self.get_context_data() itemimage_form = context['itemimage_formset'] if itemimage_form.is_valid(): itemimage_form.save() transaction.commit() return redirect(self.get_success_url()) else: transaction.rollback() return self.render_to_response( self.get_context_data(form=form) )
def load_sift(database, **kwargs): if not db.utils.table_exists('sift', schema='raw'): return cursor = connections[database].cursor() cursor.execute(db.utils.sequence_reset_sql(Sift, database)) with transaction.commit_manually(database): try: cursor.execute(''' INSERT INTO "sift" ( "variant_id", "score", "refaa", "varaa" ) ( SELECT variant.id, r.score, r.refaa, r.varaa FROM "variant" INNER JOIN "raw"."sift" r ON ("variant".md5 = r."md5") LEFT OUTER JOIN "sift" ON ("sift"."variant_id" = "variant"."id") WHERE "sift"."id" IS NULL ) ''') transaction.commit() except Exception as e: transaction.rollback() log.exception(e)
def form_valid(self, merchandise_form, metadata_formset, client_form): merchandise_form.instance.user = self.request.user # examine if the user has created more than limit merchandises if hasattr(settings, 'MERCHANDISE_LIMIT_PER_TYPE'): merchandise_count = merchandise_form.instance.__class__.objects.filter( user=self.request.user).count() if merchandise_count > settings.MERCHANDISE_LIMIT_PER_TYPE: return self.form_invalid(merchandise_form, metadata_formset, client_form, error_msg=u'用户已经超过物品创建数量上限:%s' % settings.MERCHANDISE_LIMIT_PER_TYPE) with transaction.commit_manually(): _CREATE_SUCCESS = False try: sid = transaction.savepoint() #TODO database transaction merchandise = merchandise_form.save(commit=True) # 添加Meta data to item for form in metadata_formset: data = form.cleaned_data if data: name = data['name'] value = data['value'] merchandise.metadata[name] = value transaction.savepoint_commit(sid) _CREATE_SUCCESS = True except IntegrityError, err: # 物品保存错误,返回 logger.critical(u"%r err:%s" % (self, err)) transaction.savepoint_rollback(sid) finally:
def safe_load(name, backup_path=None, using=DEFAULT_DB_ALIAS): """Creates a backup of the current state of the metadata, attempts to load the new fixture and falls back to the backup fixture if the load fails for any reason. """ _check_app() with transaction.commit_manually(using): # Create the backup fixture if backup_path: create_fixture(os.path.abspath(backup_path), using=using, silent=True) else: backup_path = create_temp_fixture(using=using, silent=True) log.info(u'Backup fixture written to {0}'.format( os.path.abspath(backup_path))) delete_metadata(using=using) try: load_fixture(name, using=using) except (DatabaseError, IntegrityError): transaction.rollback(using) log.error( u'Fixture load failed, reverting from backup: {0}'.format( backup_path)) load_fixture(os.path.abspath(backup_path), using=using) raise transaction.commit(using) return backup_path
def test_manually_managed_with_using(self): """ The commit_manually function also works with a using argument. """ with self.assertRaises(transaction.TransactionManagementError): with transaction.commit_manually(using="default"): Reporter.objects.create(first_name="Walter", last_name="Cronkite")
def load_fixture(name, using=DEFAULT_DB_ALIAS): """Progammatic way to load a fixture given some path. This does not assume the path is a fixture within some app and assumes a full path. """ if os.path.isabs(name): fixture_path = name else: fixture_path = full_fixture_path(name) with open(fixture_path) as fixture: objects = serializers.deserialize(FIXTURE_FORMAT, fixture, using=using) with transaction.commit_manually(using): for obj in objects: if (hasattr(router, "allow_migrate") and router.allow_migrate( using, obj.object.__class__)) or (hasattr( router, "allow_syncdb") and router.allow_syncdb( using, obj.object.__class__)): try: obj.save(using=using) except (DatabaseError, IntegrityError), e: transaction.rollback(using) msg = u'Could not load {0}.{1}(pk={2}): {3}'.format( obj.object._meta.app_label, obj.object._meta.object_name, obj.object.pk, e) raise e.__class__, e.__class__(msg), sys.exc_info()[2] transaction.commit(using)
def handle_noargs(self, **options): party_member = PositionTitle.objects.get(name="Party Member") member = PositionTitle.objects.get(name="Member") party = OrganisationKind.objects.get(name="Party") with transaction.commit_manually(): positions = Position.objects.filter( title=party_member, ) positions_at_parties = positions.filter( organisation__kind=party ) no_non_party_positions = (positions.count() == positions_at_parties.count()) assert no_non_party_positions, """There are some Party Member positions that point to an organisation that is not a party, please manually remove these before running this command""" updated = positions.update(title=member) self.stdout.write("Will update {0} Positions\n".format(updated)) party_member.delete() if options['commit']: transaction.commit() self.stdout.write("Updated {0} Positions\n".format(updated)) self.stdout.write("Deleted the Party Member PositionTitle\n") else: transaction.rollback()
def handle(self, **options): database = options.get('database') if options.get('force'): cohorts = list(Cohort.objects.all()) else: cohorts = list( Cohort.objects.filter( Q(allele_freq_modified=None) | Q(modified__gt=F('allele_freq_modified')))) print 'Computing for {0} cohorts'.format(len(cohorts)) for cohort in cohorts: sys.stdout.write('"{0}" ({1} samples)...'.format( cohort, cohort.count)) sys.stdout.flush() t0 = time.time() with transaction.commit_manually(database): try: cohort.compute_allele_frequencies(database) sys.stdout.write('done in {0}s\n'.format( int(time.time() - t0))) transaction.commit() except DatabaseError, e: transaction.rollback() log.exception(e)
def apply_regex_global(admin, request, queryset, fieldname): _selected_action = map(str, queryset.values_list('product_id', flat=True)) form = None if 'apply' in request.POST: form = RegexForm(request.POST) if form.is_valid(): with transaction.commit_manually(): try: for o in queryset: old_value = getattr(o, fieldname) new_value = re.sub(form.data['from_regex'], form.data['to_regex'], old_value) setattr(o, fieldname, new_value) o.save() transaction.commit() finally: transaction.rollback() admin.message_user(request, "Successfully applied a regex.") return HttpResponseRedirect(request.get_full_path()) if not form: form = RegexForm(initial={'_selected_action': _selected_action}) return render(request, "apply_regex.html", { 'form': form, 'title': 'Apply regex'} )
def completed_game(request): game_id = request.POST.get('game_id',None) winning_player = request.POST.get('winning_player',None) json_state = request.POST.get('json_state', None) screenshot_file = request.FILES.get('screenshot_file',None) if game_id and winning_player and json_state and screenshot_file: with transaction.commit_manually(): try: #Get the game using the id game = Game.objects.select_for_update().get(id=game_id) #Get the account for the new current player empous_player = EmpousUser.objects.get(id=winning_player) for player in game.players.all(): if player != empous_player: player.send_push_message(str(empous_player.first_name) + " has defeated you.", player.playable_games) game.victor = empous_player game.current_player = empous_player game.screenshot_file = screenshot_file game.json_serialized_game = json_state; game.saveAndPossiblyPush() transaction.commit() return HttpResponse(json.dumps({'success':'Game finished'}), mimetype='application/json') except Game.DoesNotExist: transaction.rollback() return HttpResponse(json.dumps({'error':'Game does not exist'}), mimetype='application/json') except EmpousUser.DoesNotExist: transaction.rollback() return HttpResponse(json.dumps({'error':'Empous User does not exist'}), mimetype='application/json') else: return HttpResponse(json.dumps({'error':'Missing necessary arguemnts'}), mimetype='application/json')
def handle(self, *args, **options): options['handle'] = self if len(args) != 2: raise CommandError('Both directory and DMA pk should be ' 'provided') if not os.path.exists(args[0]): raise CommandError('Directory "%s" does not exist' % (args[0], )) try: zone = int(args[1]) except ValueError: raise CommandError('DMA pk "%s" is not an integer number' % (args[1], )) if not DMA.objects.filter(pk=zone).exists(): raise CommandError('No DMA exists in db with pk=%s' % (args[1], )) directory = args[0] file_list = [ f for f in listdir(args[0]) if os.path.isfile(os.path.join(args[0], f)) and f.split('.')[-1].lower() == 'csv' ] # maybe improve transaction control, is this has a # meaning, as this command is only for demonstration - concept # edition of the iWidget with transaction.commit_manually(): try: for file in file_list: process_file(file, directory, zone) create_dma_series(zone) transaction.commit() except: transaction.rollback() raise
def load_fixture(name, using=DEFAULT_DB_ALIAS): """Progammatic way to load a fixture given some path. This does not assume the path is a fixture within some app and assumes a full path. """ if os.path.isabs(name): fixture_path = name else: fixture_path = full_fixture_path(name) with open(fixture_path) as fixture: objects = serializers.deserialize(FIXTURE_FORMAT, fixture, using=using) with transaction.commit_manually(using): for obj in objects: if router.allow_syncdb(using, obj.object.__class__): try: obj.save(using=using) except (DatabaseError, IntegrityError), e: transaction.rollback(using) msg = u"Could not load {app_label}.{object_name}(pk={pk}): {error_msg}".format( app_label=obj.object._meta.app_label, object_name=obj.object._meta.object_name, pk=obj.object.pk, error_msg=e, ) raise e.__class__, e.__class__(msg), sys.exc_info()[2] transaction.commit(using)
def import_grades_comment(self, sheet): x, header, inserted, updated = self.import_prep(sheet) while x < sheet.nrows: with transaction.commit_manually(): try: name = None row = sheet.row(x) items = zip(header, row) model = None created = False comment = "" for (name, value) in items: is_ok, name, value = self.sanitize_item(name, value) if is_ok: if name == "id": model, created = GradeComment.objects.get_or_create(id=value) elif name == "comment": model.comment = value model.save() if created: self.log_and_commit(model, addition=True) inserted += 1 else: self.log_and_commit(model, addition=False) updated += 1 except: self.handle_error(row, name, sys.exc_info(), sheet.name) x += 1 return inserted, updated
def form_valid(self, form): success = True with transaction.commit_manually(): self.object = form.save() if defaults.PYBB_ATTACHMENT_ENABLE: aformset = AttachmentFormSet(self.request.POST, self.request.FILES, instance=self.object) if aformset.is_valid(): aformset.save() else: success = False else: aformset = AttachmentFormSet() if self.object.topic.poll_type != Topic.POLL_TYPE_NONE and self.object.topic.head == self.object: pollformset = PollAnswerFormSet(self.request.POST, instance=self.object.topic) if pollformset.is_valid(): pollformset.save() else: success = False else: self.object.topic.poll_question = None self.object.topic.save() self.object.topic.poll_answers.all().delete() pollformset = PollAnswerFormSet() if success: transaction.commit() return super(ModelFormMixin, self).form_valid(form) else: transaction.rollback() return self.render_to_response(self.get_context_data(form=form, aformset=aformset, pollformset=pollformset))
def handle(self, *args, **options): using = options.get('database') load_1000g = self.load_1000g if options.get('1000g') else None load_polyphen2 = \ self.load_polyphen2 if options.get('polyphen2') else None load_evs = self.load_evs if options.get('evs') else None load_sift = self.load_sift if options.get('sift') else None truncate = options.get('truncate') cursor = connections[using].cursor() for handler in (load_1000g, load_polyphen2, load_evs, load_sift): if not handler: continue with transaction.commit_manually(using): try: log.debug('Loading {0}...'.format(handler.short_name)) count = handler(cursor, truncate) log.debug('{0} rows copied'.format(count)) transaction.commit() except Exception, e: transaction.rollback() log.exception(e.message)
def _add_deadlines(self): new_deadline = self.CONTENT['deadline'] text = self.CONTENT['text'] deadlines = [] with transaction.commit_manually(): try: groups = self._query_creategroups() if len(groups) == 0: raise BadRequestFieldError('createmode', _('The given option did not match any groups.')) for group in groups: deadline = Deadline(assignment_group=group) deadline.deadline = new_deadline deadline.text = text deadline.full_clean() deadline.save() deadlines.append(deadline) logger.info('User=%s created Deadline id=%s (%s)', self.user, deadline.id, deadline.deadline) except ValidationError as e: transaction.rollback() raise ValidationErrorResponse(e) except Exception as e: transaction.rollback() raise else: transaction.commit() return deadlines
def handle(self, *args, **options): options['handle'] = self if len(args)!=2: raise CommandError('Both directory and DMA pk should be ' 'provided') if not os.path.exists(args[0]): raise CommandError('Directory "%s" does not exist'%( args[0],)) try: zone = int(args[1]) except ValueError: raise CommandError('DMA pk "%s" is not an integer number'%( args[1],)) if not DMA.objects.filter(pk=zone).exists(): raise CommandError('No DMA exists in db with pk=%s'%( args[1],)) directory = args[0] file_list = [ f for f in listdir(args[0]) if os.path.isfile( os.path.join(args[0], f)) and f.split('.')[-1].lower()=='csv'] # maybe improve transaction control, is this has a # meaning, as this command is only for demonstration - concept # edition of the iWidget with transaction.commit_manually(): try: for file in file_list: process_file(file, directory, zone) create_dma_series(zone) transaction.commit() except: transaction.rollback() raise
def main(): mojang_status = _get_mojang_status() durations = [] for server in Server.objects.filter(online=True): with transaction.commit_manually(): start = int(round(time.time() * 1000)) try: _query_server(server, mojang_status) except: transaction.rollback() rollbar.report_exc_info() else: transaction.commit() durations.append( (server.id, int(round(time.time() * 1000)) - start)) extra_data = { 'server.%d.ms' % server_id: duration for server_id, duration in durations } extra_data['login'] = mojang_status.login extra_data['session'] = mojang_status.session rollbar.report_message('Server queries complete', 'debug', extra_data=extra_data)
def handle(self, *args, **options): if len(args) != 1: raise CommandError('Usage index_team_videos <team-slug>') try: team = Team.objects.get(slug=args[0]) except Team.DoesNotExist: raise CommandError('Team with slug %r not found' % (args[0], )) video_index = site.get_index(Video) team_video_index = site.get_index(TeamVideo) self.stdout.write("Fetching videos\n") video_list = list( TeamVideo.objects.filter(team=team).select_related('video')) start_time = time.time() self.stdout.write("Indexing") self.stdout.flush() with transaction.commit_manually(): for team_video in video_list: video_index.update_object(team_video.video) team_video_index.update_object(team_video) self.stdout.write(".") self.stdout.flush() # commit after each pass to make sure that we aren't keeping # open any database locks transaction.commit() end_time = time.time() self.stdout.write("\ndone indexed %s videos in %0.1f seconds\n" % (len(video_list), end_time - start_time))
def import_grades_comment(self, sheet): x, header, inserted, updated = self.import_prep(sheet) while x < sheet.nrows: with transaction.commit_manually(): try: name = None row = sheet.row(x) items = zip(header, row) model = None created = False comment = "" for (name, value) in items: is_ok, name, value = self.sanitize_item(name, value) if is_ok: if name == "id": model, created = GradeComment.objects.get_or_create( id=value) elif name == "comment": model.comment = value model.save() if created: self.log_and_commit(model, addition=True) inserted += 1 else: self.log_and_commit(model, addition=False) updated += 1 except: self.handle_error(row, name, sys.exc_info(), sheet.name) x += 1 return inserted, updated
def run(self): if self.status != TX_STATUS["pending"]: raise with db_tx.commit_manually(): self.status = TX_STATUS["running"] self.save() try: source = self.source.get_concrete() self.result = source.refresh() self.validate() except Exception as ex: logger.error("Error for transaction %s: %s" % (self.tx_id, ex.message)) self.status = TX_STATUS["failure"] self.result = { "message": "Error transforming data: %s" % ex.message } self.save() db_tx.commit() return self
def safe_load(name, backup_path=None, using=DEFAULT_DB_ALIAS): """Creates a backup of the current state of the metadata, attempts to load the new fixture and falls back to the backup fixture if the load fails for any reason. """ _check_app() with transaction.commit_manually(using): # Create the backup fixture if backup_path: create_fixture(os.path.abspath(backup_path), using=using, silent=True) else: backup_path = create_temp_fixture(using=using, silent=True) log.info(u'Backup fixture written to {0}'.format(os.path.abspath( backup_path))) delete_metadata(using=using) try: load_fixture(name, using=using) except (DatabaseError, IntegrityError): transaction.rollback(using) log.error(u'Fixture load failed, reverting from backup: {0}' .format(backup_path)) load_fixture(backup_path, using=using) raise transaction.commit(using) return backup_path
def fetch(self, verbose=False, log=False): """ Compares and updates all Legislator objects against the data contained within the Sunlight Foundation's Congress API. The entire operation is enclosed in a transaction that is only committed if no exceptions are raised. """ if verbose and log: log.write("Fetching Legislators\n") with transaction.commit_manually(): try: for legislator in congress.legislators(): legislator_obj = self.get_or_create(bioguide_id=legislator["bioguide_id"])[0] for name, value in legislator.iteritems(): setattr(legislator_obj, name, value) legislator_obj.save() if verbose and log: log.write("- %s\n" % legislator_obj.fullname) except Exception as e: transaction.rollback() raise e else: transaction.commit()
def handle(self, *args, **options): commit = options.get('commit') self.debug = options.get('debug') if not USE_ANONYMIZE: raise Exception("Anonymize feature is not activated") self.survey_name = getattr(settings, 'IFN_ANONYMIZE_SURVEY', None) if self.survey_name is None: raise Exception( "Anonymize survey is not configured, please define IFN_ANONYMIZE_SURVEY in settings" ) self.data_tables = self.get_data_tables() print("found %d tables" % len(self.data_tables)) print(','.join(self.data_tables)) commited = False with transaction.commit_manually(): try: for req in self.get_ano_requests(): user_id = req['user'] if req['action'] == 1: action = ACTION_DELETE action_label = "Remove" else: action = ACTION_ANONYMZE action_label = "Anonymize" print("Handling request %d User %d for %s" % (req['id'], user_id, action_label)) try: user = User.objects.get(id=user_id) except User.DoesNotExist: print("User %d not found ?" % (user_id)) continue self.anonymize(user) if action == ACTION_DELETE: self.delete(user_id) a = AnonymizedUser() a.action = action a.user = user a.request_id = req['id'] a.save() except Exception as e: transaction.rollback() raise if commit: transaction.commit() commited = True else: transaction.rollback() if commited: print("Changed has been made on the database") else: print( "No change has been made, add --commit option to make the changes" )
def test_manually_managed(self): """ You can manually manage transactions if you really want to, but you have to remember to commit/rollback. """ manually_managed = transaction.commit_manually(self.manually_managed) manually_managed() self.assertEqual(Reporter.objects.count(), 1)
def run(self): # FIXME: tests for changeset retrieval, etc. try: for lid in self.dataset.layerindataset_set.all(): layer = lid.layer table_name = "%s_update_%s" % (layer.name, self.to_version.replace("-", "_")) import_proc = subprocess.Popen([ settings.LINZ2OSM_SCRIPT_ROOT + '/load_lds_dataset.sh', 'update', self.dataset.database_name, layer.wfs_type_name, table_name, settings.LINZ_DATA_SERVICE_API_KEY, 'from:%s;to:%s' % (self.from_version, self.to_version), urllib.quote(layer.wfs_cql_filter) ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) import_proc.wait() import_output = import_proc.stdout.read() print import_output if import_proc.returncode != 0: raise DatasetUpdateError("Import of %s failed with return code %d and output:\n%s" % (layer.name, import_proc.returncode, import_output)) with transaction.commit_manually(): with transaction.commit_manually(using=self.dataset.name): try: osm.apply_changeset_to_dataset(self, table_name, lid) except: print "ERROR - ROLLBACK" transaction.rollback(using=self.dataset.name) transaction.rollback() raise else: transaction.commit(using=self.dataset.name) transaction.commit() except DatasetUpdateError as e: self.error = unicode(e) except Exception as e: print "Unexpected other exception:", sys.exc_info()[0] self.error = unicode(e) raise else: self.complete = True self.dataset.version = self.to_version self.dataset.save() finally: self.save()
def test_manually_managed_with_using(self): """ The commit_manually function also works with a using argument. """ using_manually_managed_mistake = transaction.commit_manually( using='default')(self.manually_managed_mistake) self.assertRaises(transaction.TransactionManagementError, using_manually_managed_mistake)
def test_manually_managed_mistake(self): """ If you forget, you'll get bad errors. """ manually_managed_mistake = transaction.commit_manually( self.manually_managed_mistake) self.assertRaises(transaction.TransactionManagementError, manually_managed_mistake)
def save(self, *args, **kwargs): # Avoid infinite recursion if creating from get_or_create() call below. if 'force_insert' in kwargs and kwargs['force_insert'] is True: super(CohortMembership, self).save(*args, **kwargs) return self.full_clean(validate_unique=False) # This loop has been created to allow for optimistic locking, and retrial in case of losing a race condition. # The limit is 2, since select_for_update ensures atomic updates. Creation is the only possible race condition. max_retries = 2 success = False for __ in range(max_retries): # The following 2 "transaction" lines force a fresh read, they can be removed once we're on django 1.8 # http://stackoverflow.com/questions/3346124/how-do-i-force-django-to-ignore-any-caches-and-reload-data with transaction.commit_manually(): transaction.commit() with transaction.commit_on_success(): try: saved_membership, created = CohortMembership.objects.select_for_update( ).get_or_create( user__id=self.user.id, # pylint: disable=E1101 course_id=self.course_id, defaults={ 'course_user_group': self.course_user_group, 'user': self.user }) except IntegrityError: # This can happen if simultaneous requests try to create a membership transaction.rollback() continue if not created: if saved_membership.course_user_group == self.course_user_group: raise ValueError( "User {user_name} already present in cohort {cohort_name}" .format( user_name=self.user.username, # pylint: disable=E1101 cohort_name=self.course_user_group.name)) self.previous_cohort = saved_membership.course_user_group self.previous_cohort_name = saved_membership.course_user_group.name self.previous_cohort_id = saved_membership.course_user_group.id self.previous_cohort.users.remove(self.user) saved_membership.course_user_group = self.course_user_group self.course_user_group.users.add(self.user) # pylint: disable=E1101 #note: in django 1.8, we can call save with updated_fields=['course_user_group'] super(CohortMembership, saved_membership).save() success = True break if not success: raise IntegrityError( "Unable to save membership after {} tries, aborting.".format( max_retries))
def handle(self, **options): if not self.targets: log.info('No targets defined. Nothing to do.') return source = options.get('source') drop = options.get('drop') create = options.get('create') load = options.get('load') using = options.get('database') connection = connections[using] cursor = connection.cursor() # If no operation is supplied, assume all operations if not (drop or create or load): drop = create = load = True # Perform checks ahead of time to prevent failing in an # inconsistent state if not os.path.exists(os.path.join(source, 'MANIFEST')): raise CommandError('No MANIFEST in the source directory: {0}' .format(source)) targets = OrderedDict() parser = ConfigParser() parser.read([os.path.join(source, 'MANIFEST')]) for target in parser.sections(): if target not in self.targets: log.warn('Unknown target "{0}", skipping...'.format(target)) continue options = dict(parser.items(target)) files = [] for fn in shlex.split(options['files']): path = os.path.join(source, fn) if not os.path.exists(path): raise CommandError('No file named {0} exists'.format(fn)) files.append(path) targets[target] = (files, options) if not targets: log.info('No targets defined in MANIFEST. Nothing to do.') return # Manually commit along the way with transaction.commit_manually(using=using): if drop: self.drop_tables(cursor, targets) if create: self.create_tables(cursor, targets) if load: self.load_tables(cursor, targets) transaction.commit()
def handle(self, **options): if not self.targets: log.info('No targets defined. Nothing to do.') return source = options.get('source') drop = options.get('drop') create = options.get('create') load = options.get('load') using = options.get('database') connection = connections[using] cursor = connection.cursor() # If no operation is supplied, assume all operations if not (drop or create or load): drop = create = load = True # Perform checks ahead of time to prevent failing in an # inconsistent state if not os.path.exists(os.path.join(source, 'MANIFEST')): raise CommandError( 'No MANIFEST in the source directory: {0}'.format(source)) targets = OrderedDict() parser = ConfigParser() parser.read([os.path.join(source, 'MANIFEST')]) for target in parser.sections(): if target not in self.targets: log.warn('Unknown target "{0}", skipping...'.format(target)) continue options = dict(parser.items(target)) files = [] for fn in shlex.split(options['files']): path = os.path.join(source, fn) if not os.path.exists(path): raise CommandError('No file named {0} exists'.format(fn)) files.append(path) targets[target] = (files, options) if not targets: log.info('No targets defined in MANIFEST. Nothing to do.') return # Manually commit along the way with transaction.commit_manually(using=using): if drop: self.drop_tables(cursor, targets) if create: self.create_tables(cursor, targets) if load: self.load_tables(cursor, targets) transaction.commit()
def test_manually_managed(self): """ You can manually manage transactions if you really want to, but you have to remember to commit/rollback. """ with transaction.commit_manually(): Reporter.objects.create(first_name="Libby", last_name="Holtzman") transaction.commit() self.assertEqual(Reporter.objects.count(), 1)
def _refresh_power_devices(self): # Ensure that we have a fresh view of the DB with transaction.commit_manually(): transaction.commit() with self._lock: for device in PowerControlDevice.objects.all(): if device.sockaddr not in self._power_devices: self._power_devices[device.sockaddr] = device
def root_resource_ids(self, plugin): """Return the PK of all StorageResourceRecords for 'plugin' which have no parents""" from chroma_core.lib.storage_plugin.manager import storage_plugin_manager # We will be polling, to need to commit to see new data with transaction.commit_manually(): transaction.commit() ids = storage_plugin_manager.get_scannable_resource_ids(plugin) transaction.commit() return ids