def post(self, request): if not (is_active_superuser(request) and request.access.has_permission('broadcasts.admin')): return self.respond(status=401) validator = AdminBroadcastValidator(data=request.DATA) if not validator.is_valid(): return self.respond(validator.errors, status=400) result = validator.object with transaction.atomic(): broadcast = Broadcast.objects.create( title=result['title'], message=result['message'], link=result['link'], is_active=result.get('isActive') or False, date_expires=result.get('expiresAt'), ) logger.info('broadcasts.create', extra={ 'ip_address': request.META['REMOTE_ADDR'], 'user_id': request.user.id, 'broadcast_id': broadcast.id, }) if result.get('hasSeen'): try: with transaction.atomic(): BroadcastSeen.objects.create( broadcast=broadcast, user=request.user, ) except IntegrityError: pass return self.respond(self._serialize_objects(broadcast, request))
def create_contributions(apps, schema_editor): Word = apps.get_model("linguistics", "Word") Template = apps.get_model("linguistics", "Template") Contribution = apps.get_model("linguistics", "Contribution") for word in Word.objects.all(): if word.author_id is None: continue if word.state == 1: continue with transaction.atomic(): try: Contribution.objects.create( account_id=word.author_id, state=word.state, type=0, source=0, entity_id=word.id ) except: pass for template in Template.objects.all(): if template.author_id is None: continue if template.state == 1: continue with transaction.atomic(): try: Contribution.objects.create( account_id=template.author_id, state=template.state, type=1, source=0, entity_id=template.id ) except: pass
def install_vocabularies(): Vocabulary.objects.all().delete() VocabularyClass.objects.all().delete() VocabularyProperty.objects.all().delete() # load vocabularies vocabularies_fp = open(os.path.join(os.path.dirname(__file__), 'data/vocabularies.json')) data = json.load(vocabularies_fp) n_of_vocabularies = 0 with transaction.atomic(): # atomic transactions vastly improve performence for v in data: vocabulary = Vocabulary.objects.create(pk=v['pk'], category=v['fields']['category'], version=v['fields']['version'], votes=v['fields']['votes'], originalUrl=v['fields']['originalUrl'], description=v['fields']['description'], title=v['fields']['title'], downloads=v['fields']['downloads'], lodRanking=v['fields']['lodRanking'], preferredNamespacePrefix=v['fields']['preferredNamespacePrefix'], datePublished=v['fields']['datePublished'], downloadUrl=v['fields']['downloadUrl'], score=v['fields']['score'], uploader=User.objects.get(pk=v['fields']['uploader']), dateModified=v['fields']['dateModified'], dateCreated=v['fields']['dateCreated'], preferredNamespaceUri=v['fields']['preferredNamespaceUri'], example=v['fields']['example'], prevent_default_make=True) n_of_vocabularies += 1 vocabulary.prevent_default_make = False # reset to false so it can be updated vocabularies_fp.close() # load classes classes_fp = open(os.path.join(os.path.dirname(__file__), 'data/classes.json')) data = json.load(classes_fp) n_of_classes = 0 with transaction.atomic(): for c in data: VocabularyClass.objects.create(pk=c['pk'], description=c['fields']['description'], uri=c['fields']['uri'], label=c['fields']['label'], vocabulary=Vocabulary.objects.get(pk=c['fields']['vocabulary'])) n_of_classes += 1 classes_fp.close() # load properties properties_fp = open(os.path.join(os.path.dirname(__file__), 'data/properties.json')) data = json.load(properties_fp) n_of_properties = 0 with transaction.atomic(): for p in data: VocabularyProperty.objects.create(pk=p['pk'], description=p['fields']['description'], uri=p['fields']['uri'], vocabulary=Vocabulary.objects.get(pk=p['fields']['vocabulary']), label=p['fields']['label'], domain=p['fields']['domain'], range=p['fields']['range'], parent_uri=p['fields']['parent_uri']) n_of_properties += 1 properties_fp.close() return HttpResponse(json.dumps({ 'n_of_vocabularies': n_of_vocabularies, 'n_of_classes': n_of_classes, 'n_of_properties': n_of_properties }), status=200, content_type="application/json")
def kill(self): with transaction.atomic(): p = type(self).objects.get(pk=self.pk) p.not_needed_anymore = True p.save() save_lives = not self.finished logger().info("Killing pool #{}".format(self.id)) if self.appliances: for appliance in self.appliances: if ( save_lives and appliance.ready and appliance.leased_until is None and appliance.marked_for_deletion is False and not appliance.managed_providers): with transaction.atomic(): with appliance.kill_lock: appliance.appliance_pool = None appliance.datetime_leased = None appliance.save() self.total_count -= 1 if self.total_count < 0: self.total_count = 0 # Protection against stupidity self.save() appliance.set_status( "The appliance was taken out of dying pool {}".format(self.id)) else: Appliance.kill(appliance) else: # No appliances, so just delete it self.delete()
def test__init(self): rated_by = TestUser() # all fields rcpe1 = TestRecipe() r1 = Rating.objects.create(recipe=rcpe1, rated_by=rated_by, vote=1) self.assertIsNotNone(r1.last_updated) # recipe and rated_by should be unique with transaction.atomic(): self.assertRaises(IntegrityError, Rating.objects.create, recipe=rcpe1, rated_by=rated_by, vote=1) self.assert_(Rating.objects.create(recipe=rcpe1, rated_by=TestUser(), vote=1)) self.assert_(Rating.objects.create(recipe=TestRecipe(), rated_by=rated_by, vote=-1)) # recipe is required with transaction.atomic(): self.assertRaises(IntegrityError, Rating.objects.create, rated_by=rated_by, vote=-1) # rated_by is required with transaction.atomic(): self.assertRaises(IntegrityError, Rating.objects.create, recipe=rcpe1, vote=1) # vote is required with transaction.atomic(): self.assertRaises(IntegrityError, Rating.objects.create, recipe=TestRecipe(), rated_by=TestUser())
def test_hooks_cleared_after_successful_commit(self, track): with atomic(): track.do(1) with atomic(): track.do(2) track.assert_done([1, 2]) # not [1, 1, 2]
def test_executes_only_after_final_transaction_committed(self, track): with atomic(): with atomic(): track.do(1) track.assert_notified([]) track.assert_notified([]) track.assert_done([1])
def test_wrap_callable_instance(self): # Regression test for #20028 class Callable(object): def __call__(self): pass # Must not raise an exception transaction.atomic(Callable())
def test_nested_commit_commit(self): with transaction.atomic(): Reporter.objects.create(first_name="Tintin") with transaction.atomic(): Reporter.objects.create(first_name="Archibald", last_name="Haddock") self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Archibald Haddock>', '<Reporter: Tintin>'])
def test_float_validates_object(self): instance = FloatModel(size=2.5) # Try setting float field to unsaved object instance.size = instance with transaction.atomic(): with self.assertRaises(TypeError): instance.save() # Set value to valid and save instance.size = 2.5 instance.save() self.assertTrue(instance.id) # Set field to object on saved instance instance.size = instance msg = ( "Tried to update field model_fields.FloatModel.size with a model " "instance, <FloatModel: FloatModel object>. Use a value " "compatible with FloatField." ) with transaction.atomic(): with self.assertRaisesMessage(TypeError, msg): instance.save() # Try setting field to object on retrieved object obj = FloatModel.objects.get(pk=instance.id) obj.size = obj with self.assertRaises(TypeError): obj.save()
def link_kamervragen_and_antwoorden(): logger.info('BEGIN') from django.db.utils import IntegrityError kamerantwoorden = Kamerantwoord.objects.all() for kamerantwoord in kamerantwoorden: kamervragen = Kamervraag.objects.filter(vraagnummer=kamerantwoord.vraagnummer) if kamervragen: kamervraag = kamervragen[0] try: with transaction.atomic(): kamervraag.kamerantwoord = kamerantwoord kamervraag.save() except IntegrityError as error: logger.error('kamervraag: ' + str(kamervraag.id)) logger.error('kamerantwoord: ' + str(kamerantwoord.id)) logger.exception(error) mededelingen = KamervraagMededeling.objects.all() for mededeling in mededelingen: kamervragen = Kamervraag.objects.filter(vraagnummer=mededeling.vraagnummer) if kamervragen: kamervraag = kamervragen[0] try: with transaction.atomic(): mededeling.kamervraag = kamervraag mededeling.save() except IntegrityError as error: logger.error('mededeling: ' + str(mededeling.id)) logger.error('kamervraag: ' + str(kamervraag.id)) logger.exception(error) logger.info('END')
def get_model_object( self, obj_type, **conditions ): """(DBTools, <arbitrary Django model object>) -> <unsaved arbitrary Django model object> | <saved arbitrary Django model object> | Error """ try: with transaction.atomic(): return obj_type.objects.get( **conditions ) except IntegrityError: try: with transaction.atomic(): return obj_type.objects.create( **conditions ) except IntegrityError: if not len( obj_type.objects.filter( **conditions ) ): raise NonUniqueError( ) elif len( obj_type.objects.filter( **conditions ) ) > 1: raise ClassNotFoundError( ) else: raise ObjectUnreadyError() if not len( obj_type.objects.filter( **conditions ) ): return obj_type( **conditions ) elif len( obj_type.objects.filter( **conditions ) ) == 1: return obj_type.objects.get( **conditions ) else: raise NonUniqueError( )
def test_implicit_savepoint_rollback(self): """MySQL implicitly rolls back savepoints when it deadlocks (#22291).""" other_thread_ready = threading.Event() def other_thread(): try: with transaction.atomic(): Reporter.objects.create(id=1, first_name="Tintin") other_thread_ready.set() # We cannot synchronize the two threads with an event here # because the main thread locks. Sleep for a little while. time.sleep(1) # 2) ... and this line deadlocks. (see below for 1) Reporter.objects.exclude(id=1).update(id=2) finally: # This is the thread-local connection, not the main connection. connection.close() other_thread = threading.Thread(target=other_thread) other_thread.start() other_thread_ready.wait() with self.assertRaisesMessage(OperationalError, 'Deadlock found'): # Double atomic to enter a transaction and create a savepoint. with transaction.atomic(): with transaction.atomic(): # 1) This line locks... (see above for 2) Reporter.objects.create(id=1, first_name="Tintin") other_thread.join()
def test_merged_rollback_rollback(self): with six.assertRaisesRegex(self, Exception, "Oops"): with transaction.atomic(): Reporter.objects.create(last_name="Tintin") with six.assertRaisesRegex(self, Exception, "Oops"): with transaction.atomic(savepoint=False): Reporter.objects.create(first_name="Haddock") raise Exception("Oops, that's his last name") raise Exception("Oops, that's his first name") self.assertQuerysetEqual(Reporter.objects.all(), []) self.assertAtomicSignalCalls( # Enter atomic transaction block. enter_block_atomic_signal_call_sequence(True) + # Create Reporter. create_model_atomic_signal_call_sequence() + # Enter nested atomic transaction block. enter_block_atomic_signal_call_sequence(False, savepoint=False) + # Create Reporter. create_model_atomic_signal_call_sequence() + # Leave nested atomic transaction block. leave_block_atomic_signal_call_sequence(False, True, savepoint=False) + # Leave atomic transaction block. leave_block_atomic_signal_call_sequence(True, False) )
def test_prevent_rollback(self): with transaction.atomic(): Reporter.objects.create(first_name="Tintin") sid = transaction.savepoint() # trigger a database error inside an inner atomic without savepoint with self.assertRaises(DatabaseError): with transaction.atomic(savepoint=False): connection.cursor().execute( "SELECT no_such_col FROM transactions_reporter" ) # prevent atomic from rolling back since we're recovering manually self.assertTrue(transaction.get_rollback()) transaction.set_rollback(False) transaction.savepoint_rollback(sid) self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Tintin>']) self.assertAtomicSignalCalls( # Enter atomic transaction block. enter_block_atomic_signal_call_sequence(True) + # Create Reporter. create_model_atomic_signal_call_sequence() + # Enter and leave atomic transaction block. enter_block_atomic_signal_call_sequence(False, savepoint=False) + leave_block_atomic_signal_call_sequence(False, False, savepoint=False) + # Leave atomic transaction with recovered rollback. leave_block_atomic_signal_call_sequence(True, True) )
def judge(self): self.submission.judge_start_time = int(time.time() * 1000) with transaction.atomic(): judge_server = self.choose_judge_server() # 如果没有合适的判题服务器,就放入等待队列中等待判题 if not judge_server: JudgeWaitingQueue.objects.create(submission_id=self.submission.id, time_limit=self.time_limit, memory_limit=self.memory_limit, test_case_id=self.test_case_id) return judge_server.use_judge_instance() try: s = TimeoutServerProxy("http://" + judge_server.ip + ":" + str(judge_server.port), timeout=20) data = s.run(judge_server.token, self.submission.id, self.submission.language, self.submission.code, self.time_limit, self.memory_limit, self.test_case_id) # 编译错误 if data["code"] == 1: self.submission.result = result["compile_error"] self.submission.info = data["data"]["error"] # system error elif data["code"] == 2: self.submission.result = result["system_error"] self.submission.info = data["data"]["error"] elif data["code"] == 0: self.submission.result = data["data"]["result"] self.submission.info = json.dumps(data["data"]["info"]) self.submission.accepted_answer_time = data["data"]["accepted_answer_time"] except Exception as e: self.submission.result = result["system_error"] self.submission.info = str(e) finally: with transaction.atomic(): judge_server.release_judge_instance() self.submission.judge_end_time = int(time.time() * 1000) self.submission.save() if self.submission.contest_id: self.update_contest_problem_status() else: self.update_problem_status() with transaction.atomic(): waiting_submissions = JudgeWaitingQueue.objects.select_for_update().all() if waiting_submissions.exists(): # 防止循环依赖 from submission.tasks import _judge waiting_submission = waiting_submissions.first() submission = Submission.objects.get(id=waiting_submission.submission_id) waiting_submission.delete() _judge.delay(submission, time_limit=waiting_submission.time_limit, memory_limit=waiting_submission.memory_limit, test_case_id=waiting_submission.test_case_id, is_waiting_task=True)
def test_merged_commit_commit(self): with transaction.atomic(): Reporter.objects.create(first_name="Tintin") with transaction.atomic(savepoint=False): Reporter.objects.create(first_name="Archibald", last_name="Haddock") self.assertQuerysetEqual(Reporter.objects.all(), ['<Reporter: Archibald Haddock>', '<Reporter: Tintin>']) self.assertAtomicSignalCalls( # Enter atomic transaction block. enter_block_atomic_signal_call_sequence(True) + # Create Reporter. create_model_atomic_signal_call_sequence() + # Enter nested atomic transaction block. enter_block_atomic_signal_call_sequence(False, savepoint=False) + # Create Reporter. create_model_atomic_signal_call_sequence() + # Leave nested atomic transaction block with caught eexception. leave_block_atomic_signal_call_sequence(False, True, savepoint=False) + # Leave atomic transaction block with exception. leave_block_atomic_signal_call_sequence(True, True) )
def updatejobparams(name=None, form=None, forcelevel=None): if form is None or name is None: return if 'descr' in form.changed_data: # print "Update description" with transaction.atomic(): updateDIRJobDescr(name=name, descr=form.cleaned_data['descr']) if 'storage' in form.changed_data: # print "Update storage" with transaction.atomic(): updateJobStorage(name=name, storage=form.cleaned_data['storage']) if 'client' in form.changed_data: # print "Update client" with transaction.atomic(): updateJobClient(name=name, client=form.cleaned_data['client']) if 'backupsch' in form.changed_data or 'starttime' in form.changed_data or \ 'scheduleweek' in form.changed_data or 'schedulemonth' in form.changed_data or \ 'backuprepeat' in form.changed_data or 'backuplevel' in form.changed_data: # update Schedule with transaction.atomic(): updateJobSchedule(jobname=name, data=form.cleaned_data, forcelevel=forcelevel) if 'retention' in form.changed_data: # update retention with transaction.atomic(): updateJobRetention(name=name, retention=form.cleaned_data['retention'])
def do_update(self, limit): indexupdates = set() unit_ids = set() source_unit_ids = set() # Grab all updates from the database with transaction.atomic(): updates = IndexUpdate.objects.filter(to_delete=False) for update in updates[:limit].iterator(): indexupdates.add(update.pk) unit_ids.add(update.unitid) if update.source: source_unit_ids.add(update.unitid) # Filter matching units units = Unit.objects.filter( id__in=unit_ids ) source_units = Unit.objects.filter( id__in=source_unit_ids ) # Udate index update_index(units, source_units) # Delete processed updates with transaction.atomic(): IndexUpdate.objects.filter(id__in=indexupdates).delete()
def test_hooks_cleared_after_successful_commit(self): with transaction.atomic(): self.do(1) with transaction.atomic(): self.do(2) self.assertDone([1, 2]) # not [1, 1, 2]
def test_executes_only_after_final_transaction_committed(self): with transaction.atomic(): with transaction.atomic(): self.do(1) self.assertNotified([]) self.assertNotified([]) self.assertDone([1])
def purseentry_edit_batch_save(request): id_list = request.POST.get('id_list') id_list = id_list.split(',') with transaction.atomic(): for billentry_id in id_list: form = PurseEntryEditBatchForm(request.POST) if not form.is_valid(): return JsonResponse({'success': False, 'message': '验证失败!' + get_form_error_msg(form)}) material = form.data.get('material') amount = form.data.get('amount') if amount.strip() == '': amount = 0 qty = form.data.get('qty') if qty.strip() == '': qty = 0 material_id = None try: material_id = Materiel.objects.filter(bur_num=material)[0].bur_id except: pass bill_date = form.data.get('bill_date') with transaction.atomic(): PurseEntry.objects.filter(pk=billentry_id).update( material_id=material_id, amount=amount, qty=qty, ) purse_service.markedit_by_bill_entry_id(billentry_id) purse_service.update_bill_date(bill_entry_id=billentry_id, bill_date=bill_date) return JsonResponse({'success': True, 'message': '保存成功!'})
def save_avatar(cls, relation, type, avatar=None, filename=None): from sentry.models import File if avatar: with transaction.atomic(): photo = File.objects.create( name=filename, type=cls.FILE_TYPE, ) if isinstance(avatar, six.string_types): avatar = BytesIO(avatar) photo.putfile(avatar) else: photo = None with transaction.atomic(): instance, created = cls.objects.get_or_create(**relation) if instance.file and photo: instance.file.delete() if photo: instance.file = photo instance.ident = uuid4().hex instance.avatar_type = [i for i, n in cls.AVATAR_TYPES if n == type][0] instance.save() if photo and not created: instance.clear_cached_photos() return instance
def addfriend(user_id, receiver_id): try: with transaction.atomic(): # get user u = UserDAO.get_user_by_id(user_id) r = UserDAO.get_user_by_id(receiver_id) except: return "fail" # object create new_friend = {"userid": u, "friendid": receiver_id, "datetime": datetime.datetime.now()} # addfriend_request = Addfriend.object.create(new_friend) message = {"userid": r, "type": "3", "content": u.nickname + " wants to add you"} # addfriend_message = Message.objects.create(message) # object commit try: with transaction.atomic(): result = AddFriendDAO.new_addfriend(**new_friend) # add receiver's message MessageDAO.new_message(**message) except: result = "fail" return result
def delete(self, request, args=None): ''' :argument 删除数据库连接,并删除改数据库连接相关的工单记录,执行记录,以及权限表等相关所有数据 :return: success or fail ''' try: with transaction.atomic(): con_id = DatabaseList.objects.filter(connection_name=args).first() work_id = SqlOrder.objects.filter(bundle_id=con_id.id).first() with transaction.atomic(): SqlRecord.objects.filter(workid=work_id).delete() SqlOrder.objects.filter(bundle_id=con_id.id).delete() DatabaseList.objects.filter(connection_name=args).delete() per = grained.objects.all().values('username', 'permissions') for i in per: for c in i['permissions']: if isinstance(i['permissions'][c], list) and c != 'diccon': i['permissions'][c] = list(filter(lambda x: x != args, i['permissions'][c])) grained.objects.filter(username=i['username']).update(permissions=i['permissions']) return Response('数据库信息已删除!') except Exception as e: CUSTOM_ERROR.error(f'{e.__class__.__name__}: {e}') return HttpResponse(status=500)
def delete_user_threads(sender, **kwargs): recount_forums = set() recount_threads = set() for thread in batch_delete(sender.thread_set.all(), 50): recount_forums.add(thread.forum_id) with transaction.atomic(): thread.delete() for post in batch_delete(sender.post_set.all(), 50): recount_forums.add(post.forum_id) recount_threads.add(post.thread_id) with transaction.atomic(): post.delete() if recount_threads: changed_threads_qs = Thread.objects.filter(id__in=recount_threads) for thread in batch_update(changed_threads_qs, 50): thread.synchronize() thread.save() if recount_forums: for forum in Forum.objects.filter(id__in=recount_forums): forum.synchronize() forum.save()
def record_new_project(project, user, **kwargs): if not user.is_authenticated(): user = None try: with transaction.atomic(): OrganizationOnboardingTask.objects.create( organization=project.organization, task=OnboardingTask.FIRST_PROJECT, user=user, status=OnboardingTaskStatus.COMPLETE, project_id=project.id, date_completed=timezone.now(), ) except IntegrityError: try: with transaction.atomic(): OrganizationOnboardingTask.objects.create( organization=project.organization, task=OnboardingTask.SECOND_PLATFORM, user=user, status=OnboardingTaskStatus.PENDING, project_id=project.id, date_completed=timezone.now(), ) except IntegrityError: pass
def addperms(request, rolename): group = get_object_or_404(Group, name=rolename) status = [False, False, ''] if request.method == 'POST': perms = get_system_permissions_filtered(group) form = RolesAddpermForm(perms=perms, data=request.POST) if form.is_valid(): permname = form.cleaned_data['perms'] if permname.startswith('addallperms'): (n, applabel) = permname.split('_') status = [True, True, applabel.capitalize()] allperms = Permission.objects\ .filter(content_type__model='permissions', content_type__app_label=applabel)\ .exclude(codename__icontains='_permissions') with transaction.atomic(): for perm in allperms: group.permissions.add(perm) group.save() else: with transaction.atomic(): status = [True, False, permname] perm = get_object_or_404(Permission, codename=permname, content_type__model='permissions') group.permissions.add(perm) group.save() else: status = [False, False, "Cannot validate a form: %s" % form.errors] return JsonResponse(status, safe=False)
def _update_xblocks_cache(course_key): """ Calculate the XBlock cache data for a course and update the XBlockCache table. """ from .models import XBlockCache blocks_data = _calculate_course_xblocks_data(course_key) def update_block_cache_if_needed(block_cache, block_data): """ Compare block_cache object with data and update if there are differences. """ paths = _paths_from_data(block_data['paths']) if block_cache.display_name != block_data['display_name'] or not paths_equal(block_cache.paths, paths): log.info(u'Updating XBlockCache with usage_key: %s', unicode(block_cache.usage_key)) block_cache.display_name = block_data['display_name'] block_cache.paths = paths block_cache.save() with transaction.atomic(): block_caches = XBlockCache.objects.filter(course_key=course_key) for block_cache in block_caches: block_data = blocks_data.pop(unicode(block_cache.usage_key), None) if block_data: update_block_cache_if_needed(block_cache, block_data) for block_data in blocks_data.values(): with transaction.atomic(): paths = _paths_from_data(block_data['paths']) log.info(u'Creating XBlockCache with usage_key: %s', unicode(block_data['usage_key'])) block_cache, created = XBlockCache.objects.get_or_create(usage_key=block_data['usage_key'], defaults={ 'course_key': course_key, 'display_name': block_data['display_name'], 'paths': paths, }) if not created: update_block_cache_if_needed(block_cache, block_data)
def test_delete_recipe(self): Recipe.objects.all().delete() vodka_id = Ingredient.objects.get(name='Vodka').id oj_id = Ingredient.objects.get(name='Orange Juice').id screwdriver_recipe = { 'name': 'Screwdriver', 'instructions': [ 'Fill glass with ice', 'Add 2 parts Orange Juice', 'Add 1 part Vodka', 'Shake well' ], 'ingredients': { vodka_id: 1, oj_id: 2 } } recipes = self.profile.create_recipe(screwdriver_recipe) screwdriver = recipes.get(name='Screwdriver') profile = self.profile.delete_recipe(screwdriver.id) with self.assertRaises(Recipe.DoesNotExist): with transaction.atomic(): profile.created_recipes.get(id=screwdriver.id) with self.assertRaises(Recipe.DoesNotExist): with transaction.atomic(): Recipe.objects.get(id=screwdriver.id) recipes = Recipe.objects.all() self.assertEqual(0, recipes.count())
def perform_bulk_destroy(self, objects): with transaction.atomic(): for o in objects: if hasattr(o, "snapshot"): o.snapshot() self.perform_destroy(o)
def save(self, project_id, raw=False, assume_normalized=False): # Normalize if needed if not self._normalized: if not assume_normalized: self.normalize() self._normalized = True data = self._data project = Project.objects.get_from_cache(id=project_id) project._organization_cache = Organization.objects.get_from_cache( id=project.organization_id ) # Check to make sure we're not about to do a bunch of work that's # already been done if we've processed an event with this ID. (This # isn't a perfect solution -- this doesn't handle ``EventMapping`` and # there's a race condition between here and when the event is actually # saved, but it's an improvement. See GH-7677.) try: event = Event.objects.get(project_id=project.id, event_id=data["event_id"]) except Event.DoesNotExist: pass else: # Make sure we cache on the project before returning event._project_cache = project logger.info( "duplicate.found", exc_info=True, extra={ "event_uuid": data["event_id"], "project_id": project.id, "platform": data.get("platform"), "model": Event.__name__, }, ) return event # Pull out the culprit culprit = self.get_culprit() # Pull the toplevel data we're interested in level = data.get("level") # TODO(mitsuhiko): this code path should be gone by July 2018. # This is going to be fine because no code actually still depends # on integers here. When we need an integer it will be converted # into one later. Old workers used to send integers here. if level is not None and isinstance(level, six.integer_types): level = LOG_LEVELS[level] transaction_name = data.get("transaction") logger_name = data.get("logger") release = data.get("release") dist = data.get("dist") environment = data.get("environment") recorded_timestamp = data.get("timestamp") # We need to swap out the data with the one internal to the newly # created event object event = self._get_event_instance(project_id=project_id) self._data = data = event.data.data event._project_cache = project date = event.datetime platform = event.platform event_id = event.event_id if transaction_name: transaction_name = force_text(transaction_name) # Right now the event type is the signal to skip the group. This # is going to change a lot. if event.get_event_type() == "transaction": issueless_event = True else: issueless_event = False # Some of the data that are toplevel attributes are duplicated # into tags (logger, level, environment, transaction). These are # different from legacy attributes which are normalized into tags # ahead of time (site, server_name). setdefault_path(data, "tags", value=[]) set_tag(data, "level", level) if logger_name: set_tag(data, "logger", logger_name) if environment: set_tag(data, "environment", environment) if transaction_name: set_tag(data, "transaction", transaction_name) if release: # dont allow a conflicting 'release' tag pop_tag(data, "release") release = Release.get_or_create(project=project, version=release, date_added=date) set_tag(data, "sentry:release", release.version) if dist and release: dist = release.add_dist(dist, date) # dont allow a conflicting 'dist' tag pop_tag(data, "dist") set_tag(data, "sentry:dist", dist.name) else: dist = None event_user = self._get_event_user(project, data) if event_user: # dont allow a conflicting 'user' tag pop_tag(data, "user") set_tag(data, "sentry:user", event_user.tag_value) # At this point we want to normalize the in_app values in case the # clients did not set this appropriately so far. grouping_config = load_grouping_config( get_grouping_config_dict_for_event_data(data, project) ) normalize_stacktraces_for_grouping(data, grouping_config) for plugin in plugins.for_project(project, version=None): added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False) if added_tags: # plugins should not override user provided tags for key, value in added_tags: if get_tag(data, key) is None: set_tag(data, key, value) for path, iface in six.iteritems(event.interfaces): for k, v in iface.iter_tags(): set_tag(data, k, v) # Get rid of ephemeral interface data if iface.ephemeral: data.pop(iface.path, None) # The active grouping config was put into the event in the # normalize step before. We now also make sure that the # fingerprint was set to `'{{ default }}' just in case someone # removed it from the payload. The call to get_hashes will then # look at `grouping_config` to pick the right parameters. data["fingerprint"] = data.get("fingerprint") or ["{{ default }}"] apply_server_fingerprinting(data, get_fingerprinting_config_for_project(project)) # Here we try to use the grouping config that was requested in the # event. If that config has since been deleted (because it was an # experimental grouping config) we fall back to the default. try: hashes = event.get_hashes() except GroupingConfigNotFound: data["grouping_config"] = get_grouping_config_dict_for_project(project) hashes = event.get_hashes() data["hashes"] = hashes # we want to freeze not just the metadata and type in but also the # derived attributes. The reason for this is that we push this # data into kafka for snuba processing and our postprocessing # picks up the data right from the snuba topic. For most usage # however the data is dynamically overridden by Event.title and # Event.location (See Event.as_dict) materialized_metadata = self.materialize_metadata() event_metadata = materialized_metadata["metadata"] data.update(materialized_metadata) data["culprit"] = culprit # index components into ``Event.message`` # See GH-3248 event.message = self.get_search_message(event_metadata, culprit) received_timestamp = event.data.get("received") or float(event.datetime.strftime("%s")) if not issueless_event: # The group gets the same metadata as the event when it's flushed but # additionally the `last_received` key is set. This key is used by # _save_aggregate. group_metadata = dict(materialized_metadata) group_metadata["last_received"] = received_timestamp kwargs = { "platform": platform, "message": event.message, "culprit": culprit, "logger": logger_name, "level": LOG_LEVELS_MAP.get(level), "last_seen": date, "first_seen": date, "active_at": date, "data": group_metadata, } if release: kwargs["first_release"] = release try: group, is_new, is_regression = self._save_aggregate( event=event, hashes=hashes, release=release, **kwargs ) except HashDiscarded: event_discarded.send_robust(project=project, sender=EventManager) metrics.incr( "events.discarded", skip_internal=True, tags={"organization_id": project.organization_id, "platform": platform}, ) raise else: event_saved.send_robust(project=project, event_size=event.size, sender=EventManager) event.group = group else: group = None is_new = False is_regression = False event_saved.send_robust(project=project, event_size=event.size, sender=EventManager) # store a reference to the group id to guarantee validation of isolation event.data.bind_ref(event) environment = Environment.get_or_create(project=project, name=environment) if group: group_environment, is_new_group_environment = GroupEnvironment.get_or_create( group_id=group.id, environment_id=environment.id, defaults={"first_release": release if release else None}, ) else: is_new_group_environment = False if release: ReleaseEnvironment.get_or_create( project=project, release=release, environment=environment, datetime=date ) ReleaseProjectEnvironment.get_or_create( project=project, release=release, environment=environment, datetime=date ) if group: grouprelease = GroupRelease.get_or_create( group=group, release=release, environment=environment, datetime=date ) counters = [(tsdb.models.project, project.id)] if group: counters.append((tsdb.models.group, group.id)) if release: counters.append((tsdb.models.release, release.id)) tsdb.incr_multi(counters, timestamp=event.datetime, environment_id=environment.id) frequencies = [ # (tsdb.models.frequent_projects_by_organization, { # project.organization_id: { # project.id: 1, # }, # }), # (tsdb.models.frequent_issues_by_project, { # project.id: { # group.id: 1, # }, # }) ] if group: frequencies.append( (tsdb.models.frequent_environments_by_group, {group.id: {environment.id: 1}}) ) if release: frequencies.append( (tsdb.models.frequent_releases_by_group, {group.id: {grouprelease.id: 1}}) ) if frequencies: tsdb.record_frequency_multi(frequencies, timestamp=event.datetime) if group: UserReport.objects.filter(project=project, event_id=event_id).update( group=group, environment=environment ) # save the event try: with transaction.atomic(using=router.db_for_write(Event)): event.save() except IntegrityError: logger.info( "duplicate.found", exc_info=True, extra={ "event_uuid": event_id, "project_id": project.id, "group_id": group.id if group else None, "model": Event.__name__, }, ) return event if event_user: counters = [ (tsdb.models.users_affected_by_project, project.id, (event_user.tag_value,)) ] if group: counters.append( (tsdb.models.users_affected_by_group, group.id, (event_user.tag_value,)) ) tsdb.record_multi(counters, timestamp=event.datetime, environment_id=environment.id) if release: if is_new: buffer.incr( ReleaseProject, {"new_groups": 1}, {"release_id": release.id, "project_id": project.id}, ) if is_new_group_environment: buffer.incr( ReleaseProjectEnvironment, {"new_issues_count": 1}, { "project_id": project.id, "release_id": release.id, "environment_id": environment.id, }, ) if not raw: if not project.first_event: project.update(first_event=date) first_event_received.send_robust(project=project, event=event, sender=Project) eventstream.insert( group=group, event=event, is_new=is_new, is_regression=is_regression, is_new_group_environment=is_new_group_environment, primary_hash=hashes[0], # We are choosing to skip consuming the event back # in the eventstream if it's flagged as raw. # This means that we want to publish the event # through the event stream, but we don't care # about post processing and handling the commit. skip_consume=raw, ) metrics.timing("events.latency", received_timestamp - recorded_timestamp) metrics.timing("events.size.data.post_save", event.size) return event
def _add_items(self, source_field_name, target_field_name, *objs): # source_field_name: the PK fieldname in join table for the source object # target_field_name: the PK fieldname in join table for the target object # *objs - objects to add. Either object instances, or primary keys of object instances. # If there aren't any objects, there is nothing to do. from django.db.models import Model if objs: new_ids = set() for obj in objs: if isinstance(obj, self.model): if not router.allow_relation(obj, self.instance): raise ValueError( 'Cannot add "%r": instance is on database "%s", value is on database "%s"' % (obj, self.instance._state.db, obj._state.db) ) fk_val = self.through._meta.get_field( target_field_name).get_foreign_related_value(obj)[0] if fk_val is None: raise ValueError( 'Cannot add "%r": the value for field "%s" is None' % (obj, target_field_name) ) new_ids.add(fk_val) elif isinstance(obj, Model): raise TypeError( "'%s' instance expected, got %r" % (self.model._meta.object_name, obj) ) else: new_ids.add(obj) db = router.db_for_write(self.through, instance=self.instance) vals = (self.through._default_manager.using(db) .values_list(target_field_name, flat=True) .filter(**{ source_field_name: self.related_val[0], '%s__in' % target_field_name: new_ids, })) new_ids = new_ids - set(vals) with transaction.atomic(using=db, savepoint=False): if self.reverse or source_field_name == self.source_field_name: # Don't send the signal when we are inserting the # duplicate data row for symmetrical reverse entries. signals.m2m_changed.send( sender=self.through, action='pre_add', instance=self.instance, reverse=self.reverse, model=self.model, pk_set=new_ids, using=db, ) # Add the ones that aren't there already self.through._default_manager.using(db).bulk_create([ self.through(**{ '%s_id' % source_field_name: self.related_val[0], '%s_id' % target_field_name: obj_id, }) for obj_id in new_ids ]) if self.reverse or source_field_name == self.source_field_name: # Don't send the signal when we are inserting the # duplicate data row for symmetrical reverse entries. signals.m2m_changed.send( sender=self.through, action='post_add', instance=self.instance, reverse=self.reverse, model=self.model, pk_set=new_ids, using=db, )
def create(self, validated_data): active_state_signature_data = validated_data.pop('debit_signature') wallet = validated_data.pop('wallet') recipient = validated_data.pop('recipient') # get current eon current_eon = LocalViewInterface.latest().eon_number() # transfer eon should be the current eon number if validated_data.pop('eon_number') != current_eon: raise serializers.ValidationError( detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC) # TODO refactor this such that the recipient is only locked after the sender's details are verified wallets = sorted([wallet, recipient], key=lambda w: w.trail_identifier) with RootCommitment.read_write_lock( suffix=current_eon, auto_renewal=False), wallets[0].lock( auto_renewal=False), wallets[1].lock(auto_renewal=False): if RootCommitment.objects.filter(eon_number=current_eon + 1).exists(): raise serializers.ValidationError( detail='', code=ErrorCode.EON_NUMBER_OUT_OF_SYNC) transfer = Transfer(wallet=wallet, amount=validated_data.pop('amount'), eon_number=current_eon, recipient=recipient, nonce=validated_data.pop('nonce'), passive=True) wallet_view_context = WalletTransferContext(wallet=wallet, transfer=transfer) recipient_view_context = WalletTransferContext(wallet=recipient, transfer=transfer) # Minimal SLA if not wallet.is_sla_exempt() and not recipient.is_sla_exempt(): if not wallet.has_valid_sla(): sender_transfers_list = wallet_view_context.authorized_transfers_list( only_appended=False, force_append=True) if len(sender_transfers_list) > settings.SLA_THRESHOLD: raise serializers.ValidationError( detail='', code=ErrorCode.DEBIT_WALLET_EXCEEDED_SLA) elif not recipient.has_valid_sla(): recipient_transfers_list = recipient_view_context.authorized_transfers_list( only_appended=False, force_append=True) if len(recipient_transfers_list) > settings.SLA_THRESHOLD: raise serializers.ValidationError( detail='', code=ErrorCode.CREDIT_WALLET_EXCEEDED_SLA) # Ensure sender log consistency can_append_to_sender_log = wallet_view_context.can_schedule_transfer( ) if can_append_to_sender_log is not True: raise serializers.ValidationError( detail='Sender: {}'.format(can_append_to_sender_log), code=ErrorCode.DEBIT_WALLET_CANNOT_ADD_TRANSACTION) # Ensure recipient log consistency can_append_to_recipient_log = recipient_view_context.can_schedule_transfer( ) if can_append_to_recipient_log is not True: raise serializers.ValidationError( detail='Recipient: {}'.format(can_append_to_recipient_log), code=ErrorCode.CREDIT_WALLET_CANNOT_ADD_TRANSACTION) # Ensure transfer consistency can_spend, currently_available_funds = wallet_view_context.can_send_transfer( current_eon_number=current_eon, using_only_appended_funds=False) if can_spend is not True: raise serializers.ValidationError( detail=can_spend, code=ErrorCode.DEBIT_WALLET_OVERSPENDING) # Validate data concise_balance_marker_signature_data = validated_data.pop( 'debit_balance_signature') concise_balance_marker_amount = validated_data.pop('debit_balance') if concise_balance_marker_amount > currently_available_funds - transfer.amount: raise serializers.ValidationError( detail='', code=ErrorCode.DEBIT_WALLET_BALANCE_MARKER_EXCEED_BALANCE) concise_balance_marker = MinimumAvailableBalanceMarker( wallet=wallet, eon_number=transfer.eon_number, amount=concise_balance_marker_amount) concise_balance_marker_checksum = hex_value( concise_balance_marker.checksum()) concise_balance_marker_signature = Signature( wallet=transfer.wallet, checksum=concise_balance_marker_checksum, value=concise_balance_marker_signature_data.get('value')) if not concise_balance_marker_signature.is_valid(): raise serializers.ValidationError( detail='', code=ErrorCode.INVALID_DEBIT_BALANCE_SIGNATURE) tx_set_tree = wallet_view_context.optimized_authorized_transfers_tree( ) tx_set_hash = hex_value(tx_set_tree.root_hash()) transfer_index = tx_set_tree.merkle_tree_nonce_map.get( transfer.nonce) transfer_proof = tx_set_tree.proof(transfer_index) highest_spendings, highest_gains = wallet_view_context.off_chain_actively_sent_received_amounts( eon_number=transfer.eon_number, only_appended=False) active_state = ActiveState(wallet=wallet, updated_spendings=highest_spendings + transfer.amount, updated_gains=highest_gains, tx_set_hash=tx_set_hash, tx_set_proof_hashes=transfer_proof, tx_set_index=transfer_index, eon_number=transfer.eon_number) checksum = hex_value(active_state.checksum()) active_state_signature = Signature( wallet=transfer.wallet, checksum=checksum, value=active_state_signature_data.get('value')) if not active_state_signature.is_valid(): raise serializers.ValidationError( detail='', code=ErrorCode.INVALID_DEBIT_SIGNATURE) transfer.position = recipient_view_context.off_chain_passively_received_amount( eon_number=transfer.eon_number, only_appended=False) # locking context covers saving the state as well to make sure checkpoint creation is consistent with transaction.atomic(): Signature.objects.bulk_create( [concise_balance_marker_signature, active_state_signature]) concise_balance_marker.signature = concise_balance_marker_signature concise_balance_marker.save() active_state.wallet_signature = active_state_signature active_state.operator_signature = active_state.sign_active_state( settings.HUB_OWNER_ACCOUNT_ADDRESS, settings.HUB_OWNER_ACCOUNT_KEY) active_state.save() transfer.sender_active_state = active_state transfer.sender_balance_marker = concise_balance_marker # cache transfer index in sender active set transfer.sender_merkle_index = transfer_index # transfer.sender_merkle_root_cache = tx_set_hash # cache active set merkle mountains height array and hash array for recipient active set transfer.sender_merkle_hash_cache, transfer.sender_merkle_height_cache = tx_set_tree.merkle_cache_stacks( ) transfer.complete = True transfer.appended = True transfer.processed = True transfer.save() if transfer.appended: operator_celery.send_task('auditor.tasks.on_transfer_confirmation', args=[transfer.id]) return transfer
def sync(remote_pk, repository_pk): """ Sync Collections with ``remote_pk``, and save a new RepositoryVersion for ``repository_pk``. Args: remote_pk (str): The remote PK. repository_pk (str): The repository PK. Raises: ValueError: If the remote does not specify a URL to sync or a ``whitelist`` of Collections to sync. """ remote = CollectionRemote.objects.get(pk=remote_pk) repository = Repository.objects.get(pk=repository_pk) if not remote.url: raise ValueError(_("A CollectionRemote must have a 'url' specified to synchronize.")) if not remote.whitelist: raise ValueError(_("A CollectionRemote must have a 'whitelist' specified to synchronize.")) repository_spec_strings = remote.whitelist.split(' ') def nowhere(*args, **kwargs): pass collections_pks = [] with RepositoryVersion.create(repository) as new_version: with tempfile.TemporaryDirectory() as temp_ansible_path: # workaround: mazer logs errors without this dir https://pulp.plan.io/issues/4999 os.mkdir(temp_ansible_path + os.sep + 'ansible_collections') galaxy_context = GalaxyContext( collections_path=temp_ansible_path, server={ 'url': remote.url, 'ignore_certs': False, }, ) install_repository_specs_loop( display_callback=nowhere, galaxy_context=galaxy_context, repository_spec_strings=repository_spec_strings, ) content_walk_generator = os.walk(temp_ansible_path) for dirpath, dirnames, filenames in content_walk_generator: if 'MANIFEST.json' in filenames: with open(dirpath + os.path.sep + 'MANIFEST.json') as manifest_file: manifest_data = json.load(manifest_file) info = manifest_data['collection_info'] filename = '{namespace}-{name}-{version}'.format( namespace=info['namespace'], name=info['name'], version=info['version'], ) tarfile_path = temp_ansible_path + os.path.sep + filename + '.tar.gz' with tarfile.open(name=tarfile_path, mode='w|gz') as newtar: newtar.add(dirpath, arcname=filename) with transaction.atomic(): collection, created = Collection.objects.get_or_create( namespace=info['namespace'], name=info['name'], version=info['version'] ) if created: artifact = Artifact.init_and_validate(newtar.name) artifact.save() ContentArtifact.objects.create( artifact=artifact, content=collection, relative_path=collection.relative_path, ) collections_pks.append(collection) collections = Collection.objects.filter(pk__in=collections_pks) new_version.add_content(collections)
def view(request): ex = None data = {'title': u'Sesiones de Caja'} addUserData(request, data) usuario = request.user if not data['es_cajero']: return HttpResponseRedirect("/?info=El usuario no tiene permisos como cajero") data['cajero'] = cajero = usuario.cajero_set.all()[0] if request.method == 'POST': action = request.POST['action'] if action == 'abrirsesion': f = SesionCajaForm(request.POST) if f.is_valid(): try: with transaction.atomic(): sc = SesionCaja(cajero=cajero, fecha=datetime.now(), fondo=f.cleaned_data['fondo'], abierta=True) sc.save() salva_auditoria(request, sc, ACCION_MODIFICAR, 'Abierta sesion de caja: ' + sc.__str__()) return ok_json() except Exception: return bad_json(error=1) else: return bad_json(error=1) elif action == 'closesesion': sc = SesionCaja.objects.get(pk=request.POST['id']) f = CierreSesionCajaForm(request.POST) if f.is_valid(): try: with transaction.atomic(): sc.bill100 = f.cleaned_data['bill100'] sc.bill50 = f.cleaned_data['bill50'] sc.bill20 = f.cleaned_data['bill20'] sc.bill10 = f.cleaned_data['bill10'] sc.bill5 = f.cleaned_data['bill5'] sc.bill2 = f.cleaned_data['bill2'] sc.bill1 = f.cleaned_data['bill1'] sc.enmonedas = f.cleaned_data['enmonedas'] sc.cheque = f.cleaned_data['cheque'] sc.deposito = f.cleaned_data['deposito'] sc.transferencia = f.cleaned_data['transferencia'] sc.abierta = False sc.fechacierre = datetime.now() sc.horacierre = datetime.now().time() sc.save() salva_auditoria(request, sc, ACCION_MODIFICAR, 'Cerrada sesion de caja: {}'.format(sc)) return ok_json() except Exception: return bad_json(error=1) else: return bad_json(error=1) return bad_json(error=0) else: if 'action' in request.GET: action = request.GET['action'] if action == 'addsesion': try: data['title'] = u'Abrir Sesión de Caja' data['form'] = SesionCajaForm(initial={'fondo': '0.00'}) return render_to_response("caja/adicionarbs.html", data) except Exception as ex: pass elif action == 'closesesion': try: data['title'] = u'Cierre Sesión de Caja' data['sesion'] = SesionCaja.objects.get(pk=request.GET['id']) data['form'] = CierreSesionCajaForm(initial={'bill100': 0, 'bill50': 0, 'bill20': 0, 'bill10': 0, 'bill5': 0, 'bill2': 0, 'bill1': 0, 'enmonedas': 0.00, 'cheque': 0.00, 'deposito': 0.00, 'transferencia': 0.00, 'total': 0.00}) return render_to_response("caja/cerrarsesionbs.html", data) except Exception as ex: pass return HttpResponseRedirect(url_back(request, mensaje_excepcion(ex.args[0]))) else: sesiones = cajero.sesiones_caja() paging = MiPaginador(sesiones, 30) p = 1 try: if 'page' in request.GET: p = int(request.GET['page']) sesionespagina = paging.page(p) except: sesionespagina = paging.page(1) data['paging'] = paging data['rangospaging'] = paging.rangos_paginado(p) data['page'] = sesionespagina data['sesiones'] = sesionespagina.object_list return render_to_response("caja/sesionesbs.html", data)
def process_free_opinion_result(self, row_pk, cnt): """Process a single result from the free opinion report""" result = PACERFreeDocumentRow.objects.get(pk=row_pk) result.court = Court.objects.get(pk=map_pacer_to_cl_id(result.court_id)) result.case_name = harmonize(result.case_name) result.case_name_short = cnt.make_case_name_short(result.case_name) row_copy = copy.copy(result) # If we don't do this, the doc's date_filed becomes the docket's # date_filed. Bad. delattr(row_copy, 'date_filed') # If we don't do this, we get the PACER court id and it crashes delattr(row_copy, 'court_id') # If we don't do this, the id of result tries to smash that of the docket. delattr(row_copy, 'id') try: with transaction.atomic(): docket = lookup_and_save(row_copy) if not docket: msg = "Unable to create docket for %s" % result logger.error(msg) result.error_msg = msg result.save() self.request.callbacks = None return docket.blocked, docket.date_blocked = get_blocked_status(docket) docket.save() de, de_created = DocketEntry.objects.update_or_create( docket=docket, entry_number=result.document_number, defaults={ 'date_filed': result.date_filed, 'description': result.description, }) rd, rd_created = RECAPDocument.objects.update_or_create( docket_entry=de, document_number=result.document_number, attachment_number=None, defaults={ 'pacer_doc_id': result.pacer_doc_id, 'document_type': RECAPDocument.PACER_DOCUMENT, 'is_free_on_pacer': True, }) except IntegrityError as e: msg = "Raised IntegrityError: %s" % e logger.error(msg) if self.request.retries == self.max_retries: result.error_msg = msg result.save() return raise self.retry(exc=e) except DatabaseError as e: msg = "Unable to complete database transaction:\n%s" % e logger.error(msg) result.error_msg = msg result.save() self.request.callbacks = None return if not rd_created and rd.is_available: # The item already exists and is available. Fantastic, mark it as free, # and call it a day. rd.is_free_on_pacer = True rd.save() result.delete() self.request.callbacks = None return return { 'result': result, 'rd_pk': rd.pk, 'pacer_court_id': result.court_id }
def handle_outdated_resources(plan): """ Marks repositories, importers, distributors which are no longer present in Pulp2. Delete Publications and Distributions which are no longer present in Pulp2. Args: plan(MigrationPlan): A Migration Plan """ RepoSetup.mark_changed_relations() RepoSetup.finalize() for plugin_plan in plan.get_plugin_plans(): inplan_repos = plugin_plan.get_repositories() # filter by repo type and by the repos specified in a plan repos_to_consider = plan.type_to_repo_ids[plugin_plan.type] repos_to_consider = set(inplan_repos).intersection(repos_to_consider) mongo_repo_q = mongo_Q(repo_id__in=repos_to_consider) mongo_repo_obj_ids = set(str(i.id) for i in Repository.objects(mongo_repo_q).only('id')) repo_type_q = Q(pulp2_repo_type=plugin_plan.type) inplan_repo_q = Q(pulp2_object_id__in=mongo_repo_obj_ids) Pulp2Repository.objects.filter(repo_type_q).exclude(inplan_repo_q).update(not_in_plan=True) # Mark removed or excluded importers inplan_imp_repos = plugin_plan.get_importers_repos() mongo_imp_q = mongo_Q(repo_id__in=inplan_imp_repos) mongo_imp_obj_ids = set(str(i.id) for i in Importer.objects(mongo_imp_q).only('id')) imp_types = plugin_plan.migrator.importer_migrators.keys() imp_type_q = Q(pulp2_type_id__in=imp_types) inplan_imp_q = Q(pulp2_object_id__in=mongo_imp_obj_ids) Pulp2Importer.objects.filter(imp_type_q).exclude(inplan_imp_q).update(not_in_plan=True) # Mark removed or excluded distributors inplan_dist_repos = plugin_plan.get_distributors_repos() mongo_dist_q = mongo_Q(repo_id__in=inplan_dist_repos) mongo_dist_obj_ids = set(str(i.id) for i in Distributor.objects(mongo_dist_q).only('id')) dist_types = plugin_plan.migrator.distributor_migrators.keys() dist_type_q = Q(pulp2_type_id__in=dist_types) inplan_dist_q = Q(pulp2_object_id__in=mongo_dist_obj_ids) Pulp2Distributor.objects.filter(dist_type_q).exclude(inplan_dist_q).update(not_in_plan=True) # Delete old Publications/Distributions which are no longer present in Pulp2. # It's critical to remove Distributions to avoid base_path overlap. # It makes the migration logic easier if we remove old Publications as well. # Delete criteria: # - pulp2distributor is no longer in plan # - pulp2repository content changed (repo.is_migrated=False) or it is no longer in plan repos_with_old_distributions_qs = Pulp2Repository.objects.filter( Q(is_migrated=False) | Q(not_in_plan=True) ) old_dist_query = Q(pulp3_distribution__isnull=False) | Q(pulp3_publication__isnull=False) old_dist_query &= Q(pulp2_repos__in=repos_with_old_distributions_qs) | Q(not_in_plan=True) with transaction.atomic(): pulp2distributors_with_old_distributions_qs = Pulp2Distributor.objects.filter( old_dist_query ) pulp2distributors_with_old_distributions_qs.update( is_migrated=False ) # If publication is shared by multiple distributions, on the corresponding distributors # flip the flag to false so the affected distributions will be updated with the new # publication Pulp2Distributor.objects.filter( pulp3_publication__in=Publication.objects.filter( pulp2distributor__in=pulp2distributors_with_old_distributions_qs ) ).update(is_migrated=False) # Delete outdated publications Publication.objects.filter( pulp2distributor__in=pulp2distributors_with_old_distributions_qs).delete() # Delete outdated distributions BaseDistribution.objects.filter( pulp2distributor__in=pulp2distributors_with_old_distributions_qs).delete() # Remove relations to the pulp2repository in case the relation changed. # Pulp2Distributors with is_migrated=false is handled and re-added properly at # migration stage. # NOTE: this needs to be removed last, the queries above use this relation. not_migrated_dists = Pulp2Distributor.objects.filter(is_migrated=False).only('pulp_id') Pulp2Distributor.pulp2_repos.through.objects.filter( pulp2distributor__in=not_migrated_dists ).delete()
def _save_aggregate(self, event, hashes, release, **kwargs): project = event.project # attempt to find a matching hash all_hashes = self._find_hashes(project, hashes) try: existing_group_id = (h[0] for h in all_hashes if h[0]).next() except StopIteration: existing_group_id = None # XXX(dcramer): this has the opportunity to create duplicate groups # it should be resolved by the hash merging function later but this # should be better tested/reviewed if existing_group_id is None: kwargs['score'] = ScoreClause.calculate(1, kwargs['last_seen']) with transaction.atomic(): short_id = project.next_short_id() group, group_is_new = Group.objects.create(project=project, short_id=short_id, **kwargs), True else: group = Group.objects.get(id=existing_group_id) group_is_new = False # If all hashes are brand new we treat this event as new is_new = False new_hashes = [h[1] for h in all_hashes if h[0] is None] if new_hashes: affected = GroupHash.objects.filter( project=project, hash__in=new_hashes, group__isnull=True, ).update(group=group, ) if affected != len(new_hashes): self._ensure_hashes_merged(group, new_hashes) elif group_is_new and len(new_hashes) == len(all_hashes): is_new = True # XXX(dcramer): it's important this gets called **before** the aggregate # is processed as otherwise values like last_seen will get mutated can_sample = should_sample( event.data.get('received') or float(event.datetime.strftime('%s')), group.data.get('last_received') or float(group.last_seen.strftime('%s')), group.times_seen, ) if not is_new: is_regression = self._process_existing_aggregate( group=group, event=event, data=kwargs, release=release, ) else: is_regression = False # Determine if we've sampled enough data to store this event if is_new or is_regression: is_sample = False else: is_sample = can_sample tsdb.incr_multi([ (tsdb.models.group, group.id), (tsdb.models.project, project.id), ], timestamp=event.datetime) tsdb.record_frequency_multi([ (tsdb.models.frequent_projects_by_organization, { project.organization_id: { project.id: 1, }, }), (tsdb.models.frequent_issues_by_project, { project.id: { group.id: 1, }, }), ], timestamp=event.datetime) return group, is_new, is_regression, is_sample
def set_session(self, session): with transaction.atomic(): RoomToSession.objects.filter(room_name=self.name).delete() if session: RoomToSession.objects.create(room_name=self.name, session=session)
def test_fk_assignment_and_related_object_cache(self): # Tests of ForeignKey assignment and the related-object cache (see #6886). p = Parent.objects.create(name="Parent") c = Child.objects.create(name="Child", parent=p) # Look up the object again so that we get a "fresh" object. c = Child.objects.get(name="Child") p = c.parent # Accessing the related object again returns the exactly same object. self.assertIs(c.parent, p) # But if we kill the cache, we get a new object. del c._parent_cache self.assertIsNot(c.parent, p) # Assigning a new object results in that object getting cached immediately. p2 = Parent.objects.create(name="Parent 2") c.parent = p2 self.assertIs(c.parent, p2) # Assigning None succeeds if field is null=True. p.bestchild = None self.assertIsNone(p.bestchild) # bestchild should still be None after saving. p.save() self.assertIsNone(p.bestchild) # bestchild should still be None after fetching the object again. p = Parent.objects.get(name="Parent") self.assertIsNone(p.bestchild) # Assigning None will not fail: Child.parent is null=False. setattr(c, "parent", None) # You also can't assign an object of the wrong type here with self.assertRaises(ValueError): setattr(c, "parent", First(id=1, second=1)) # You can assign None to Child.parent during object creation. Child(name='xyzzy', parent=None) # But when trying to save a Child with parent=None, the database will # raise IntegrityError. with self.assertRaises(IntegrityError), transaction.atomic(): Child.objects.create(name='xyzzy', parent=None) # Creation using keyword argument should cache the related object. p = Parent.objects.get(name="Parent") c = Child(parent=p) self.assertIs(c.parent, p) # Creation using keyword argument and unsaved related instance (#8070). p = Parent() msg = "save() prohibited to prevent data loss due to unsaved related object 'parent'." with self.assertRaisesMessage(ValueError, msg): Child.objects.create(parent=p) msg = "save() prohibited to prevent data loss due to unsaved related object 'parent'." with self.assertRaisesMessage(ValueError, msg): ToFieldChild.objects.create(parent=p) # Creation using attname keyword argument and an id will cause the # related object to be fetched. p = Parent.objects.get(name="Parent") c = Child(parent_id=p.id) self.assertIsNot(c.parent, p) self.assertEqual(c.parent, p)
def pre_migrate_all_without_content(plan): """ Pre-migrate repositories, relations to their contents, importers and distributors. Look at the last updated times in the pulp2to3 tables for repositories/importers/distributors: * pulp2_last_unit_added or pulp2_last_unit_removed for repositories * pulp2_last_updated for importers and distributors Query empty-never-had-content repos (can't filter them out in any way) and repos for which there were: * content changes since the last run * importer changes since the last run * distributor changes since the last run Query in order of last_unit_added for the case when pre-migration is interrupted before we are done with repositories. Args: plan(MigrationPlan): A Migration Plan """ _logger.debug('Pre-migrating Pulp 2 repositories') with ProgressReport(message='Processing Pulp 2 repositories, importers, distributors', code='processing.repositories', total=0) as pb: for plugin_plan in plan.get_plugin_plans(): repos = plugin_plan.get_repositories() importers_repos = plugin_plan.get_importers_repos() distributors_repos = plugin_plan.get_distributors_repos() importer_types = list(plugin_plan.migrator.importer_migrators.keys()) distributor_migrators = plugin_plan.migrator.distributor_migrators distributor_types = list(distributor_migrators.keys()) # figure out which repos/importers/distributors have been updated since the last run epoch = datetime.utcfromtimestamp(0) repo_type_q = Q(pulp2_repo_type=plugin_plan.type) imp_type_q = Q(pulp2_type_id__in=importer_types) dist_type_q = Q(pulp2_type_id__in=distributor_types) plugin_pulp2repos = Pulp2Repository.objects.filter(repo_type_q) repo_premigrated_last_by_added = plugin_pulp2repos.aggregate( Max('pulp2_last_unit_added') )['pulp2_last_unit_added__max'] or epoch repo_premigrated_last_by_removed = plugin_pulp2repos.aggregate( Max('pulp2_last_unit_removed') )['pulp2_last_unit_removed__max'] or epoch imp_premigrated_last = Pulp2Importer.objects.filter(imp_type_q).aggregate( Max('pulp2_last_updated') )['pulp2_last_updated__max'] or epoch dist_premigrated_last = Pulp2Distributor.objects.filter(dist_type_q).aggregate( Max('pulp2_last_updated') )['pulp2_last_updated__max'] or epoch is_content_added_q = mongo_Q(last_unit_added__gte=repo_premigrated_last_by_added) is_content_removed_q = mongo_Q(last_unit_removed__gte=repo_premigrated_last_by_removed) is_new_enough_repo_q = is_content_added_q | is_content_removed_q is_empty_repo_q = mongo_Q(last_unit_added__exists=False) is_new_enough_imp_q = mongo_Q(last_updated__gte=imp_premigrated_last) is_new_enough_dist_q = mongo_Q(last_updated__gte=dist_premigrated_last) repo_repo_id_q = mongo_Q(repo_id__in=repos) imp_repo_id_q = mongo_Q(repo_id__in=importers_repos) dist_repo_id_q = mongo_Q(repo_id__in=distributors_repos) updated_importers = Importer.objects( imp_repo_id_q & is_new_enough_imp_q ).only('repo_id') updated_imp_repos = set(imp.repo_id for imp in updated_importers) updated_distributors = Distributor.objects( dist_repo_id_q & is_new_enough_dist_q ).only('repo_id') updated_dist_repos = set(dist.repo_id for dist in updated_distributors) updated_impdist_repos = updated_imp_repos | updated_dist_repos mongo_updated_repo_q = repo_repo_id_q & (is_new_enough_repo_q | is_empty_repo_q) mongo_updated_imp_dist_repo_q = mongo_Q(repo_id__in=updated_impdist_repos) mongo_repo_qs = Repository.objects( mongo_updated_repo_q | mongo_updated_imp_dist_repo_q ).order_by('last_unit_added') pb.total += mongo_repo_qs.count() pb.save() for repo_data in mongo_repo_qs.only('id', 'repo_id', 'last_unit_added', 'last_unit_removed', 'description'): repo_id = repo_data.repo_id with transaction.atomic(): if repo_id in repos: pre_migrate_repo(repo_data, plan.repo_id_to_type) if repo_id in importers_repos: pre_migrate_importer(repo_id, importer_types) if repo_id in distributors_repos: pre_migrate_distributor(repo_id, distributor_migrators) pb.increment()
def update_api(request): """ 修改接口信息 project_id 项目ID api_id 接口ID first_group_id 一级分组ID second_group_id 二级分组ID name 接口名称 httpType HTTP/HTTPS requestType 请求方式 address 请求地址 headDict 头文件 requestParameterType 参数请求格式 requestList 请求参数列表 responseList 返回参数列表 mockStatus mockhttp状态 code mock代码 description 描述 :return: """ data = json.loads(request.body) if not data["project_id"] or not data["first_group_id"] or not data["name"] or not data["httpType"] or not \ data["requestType"] or not data["address"] or not data["requestParameterType"] or not data["status"] or \ not data["api_id"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if not isinstance(data["project_id"], int) or not isinstance(data["first_group_id"], int) or \ not isinstance(data["api_id"], int): return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["status"] not in ["True", "False"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if not isinstance(data["project_id"], int) or not isinstance( data["first_group_id"], int): return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["httpType"] not in ["HTTP", "HTTPS"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["requestType"] not in ["POST", "GET", "PUT", "DELETE"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["requestParameterType"] not in ["form-data", "raw", "Restful"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) obj = Project.objects.filter(id=data["project_id"]) if obj: obm = ApiInfo.objects.filter(id=data["api_id"], project=data["project_id"]) if obm: obi = ApiInfo.objects.filter( name=data["name"], project=data["project_id"]).exclude(id=data["api_id"]) if len(obi) == 0: try: with transaction.atomic(): first_group = ApiGroupLevelFirst.objects.filter( id=data["first_group_id"], project=data["project_id"]) if len(first_group) == 0: return JsonResponse( code_msg=GlobalStatusCode.group_not_exist()) if data["first_group_id"] and data["second_group_id"]: if not isinstance(data["second_group_id"], int): return JsonResponse(code_msg=GlobalStatusCode. parameter_wrong()) second_group = ApiGroupLevelSecond.objects.filter( id=data["second_group_id"], apiGroupLevelFirst=data["first_group_id"]) if len(second_group) == 0: return JsonResponse(code_msg=GlobalStatusCode. group_not_exist()) try: obm.update( project=Project.objects.get( id=data["project_id"]), apiGroupLevelFirst=ApiGroupLevelFirst. objects.get(id=data["first_group_id"]), apiGroupLevelSecond=ApiGroupLevelSecond. objects.get(id=data["second_group_id"]), name=data["name"], httpType=data["httpType"], requestType=data["requestType"], apiAddress=data["address"], requestParameterType=data[ "requestParameterType"], mockCode=data["mockStatus"], data=data["code"], status=data["status"], userUpdate=User.objects.get( id=request.user.pk), description=data["description"]) except KeyError: return JsonResponse(code_msg=GlobalStatusCode. parameter_wrong()) else: return JsonResponse( code_msg=GlobalStatusCode.parameter_wrong()) if len(data["headDict"]): _list = [] for j in data["headDict"]: try: _list.append(j["id"]) except KeyError: pass parameter = ApiHead.objects.filter( api=data["api_id"]) for n in parameter: if n.pk not in _list: n.delete() for i in data["headDict"]: if i["name"]: try: ApiHead.objects.filter(id=i["id"], api=data["api_id"]).\ update(name=i["name"], value=i["value"]) except KeyError: ApiHead(api=ApiInfo.objects.get( id=data["api_id"]), name=i["name"], value=i["value"]).save() if data["requestParameterType"] == "form-data": ApiParameterRaw.objects.filter( api=data["api_id"]).delete() if len(data["requestList"]): _list = [] for j in data["requestList"]: try: _list.append(j["id"]) except KeyError: pass parameter = ApiParameter.objects.filter( api=data["api_id"]) for n in parameter: if n.pk not in _list: n.delete() for i in data["requestList"]: try: if i["name"]: try: ApiParameter.objects.filter(id=i["id"], api=data["api_id"]).\ update(name=i["name"], value=i["value"], required=i["required"], restrict=i["restrict"], _type=i["_type"], description=i["description"]) except KeyError: ApiParameter( api=ApiInfo.objects.get( id=data["api_id"]), name=i["name"], value=i["value"], required=i["required"], _type=i["_type"], description=i[ "description"]).save() except KeyError: logging.exception("Error") return JsonResponse( code_msg=GlobalStatusCode. parameter_wrong()) else: ApiParameterRaw.objects.filter( api=data["api_id"]).delete() ApiParameter.objects.filter( api=data["api_id"]).delete() if data["requestList"]: ApiParameterRaw( api=ApiInfo.objects.get(id=data["api_id"]), data=data["requestList"]).save() if len(data["responseList"]): _list = [] for j in data["responseList"]: try: _list.append(j["id"]) except KeyError: pass parameter = ApiResponse.objects.filter( api=data["api_id"]) for n in parameter: if n.pk not in _list: n.delete() for i in data["responseList"]: if i["name"]: try: ApiResponse.objects.filter(id=i["id"], api=data["api_id"]).\ update(name=i["name"], value=i["value"], required=i["required"], _type=i["_type"], description=i["description"]) except KeyError: ApiResponse( api=ApiInfo.objects.get( id=data["api_id"]), name=i["name"], value=i["value"], required=i["required"], _type=i["_type"], description=i["description"]).save( ) record_dynamic(data["project_id"], "修改", "接口", "修改接口“%s”" % data["name"]) api_record = ApiOperationHistory( apiInfo=ApiInfo.objects.get(id=data["api_id"]), user=User.objects.get(id=request.user.pk), description="修改接口\"%s\"" % data["name"]) api_record.save() return JsonResponse( code_msg=GlobalStatusCode.success()) except Exception as e: logging.exception("ERROR") logging.error(e) return JsonResponse(code_msg=GlobalStatusCode.fail()) else: return JsonResponse(code_msg=GlobalStatusCode.api_is_exist()) else: return JsonResponse(code_msg=GlobalStatusCode.api_not_exist()) else: return JsonResponse(code_msg=GlobalStatusCode.project_not_exist())
def save(self, project, raw=False): from sentry.tasks.post_process import index_event_tags project = Project.objects.get_from_cache(id=project) data = self.data.copy() # First we pull out our top-level (non-data attr) kwargs event_id = data.pop('event_id') message = data.pop('message') level = data.pop('level') culprit = data.pop('culprit', None) logger_name = data.pop('logger', None) server_name = data.pop('server_name', None) site = data.pop('site', None) checksum = data.pop('checksum', None) fingerprint = data.pop('fingerprint', None) platform = data.pop('platform', None) release = data.pop('release', None) environment = data.pop('environment', None) # unused time_spent = data.pop('time_spent', None) if not culprit: culprit = generate_culprit(data, platform=platform) date = datetime.fromtimestamp(data.pop('timestamp')) date = date.replace(tzinfo=timezone.utc) kwargs = { 'message': message, 'platform': platform, } event = Event(project_id=project.id, event_id=event_id, data=data, time_spent=time_spent, datetime=date, **kwargs) tags = data.get('tags') or [] tags.append(('level', LOG_LEVELS[level])) if logger_name: tags.append(('logger', logger_name)) if server_name: tags.append(('server_name', server_name)) if site: tags.append(('site', site)) if release: # TODO(dcramer): we should ensure we create Release objects tags.append(('sentry:release', release)) if environment: tags.append(('environment', environment)) for plugin in plugins.for_project(project, version=None): added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False) if added_tags: tags.extend(added_tags) event_user = self._get_event_user(project, data) if event_user: tags.append(('sentry:user', event_user.tag_value)) # XXX(dcramer): we're relying on mutation of the data object to ensure # this propagates into Event data['tags'] = tags data['fingerprint'] = fingerprint or ['{{ default }}'] # Get rid of ephemeral interface data for interface_class, _ in iter_interfaces(): interface = interface_class() if interface.ephemeral: data.pop(interface.get_path(), None) # prioritize fingerprint over checksum as its likely the client defaulted # a checksum whereas the fingerprint was explicit if fingerprint: hashes = map(md5_from_hash, get_hashes_from_fingerprint(event, fingerprint)) elif checksum: hashes = [checksum] else: hashes = map(md5_from_hash, get_hashes_for_event(event)) # TODO(dcramer): temp workaround for complexity data['message'] = message event_type = eventtypes.get(data.get('type', 'default'))(data) group_kwargs = kwargs.copy() group_kwargs.update({ 'culprit': culprit, 'logger': logger_name, 'level': level, 'last_seen': date, 'first_seen': date, 'data': { 'last_received': event.data.get('received') or float(event.datetime.strftime('%s')), 'type': event_type.key, # we cache the events metadata on the group to ensure its # accessible in the stream 'metadata': event_type.get_metadata(), }, }) # TODO(dcramer): temp workaround for complexity del data['message'] if release: release = Release.get_or_create( project=project, version=release, date_added=date, ) group_kwargs['first_release'] = release group, is_new, is_regression, is_sample = self._save_aggregate( event=event, hashes=hashes, release=release, **group_kwargs) event.group = group # store a reference to the group id to guarantee validation of isolation event.data.bind_ref(event) try: with transaction.atomic(using=router.db_for_write(EventMapping)): EventMapping.objects.create(project=project, group=group, event_id=event_id) except IntegrityError: self.logger.info('Duplicate EventMapping found for event_id=%s', event_id, exc_info=True) return event UserReport.objects.filter( project=project, event_id=event_id, ).update(group=group) # save the event unless its been sampled if not is_sample: try: with transaction.atomic(using=router.db_for_write(Event)): event.save() except IntegrityError: self.logger.info('Duplicate Event found for event_id=%s', event_id, exc_info=True) return event index_event_tags.delay( project_id=project.id, group_id=group.id, event_id=event.id, tags=tags, ) if event_user: tsdb.record_multi(( (tsdb.models.users_affected_by_group, group.id, (event_user.tag_value, )), (tsdb.models.users_affected_by_project, project.id, (event_user.tag_value, )), ), timestamp=event.datetime) if is_new and release: buffer.incr(Release, {'new_groups': 1}, { 'id': release.id, }) safe_execute(Group.objects.add_tags, group, tags, _with_transaction=False) if not raw: if not project.first_event: project.update(first_event=date) first_event_received.send(project=project, group=group, sender=Project) post_process_group.delay( group=group, event=event, is_new=is_new, is_sample=is_sample, is_regression=is_regression, ) else: self.logger.info( 'Raw event passed; skipping post process for event_id=%s', event_id) # TODO: move this to the queue if is_regression and not raw: regression_signal.send_robust(sender=Group, instance=group) return event
def _perform_update_iteration(self): """Performs a single iteration of the database update """ # Retrieve 500 job executions that need to be updated and get job IDs job_ids = set() for job_exe in JobExecution.objects.filter(status__isnull=False).only('id', 'job_id')[:500]: job_ids.add(job_exe.job_id) # Retrieve all job executions for those jobs in sorted order job_exe_count = 0 current_job_id = None current_exe_num = 1 exe_num_dict = {} # {exe_num: [job_exe.id]} job_exe_end_models = [] job_exe_output_models = [] job_exe_qry = JobExecution.objects.select_related('job').filter(job_id__in=job_ids) for job_exe in job_exe_qry.defer('resources', 'configuration', 'stdout', 'stderr').order_by('job_id', 'id'): job_exe_count += 1 if job_exe.job_id == current_job_id: current_exe_num += 1 else: current_job_id = job_exe.job_id current_exe_num = 1 # This job_exe model needs to be updated with its exe_num if current_exe_num in exe_num_dict: exe_num_dict[current_exe_num].append(job_exe.id) else: exe_num_dict[current_exe_num] = [job_exe.id] if job_exe.status in ['COMPLETED', 'FAILED', 'CANCELED']: # Create corresponding job_exe_end model job_exe_end = JobExecutionEnd() job_exe_end.job_exe_id = job_exe.id job_exe_end.job_id = job_exe.job_id job_exe_end.job_type_id = job_exe.job.job_type_id job_exe_end.exe_num = current_exe_num # Create task results from job_exe task fields task_list = [] if job_exe.pre_started: pre_task_dict = {'task_id': '%s_%s' % (job_exe.get_cluster_id(), 'pre'), 'type': 'pre', 'was_launched': True, 'was_started': True, 'started': datetime_to_string(job_exe.pre_started)} if job_exe.pre_completed: pre_task_dict['ended'] = datetime_to_string(job_exe.pre_completed) if job_exe.pre_exit_code is not None: pre_task_dict['exit_code'] = job_exe.pre_exit_code task_list.append(pre_task_dict) if job_exe.job_started: job_task_dict = {'task_id': '%s_%s' % (job_exe.get_cluster_id(), 'job'), 'type': 'main', 'was_launched': True, 'was_started': True, 'started': datetime_to_string(job_exe.job_started)} if job_exe.job_completed: job_task_dict['ended'] = datetime_to_string(job_exe.job_completed) if job_exe.job_exit_code is not None: job_task_dict['exit_code'] = job_exe.job_exit_code task_list.append(job_task_dict) if job_exe.post_started: post_task_dict = {'task_id': '%s_%s' % (job_exe.get_cluster_id(), 'post'), 'type': 'post', 'was_launched': True, 'was_started': True, 'started': datetime_to_string(job_exe.post_started)} if job_exe.post_completed: post_task_dict['ended'] = datetime_to_string(job_exe.post_completed) if job_exe.post_exit_code is not None: post_task_dict['exit_code'] = job_exe.post_exit_code task_list.append(post_task_dict) task_results = TaskResults({'tasks': task_list}) job_exe_end.task_results = task_results.get_dict() job_exe_end.status = job_exe.status job_exe_end.error_id = job_exe.error_id job_exe_end.node_id = job_exe.node_id job_exe_end.queued = job_exe.queued job_exe_end.started = job_exe.started job_exe_end.seed_started = task_results.get_task_started('main') job_exe_end.seed_ended = task_results.get_task_ended('main') job_exe_end.ended = job_exe.ended job_exe_end_models.append(job_exe_end) if job_exe.status == 'COMPLETED': # Create corresponding job_exe_output model job_exe_output = JobExecutionOutput() job_exe_output.job_exe_id = job_exe.id job_exe_output.job_id = job_exe.job_id job_exe_output.job_type_id = job_exe.job.job_type_id job_exe_output.exe_num = current_exe_num job_exe_output.output = job_exe.results job_exe_output_models.append(job_exe_output) # Update/create models in an atomic transaction with transaction.atomic(): for exe_num, job_exe_ids in exe_num_dict.items(): JobExecution.objects.filter(id__in=job_exe_ids).update(exe_num=exe_num, status=None, error_id=None, command_arguments=None, environment=None, cpus_scheduled=None, mem_scheduled=None, disk_out_scheduled=None, disk_total_scheduled=None, pre_started=None, pre_completed=None, pre_exit_code=None, job_started=None, job_completed=None, job_exit_code=None, job_metrics=None, post_started=None, post_completed=None, post_exit_code=None, stdout=None, stderr=None, results_manifest=None, results=None, ended=None, last_modified=None) JobExecutionEnd.objects.bulk_create(job_exe_end_models) JobExecutionOutput.objects.bulk_create(job_exe_output_models) logger.info('Updated %d job executions', job_exe_count) self._updated_job_exe += job_exe_count percent = (float(self._updated_job_exe) / float(self._total_job_exe)) * 100.00 print 'Completed %s of %s job executions (%.1f%%)' % (self._updated_job_exe, self._total_job_exe, percent)
def harvest(self, log_id=None, ignore_disabled=False, ingest=True, exhaust=True, superfluous=False, force=False, limit=None): """Complete the harvest of the given HarvestLog or next the next available HarvestLog. Args: log_id (int, optional): Harvest the given log. Defaults to None. If the given log cannot be locked, the task will retry indefinitely. If the given log belongs to a disabled or deleted Source or SourceConfig, the task will fail. ingest (bool, optional): Whether or not to start the full ingest process for harvested data. Defaults to True. exhaust (bool, optional): Whether or not to start another harvest task if one is found. Defaults to True. Used to prevent a backlog of harvests. If we have a valid job, spin off another task to eat through the rest of the queue. superfluous (bool, optional): Re-ingest Rawdata that we've already collected. Defaults to False. """ qs = HarvestLog.objects.all() if log_id is not None: logger.debug('Loading harvest log %d', log_id) qs = qs.filter(id=log_id) else: logger.debug( 'log_id was not specified, searching for an available log.') if not ignore_disabled: qs = qs.exclude(source_config__disabled=True, ).exclude( source_config__source__is_deleted=True) qs = qs.filter( status__in=HarvestLog.READY_STATUSES, end_date__lte=timezone.now().date(), source_config__harvest_after__lte=timezone.now().time(), ).unlocked('source_config') with qs.lock_first('source_config') as log: if log is None and log_id is None: logger.warning('No HarvestLogs are currently available') return None if log is None and log_id is not None: # If an id was given to us, we should have gotten a log log = HarvestLog.objects.get(id=log_id) # Force the failure raise Exception('Failed to load {} but then found {!r}.'.format( log_id, log)) # Should never be reached if self.request.id: # Additional attributes for the celery backend # Allows for better analytics of currently running tasks self.update_state( meta={ 'log_id': log.id, 'source': log.source_config.source.long_title, 'source_config': log.source_config.label, }) log.task_id = self.request.id HarvestLog.objects.filter(id=log.id).update( task_id=self.request.id) if log.completions > 0 and log.status == HarvestLog.STATUS.succeeded and not superfluous: log.skip(HarvestLog.SkipReasons.duplicated) logger.warning( '%r has already been harvested. Force a re-run with superfluous=True', log) return None elif log.completions > 0 and log.status == HarvestLog.STATUS.succeeded: logger.info( '%r has already been harvested. Re-running superfluously', log) if log.harvester_version < log.source_config.harvester.version: # If a harvest log has an outdated harvester_version but has not been run before, we can go ahead and upgrade it. # Otherwise, mark it obsolete and skip it. if log.completions > 0: log.skip(HarvestLog.SkipReasons.obsolete) logger.warning( '%r is outdated but has previously completed, skipping...', log) return None try: # Attempt to upgrade the log with transaction.atomic(): log.harvester_version = log.source_config.harvester.version log.save() except IntegrityError: # Sometimes a new harvest log will already be generated for one reason or another. # We can safely mark this log obsolete log.skip(HarvestLog.SkipReasons.obsolete) logger.warning( 'A newer version of %r already exists, skipping...', log) return None logger.warning( '%r has been updated to the latest harvester version, %s', log, log.harvester_version) if exhaust and log_id is None: if force: logger.warning('propagating force=True until queue exhaustion') logger.debug('Spawning another harvest task') res = harvest.apply_async(self.request.args, self.request.kwargs) logger.info('Spawned %r', res) logger.info('Harvesting %r', log) try: for datum in log.source_config.get_harvester().harvest_from_log( log, force=force, ignore_disabled=ignore_disabled, limit=limit): if ingest and (datum.created or superfluous): transform.apply_async((datum.id, )) except HarvesterConcurrencyError as e: # If log_id has been specified there's a chance that # the advisory lock was not, in fact, acquired. If so retry indefinitely to preserve # existing functionality # Use random to add jitter to help break up locking issues # Kinda hacky, allow a stupidly large number of retries as there is no options for infinite raise self.retry( exc=e, max_retries=99999, countdown=(random.random() + 1) * min(settings.CELERY_RETRY_BACKOFF_BASE**self.request.retries, 60 * 15))
def task(request): user_id = request.session['user_id'] if request.method == 'POST': # This part will only execute after submitting annotation answer for given data instance data_class_id = request.POST['data_class_id'] data_instance = request.POST['DataInstance'] task_id = request.POST['task_id'] try: with transaction.atomic(): annotating_data_instance = MediaDataInstance.objects.get(taskID_id=task_id, media=data_instance) # system will check whether the data instance is belongs to this user for more accuracy. if (annotating_data_instance.WhoIsViewing==user_id) and (annotating_data_instance.IsViewing==True) and (annotating_data_instance.NumberOfAnnotations < Task.objects.get(id=task_id).requiredNumofAnnotations): # for extra protection.Can be removed if nessasary # create data annotation result and save it in the database data_annotation_result = DataAnnotationResult(TaskID=Task.objects.get(id=task_id), DataInstance=annotating_data_instance, ClassID=data_class_id, UserID=user_id) data_annotation_result.save() # release the lock for submitted data instance annotating_data_instance.IsViewing = False annotating_data_instance.WhoIsViewing = 0 annotating_data_instance.NumberOfAnnotations += 1 annotating_data_instance.save() return redirect('/DoDataAnnotationTask/Task?task_id=' + str(task_id)) else: return redirect('/DoDataAnnotationTask/Task?task_id=' + str(task_id)) except DatabaseError: print('DatabaseError in task() annotation submission Image Data Annotation') return redirect('/UserManagement/MyTasks/') else: # this part is responsible for giving data instance to annotate. This will execute after clicking a task, submitting an annotation or skipping a data instance try: task_id = request.GET['task_id'] if len(ContributorTask.objects.filter(User_id=user_id, Task_id=task_id)) == 0:# check whether this user is registerd for this task return redirect('/UserManagement/MyTasks/') data_instance_annotation_times = int(Task.objects.get(id=task_id).requiredNumofAnnotations) # Take required number of annotations for the task annotated_data_instances = DataAnnotationResult.objects.filter(TaskID_id=task_id, UserID=user_id).order_by('-LastUpdate')# Take submitted annotation for this task by this user so that we can prevent being annotate same data instance by the same user. data_instances_to_exclude = [] # add those data instance to this list for i in annotated_data_instances: data_instances_to_exclude += [i.DataInstance.id] try:# there is an option to skip data instances. here skipped data instances are addded to excluding data instances list so that annotator will not get those data instances. skip_instance=request.GET['skip_instance'] skip_instance_object = MediaDataInstance.objects.get(taskID_id=task_id, media=skip_instance) data_instances_to_exclude += [skip_instance_object.id] skip_instance_request =True except: skip_instance_request =False try: with transaction.atomic(): data_annotation = MediaDataInstance.objects.filter(taskID_id=task_id,IsViewing=False,NumberOfAnnotations__lt=data_instance_annotation_times).exclude(id__in=data_instances_to_exclude) # take data instances which are not viewing and number of annotations are less than required number of annotations while excluding the data instances which have annotated already by this user if len(data_annotation) > 0: data_instance = random.choice(data_annotation) # choose one data instance randomly and put a lock to that data instance before giving. data_instance_about_to_annotate = MediaDataInstance.objects.get(taskID_id=task_id, media=data_instance.media) data_instance_about_to_annotate.IsViewing=True data_instance_about_to_annotate.WhoIsViewing=user_id data_instance_about_to_annotate.save() if len(annotated_data_instances) > 0: return render(request, 'DoDataAnnotationTask/DataAnnotationTask.html', {'data_instance_available': True, 'task_object': Task.objects.get(id=task_id), 'data_classes': Cateogary.objects.filter(taskID_id=task_id), 'data_instance': data_instance, 'first_name':Profile.objects.get(user=request.user).first_name, 'task_id': task_id, 'annotated_data_instances_available': True, 'annotated_data_instances': annotated_data_instances}) else: return render(request, 'DoDataAnnotationTask/DataAnnotationTask.html', {'data_instance_available': True, 'task_object': Task.objects.get(id=task_id), 'data_classes': Cateogary.objects.filter(taskID_id=task_id), 'data_instance': data_instance, 'first_name':Profile.objects.get(user=request.user).first_name, 'task_id': task_id, 'annotated_data_instances_available': False}) elif len(data_annotation)==0 and skip_instance_request: # if contributor wants to skip the only remaining data instance, same data instance will be given again and again data_instance = MediaDataInstance.objects.get(taskID_id=task_id, id=skip_instance_object.id) data_instance_about_to_annotate = data_instance data_instance_about_to_annotate.IsViewing=True data_instance_about_to_annotate.WhoIsViewing=user_id data_instance_about_to_annotate.save() if len(annotated_data_instances) > 0: return render(request, 'DoDataAnnotationTask/DataAnnotationTask.html', {'data_instance_available': True, 'task_object': Task.objects.get(id=task_id), 'data_classes': Cateogary.objects.filter(taskID_id=task_id), 'data_instance': data_instance, 'first_name':Profile.objects.get(user=request.user).first_name, 'task_id': task_id, 'annotated_data_instances_available': True, 'annotated_data_instances': annotated_data_instances}) else: return render(request, 'DoDataAnnotationTask/DataAnnotationTask.html', {'data_instance_available': True, 'task_object': Task.objects.get(id=task_id), 'data_classes': Cateogary.objects.filter(taskID_id=task_id), 'data_instance': data_instance, 'first_name':Profile.objects.get(user=request.user).first_name, 'task_id': task_id, 'annotated_data_instances_available': False}) else: # if all the data instances have been annotated, system will automatically change the status of the task as 'completed' remaining_data_instances = MediaDataInstance.objects.filter(taskID_id=task_id,NumberOfAnnotations__lt=data_instance_annotation_times) if len(remaining_data_instances)==0: completed_task = Task.objects.get(id=task_id) if completed_task.status=='inprogress': completed_task.status = 'completed' completed_task.save() if len(annotated_data_instances) > 0: return render(request, 'DoDataAnnotationTask/DataAnnotationTask.html', {'data_instance_available': False, 'task_object': Task.objects.get(id=task_id), 'task_id': task_id, 'first_name':Profile.objects.get(user=request.user).first_name, 'annotated_data_instances_available': True, 'annotated_data_instances': annotated_data_instances}) else: return render(request, 'DoDataAnnotationTask/DataAnnotationTask.html', {'data_instance_available': False, 'task_object': Task.objects.get(id=task_id), 'task_id': task_id, 'first_name':Profile.objects.get(user=request.user).first_name, 'annotated_data_instances_available': False,}) except DatabaseError: print('DatabaseError in task() giving data instance to annotator Image data annotation') return redirect('/UserManagement/MyTasks/') except: print('Failure in task() Image data annotation ') return redirect('/UserManagement/MyTasks/')
def add_api(request): """ 新增接口信息 project_id 项目ID first_group_id 一级分组ID second_group_id 二级分组ID name 接口名称 httpType HTTP/HTTPS requestType 请求方式 address 请求地址 headDict 头文件 requestParameterType 参数请求格式 requestList 请求参数列表 responseList 返回参数列表 mockStatus mockhttp状态 code mock代码 description 描述 :return: """ data = json.loads(request.body) if not data["project_id"] or not data["first_group_id"] or not data["name"] or not data["httpType"] or not \ data["requestType"] or not data["address"] or not data["requestParameterType"] or not data["status"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["status"] not in ["True", "False"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if not isinstance(data["project_id"], int) or not isinstance( data["first_group_id"], int): return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["httpType"] not in ["HTTP", "HTTPS"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["requestType"] not in ["POST", "GET", "PUT", "DELETE"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) if data["requestParameterType"] not in ["form-data", "raw", "Restful"]: return JsonResponse(code_msg=GlobalStatusCode.parameter_wrong()) obj = Project.objects.filter(id=data["project_id"]) if obj: obi = ApiInfo.objects.filter(name=data["name"], project=data["project_id"]) if obi: return JsonResponse(code_msg=GlobalStatusCode.name_repetition()) else: try: with transaction.atomic(): first_group = ApiGroupLevelFirst.objects.filter( id=data["first_group_id"], project=data["project_id"]) if len(first_group) == 0: return JsonResponse( code_msg=GlobalStatusCode.group_not_exist()) if data["first_group_id"] and data["second_group_id"]: if not isinstance(data["second_group_id"], int): return JsonResponse( code_msg=GlobalStatusCode.parameter_wrong()) second_group = ApiGroupLevelSecond.objects.filter( id=data["second_group_id"], apiGroupLevelFirst=data["first_group_id"]) if len(second_group) == 0: return JsonResponse( code_msg=GlobalStatusCode.group_not_exist()) try: oba = ApiInfo( project=Project.objects.get( id=data["project_id"]), apiGroupLevelFirst=ApiGroupLevelFirst.objects. get(id=data["first_group_id"]), apiGroupLevelSecond=ApiGroupLevelSecond. objects.get(id=data["second_group_id"]), name=data["name"], httpType=data["httpType"], status=data["status"], requestType=data["requestType"], apiAddress=data["address"], requestParameterType=data[ "requestParameterType"], mockCode=data["mockStatus"], data=data["code"], userUpdate=User.objects.get( id=request.user.pk), description=data["desc"]) except KeyError: return JsonResponse( code_msg=GlobalStatusCode.parameter_wrong()) else: return JsonResponse( code_msg=GlobalStatusCode.parameter_wrong()) oba.save() if len(data["headDict"]): for i in data["headDict"]: try: if i["name"]: ApiHead(api=ApiInfo.objects.get(id=oba.pk), name=i["name"], value=i["value"]).save() except KeyError: logging.exception("Error") return JsonResponse(GlobalStatusCode.fail()) if data["requestParameterType"] == "form-data": if len(data["requestList"]): for i in data["requestList"]: try: # i = i.replace("true", "True").replace("false", "False") if i["name"]: ApiParameter( api=ApiInfo.objects.get(id=oba.pk), name=i["name"], value=i["value"], required=i["required"], _type=i["_type"], restrict=i["restrict"], description=i["description"]).save( ) except KeyError: logging.exception("Error") return JsonResponse( code_msg=GlobalStatusCode.fail()) else: if len(data["requestList"]): ApiParameterRaw(api=ApiInfo.objects.get(id=oba.pk), data=data["requestList"]).save() if len(data["responseList"]): for i in data["responseList"]: try: # i = i.replace("true", "True").replace("false", "False") if i["name"]: ApiResponse( api=ApiInfo.objects.get(id=oba.pk), name=i["name"], value=i["value"], required=i["required"], _type=i["_type"], description=i["description"]).save() except KeyError: logging.exception("Error") return JsonResponse( code_msg=GlobalStatusCode.fail()) record_dynamic(data["project_id"], "新增", "接口", "新增接口“%s”" % data["name"]) api_record = ApiOperationHistory( apiInfo=ApiInfo.objects.get(id=oba.pk), user=User.objects.get(id=request.user.pk), description="新增接口\"%s\"" % data["name"]) api_record.save() return JsonResponse(data={"api_id": oba.pk}, code_msg=GlobalStatusCode.success()) except Exception as e: logging.exception("error") logging.error(e) return JsonResponse(code_msg=GlobalStatusCode.fail()) else: return JsonResponse(code_msg=GlobalStatusCode.project_not_exist())
def handle(self, *args, **options): if settings.ARCHIVES_SEARCH_PLAINTEXT: proto = "http" else: proto = "https" r = requests.get('{0}://{1}/listinfo/'.format( proto, settings.ARCHIVES_SEARCH_SERVER)) j = r.json() allgroups = list(set([l['group'] for l in j])) with transaction.atomic(): curs = connection.cursor() # Add any groups necessary curs.execute( "INSERT INTO lists_mailinglistgroup (groupname, sortkey) SELECT n,50 FROM UNNEST(%s) n(n) WHERE NOT EXISTS (SELECT 1 FROM lists_mailinglistgroup WHERE groupname=n) RETURNING groupname", (allgroups, )) for n, in curs.fetchall(): print "Added group %s" % n # Add and update lists for l in j: curs.execute( "SELECT id FROM lists_mailinglist WHERE listname=%s", (l['name'], )) if curs.rowcount == 0: curs.execute( "INSERT INTO lists_mailinglist (listname, group_id, active, description, shortdesc) VALUES (%s, (SELECT id FROM lists_mailinglistgroup WHERE groupname=%s), %s, %s, %s)", (l['name'], l['group'], l['active'], l['description'], l['shortdesc'])) print "Added list %s" % l['name'] else: curs.execute( "UPDATE lists_mailinglist SET group_id=(SELECT id FROM lists_mailinglistgroup WHERE groupname=%s), active=%s, description=%s, shortdesc=%s WHERE listname=%s AND NOT (group_id=(SELECT id FROM lists_mailinglistgroup WHERE groupname=%s) AND active=%s AND description=%s AND shortdesc=%s) RETURNING listname", ( l['group'], l['active'], l['description'], l['shortdesc'], l['name'], l['group'], l['active'], l['description'], l['shortdesc'], )) for n, in curs.fetchall(): print "Updated list %s" % n # Delete any lists that shouldn't exist anymore (this is safe because we don't keep any data about them, # so they are trivial to add back) curs.execute( "DELETE FROM lists_mailinglist WHERE NOT listname=ANY(%s) RETURNING listname", ([l['name'] for l in j], )) for n, in curs.fetchall(): print "Deleted list %s" % n # Delete listgroups curs.execute( "DELETE FROM lists_mailinglistgroup WHERE NOT groupname=ANY(%s) RETURNING groupname", (allgroups, )) for n, in curs.fetchall(): print "Deleted group %s" % n if options['dryrun']: raise CommandError("Dry run, rolling back")
def post(self, request, *args, **kwargs): if self.keyword_form.is_valid(): with transaction.atomic(): self.keyword.keyword = self.keyword_form.cleaned_data[ 'keyword'] self.keyword.description = self.keyword_form.cleaned_data[ 'description'] self.keyword.delimiter = self.keyword_form.cleaned_data[ 'delimiter'] self.keyword.override_open_sessions = self.keyword_form.cleaned_data[ 'override_open_sessions'] self.keyword.initiator_doc_type_filter = [] if self.keyword_form.cleaned_data[ 'allow_keyword_use_by'] == 'users': self.keyword.initiator_doc_type_filter.append( 'CommCareUser') if self.keyword_form.cleaned_data[ 'allow_keyword_use_by'] == 'cases': self.keyword.initiator_doc_type_filter.append( 'CommCareCase') self.keyword.save() self.keyword.keywordaction_set.all().delete() if self.keyword_form.cleaned_data[ 'sender_content_type'] != NO_RESPONSE: app_id, form_unique_id = split_combined_id( self.keyword_form. cleaned_data['sender_app_and_form_unique_id']) self.keyword.keywordaction_set.create( recipient=KeywordAction.RECIPIENT_SENDER, action=self.keyword_form. cleaned_data['sender_content_type'], message_content=self.keyword_form. cleaned_data['sender_message'], app_id=app_id, form_unique_id=form_unique_id, ) if self.process_structured_message: app_id, form_unique_id = split_combined_id( self.keyword_form. cleaned_data['structured_sms_app_and_form_unique_id']) self.keyword.keywordaction_set.create( recipient=KeywordAction.RECIPIENT_SENDER, action=KeywordAction.ACTION_STRUCTURED_SMS, app_id=app_id, form_unique_id=form_unique_id, use_named_args=self.keyword_form. cleaned_data['use_named_args'], named_args=self.keyword_form. cleaned_data['named_args'], named_args_separator=self.keyword_form. cleaned_data['named_args_separator'], ) if self.keyword_form.cleaned_data[ 'other_recipient_content_type'] != NO_RESPONSE: app_id, form_unique_id = split_combined_id( self.keyword_form. cleaned_data['other_recipient_app_and_form_unique_id']) self.keyword.keywordaction_set.create( recipient=self.keyword_form. cleaned_data['other_recipient_type'], recipient_id=self.keyword_form. cleaned_data['other_recipient_id'], action=self.keyword_form. cleaned_data['other_recipient_content_type'], message_content=self.keyword_form. cleaned_data['other_recipient_message'], app_id=app_id, form_unique_id=form_unique_id, ) return HttpResponseRedirect( reverse(KeywordsListView.urlname, args=[self.domain])) return self.get(request, *args, **kwargs)
def insert_plaintext(self, hashfile): with transaction.atomic(): locked = False while not locked: try: # Lock: prevent cracked file from being processed hashfile_lock = Lock.objects.select_for_update().filter(hashfile_id=hashfile.id, lock_ressource="hashfile")[0] locked = True except OperationalError as e: continue hashfile_path = os.path.join(os.path.dirname(__file__), "..", "Files", "Hashfiles", hashfile.hashfile) if os.path.exists(hashfile_path): try: batch_create_list = [] for index, line in enumerate(open(hashfile_path, encoding='utf-8')): if index < hashfile.cracked_count: continue line = line.strip() password = line.split(":")[-1] if hashfile.username_included: username = line.split(":")[0] password_hash = "" else: username = None password_hash = "" pass_len, pass_charset, _, pass_mask, _ = analyze_password(password) h = Hash( hashfile=hashfile, hash_type=hashfile.hash_type, username=username, password=password, hash=password_hash, password_len=pass_len, password_charset=pass_charset, password_mask=pass_mask, ) batch_create_list.append(h) if len(batch_create_list) >= 100000: hashfile.line_count += len(batch_create_list) hashfile.cracked_count += len(batch_create_list) while len(batch_create_list) != 0: Hash.objects.bulk_create(batch_create_list[:1000]) batch_create_list = batch_create_list[1000:] hashfile.save() hashfile.line_count += len(batch_create_list) hashfile.cracked_count += len(batch_create_list) while len(batch_create_list) != 0: Hash.objects.bulk_create(batch_create_list[:1000]) batch_create_list = batch_create_list[1000:] hashfile.save() except Exception as e: traceback.print_exc() # Crackedfile processing if over, remove lock del hashfile_lock
def Collector__delete(self): # NOTE: Original Django code. Only changes are marked as ADDED. # sort instance collections for model, instances in self.data.items(): self.data[model] = sorted(instances, key=attrgetter("pk")) # if possible, bring the models in an order suitable for databases that # don't support transactions or cannot defer constraint checks until the # end of a transaction. self.sort() # number of objects deleted for each model label deleted_counter = Counter() with transaction.atomic(using=self.using, savepoint=False): # ADDED: Chunk deletion of larger amounts of related objects (+6 lines) for obj_queryset in self.chunk_deletes: exists = True while exists: values_list = obj_queryset.values_list('pk', flat=True)[:DELETION_MAX_CHUNK] obj_queryset.model.objects.filter(pk__in=values_list).delete() exists = obj_queryset.exists() # send pre_delete signals for model, obj in self.instances_with_model(): if not model._meta.auto_created: signals.pre_delete.send( sender=model, instance=obj, using=self.using ) # fast deletes for qs in self.fast_deletes: count = qs._raw_delete(using=self.using) deleted_counter[qs.model._meta.label] += count # update fields for model, instances_for_fieldvalues in self.field_updates.items(): for (field, value), instances in instances_for_fieldvalues.items(): query = sql.UpdateQuery(model) query.update_batch([obj.pk for obj in instances], {field.name: value}, self.using) # reverse instance collections for instances in self.data.values(): instances.reverse() # delete instances for model, instances in self.data.items(): query = sql.DeleteQuery(model) pk_list = [obj.pk for obj in instances] count = query.delete_batch(pk_list, self.using) deleted_counter[model._meta.label] += count if not model._meta.auto_created: for obj in instances: signals.post_delete.send( sender=model, instance=obj, using=self.using ) # update collected instances for model, instances_for_fieldvalues in self.field_updates.items(): for (field, value), instances in instances_for_fieldvalues.items(): for obj in instances: setattr(obj, field.attname, value) for model, instances in self.data.items(): for instance in instances: setattr(instance, model._meta.pk.attname, None) return sum(deleted_counter.values()), dict(deleted_counter)
def setup_lang(sender, **kwargs): """Hook for creating basic set of languages on database migration.""" if sender.label == 'lang': with transaction.atomic(): Language.objects.setup(False)
def compare_potfile(self, hashfile, potfile=None): if not potfile: potfile = self.get_potfile() with transaction.atomic(): locked = False while not locked: try: # Lock: lock all the potfiles, this way only one instance of hashcat will be running at a time, the --left option eats a lot of RAM... potfile_locks = list(Lock.objects.select_for_update().filter(lock_ressource="potfile")) # Lock: prevent hashes file from being processed hashfile_lock = Lock.objects.select_for_update().filter(hashfile_id=hashfile.id, lock_ressource="hashfile")[0] locked = True except OperationalError as e: continue hashfile_path = os.path.join(os.path.dirname(__file__), "..", "Files", "Hashfiles", hashfile.hashfile) # trick to allow multiple instances of hashcat session_name = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(12)) cracked_file = tempfile.NamedTemporaryFile(delete=False) # is there a way to combine --show and --remove in hashcat ? # Get cracked hashes cmd_line = [self.get_binary(), '--show', '-m', str(hashfile.hash_type), hashfile_path, '-o', cracked_file.name, '--session', session_name] cmd_line += ['--outfile-format', '3'] if potfile: cmd_line += ['--potfile-path', potfile] print("%s: Command: %s" % (hashfile.name, " ".join(cmd_line))) p = subprocess.Popen(cmd_line, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) p.wait() # Remove cracked hashes from list f = tempfile.NamedTemporaryFile(delete=False) f.close() cmd_line = [self.get_binary(), '--left', '-m', str(hashfile.hash_type), hashfile_path, '-o', f.name, '--session', session_name] cmd_line += ['--outfile-format', '1'] if potfile: cmd_line += ['--potfile-path', potfile] print("%s: Command: %s" % (hashfile.name, " ".join(cmd_line))) p = subprocess.Popen(cmd_line, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) p.wait() copyfile(f.name, hashfile_path) os.remove(f.name) # hashcat over, remove lock on potfile and hashfile del potfile_locks del hashfile_lock if os.path.exists(cracked_file.name): start = time.perf_counter() cursor = connection.cursor() tmp_table_name = "tmp_table_%s" % ''.join(random.choice(string.ascii_lowercase+string.digits) for i in range(10)) try: # create temporary table cursor.execute("BEGIN;") cursor.execute("CREATE TEMPORARY TABLE " + tmp_table_name + " (hash char(190) PRIMARY KEY, password varchar(190) NOT NULL, pass_len INTEGER, pass_charset varchar(190), pass_mask varchar(190));") cursor.execute("SET unique_checks=0;") bulk_insert_list = [] nb_insert = 0 for index, line in enumerate(open(cracked_file.name, encoding='utf-8')): line = line.strip() password = line.split(":")[-1] password_hash = ":".join(line.split(":")[0:-1]) pass_len, pass_charset, _, pass_mask, _ = analyze_password(password) bulk_insert_list += [password_hash, password, pass_len, pass_charset, pass_mask] nb_insert += 1 if nb_insert >= 1000: cursor.execute("INSERT INTO " + tmp_table_name + " VALUES " + ", ".join(["(%s, %s, %s, %s, %s)"]*nb_insert) + ";", bulk_insert_list) bulk_insert_list = [] nb_insert = 0 # insert into table every 100K rows will prevent MySQL from raising "The number of locks exceeds the lock table size" if index % 100000 == 0: print(index) cursor.execute("UPDATE " + tmp_table_name + " b JOIN Hashcat_hash a ON a.hash = b.hash AND a.hash_type=%s SET a.password = b.password, a.password_len = b.pass_len, a.password_charset = b.pass_charset, a.password_mask = b.pass_mask;", [hashfile.hash_type]) cursor.execute("DELETE FROM " + tmp_table_name + ";") cursor.execute("COMMIT;") if len(bulk_insert_list) != 0: cursor.execute("INSERT INTO " + tmp_table_name + " VALUES " + ", ".join(["(%s, %s, %s, %s, %s)"]*nb_insert) + ";", bulk_insert_list) cursor.execute("UPDATE " + tmp_table_name + " b JOIN Hashcat_hash a ON a.hash = b.hash AND a.hash_type=%s SET a.password = b.password, a.password_len = b.pass_len, a.password_charset = b.pass_charset, a.password_mask = b.pass_mask;", [hashfile.hash_type]) cursor.execute("COMMIT;") except Exception as e: traceback.print_exc() finally: cursor.execute("SET unique_checks=1;") cursor.execute("DROP TABLE %s;" % tmp_table_name) cursor.execute("COMMIT;") cursor.close() hashfile.cracked_count = Hash.objects.filter(hashfile_id=hashfile.id, password__isnull=False).count() hashfile.save() end = time.perf_counter() print("Update password time: %fs" % (end-start,)) os.remove(cracked_file.name)
def post(self, request): """ Creates a new CCX course for a given Master Course. Args: request (Request): Django request object. Return: A JSON serialized representation a newly created CCX course. """ master_course_id = request.data.get('master_course_id') master_course_object, master_course_key, error_code, http_status = get_valid_course( master_course_id, advanced_course_check=True ) if master_course_object is None: return Response( status=http_status, data={ 'error_code': error_code } ) # validating the rest of the input valid_input, field_errors = get_valid_input(request.data) if field_errors: return Response( status=status.HTTP_400_BAD_REQUEST, data={ 'field_errors': field_errors } ) try: coach = User.objects.get(email=valid_input['coach_email']) except User.DoesNotExist: return Response( status=status.HTTP_404_NOT_FOUND, data={ 'error_code': 'coach_user_does_not_exist' } ) with transaction.atomic(): ccx_course_object = CustomCourseForEdX( course_id=master_course_object.id, coach=coach, display_name=valid_input['display_name']) ccx_course_object.save() # Make sure start/due are overridden for entire course start = TODAY().replace(tzinfo=pytz.UTC) override_field_for_ccx(ccx_course_object, master_course_object, 'start', start) override_field_for_ccx(ccx_course_object, master_course_object, 'due', None) # Enforce a static limit for the maximum amount of students that can be enrolled override_field_for_ccx( ccx_course_object, master_course_object, 'max_student_enrollments_allowed', valid_input['max_students_allowed'] ) # Hide anything that can show up in the schedule hidden = 'visible_to_staff_only' for chapter in master_course_object.get_children(): override_field_for_ccx(ccx_course_object, chapter, hidden, True) for sequential in chapter.get_children(): override_field_for_ccx(ccx_course_object, sequential, hidden, True) for vertical in sequential.get_children(): override_field_for_ccx(ccx_course_object, vertical, hidden, True) # make the coach user a coach on the master course make_user_coach(coach, master_course_key) # pull the ccx course key ccx_course_key = CCXLocator.from_course_locator(master_course_object.id, ccx_course_object.id) # enroll the coach in the newly created ccx email_params = get_email_params( master_course_object, auto_enroll=True, course_key=ccx_course_key, display_name=ccx_course_object.display_name ) enroll_email( course_id=ccx_course_key, student_email=coach.email, auto_enroll=True, email_students=True, email_params=email_params, ) # assign coach role for the coach to the newly created ccx assign_coach_role_to_ccx(ccx_course_key, coach, master_course_object.id) serializer = self.get_serializer(ccx_course_object) return Response( status=status.HTTP_201_CREATED, data=serializer.data )
def insert_hashes(self, hashfile): with transaction.atomic(): locked = False while not locked: try: # Lock: prevent cracked file from being processed hashfile_lock = Lock.objects.select_for_update().filter(hashfile_id=hashfile.id, lock_ressource="hashfile")[0] locked = True except OperationalError as e: continue hashfile_path = os.path.join(os.path.dirname(__file__), "..", "Files", "Hashfiles", hashfile.hashfile) if os.path.exists(hashfile_path): try: # 1 - import hashfile to database print("importing hashfile to database") start = time.perf_counter() batch_create_list = [] hash_count = 0 for index, line in enumerate(open(hashfile_path, encoding='utf-8')): try: line = line.strip() if hashfile.username_included: username = line.split(":")[0] password_hash = ":".join(line.split(":")[1:]) else: username = None password_hash = line except IndexError: continue h = Hash( hashfile=hashfile, hash_type=hashfile.hash_type, username=username, hash=password_hash, password=None, password_len=None, password_charset=None, password_mask=None, ) batch_create_list.append(h) if len(batch_create_list) >= 100000: print(index) hashfile.line_count += len(batch_create_list) while len(batch_create_list) != 0: Hash.objects.bulk_create(batch_create_list[:1000]) batch_create_list = batch_create_list[1000:] hashfile.save() hashfile.line_count += len(batch_create_list) while len(batch_create_list) != 0: Hash.objects.bulk_create(batch_create_list[:1000]) batch_create_list = batch_create_list[1000:] hashfile.save() end = time.perf_counter() print("Inserted hashes in : %fs" % (end-start,)) # 2 - if username in hashfile, delete file and create one with only the hashes, # --username takes a lot of RAM with hashcat, this method is better when processing huge hashfiles start = time.perf_counter() if hashfile.username_included: os.remove(hashfile_path) tmpfile_name = ''.join([random.choice(string.ascii_lowercase) for i in range(16)]) tmpfile_path = os.path.join(os.path.dirname(__file__), "..", "Files", "tmp", tmpfile_name) cursor = connection.cursor() cursor.execute("SELECT DISTINCT hash FROM Hashcat_hash WHERE hashfile_id=%s INTO OUTFILE %s", [hashfile.id, tmpfile_path]) cursor.close() copyfile(tmpfile_path, hashfile_path) os.remove(tmpfile_path) end = time.perf_counter() print("Wrote hashfile on disk in : %fs" % (end-start,)) except Exception as e: traceback.print_exc() else: print("Error: hashfile doesn't exists") # Crackedfile processing if over, remove lock del hashfile_lock
def mark_order_as_failed(order, charge_failure_reason): logger.info('mark_order_as_failed', order=order.order_id, charge_failure_reason=charge_failure_reason) with transaction.atomic(): order.mark_as_failed(charge_failure_reason)
def patch(self, request, ccx_course_id=None): """ Modifies a CCX course. Args: request (Request): Django request object. ccx_course_id (string): URI element specifying the CCX course location. """ ccx_course_object, ccx_course_key, error_code, http_status = self.get_object(ccx_course_id, is_ccx=True) if ccx_course_object is None: return Response( status=http_status, data={ 'error_code': error_code } ) master_course_id = request.data.get('master_course_id') if master_course_id is not None and unicode(ccx_course_object.course_id) != master_course_id: return Response( status=status.HTTP_403_FORBIDDEN, data={ 'error_code': 'master_course_id_change_not_allowed' } ) valid_input, field_errors = get_valid_input(request.data, ignore_missing=True) if field_errors: return Response( status=status.HTTP_400_BAD_REQUEST, data={ 'field_errors': field_errors } ) with transaction.atomic(): # update the display name if 'display_name' in valid_input: ccx_course_object.display_name = valid_input['display_name'] # check if the coach has changed and in case update it old_coach = None if 'coach_email' in valid_input: try: coach = User.objects.get(email=valid_input['coach_email']) except User.DoesNotExist: return Response( status=status.HTTP_404_NOT_FOUND, data={ 'error_code': 'coach_user_does_not_exist' } ) if ccx_course_object.coach.id != coach.id: old_coach = ccx_course_object.coach ccx_course_object.coach = coach ccx_course_object.save() # update the overridden field for the maximum amount of students if 'max_students_allowed' in valid_input: override_field_for_ccx( ccx_course_object, ccx_course_object.course, 'max_student_enrollments_allowed', valid_input['max_students_allowed'] ) # if the coach has changed, update the permissions if old_coach is not None: # get the master course key and master course object master_course_object, master_course_key, _, _ = get_valid_course(unicode(ccx_course_object.course_id)) # make the new ccx coach a coach on the master course make_user_coach(coach, master_course_key) # enroll the coach in the ccx email_params = get_email_params( master_course_object, auto_enroll=True, course_key=ccx_course_key, display_name=ccx_course_object.display_name ) enroll_email( course_id=ccx_course_key, student_email=coach.email, auto_enroll=True, email_students=True, email_params=email_params, ) # enroll the coach to the newly created ccx assign_coach_role_to_ccx(ccx_course_key, coach, master_course_object.id) return Response( status=status.HTTP_204_NO_CONTENT, )
def refund_item(item, reason): logger.info('refund_item', item=item.ticket_id) stripe_refund = stripe_integration.refund_item(item) with transaction.atomic(): Refund.objects.create_for_item(item, reason, stripe_refund.id, stripe_refund.created)
def mark_order_as_errored_after_charge(order, charge_id): logger.info('mark_order_as_errored_after_charge', order=order.order_id, charge_id=charge_id) with transaction.atomic(): order.mark_as_errored_after_charge(charge_id)