def test_best_for_without_plan_and_environment_options_returns_None(self): plan = factory.PlanFactory() environment = plan.environments.all()[0] self.assertIsNone( DatabaseInfra.best_for(plan=plan, environment=environment, name="test"))
def clone_infra(plan, environment, name, team, project, description, task=None, clone=None): if not plan.provider == plan.CLOUDSTACK: dbinfra = DatabaseInfra.best_for( plan=plan, environment=environment, name=name) if dbinfra: database = Database.provision(databaseinfra=dbinfra, name=name) database.team = team database.description = description database.project = project database.save() return build_dict(databaseinfra=dbinfra, database=database, created=True) return build_dict(databaseinfra=None, created=False) workflow_dict = build_dict(name=slugify(name), plan=plan, environment=environment, steps=get_clone_settings(plan.engine_type.name), qt=get_vm_qt(plan=plan, ), dbtype=str(plan.engine_type), team=team, project=project, description=description, clone=clone ) start_workflow(workflow_dict=workflow_dict, task=task) return workflow_dict
def clean(self): cleaned_data = super(DatabaseForm, self).clean() databaseinfra = DatabaseInfra.best_for(cleaned_data['plan']) if not databaseinfra: raise forms.ValidationError(_("Sorry. I have no infra-structure to allocate this database. Try select another plan.")) cleaned_data['databaseinfra'] = databaseinfra return cleaned_data
def clone_infra(plan, environment, name, team, backup_hour, maintenance_window, maintenance_day, project, description, subscribe_to_email_events, task=None, clone=None): if not plan.provider == plan.CLOUDSTACK: infra = DatabaseInfra.best_for( plan=plan, environment=environment, name=name, backup_hour=backup_hour, maintenance_window=maintenance_window, maintenance_day=maintenance_day, ) if infra: database = Database.provision(databaseinfra=infra, name=name) database.team = team database.description = description database.project = project database.save() return build_dict( databaseinfra=infra, database=database, created=True, subscribe_to_email_events=subscribe_to_email_events) return build_dict(databaseinfra=None, created=False, subscribe_to_email_events=subscribe_to_email_events) workflow_dict = build_dict( name=slugify(name), plan=plan, environment=environment, steps=get_clone_settings(plan.replication_topology.class_path), qt=get_vm_qt(plan=plan), dbtype=str(plan.engine_type), team=team, backup_hour=backup_hour, maintenance_window=maintenance_window, maintenance_day=maintenance_day, project=project, description=description, clone=clone, subscribe_to_email_events=subscribe_to_email_events, ) start_workflow(workflow_dict=workflow_dict, task=task) return workflow_dict
def provision(cls, name, plan): # create new databaseinfra LOG.debug("provisioning databaseinfra with plan: %s | name: %s", plan, name) database = Database() database.databaseinfra = DatabaseInfra.best_for(plan) database.name = name database.save() return database
def clean(self): cleaned_data = super(DatabaseForm, self).clean() # if there is an instance, that means that we are in a edit page and therefore # it should return the default cleaned_data if self.instance and self.instance.id: return cleaned_data # TODO: change model field to blank=False if 'team' in cleaned_data: team = cleaned_data['team'] LOG.debug("team: %s" % team) if not team: LOG.warning("No team specified in database form") self._errors["team"] = self.error_class([_("Team: This field is required.")]) if not self.is_valid(): raise forms.ValidationError(self.errors) if len(cleaned_data['name']) > 64: self._errors["name"] = self.error_class([_("Database name too long")]) if 'plan' in cleaned_data: plan = cleaned_data.get('plan', None) if not plan: self._errors["plan"] = self.error_class([_("Plan: This field is required.")]) if 'environment' in cleaned_data: environment = cleaned_data.get('environment', None) if not environment or environment not in plan.environments.all(): raise forms.ValidationError(_("Invalid plan for selected environment.")) #validate if the team has available resources dbs = team.databases_in_use_for(environment) database_alocation_limit = team.database_alocation_limit LOG.debug("dbs: %s | type: %s" % (dbs, type(dbs))) if (database_alocation_limit != 0 and len(dbs) >= database_alocation_limit): LOG.warning("The database alocation limit of %s has been exceeded for the selected team %s => %s" % (database_alocation_limit, team, list(dbs))) self._errors["team"] = self.error_class([_("The database alocation limit of %s has been exceeded for the selected team: %s") % (database_alocation_limit, list(dbs))]) cleaned_data['databaseinfra'] = DatabaseInfra.best_for(plan, environment) if not cleaned_data['databaseinfra']: raise forms.ValidationError(_("Sorry. I have no infra-structure to allocate this database. Try select another plan.")) for infra in DatabaseInfra.objects.filter(environment=environment,plan=plan): if infra.databases.filter(name=cleaned_data['name']): self._errors["name"] = self.error_class([_("this name already exists in the selected environment")]) del cleaned_data["name"] if 'name' in cleaned_data and cleaned_data['name'] in cleaned_data['databaseinfra'].get_driver().RESERVED_DATABASES_NAME: raise forms.ValidationError(_("%s is a reserved database name" % cleaned_data['name'])) return cleaned_data
def test_best_for_with_only_one_datainfra_per_plan_and_environment(self): plan = factory.PlanFactory() environment = plan.environments.all()[0] datainfra = factory.DatabaseInfraFactory(plan=plan, environment=environment) factory.InstanceFactory(address="127.0.0.1", port=27017, databaseinfra=datainfra) self.assertEqual( datainfra, DatabaseInfra.best_for(plan=plan, environment=environment, name="test"))
def test_best_for_with_only_over_capacity_datainfra_returns_None(self): """tests database infra capacity""" NUMBER_OF_DATABASES_TO_TEST = 4 plan = factory.PlanFactory() environment = plan.environments.all()[0] datainfra = factory.DatabaseInfraFactory( plan=plan, environment=environment, capacity=NUMBER_OF_DATABASES_TO_TEST) factory.InstanceFactory(address="127.0.0.1", port=27017, databaseinfra=datainfra) for i in range(NUMBER_OF_DATABASES_TO_TEST): self.assertEqual( datainfra, DatabaseInfra.best_for(plan=plan, environment=environment, name="test")) factory_logical.DatabaseFactory(databaseinfra=datainfra) self.assertIsNone( DatabaseInfra.best_for(plan=plan, environment=environment, name="test"))
def make_infra( plan, environment, name, team, project, description, contacts, subscribe_to_email_events=True, task=None, ): if not plan.provider == plan.CLOUDSTACK: dbinfra = DatabaseInfra.best_for(plan=plan, environment=environment, name=name) if dbinfra: database = Database.provision(databaseinfra=dbinfra, name=name) database.team = team database.description = description database.project = project database.subscribe_to_email_events = subscribe_to_email_events database.contacts = contacts database.save() return build_dict(databaseinfra=dbinfra, database=database, created=True) return build_dict(databaseinfra=None, created=False) workflow_dict = build_dict( name=slugify(name), plan=plan, environment=environment, steps=get_deploy_settings(plan.replication_topology.class_path), qt=get_vm_qt(plan=plan, ), dbtype=str(plan.engine_type), team=team, project=project, description=description, subscribe_to_email_events=subscribe_to_email_events, contacts=contacts, ) start_workflow(workflow_dict=workflow_dict, task=task) return workflow_dict
def create(self, request): serializer = self.get_serializer(data=request.DATA, files=request.FILES) if serializer.is_valid(): self.pre_save(serializer.object) data = serializer.restore_fields(request.DATA, request.FILES) LOG.info("Plano %s" % data['plan']) plan = data['plan'] if plan.provider == plan.CLOUDSTACK: from notification.tasks import create_database result = create_database.delay(data['name'], data['plan'], data['environment'], data['team'], data['project'], data['description'], request.user) #data = serializer.to_native(self.object) #self.post_save(self.object, created=True) headers = self.get_success_headers(data) task_url = Site.objects.get_current().domain + '/api/task?task_id=%s' % str(result.id) return Response({"task":task_url}, status=status.HTTP_201_CREATED, headers=headers) else: self.pre_save(serializer.object) data = serializer.restore_fields(request.DATA, request.FILES) databaseinfra = DatabaseInfra.best_for(data['plan'], data['environment'], data['name']) self.object = models.Database.provision(data['name'], databaseinfra) self.object.team = data['team'] self.object.project = data['project'] self.object.description = data['description'] self.object.save() data = serializer.to_native(self.object) self.post_save(self.object, created=True) headers = self.get_success_headers(data) return Response(data, status=status.HTTP_201_CREATED, headers=headers) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def clean(self): cleaned_data = super(DatabaseForm, self).clean() # TODO: change model field to blank=False if 'team' in cleaned_data: team = cleaned_data['team'] LOG.debug("team: %s" % team) if not team: LOG.warning = "No team specified in database form" self._errors["team"] = self.error_class([_("Team: This field is required.")]) # if there is an instance, that it means that we are in a edit page and therefore # it should return the default cleaned_data if self.instance and self.instance.id: return cleaned_data if not self.is_valid(): raise forms.ValidationError(self.errors) if len(cleaned_data['name']) > 64: self._errors["name"] = self.error_class([_("Database name too long")]) plan = cleaned_data['plan'] environment = cleaned_data.get('environment', None) if not environment or environment not in plan.environments.all(): raise forms.ValidationError(_("Invalid plan for selected environment.")) cleaned_data['databaseinfra'] = DatabaseInfra.best_for(plan, environment) if not cleaned_data['databaseinfra']: raise forms.ValidationError(_("Sorry. I have no infra-structure to allocate this database. Try select another plan.")) for infra in DatabaseInfra.objects.filter(environment=environment,plan=plan): if infra.databases.filter(name=cleaned_data['name']): self._errors["name"] = self.error_class([_("this name already exists in the selected environment")]) del cleaned_data["name"] if 'name' in cleaned_data and cleaned_data['name'] in cleaned_data['databaseinfra'].get_driver().RESERVED_DATABASES_NAME: raise forms.ValidationError(_("%s is a reserved database name" % cleaned_data['name'])) return cleaned_data
def service_add(request, engine_name=None, engine_version=None): """ Responds to tsuru's service_add call. Creates a new databaseinfra. Return codes: 201: when the databaseinfra is successfully created. You don’t need to include any content in the response body. 500: in case of any failure in the creation process. Make sure you include an explanation for the failure in the response body. """ LOG.info("service_add for %s(%s)" % (engine_name, engine_version)) LOG.debug("request DATA: %s" % request.DATA) LOG.debug("request QUERY_PARAMS: %s" % request.QUERY_PARAMS) LOG.debug("request content-type: %s" % request.content_type) # LOG.debug("request meta: %s" % request.META) engine = __check_service_availability(engine_name, engine_version) if not engine: return Response(data={ "error": "endpoint not available for %s(%s)" % (engine_name, engine_version) }, status=500) data = request.DATA service_name = data.get('name', None) LOG.info("creating service %s" % (service_name)) try: databaseinfra = DatabaseInfra.provision(engine=engine, name=service_name) return Response( { "hostname": databaseinfra.instance.address, "engine_type": engine.name, "version": engine.version, "databaseinfra_name": databaseinfra.name }, status=201) except Exception, e: LOG.error("error provisioning databaseinfra %s: %s" % (service_name, e))
def make_infra(plan, environment, name,task=None): if not plan.provider == plan.CLOUDSTACK: dbinfra = DatabaseInfra.best_for(plan= plan, environment= environment, name= name) if dbinfra: return build_dict(databaseinfra= dbinfra, created=True) return build_dict(databaseinfra=None, created= False) workflow_dict = build_dict(name= name, plan= plan, environment= environment, steps= get_engine_steps(engine= str(plan.engine_type)), qt= get_vm_qt(plan= plan, ), MYSQL = MYSQL, MONGODB = MONGODB, enginecod = get_engine(engine= str(plan.engine_type)) ) start_workflow(workflow_dict= workflow_dict, task=task) return workflow_dict
def clone_infra(plan, environment, name, team, project, description, task=None, clone=None): if not plan.provider == plan.CLOUDSTACK: dbinfra = DatabaseInfra.best_for(plan=plan, environment=environment, name=name) if dbinfra: database = Database.provision(databaseinfra=dbinfra, name=name) database.team = team database.description = description database.project = project database.save() return build_dict(databaseinfra=dbinfra, database=database, created=True) return build_dict(databaseinfra=None, created=False) workflow_dict = build_dict(name=slugify(name), plan=plan, environment=environment, steps=get_clone_settings(plan.engine_type.name), qt=get_vm_qt(plan=plan, ), dbtype=str(plan.engine_type), team=team, project=project, description=description, clone=clone) start_workflow(workflow_dict=workflow_dict, task=task) return workflow_dict
def make_infra(plan, environment, name,task=None): if not plan.provider == plan.CLOUDSTACK: dbinfra = DatabaseInfra.best_for(plan= plan, environment= environment, name= name) if dbinfra: return build_dict(databaseinfra= dbinfra, created=True) return build_dict(databaseinfra=None, created= False) workflow_dict = build_dict(name= slugify(name), plan= plan, environment= environment, steps= get_engine_steps(engine= str(plan.engine_type)), qt= get_vm_qt(plan= plan, ), MYSQL = MYSQL, MONGODB = MONGODB, REDIS = REDIS, enginecod = get_engine(engine= str(plan.engine_type)) ) start_workflow(workflow_dict= workflow_dict, task=task) return workflow_dict
def provision(cls, name, plan, environment): # create new databaseinfra LOG.debug("provisioning databaseinfra with name %s, plan %s and environment %s", name, plan, environment) if not isinstance(plan, Plan): raise ValidationError('Invalid plan type %s - %s' % (type(plan), plan)) if not isinstance(environment, Environment): raise ValidationError('Invalid environment type %s - %s' % (type(environment), environment)) datainfra = DatabaseInfra.best_for(plan, environment) if not datainfra: raise NoDatabaseInfraCapacity() database = Database() database.databaseinfra = datainfra database.name = name database.full_clean() database.save() # refresh object from database database = Database.objects.get(pk=database.pk) return database
def service_add(request, engine_name=None, engine_version=None): """ Responds to tsuru's service_add call. Creates a new databaseinfra. Return codes: 201: when the databaseinfra is successfully created. You don’t need to include any content in the response body. 500: in case of any failure in the creation process. Make sure you include an explanation for the failure in the response body. """ LOG.info("service_add for %s(%s)" % (engine_name, engine_version)) LOG.debug("request DATA: %s" % request.DATA) LOG.debug("request QUERY_PARAMS: %s" % request.QUERY_PARAMS) LOG.debug("request content-type: %s" % request.content_type) # LOG.debug("request meta: %s" % request.META) engine = __check_service_availability(engine_name, engine_version) if not engine: return Response(data={"error": "endpoint not available for %s(%s)" % (engine_name, engine_version)}, status=500) data = request.DATA service_name = data.get("name", None) LOG.info("creating service %s" % (service_name)) try: databaseinfra = DatabaseInfra.provision(engine=engine, name=service_name) return Response( { "hostname": databaseinfra.instance.address, "engine_type": engine.name, "version": engine.version, "databaseinfra_name": databaseinfra.name, }, status=201, ) except Exception, e: LOG.error("error provisioning databaseinfra %s: %s" % (service_name, e))
def test_best_for_with_only_two_datainfra_per_plan_and_environment_returns_rounding_robin_them( self): plan = factory.PlanFactory() environment = plan.environments.all()[0] datainfra1 = factory.DatabaseInfraFactory(plan=plan, environment=environment, capacity=10) factory.InstanceFactory(address="127.0.0.1", port=27017, databaseinfra=datainfra1) datainfra2 = factory.DatabaseInfraFactory(plan=plan, environment=environment, capacity=10) factory.InstanceFactory(address="127.0.0.2", port=27017, databaseinfra=datainfra2) for i in range(10): should_choose = (datainfra1, datainfra2)[i % 2] choosed = DatabaseInfra.best_for(plan=plan, environment=environment, name="test") self.assertEqual(should_choose, choosed) database = factory_logical.DatabaseFactory(databaseinfra=choosed) self.assertEqual(choosed, database.databaseinfra)
def get_or_create_infra(base_name, plan, environment, retry_from=None): if retry_from: infra = retry_from.infra base_name['infra'] = infra.name base_name['name_prefix'] = infra.name_prefix base_name['name_stamp'] = infra.name_stamp else: infra = DatabaseInfra() infra.name = base_name['infra'] infra.name_prefix = base_name['name_prefix'] infra.name_stamp = base_name['name_stamp'] infra.last_vm_created = 0 infra.engine = plan.engine infra.plan = plan infra.disk_offering = plan.disk_offering infra.environment = environment infra.capacity = 1 infra.per_database_size_mbytes = plan.max_db_size infra.save() driver = infra.get_driver() user, password, key = driver.build_new_infra_auth() infra.user = user infra.password = password infra.database_key = key infra.save() return infra
def do(self, workflow_dict): try: workflow_dict['names'] = gen_infra_names( name=workflow_dict['name'], qt=workflow_dict['qt']) databaseinfra = DatabaseInfra() databaseinfra.name = workflow_dict['names']['infra'] databaseinfra.name_prefix = workflow_dict['names']['name_prefix'] databaseinfra.name_stamp = workflow_dict['names']['name_stamp'] databaseinfra.last_vm_created = 0 credentials = get_engine_credentials(engine=str(workflow_dict['plan'].engine_type), environment=workflow_dict['environment']) databaseinfra.user = credentials.user databaseinfra.password = credentials.password databaseinfra.engine = workflow_dict['plan'].engine databaseinfra.plan = workflow_dict['plan'] databaseinfra.disk_offering = workflow_dict['plan'].disk_offering databaseinfra.environment = workflow_dict['environment'] databaseinfra.capacity = 1 databaseinfra.per_database_size_mbytes = workflow_dict[ 'plan'].max_db_size databaseinfra.save() LOG.info("DatabaseInfra created!") workflow_dict['databaseinfra'] = databaseinfra return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False
def get_or_create_infra(base_name, plan, environment, backup_hour=None, maintenance_window=None, maintenance_day=None, retry_from=None, pool=None): if retry_from: infra = retry_from.infra base_name['infra'] = infra.name base_name['name_prefix'] = infra.name_prefix base_name['name_stamp'] = infra.name_stamp else: random_backup_hour, random_maintenance_hour, random_maintenance_day = ( DatabaseForm.randomize_backup_and_maintenance_hour()) infra = DatabaseInfra() infra.name = base_name['infra'] infra.name_prefix = base_name['name_prefix'] infra.name_stamp = base_name['name_stamp'] infra.last_vm_created = 0 infra.engine = plan.engine infra.plan = plan infra.disk_offering = plan.disk_offering infra.disk_offering_type = plan.disk_offering_type infra.environment = environment infra.capacity = 1 infra.per_database_size_mbytes = plan.max_db_size infra.backup_hour = backup_hour or random_backup_hour infra.maintenance_window = (maintenance_window or random_maintenance_hour) infra.maintenance_day = maintenance_day or random_maintenance_day infra.engine_patch = plan.engine.default_engine_patch infra.pool = pool infra.save() driver = infra.get_driver() user, password, key = driver.build_new_infra_auth() infra.user = user infra.password = password infra.database_key = key infra.save() return infra
def do(self, workflow_dict): try: workflow_dict['names'] = gen_infra_names( name=workflow_dict['name'], qt=workflow_dict['qt']) databaseinfra = DatabaseInfra() databaseinfra.name = workflow_dict['names']['infra'] databaseinfra.user = '' databaseinfra.password = make_db_random_password() databaseinfra.engine = workflow_dict['plan'].engine databaseinfra.plan = workflow_dict['plan'] databaseinfra.disk_offering = workflow_dict['plan'].disk_offering databaseinfra.environment = workflow_dict['environment'] databaseinfra.capacity = 1 databaseinfra.per_database_size_mbytes = workflow_dict[ 'plan'].max_db_size databaseinfra.save() LOG.info("DatabaseInfra created!") workflow_dict['databaseinfra'] = databaseinfra return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False
def get_or_create_infra(base_name, plan, environment, retry_from=None): if retry_from: infra = retry_from.infra base_name['infra'] = infra.name base_name['name_prefix'] = infra.name_prefix base_name['name_stamp'] = infra.name_stamp else: infra = DatabaseInfra() infra.name = base_name['infra'] infra.name_prefix = base_name['name_prefix'] infra.name_stamp = base_name['name_stamp'] infra.last_vm_created = 0 infra.password = make_db_random_password() infra.engine = plan.engine infra.plan = plan infra.disk_offering = plan.disk_offering infra.environment = environment infra.capacity = 1 infra.per_database_size_mbytes = plan.max_db_size infra.save() return infra
def do(self, workflow_dict): try: workflow_dict['names'] = gen_infra_names( name=workflow_dict['name'], qt=workflow_dict['qt']) databaseinfra = DatabaseInfra() databaseinfra.name = workflow_dict['names']['infra'] if workflow_dict['enginecod'] == workflow_dict['REDIS']: databaseinfra.user = '' databaseinfra.password = make_db_random_password() else: credentials = get_engine_credentials(engine=str(workflow_dict['plan'].engine_type), environment=workflow_dict['environment']) databaseinfra.user = credentials.user databaseinfra.password = credentials.password databaseinfra.engine = workflow_dict[ 'plan'].engine_type.engines.all()[0] databaseinfra.plan = workflow_dict['plan'] databaseinfra.environment = workflow_dict['environment'] databaseinfra.capacity = 1 databaseinfra.per_database_size_mbytes = workflow_dict['plan'].max_db_size databaseinfra.save() LOG.info("DatabaseInfra created!") workflow_dict['databaseinfra'] = databaseinfra return True except Exception, e: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: workflow_dict['names'] = gen_infra_names( name=workflow_dict['name'], qt=workflow_dict['qt']) databaseinfra = DatabaseInfra() databaseinfra.name = workflow_dict['names']['infra'] if workflow_dict['enginecod'] == workflow_dict['REDIS']: databaseinfra.user = '' databaseinfra.password = make_db_random_password() else: credentials = get_engine_credentials( engine=str(workflow_dict['plan'].engine_type), environment=workflow_dict['environment']) databaseinfra.user = credentials.user databaseinfra.password = credentials.password databaseinfra.engine = workflow_dict[ 'plan'].engine_type.engines.all()[0] databaseinfra.plan = workflow_dict['plan'] databaseinfra.environment = workflow_dict['environment'] databaseinfra.capacity = 1 databaseinfra.per_database_size_mbytes = workflow_dict[ 'plan'].max_db_size databaseinfra.save() LOG.info("DatabaseInfra created!") workflow_dict['databaseinfra'] = databaseinfra return True except Exception, e: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: workflow_dict['names'] = gen_infra_names( name=workflow_dict['name'], qt=workflow_dict['qt']) databaseinfra = DatabaseInfra() databaseinfra.name = workflow_dict['names']['infra'] databaseinfra.user = '' databaseinfra.password = make_db_random_password() databaseinfra.engine = workflow_dict[ 'plan'].engine databaseinfra.plan = workflow_dict['plan'] databaseinfra.environment = workflow_dict['environment'] databaseinfra.capacity = 1 databaseinfra.per_database_size_mbytes = workflow_dict[ 'plan'].max_db_size databaseinfra.save() LOG.info("DatabaseInfra created!") workflow_dict['databaseinfra'] = databaseinfra return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False