def create_service(self, params=None):
can_modify = self.update
immutable = params.pop('immutable', False)
service_list = self.client.service.list()
service_list = filter(lambda service: service['localized_name'] == params['localized_name'],
service_list['items'])
service_info = next(service_list, None)
if not service_info:
service_info = self.client.service.create(**params)
logbook.info('Service "{}" created with id={}'.format(service_info['localized_name']['en'],
service_info['service_id']))
can_modify = True
else:
diff = self.compare_fields(service_info, params, ['localized_name', 'description'])
if service_info['measure']['measure_id'] != params['measure']:
diff.append(('measure', service_info['measure']['measure_id'], params['measure']))
if diff:
diff_str = self.diff_to_str('Service <{}>'.format(service_info['service_id']), diff)
logbook.info(diff_str)
if not service_info['mutable']:
logbook.warning('Service is immutable')
if service_info['mutable'] and self.update:
self.client.service.update(service_info['service_id'], **params)
if immutable and can_modify:
self.client.service.immutable(service_info['service_id'])
def tokens(self):
"""Handle auth requests, patch service catalog endpoint urls"""
response = self.make_requests_request(bottle.request, urljoin(self.auth_url, 'tokens'))
if response.status_code != 200 or not self.patch_service_catalog:
logbook.info('Proxing tokens request to openstack without patching ({})', response.status_code)
return self.make_bottle_response(response)
try:
parsed = response.json()
for service_dict in parsed.get('access', {}).get('serviceCatalog', []):
service_name = service_dict['name']
endpoint = service_dict['endpoints'][0]
for item in endpoint:
if item.endswith('URL'):
name = service_name+'_'+item[:-3]
self.service_mapping[name] = endpoint[item] # e.g. nova_public, keystone_admin
endpoint[item] = self.urljoin(self.fake_auth_url, 'mock', name) + '/'
dump = json.dumps(parsed)
except Exception:
logbook.exception('Error while patching service catalog')
logbook.warning('Tokens content: {}', response.content)
raise
logbook.debug('service mapping is: {}', self.service_mapping)
headers = self.filter_headers(response.headers)
headers['Content-Length'] = len(dump)
return bottle.HTTPResponse(dump, response.status_code, headers)
def create_tariff(self, params=None):
messages = []
can_modify = self.update
immutable = params.pop('immutable', False)
default = params.pop('default', False)
parsed_services = []
for service_id, price in params.pop('services', {}).items():
flavor = self.get_flavor(service_id)
if flavor:
service_id = flavor['service_id']
parsed_services.append({'service_id': service_id, 'price': price})
params['services'] = parsed_services
tariff_list = self.client.tariff.list(
name=params['localized_name']['en'])
if tariff_list['total'] == 0:
tariff_info = self.client.tariff.create(**params)
logbook.info('Tariff "{}" created with id={}'.format(
tariff_info['localized_name']['en'], tariff_info['tariff_id']))
can_modify = True
else:
tariff_info = tariff_list['items'][0]
diff = self.compare_fields(
tariff_info, params,
['localized_name', 'description', 'currency'])
for service_config in params['services']:
service_id, price = service_config['service_id'], str(
service_config['price'])
service = find_first(
tariff_info['services'],
lambda srv: service_id == srv['service']['service_id'])
if not service:
messages.append(
'Service "{}" (id:{}) not in tariff service list'.
format(service['service']['localized_name']['en'],
service_id))
elif Decimal(price) != Decimal(service['price']):
diff.append(('service price ({})'.format(
service['service']['localized_name']['en']),
service['price'], price))
if diff or messages:
diff_str = self.diff_to_str(
'Tariff "{}" (id:{})'.format(
tariff_info['localized_name']['en'],
tariff_info['tariff_id']), diff)
logbook.info(diff_str)
for message in messages:
logbook.info(message)
if not tariff_info['mutable']:
logbook.warning('Tariff is immutable')
if tariff_info['mutable'] and self.update:
self.client.tariff.update(tariff_info['tariff_id'],
**params)
if immutable and can_modify:
self.client.tariff.immutable(tariff_info['tariff_id'])
if default and can_modify:
self.client.tariff.set_default(tariff_info['tariff_id'])
def check_db_write():
from model import Option
try:
Option.set('last_check', utcnow().isoformat())
except Exception as e:
logbook.warning(e)
return str(e)
def task_os_create_tenant_and_user(customer_id, email):
# email is just for logs
customer = Customer.get_by_id(customer_id)
log.debug("task_os_create_tenant_and_user: {}", customer)
if not customer:
log.warning("Can't find customer {} {}. Possible customer was removed by system test", customer_id, email)
return
info = openstack.create_tenant_and_user(email, customer_id, customer.tariff.flavors(),
password=customer.os_user_password, enabled=True)
log.debug("Openstack info: {}", info)
# Tests can delete customer already here, therefore no updates are required
if not conf.test:
db.session.commit()
db.session.close()
customer = Customer.get_by_id(customer_id)
if customer:
customer.update_os_credentials(info['tenant_id'], info['tenant_name'],
info['user_id'], info["username"], info['password'])
send_email_os_credentials(info['email'], info['name'], info['password'], info['tenant_id'],
language=customer.locale_language())
db.session.commit()
else:
final_delete_from_os(info['tenant_id'], info['user_id'])
def create_service(self, params=None):
can_modify = self.update
immutable = params.pop('immutable', False)
service_list = self.client.service.list()
service_list = filter(
lambda service: service['localized_name'] == params[
'localized_name'], service_list['items'])
service_info = next(service_list, None)
if not service_info:
service_info = self.client.service.create(**params)
logbook.info('Service "{}" created with id={}'.format(
service_info['localized_name']['en'],
service_info['service_id']))
can_modify = True
else:
diff = self.compare_fields(service_info, params,
['localized_name', 'description'])
if service_info['measure']['measure_id'] != params['measure']:
diff.append(('measure', service_info['measure']['measure_id'],
params['measure']))
if diff:
diff_str = self.diff_to_str(
'Service <{}>'.format(service_info['service_id']), diff)
logbook.info(diff_str)
if not service_info['mutable']:
logbook.warning('Service is immutable')
if service_info['mutable'] and self.update:
self.client.service.update(service_info['service_id'],
**params)
if immutable and can_modify:
self.client.service.immutable(service_info['service_id'])
def inner(*args, **kwargs):
try:
if new_session and not conf.test:
db.session.close()
logbook.debug("Start task {} with args: {} {}", fn.__name__, args, kwargs)
try:
h = "%8x" % abs(hash(args))
except TypeError:
from pprint import pformat
h = hash(pformat(args))
h = "%8x" % abs(h)
request_id = "%s-%s" % (fn.__name__, h[0:4])
def inject_request_id(record):
record.extra['request_id'] = request_id
with logbook.Processor(inject_request_id):
res = fn(*args, **kwargs)
if auto_commit:
db.session.commit()
logbook.debug("Result of task {}: {}", fn.__name__, res)
return res
except OperationalError as operation_error:
logbook.warning("Database is down {}: {}", conf.database.uri, operation_error, exc_info=True)
logbook.error("Database is down {}: {}", conf.database.uri, operation_error)
db.session.close()
current_task.retry(exc=operation_error, countdown=calc_exp_countdown())
except Exception as exc:
logbook.warning("{} failed. Retrying...", fn.__name__, exc_info=True)
current_task.retry(exc=exc, countdown=calc_exp_countdown())
def check_ceilometer():
from os_interfaces.openstack_wrapper import openstack
try:
openstack.client_ceilometer.new_samples.list(limit=10)
except Exception as e:
logbook.warning(e)
return str(e)
def clean_up_customer_service_usage(self, customer_id, end_date):
from model import Customer
customer = Customer.get_by_id(customer_id)
if not customer:
logbook.warning("Customer id '{}' not found for cleaning up services usage", customer_id)
return
customer.clean_up_service_usage(end_date)
def run_usage_collection(self, end=None):
# Run usage collection on all tenants present in Keystone.
db.session.close()
tenants = Tenant.all_active_tenants()
usage = {}
for tenant in tenants:
try:
tenant_id = tenant.tenant_id # session can be closed during next call, so we should cache tenant_id
except ObjectDeletedError as e:
logbook.warning(
"Tenant {} was removed from db (probably during cleanup after test: ",
tenant_id, e)
next_run_delay = None
with TenantMutex(tenant) as mutex:
if mutex:
logbook.debug("Processing tenant: {}", tenant_id)
tenant_usage = self.collect_usage(tenant, mutex, end)
usage[tenant_id] = tenant_usage
db.session.commit()
next_run_delay = conf.fitter.min_tenant_interval if tenant_usage else conf.fitter.tenant_interval
logbook.debug(
"Create mutex for tenant {} to prevent very often access to ceilometer. Delay: {}",
tenant, next_run_delay)
if next_run_delay and not conf.test:
mutex = TenantMutex(tenant)
mutex.acquire(ttl_ms=next_run_delay * 1000)
db.session.close()
logbook.info("Usage collection run complete.")
return usage
def service_price(self, service_id):
service_id = str(service_id)
sp = self.services_as_dict(lower=True).get(service_id.lower())
if not sp:
logbook.warning("Tariff {} don't have service {}", self, service_id)
return None
return sp.price
def get_used_quotas(customer_id):
from model import Customer
customer = Customer.get_by_id(customer_id)
if not customer:
logbook.warning("Customer id '{}' not found for getting used quotas", customer_id)
return
customer.get_used_quotas()
def create_flavor(self, params=None):
can_modify = False
immutable = params.pop('immutable', False)
flavor_id = params["flavor_id"]
flavor = self.get_flavor(flavor_id)
if not flavor:
service_info = self.client.service.create_vm(**params)
logbook.info('Flavor "{}" created with id={}'.format(service_info['localized_name']['en'],
service_info['service_id']))
can_modify = True
else:
service_info = flavor
params.pop('flavor_id')
diff = self.compare_fields(service_info, params, ['localized_name'])
diff.extend(self.compare_fields(service_info['flavor'], params, list(service_info['flavor'].keys())))
if diff:
diff_str = self.diff_to_str('Flavor "{}" (id:{})'.format(service_info['localized_name']['en'],
service_info['service_id']), diff)
logbook.info(diff_str)
if not service_info['mutable']:
logbook.warning('Service is immutable')
if service_info['mutable'] and self.update:
self.client.service.update_vm(service_info['service_id'], **params)
if immutable and can_modify:
self.client.service.immutable(service_info['service_id'])
def check_db_write():
from model import Option
try:
Option.set('last_check', utcnow().isoformat())
except Exception as e:
logbook.warning(e)
return str(e)
def get_limits(self, tenant_id, username, password):
nova_limits = self.get_nova_limits(tenant_id)
cinder_limits = defaultdict(lambda: None)
if username and password:
try:
user_auth = dict(username=username,
password=password,
tenant_id=tenant_id)
with self.change_auth(user_auth):
cinder_limits = self.client_cinder.limits.get(
)._info['absolute']
except Unauthorized as e:
log.warning(
"Can't get cinder limits for tenant {} and user: {} : {}",
tenant_id, username, e)
res = {
'gigabytes': cinder_limits['totalGigabytesUsed'],
'snapshots': cinder_limits['totalSnapshotsUsed'],
'volumes': cinder_limits['totalVolumesUsed'],
'instances': nova_limits['totalInstancesUsed'],
'cores': nova_limits['totalCoresUsed'],
'ram': nova_limits['totalRAMUsed'],
'server_groups': nova_limits['totalServerGroupsUsed'],
'floatingip': len(self.get_floating_ips(tenant_id)),
'port': len(self.get_ports(tenant_id=tenant_id))
}
return res
def wrapper(*args, **kwargs):
start_time = time.time()
request_id = self.generate_id()
def inject_request_id(record):
record.extra['request_id'] = request_id
with logbook.Processor(inject_request_id):
logbook.notice(self.request_str(), extra={"api": True})
try:
response = callback(*args, **kwargs)
except OperationalError as e:
logbook.warning("Database is down {}: {}", conf.database.uri, e, exc_info=True)
logbook.error("Database is down {}: {}", conf.database.uri, e)
response = errors.DatabaseIsDown()
except errors.BadRequest as e:
e.format_response()
response = e
except bottle.HTTPResponse as e:
response = e
except Exception as e:
if self.under_test:
import traceback
traceback.print_exc()
logbook.exception("Exception during processing request: %s %s" %
(bottle.request.method, bottle.request.path))
self.log_response(str(e), 500, time.time() - start_time)
raise
finally:
from model import db
db.session.remove()
response = self.response_process(response, time.time() - start_time)
return response
def set_assignee_due_to_comment(self, new_assignee, comment,
issue_working_state):
if new_assignee:
new_assignee_proper = self.get_assignee_login_by_name(new_assignee)
if not new_assignee_proper:
logbook.info(
u'Ignoring unknown assignee %s in comment %s by %s.' %
(new_assignee, comment.id, comment.user.login))
self.send_message(
comment.user, u'Unknown assignee',
u'(Your comment)[%s] appears to request that the assignee `%s` is set for the issue but this does not seems to be a repository collaborator.'
% (comment.url, new_assignee))
return
else:
# You can clear the assignee.
new_assignee_proper = None
if not self.user_may_set_assignee(comment.user):
logbook.warning(
u"Ignoring unathorised attempt to alter assignee by %s through comment %s."
% (comment.user.login, comment.url))
self.send_message(
comment.user, u'Unable to alter assignee',
u'(Your comment)[%s] appears to request that the assignee `%s` is set for the issue but you do not have the required authorisation.'
% (comment.url, new_assignee_proper))
else:
logbook.info("Setting assignee %s due to comment %s by %s" %
(new_assignee_proper, comment.id, comment.user.login))
self.set_assignee(new_assignee_proper, issue_working_state)
def check_openstack():
from os_interfaces.openstack_wrapper import openstack
try:
openstack.check_openstack_availability()
except Exception as e:
logbook.warning(e)
return str(e)
def transform_usage(self, tenant, usage, service, meter_info, time_label, customer):
transformed_usage = []
usage_by_resource = self.filter_and_group(usage)
transformer_name = meter_info['transformer']
transformer_args = {}
if isinstance(transformer_name, dict):
assert len(transformer_name) == 1
transformer_name, transformer_args = next(iter(transformer_name.items()))
transformer = get_transformer(transformer_name, **transformer_args)
for resource_id, entries in usage_by_resource.items():
# apply the transformer.
try:
transformed = transformer.transform_usage(service, entries, time_label)
except Exception as e:
logbook.warning("Error {} during processing usage for tenant {}, service: {}, "
"meter_info: {}, time_label: {}, resource_id: {}: {}",
e, tenant, service, meter_info, time_label, resource_id, entries)
raise
for su in transformed:
service_usage = ServiceUsage(tenant.tenant_id, su.service_id, time_label, resource_id,
customer.tariff, su.volume, su.start, su.end,
resource_name=su.resource_name)
db.session.add(service_usage)
transformed_usage.append(service_usage)
return transformed_usage
def check_db_read():
from model import User
try:
User.query.filter(User.user_id == 1).first()
except Exception as e:
logbook.warning(e)
return str(e)
def run_usage_collection(self, end=None):
# Run usage collection on all tenants present in Keystone.
db.session.close()
tenants = Tenant.all_active_tenants()
usage = {}
for tenant in tenants:
try:
tenant_id = tenant.tenant_id # session can be closed during next call, so we should cache tenant_id
except ObjectDeletedError as e:
logbook.warning("Tenant {} was removed from db (probably during cleanup after test: ", tenant_id, e)
next_run_delay = None
with TenantMutex(tenant) as mutex:
if mutex:
logbook.debug("Processing tenant: {}", tenant_id)
tenant_usage = self.collect_usage(tenant, mutex, end)
usage[tenant_id] = tenant_usage
db.session.commit()
next_run_delay = conf.fitter.min_tenant_interval if tenant_usage else conf.fitter.tenant_interval
logbook.debug("Create mutex for tenant {} to prevent very often access to ceilometer. Delay: {}",
tenant, next_run_delay)
if next_run_delay and not conf.test:
mutex = TenantMutex(tenant)
mutex.acquire(ttl_ms=next_run_delay * 1000)
db.session.close()
logbook.info("Usage collection run complete.")
return usage
def create_flavor(self, params=None):
can_modify = False
immutable = params.pop('immutable', False)
flavor_id = params["flavor_id"]
flavor = self.get_flavor(flavor_id)
if not flavor:
service_info = self.client.service.create_vm(**params)
logbook.info('Flavor "{}" created with id={}'.format(
service_info['localized_name']['en'],
service_info['service_id']))
can_modify = True
else:
service_info = flavor
params.pop('flavor_id')
diff = self.compare_fields(service_info, params,
['localized_name'])
diff.extend(
self.compare_fields(service_info['flavor'], params,
list(service_info['flavor'].keys())))
if diff:
diff_str = self.diff_to_str(
'Flavor "{}" (id:{})'.format(
service_info['localized_name']['en'],
service_info['service_id']), diff)
logbook.info(diff_str)
if not service_info['mutable']:
logbook.warning('Service is immutable')
if service_info['mutable'] and self.update:
self.client.service.update_vm(service_info['service_id'],
**params)
if immutable and can_modify:
self.client.service.immutable(service_info['service_id'])
def set_milestone_due_to_comment(self, new_milestone, comment,
issue_working_state):
if new_milestone:
new_milestone_proper = self.get_milestone_title_by_title(
new_milestone)
if not new_milestone_proper:
logbook.info(
u'Ignoring unknown milestone %s in comment %s by %s.' %
(new_milestone, comment.id, comment.user.login))
self.send_message(
comment.user, u'Unknown milestone',
u'(Your comment)[%s] appears to request that the milestone `%s` is set for the issue but this does not seems to be a valid milestone.'
% (comment.url, new_milestone))
return
else:
# You can clear the milestone.
new_milestone_proper = None
if not self.user_may_set_milestone(comment.user):
logbook.warning(
u"Ignoring unathorised attempt to alter milestone by %s through comment %s."
% (comment.user.login, comment.url))
self.send_message(
comment.user, u'Unable to alter milestone',
u'(Your comment)[%s] appears to request that the milestone `%s` is set for the issue but you do not have the required authorisation.'
% (comment.url, new_milestone_proper))
else:
logbook.info(
"Setting milestone %s due to comment %s by %s" %
(new_milestone_proper, comment.id, comment.user.login))
self.set_milestone(new_milestone_proper, issue_working_state)
def task_os_create_tenant_and_user(customer_id, email):
# email is just for logs
customer = Customer.get_by_id(customer_id)
log.debug("task_os_create_tenant_and_user: {}", customer)
if not customer:
log.warning(
"Can't find customer {} {}. Possible customer was removed by system test",
customer_id, email)
return
info = openstack.create_tenant_and_user(email,
customer_id,
customer.tariff.flavors(),
password=customer.os_user_password,
enabled=True)
log.debug("Openstack info: {}", info)
# Tests can delete customer already here, therefore no updates are required
if not conf.test:
db.session.commit()
db.session.close()
customer = Customer.get_by_id(customer_id)
if customer:
customer.update_os_credentials(info['tenant_id'], info['tenant_name'],
info['user_id'], info["username"],
info['password'])
send_email_os_credentials(info['email'],
info['name'],
info['password'],
info['tenant_id'],
language=customer.locale_language())
db.session.commit()
else:
final_delete_from_os(info['tenant_id'], info['user_id'])
def send_email(self,
send_to,
subject,
body,
from_addr=None,
cc=None,
attachments=None):
try:
mail.send_email(send_to,
subject,
body,
from_addr,
cc=cc,
attachments=attachments,
timeout=conf.email_server.timeout)
except Exception as e:
if self.request.retries >= self.max_retries - 1:
exception_to_sentry()
else:
logbook.warning(
"Exception during sending email to {} with subject {}: {} from: {}",
send_to,
subject,
e,
from_addr,
exc_info=True)
send_email.retry(exc=e)
def check_ceilometer():
from os_interfaces.openstack_wrapper import openstack
try:
openstack.client_ceilometer.new_samples.list(limit=10)
except Exception as e:
logbook.warning(e)
return str(e)
def remove_label_due_to_comment(self, remove_label, comment,
issue_working_state):
remove_label_proper = self.get_label_by_name(remove_label)
if not remove_label_proper in self.known_labels:
logbook.info(u'Ignoring unknown label %s in comment %s by %s.' %
(remove_label, comment.id, comment.user.login))
self.send_message(
comment.user, u'Unknown label',
u'(Your comment)[%s] appears to request that the label `%s` is removed from the issue but this does not seems to be a valid label.'
% (comment.url, remove_label))
return
if not self.user_may_alter_labels(comment.user):
logbook.warning(
u"Ignoring unathorised attempt to alter labels by %s through comment %s."
% (comment.user.login, comment.url))
self.send_message(
comment.user, u'Unable to alter label',
u'(Your comment)[%s] appears to request that the label `%s` is removed from the issue but you do not have the required authorisation.'
% (comment.url, remove_label_proper))
else:
logbook.info("Removing label %s due to comment %s by %s" %
(remove_label_proper, comment.id, comment.user.login))
self.remove_label(remove_label_proper, issue_working_state)
def check_openstack():
from os_interfaces.openstack_wrapper import openstack
try:
openstack.check_openstack_availability()
except Exception as e:
logbook.warning(e)
return str(e)
def get_limits(self, tenant_id, username, password):
nova_limits = self.get_nova_limits(tenant_id)
cinder_limits = defaultdict(lambda: None)
if username and password:
try:
user_auth = dict(username=username, password=password, tenant_id=tenant_id)
with self.change_auth(user_auth):
cinder_limits = self.client_cinder.limits.get()._info['absolute']
except Unauthorized as e:
log.warning("Can't get cinder limits for tenant {} and user: {} : {}",
tenant_id, username, e)
res = {
'gigabytes': cinder_limits['totalGigabytesUsed'],
'snapshots': cinder_limits['totalSnapshotsUsed'],
'volumes': cinder_limits['totalVolumesUsed'],
'instances': nova_limits['totalInstancesUsed'],
'cores': nova_limits['totalCoresUsed'],
'ram': nova_limits['totalRAMUsed'],
'server_groups': nova_limits['totalServerGroupsUsed'],
'floatingip': len(self.get_floating_ips(tenant_id)),
'port': len(self.get_ports(tenant_id=tenant_id))
}
return res
def check_db_read():
from model import User
try:
User.query.filter(User.user_id == 1).first()
except Exception as e:
logbook.warning(e)
return str(e)
def login_os_user(self, username, password):
if self._check_already_logged_in(username):
logbook.info('User "{}" is logged in already and has valid cookies.', username)
return {}
horizon_auth_login = posixpath.join(self.horizon_url, 'auth/login/')
data = {
'region': self.region,
'username': username,
'password': password,
'csrfmiddlewaretoken': self.session.cookies.get('csrftoken')
}
try:
data_without_password = data.copy()
data_without_password.pop("password")
logbook.info("Try authorize in horizon: {}, {}", horizon_auth_login, data_without_password)
horizon_auth_response = self.session.post(horizon_auth_login, data=data, verify=False, stream=False)
except requests.exceptions.RequestException as e:
logbook.warning('Request exception happens during posting data to {} for user "{}": {}',
horizon_auth_login, username, e)
raise errors.HorizonRequestError()
if horizon_auth_response.status_code != 200 or horizon_auth_response.url == horizon_auth_login:
logbook.warning('Unable to login into Horizon on {} for login = {} with status = {}: {}. Response url: {}',
horizon_auth_login, username, horizon_auth_response.status_code,
horizon_auth_response.text, horizon_auth_response.url)
raise errors.HorizonUnauthorized()
cookie = horizon_auth_response.headers.get('Set-Cookie')
logbook.debug("Horizon cookies: {}", cookie)
return http.cookies.SimpleCookie(cookie)
def payment_check(self, *args, **request_data):
"""Checks payment availability for customer
Parameters must be sent as json object.
:param Int TransactionId: Mandatory - System transaction number.
:param Numeric Amount: Mandatory - Payment amount from widget. Dot as separator, two digits after dot.
:param String Currency: Mandatory - Currency: RUB/USD/EUR/GBP from widget parameters.
:param String InvoiceId: Not mandatory - Order number from widget parameters.
:param String AccountId: Mandatory - Customer identifier from widget parameters.
:param String SubscriptionId: Not mandatory - Subscription identifier from widget parameters (for recurrent payments).
:param String Name: Not mandatory - Card holder name.
:param String Email: Payer's e-mail
:param DateTime: Mandatory - Payment creation date/time in UTC (yyyy-MM-dd HH:mm:ss).
:param String IpAddress: Not mandatory - Payer IP-address
:param String IpCountry: Not mandatory - Payer's country double-chars code (according to ISO3166-1)
:param String IpCity: Not mandatory - Payer's city
:param String IpRegion: Not mandatory - Payer's region.
:param String IpDistrict: Not mandatory - Payer's district.
:param String CardFirstSix: Mandatory - Credit card first 6 digits
:param String CardLastFour: Mandatory - Credit card last 6 digits
:param String CardType: Mandatory - Card payment system: Visa or MasterCard or Maestro
:param String CardExpDate: Mandatory - Card expiration date MM/YY
:param String Issuer: Not mandatory - Issuer bank name
:param String IssuerBankCountry: Not mandatory - Issuer bank country double-char code (according to ISO3166-1)
:param String Description: Not mandatory - Payment description from widget parameters.
:param Json Data: Not mandatory - Any json-data from widget.
:param Bit TestMode: Mandatory - Test mode flag (1 or 0)
:param String Status: Mandatory - Payment status: Completed — for single-step, Authorized — for double-step.
:return: Status code, looks like {'code': 0}
"""
logbook.info("[payment_check] Request info:{}", request_data)
short_payment_info = dict([(key, request_data.get(key)) for key in PaymentsApi.payment_info_fields])
# Common validation
validation_res, validation_info = self.validate_request(request_data, short_payment_info)
if not validation_res:
log_error("[payment_check] {} Payment info: {}", validation_info, short_payment_info)
return {'code': self.ERROR_COMMON}
# Currency validation
currency = request_data['Currency']
if not currency or currency not in conf.currency.active:
log_error("[payment_check] Invalid or incompatible currency: {}. Payment info: {}",
currency, short_payment_info)
return {'code': self.ERROR_COMMON}
# Customer validation
customer_id = request_data['AccountId']
customer = Customer.get_by_id(customer_id, False)
if not customer:
log_error("[payment_check] Customer {} not found. Payment info: {}", customer_id, short_payment_info)
return {'code': self.ERROR_COMMON}
if customer.is_test_mode():
# Payments in test mode is not allowed
logbook.warning("[payment_check] Customer {} in test mode. Payment info {}", customer, short_payment_info)
return {'code': self.ERROR_COMMON}
return {'code': self.ERROR_OK}
def check_redis_read():
from memdb import create_redis_client
redis = create_redis_client()
try:
redis.get('health_check')
except Exception as e:
logbook.warning(e)
return str(e)
def check_redis_read():
from memdb import create_redis_client
redis = create_redis_client()
try:
redis.get('health_check')
except Exception as e:
logbook.warning(e)
return str(e)
def get_used_quotas(customer_id):
from model import Customer
customer = Customer.get_by_id(customer_id)
if not customer:
logbook.warning("Customer id '{}' not found for getting used quotas",
customer_id)
return
customer.get_used_quotas()
def openstack_error_context(self, error_code:int):
self.mock_raise = error_code
logbook.warning('Enter error context: {}', error_code)
try:
yield
finally:
self.mock_raise = None
logbook.warning('Exit error context: {}', error_code)
def clean_up_customer_service_usage(self, customer_id, end_date):
from model import Customer
customer = Customer.get_by_id(customer_id)
if not customer:
logbook.warning(
"Customer id '{}' not found for cleaning up services usage",
customer_id)
return
customer.clean_up_service_usage(end_date)
def check_redis_write():
from memdb import create_redis_client
redis_health = {}
redis = create_redis_client()
try:
redis_health['redis_write'] = redis.set('health_check', utcnow().isoformat())
except Exception as e:
logbook.warning(e)
return str(e)
def del_channel(self, id):
try:
c = self.session.query(Channel).get(id)
self.session.delete(c)
info("Deleted channel with ID {0}", id)
except KeyError:
warning("Server removed channel with ID {}, "
"but we haven't seen it before!", id)
except:
pass
def check_redis_write():
from memdb import create_redis_client
redis_health = {}
redis = create_redis_client()
try:
redis_health['redis_write'] = redis.set('health_check',
utcnow().isoformat())
except Exception as e:
logbook.warning(e)
return str(e)
def send_email(self, send_to, subject, body, from_addr=None, cc=None, attachments=None):
try:
mail.send_email(send_to, subject, body, from_addr, cc=cc,
attachments=attachments, timeout=conf.email_server.timeout)
except Exception as e:
if self.request.retries >= self.max_retries - 1:
exception_to_sentry()
else:
logbook.warning("Exception during sending email to {} with subject {}: {} from: {}",
send_to, subject, e, from_addr, exc_info=True)
send_email.retry(exc=e)
def update_quota(customer_id, user_id, quotas):
from model import Customer, CustomerHistory
customer = Customer.get_by_id(customer_id)
if not customer:
logbook.warning("Customer id '{}' not found for quota update", customer_id)
return
customer = Customer.get_by_id(customer_id)
openstack.change_tenant_quotas(customer.os_tenant_id, **quotas)
CustomerHistory.quota_changed(customer, user_id, None)
def delete_snapshots(read_only):
all_snapshots = openstack.get_snapshots()
snapshots = [snapshot for snapshot in all_snapshots if not get_tenant(snapshot.project_id)]
for snapshot in snapshots:
log.info('Deleting unassigned snapshot {}, tenant: {}', snapshot, snapshot.project_id)
if not read_only:
try:
openstack.client_cinder.volume_snapshots.delete(snapshot.id)
except Exception as e:
log.warning("Can't remove snapshot {}: {}", snapshot._info, e)
return len(snapshots)
def delete_networks(read_only):
all_networks = openstack.get_networks()
networks = [network for network in all_networks if not get_tenant(network['tenant_id'])]
for network in networks:
log.info('Deleted unassigned network {}, tenant {}', network["id"], network["tenant_id"])
if not read_only:
try:
openstack.client_neutron.delete_network(network["id"])
except Exception as e:
log.warning("Can't delete network {}: {}", network, e)
return len(networks)
def calculate_usage_cost(self, usages):
from model import Service, Category, ServicePrice
from task.notifications import notify_managers_about_new_service_in_tariff
total_cost = Decimal()
tariff = self.tariff
if not tariff:
raise Exception("Tariff is not set for customer %s" % self)
services = tariff.services_as_dict(lower=True)
for usage in usages:
db.session.add(usage)
service_id = usage.service_id.lower() if isinstance(usage.service_id, str) else str(usage.service_id)
service_price = services.get(service_id)
service = Service.get_by_id(service_id)
usage.tariff_id = tariff.tariff_id
usage.customer_mode = self.customer_mode
if service is None:
logbook.error("Not found declaration service {} during calculate usage for {}",
usage.service_id, self)
continue
usage_volume = service.calculate_volume_usage(usage)
usage.usage_volume = usage_volume
if service_price is None:
if service.category_id == Category.VM:
if service.deleted:
logbook.error("Service {} not found in {} for {}. But this service is archived",
service_id, tariff, self)
else:
service_price = ServicePrice(service_id=service_id, price=Decimal(0), need_changing=True)
self.tariff.services.append(service_price)
services = tariff.services_as_dict(lower=True)
flavor_name = Service.get_by_id(service_id).flavor.flavor_id
notify_managers_about_new_service_in_tariff.delay(self.customer_id, flavor_name)
else:
logbook.warning("Service {} not found in {} for {}. Allowed services: {}",
service_id, tariff, self, services.keys())
if service_price:
usage_cost = usage_volume * service_price.price / service.hours
else:
usage_cost = Decimal(0)
total_cost += usage_cost
usage.cost = usage_cost
logbook.info("Found {} usages for customer {}. Total cost of used resources is: {}",
len(usages), self, total_cost)
return total_cost
def update_quota(customer_id, user_id, quotas):
from model import Customer, CustomerHistory
customer = Customer.get_by_id(customer_id)
if not customer:
logbook.warning("Customer id '{}' not found for quota update",
customer_id)
return
customer = Customer.get_by_id(customer_id)
openstack.change_tenant_quotas(customer.os_tenant_id, **quotas)
CustomerHistory.quota_changed(customer, user_id, None)
def reset_user_password(customer_id):
customer = Customer.get_by_id(customer_id)
if not customer:
log.warning("Can't find customer for reset user password {} {}. Possible customer was removed by system test",
customer_id)
return
password = openstack.reset_user_password(customer.os_user_id)
customer.update_os_password(password)
send_email_os_credentials(customer.email, customer.os_username, password, customer.os_tenant_id,
language=customer.locale_language())
def mock(self, service, path):
"""Handle requests to services. Proxy them to real urls or raise error."""
if self.mock_raise is not None:
raise bottle.HTTPError(self.mock_raise)
if service not in self.service_mapping:
logbook.warning('Requested unknown service: {} (mapping: {})', service, self.service_mapping)
raise bottle.HTTPError(404, 'Unknown service')
service_url = self.service_mapping[service]
url = self.urljoin(service_url, path)
if bottle.request.query:
url += '?' + bottle.request.query_string
return self.proxy_request(bottle.request, url)
def wrapper(*args, **kwargs):
# noinspection PyBroadException
try:
return fn(*args, **kwargs)
except Exception:
import traceback
import _thread
logbook.warning(u"Traceback: {}".format(traceback.format_exc()))
exception_to_sentry()
if exit_on_error:
traceback.print_exc()
_thread.interrupt_main()
return None
def get_used_quotas(self):
logbook.debug("Getting used quotas from openstack for {}", self)
if self.os_tenant_id and not self.blocked:
try:
quotas = openstack.get_limits(self.os_tenant_id, self.os_username, self.os_user_password)
except Unauthorized as e:
logbook.warning("Customer {} is not blocked but can't sign in OpenStack account. Error message: {}",
self, e)
quotas = {}
else:
quotas = {}
if quotas:
quota_cache.set(self, quotas)
return quotas
def _request_api(self, method, resource, params=None, data=None, parse_json=True, expected_status_codes=None):
if resource.startswith("http"):
url = resource
else:
url = posixpath.join(self.url_api, resource)
retry = 0
max_retry = 4
response = None
while retry < max_retry:
try:
response = self._session.request(method, url, params=params, data=data)
break
except requests.RequestException as e:
logbook.info("Mailtrap error for {} {} {}: {}", method, url, params, e)
retry += 1
if retry >= max_retry:
raise
time.sleep(1)
if expected_status_codes is None:
expected_status_codes = [requests.codes.ok]
if isinstance(expected_status_codes, int):
expected_status_codes = [expected_status_codes]
req = "%s %s" % (response.request.method, response.url)
if response.status_code not in expected_status_codes:
msg = "Mailtrap: %s, expected statuses %s, but returned: %s" % \
(req, expected_status_codes, response.status_code)
raise ValueError(msg)
if not parse_json:
return response.text
try:
j = response.json()
if self.verbose:
js = str(j)[:500]
logbook.debug("Mailtrap: {}, status: {}, response: {}", req, response.status_code, js)
except ValueError:
msg = "Mailtrap: %s, status: %s, response: %s" % (req, response.status_code, response.text)
logbook.warning(msg)
if response.status_code == requests.codes.ok:
raise ValueError(msg)
j = {}
return j
def delete_networks(read_only):
all_networks = openstack.get_networks()
networks = [
network for network in all_networks
if not get_tenant(network['tenant_id'])
]
for network in networks:
log.info('Deleted unassigned network {}, tenant {}', network["id"],
network["tenant_id"])
if not read_only:
try:
openstack.client_neutron.delete_network(network["id"])
except Exception as e:
log.warning("Can't delete network {}: {}", network, e)
return len(networks)
def delete_snapshots(read_only):
all_snapshots = openstack.get_snapshots()
snapshots = [
snapshot for snapshot in all_snapshots
if not get_tenant(snapshot.project_id)
]
for snapshot in snapshots:
log.info('Deleting unassigned snapshot {}, tenant: {}', snapshot,
snapshot.project_id)
if not read_only:
try:
openstack.client_cinder.volume_snapshots.delete(snapshot.id)
except Exception as e:
log.warning("Can't remove snapshot {}: {}", snapshot._info, e)
return len(snapshots)
def reset_user_password(customer_id):
customer = Customer.get_by_id(customer_id)
if not customer:
log.warning(
"Can't find customer for reset user password {} {}. Possible customer was removed by system test",
customer_id)
return
password = openstack.reset_user_password(customer.os_user_id)
customer.update_os_password(password)
send_email_os_credentials(customer.email,
customer.os_username,
password,
customer.os_tenant_id,
language=customer.locale_language())