def run_usage_collection(self, end=None):
# Run usage collection on all tenants present in Keystone.
db.session.close()
tenants = Tenant.all_active_tenants()
usage = {}
for tenant in tenants:
try:
tenant_id = tenant.tenant_id # session can be closed during next call, so we should cache tenant_id
except ObjectDeletedError as e:
logbook.warning(
"Tenant {} was removed from db (probably during cleanup after test: ",
tenant_id, e)
next_run_delay = None
with TenantMutex(tenant) as mutex:
if mutex:
logbook.debug("Processing tenant: {}", tenant_id)
tenant_usage = self.collect_usage(tenant, mutex, end)
usage[tenant_id] = tenant_usage
db.session.commit()
next_run_delay = conf.fitter.min_tenant_interval if tenant_usage else conf.fitter.tenant_interval
logbook.debug(
"Create mutex for tenant {} to prevent very often access to ceilometer. Delay: {}",
tenant, next_run_delay)
if next_run_delay and not conf.test:
mutex = TenantMutex(tenant)
mutex.acquire(ttl_ms=next_run_delay * 1000)
db.session.close()
logbook.info("Usage collection run complete.")
return usage
def acquire_weights(scale_obj,
sleep_time=settings.WEIGHT_MEASURE_WAIT_TIME,
measure_count=settings.WEIGHT_MEASURE_COUNT):
"""
to ensure accuracy of the weight do a number of
measures with sleep time in between
:param scale_obj: BeeHiveScale object
:type scale_obj: BeeHiveScale
:param sleep_time: how long to wait between measures
:type sleep_time: float
:param measure_count: umber of measures
:type measure_count: int
:return: list of measures
:rtype list
"""
logbook.info("start acquire weights")
_measures = []
for count in range(measure_count + 1):
weight = scale_obj.weight
logbook.info("weight: {weight} kg".format(weight=weight))
_measures.append(scale_obj.weight)
if count < measure_count:
logbook.debug("wait for {0}s".format(sleep_time))
time.sleep(sleep_time)
return _measures
def test_sanity(work_dir, source_file, disks):
cmd = ['target/raiden', 'split', source_file, str(disks)]
logbook.debug(cmd)
subprocess.check_call(cmd)
def merge():
cmd = ['target/raiden', 'merge', source_file, str(disks)]
subprocess.check_call(cmd)
logbook.debug(cmd)
restored_file = os.path.join(work_dir, 'res__source')
cmd = ['diff', source_file, restored_file]
logbook.debug(cmd)
subprocess.check_call(cmd)
merge()
@contextmanager
def move_aside(disk):
path = '{}_{}'.format(source_file, disk)
os.rename(path, path + '~')
try:
yield
finally:
os.rename(path + '~', path)
for i in xrange(disks):
with move_aside(i):
merge()
def doCleanups(self):
"""
Run cleanups for current test. This cleanups will run EVEN if setUp of test fails.
Original doCleanups will be called AFTER.
"""
outcome = self._outcome or unittest.case._Outcome()
def generic_cleanup(source):
while source:
function, args, kwargs = source.pop()
logbook.debug('Executing cleanup: {}'.format(function))
with outcome.testPartExecutor(self):
function(*(args or tuple()), **(kwargs or {}))
generic_cleanup(self._cleanup_before_force_delete)
for table in self._force_delete_order:
if table in self._cleanup_force_delete:
while self._cleanup_force_delete[table]:
prefix, field = self._cleanup_force_delete[table].pop()
logbook.debug('Executing cleanup: Force delete {} with prefix "{}"'.format(table, prefix))
with outcome.testPartExecutor(self):
self.force_delete(table, prefix, field)
generic_cleanup(self._cleanup_after_delete)
generic_cleanup(self._cleanups)
return outcome.success
def validate(self, value):
"""
This method validates given value. It doesn't catch any exception, it is a task of internal executor.
:param object value: The value we have to validate.
:return Validated value.
"""
processed_value = value
for type_validator in self.validators:
try:
processed_value = type_validator(value)
except ValidateError as exc:
if self.name != self.TOKEN_NAME:
name = self.name
if exc.subname:
name += "." + exc.subname
raise BadParameter(exc, name)
raise BadRequest(exc)
except HTTPError as exc:
raise exc
except Exception as exc:
logbook.info(u"Invalid parameter {}: {} ({})".format(
self.name, exc, type(exc)))
logbook.debug("{}", traceback.format_exc())
if hasattr(type_validator, "__class__"):
validator_name = type_validator.__class__.__name__
else:
validator_name = getattr(type_validator, "__name__",
"<unknown type>")
raise BadRequest(
_(u"Invalid parameter {}. It should have type {}"),
self.name, validator_name)
return processed_value
async def expire_snapshots(ctx, rule):
"""Expire existing snapshots for the rule.
"""
logbook.debug('Expire existing snapshots')
snapshots = await load_snapshots(ctx)
snapshots = filter_snapshots_by_rule(snapshots, rule)
snapshots = {
s['name']: pendulum.parse(s['creationTimestamp'])
for s in snapshots
}
to_keep = expire(snapshots, rule.deltas)
logbook.info('Out of {} snapshots, we want to keep {}', len(snapshots),
len(to_keep))
for snapshot_name in snapshots:
if snapshot_name in to_keep:
logbook.debug('Keeping snapshot {}', snapshot_name)
continue
if snapshot_name not in to_keep:
logbook.info('Deleting snapshot {}', snapshot_name)
result = await exec(ctx.gcloud.snapshots().delete(
snapshot=snapshot_name,
project=ctx.config['gcloud_project']).execute)
def tokens(self):
"""Handle auth requests, patch service catalog endpoint urls"""
response = self.make_requests_request(bottle.request, urljoin(self.auth_url, 'tokens'))
if response.status_code != 200 or not self.patch_service_catalog:
logbook.info('Proxing tokens request to openstack without patching ({})', response.status_code)
return self.make_bottle_response(response)
try:
parsed = response.json()
for service_dict in parsed.get('access', {}).get('serviceCatalog', []):
service_name = service_dict['name']
endpoint = service_dict['endpoints'][0]
for item in endpoint:
if item.endswith('URL'):
name = service_name+'_'+item[:-3]
self.service_mapping[name] = endpoint[item] # e.g. nova_public, keystone_admin
endpoint[item] = self.urljoin(self.fake_auth_url, 'mock', name) + '/'
dump = json.dumps(parsed)
except Exception:
logbook.exception('Error while patching service catalog')
logbook.warning('Tokens content: {}', response.content)
raise
logbook.debug('service mapping is: {}', self.service_mapping)
headers = self.filter_headers(response.headers)
headers['Content-Length'] = len(dump)
return bottle.HTTPResponse(dump, response.status_code, headers)
def main(short_break_time, long_break_time):
config_path = create_config_path()
poms_file = config_path / "Pomodoro.json"
debug(f"config_path: {config_path} poms_file: {poms_file}")
Pomodoros = []
# A single Pomodoro is 25 minutes long.
pom_length = 25
pomodoro_seconds = 60 * pom_length
completed_poms = 0
try:
with open(poms_file, 'r') as pfile:
Pomodoros = json.load(pfile)
except OSError:
info("No previously stored Pomodoros! Welcome!")
Pom = {}
try:
info(
"Pomodoro timer starting! Hit any key when done. Hit 'Ctrl-c' to log an interrupt."
)
while True:
start_time = datetime.now()
Pom['start_time'] = str(start_time)
info(f"Starting next Pomodoro at: {start_time}")
time.sleep(pomodoro_seconds)
completed_poms = completed_poms + 1
info(f"{completed_poms} pomodoros complete!")
print('\a')
break_time = decide_break_time(completed_poms, short_break_time,
long_break_time)
info(f"Well done! Take a {break_time} minute break!")
time.sleep(break_time)
back_to_work()
except KeyboardInterrupt:
end_time = datetime.now()
elapsed = end_time - start_time
pom_seconds = elapsed.seconds
info(
"Ending session. Hit 'i' if this was an interrupt. Anything else to count this pom."
)
interrupt_prompt = readchar.readchar()
if interrupt_prompt.lower() is not 'i':
finished_one_pom = start_time + timedelta(minutes=pom_length)
if datetime.now() > finished_one_pom:
completed_poms = completed_poms + 1
info(f"Pom seconds: {pom_seconds} Finished poms: {completed_poms}")
Pom['end_time'] = str(end_time)
Pom['elapsed'] = str(elapsed)
Pom['completed'] = completed_poms
Pomodoros.append(Pom)
with open(poms_file, 'w') as pfile:
json.dump(Pomodoros, pfile)
def task_os_create_tenant_and_user(customer_id, email):
# email is just for logs
customer = Customer.get_by_id(customer_id)
log.debug("task_os_create_tenant_and_user: {}", customer)
if not customer:
log.warning(
"Can't find customer {} {}. Possible customer was removed by system test",
customer_id, email)
return
info = openstack.create_tenant_and_user(email,
customer_id,
customer.tariff.flavors(),
password=customer.os_user_password,
enabled=True)
log.debug("Openstack info: {}", info)
# Tests can delete customer already here, therefore no updates are required
if not conf.test:
db.session.commit()
db.session.close()
customer = Customer.get_by_id(customer_id)
if customer:
customer.update_os_credentials(info['tenant_id'], info['tenant_name'],
info['user_id'], info["username"],
info['password'])
send_email_os_credentials(info['email'],
info['name'],
info['password'],
info['tenant_id'],
language=customer.locale_language())
db.session.commit()
else:
final_delete_from_os(info['tenant_id'], info['user_id'])
def iter_api_posts(start=None):
"""Iterates over all posts that the api provides,
starting at the post witht he given id.
"""
at_end = False
while not at_end and start != 1:
# build url for next page
url = "http://pr0gramm.com/api/items/get?flags=3"
if start is not None:
url += "&older=%d" % start
# perform api request
#: :type: requests.Response
logbook.debug("requesting api page {}", url)
response = requests.get(url)
response.raise_for_status()
# parse response
data = response.json()
at_end = data["atEnd"]
# create posts
for item in data.get("items", ()):
post = Post(item["id"], item["created"], item["user"],
item["flags"], item["image"])
start = post.id
yield post
def change_auth(self, users_auth):
"""Change auth to User's credentials.
Use this to do something by user's privileges.
"""
self.__auth = {
'username': users_auth['username'],
'password': users_auth['password'],
'auth_url': users_auth.get(
'auth_url',
conf.openstack.auth.auth_url), # Use this from conf here?
'tenant_id': users_auth['tenant_id']
}
self.__client_nova = None
self.__client_glance = None
self.__client_cinder = None
self.__client_neutron = None
self.__client_keystone = None
self.__client_ceilometer = None
self.__auth_session = None
log.debug("Change openstack auth to {}", users_auth["username"])
try:
yield
finally:
self.__client_nova = None
self.__client_cinder = None
self.__client_neutron = None
self.__client_keystone = None
self.__client_ceilometer = None
self.__auth_session = None
self.__init__(conf.openstack)
def search_email(cls, regexp: str, prefix:str=None, timeout: int=60, dotall:bool=False):
"""
Search an email in mailtrap mailbox with prefix, which body matches regexp (re.search).
Raise AssertionError if none matched.
:param regexp regexp: regular expression to search in email body.
:param str prefix: email address prefix. Current test prefix used if None.
:param int timeout: searching timeout.
:param bool dotall: use re.DOTALL.
:return: match object.
"""
if prefix is None:
prefix = cls.get_test_prefix()
mailtrap_api = MailTrapApi.create_default()
regexp = re.compile(regexp, re.DOTALL if dotall else 0)
for r in cls.retries(timeout=timeout):
with r:
messages = mailtrap_api.get_messages(prefix=prefix)
for message in messages:
match = regexp.search(message['text_body'])
if match:
logbook.debug('Got match in: {}'.format(message['text_body']))
return match
else:
raise AssertionError('Emails with prefix {} matching pattern "{}" not found. {} tried.'.format(
prefix, regexp.pattern, len(messages)))
def client_keystone(self):
if not self.__client_keystone:
log.debug("Creating keystone client")
self.__client_keystone = ksclient.Client(
session=self.auth_session,
connect_retries=self.connect_retries,
timeout=self.request_timeout)
return self.__client_keystone
def get_report(self, report_id):
res = self.redis.get(self.key(report_id))
if res:
logbook.debug("Extract report {}. Size: {}", report_id, len(res))
else:
logbook.debug("Report for {} not found. Key: {}", report_id,
self.key(report_id))
return res
def merge():
cmd = ['target/raiden', 'merge', source_file, str(disks)]
subprocess.check_call(cmd)
logbook.debug(cmd)
restored_file = os.path.join(work_dir, 'res__source')
cmd = ['diff', source_file, restored_file]
logbook.debug(cmd)
subprocess.check_call(cmd)
def get_tenant(self, tenant_id):
"""Returns tenant with given tenant_id"""
try:
tenants = self.client_keystone.tenants.get(tenant_id)
except NotFound as e:
log.debug(e)
return None
return tenants
def _tests_send_email(email,
subject,
from_addr,
body,
attachments=None,
cc=None):
msg = Message(email, subject, from_addr, body, attachments or [], cc or [])
logbook.debug("Send test email to {} (cc: {}) with subject {} from: {}",
email, cc, subject, from_addr)
outbox.append(msg)
def source_file_fixture(work_dir):
path = os.path.join(work_dir, 'source')
logbook.debug('Creating the source file at {}', path)
with open(path, 'wb') as f:
for _ in xrange(1 * 1024):
f.write(bytes(random.randrange(0, 0xff)))
logbook.debug('Done creating the source file')
return path
def handle_rcpt(self, server, line, ctx):
match = _RECIPIENT_REGEX.match(line)
if not match:
logbook.error("Invalid recipient line received: {!r}", line)
server.send(_SYNTAX_ERROR)
return
recipient = match.groups()[0]
logbook.debug('Will send to {!r}', recipient)
ctx.recipients.append(recipient)
server.send(_OK)
def _log_start(self):
message = self.cstr + ' Start'
if self.client.loggedin:
message += ' "{}"'.format(self.client.email)
message += ' {} {} ({})'.format(self.method, self.url,
self.expected_status)
logbook.debug(message)
if len(self.kwargs) > 0:
logbook.debug(self.cstr +
' Params {}'.format(json.dumps(self.kwargs)))
def cleanup(self, mailbox=None, created_after=None, search=None, prefix=None):
try:
messages = self.get_messages(mailbox, created_after, search, prefix)
except Exception as e:
logbook.error("Failed to cleanup mailtrap: {}", e)
return
logbook.debug("Deleting {} messages by filter {}".format(len(messages), search))
for message in messages:
logbook.debug("Mailtrap deleting: {}", self.short_message_str(message))
self._delete(posixpath.join(self._mailbox_url(mailbox), str(message["id"])))
def calculate_signature(self, timestamp, method, path, parameters):
timestamp = str(int(timestamp))
canonical_str = self.canonical_str(timestamp, method, path, parameters)
logbook.debug("Canonical string is {}", canonical_str)
signature = hmac.new(self.secret_key.encode("utf-8"), canonical_str,
self.ALGORITHM).hexdigest()
logbook.debug("Signature: {}", signature)
return signature + timestamp
def auth_session(self):
if not self.__auth_session:
log.debug(
"Creating auth session with args {}",
{k: v
for k, v in self.__auth.items() if k != 'password'})
plugin = Password(**self.__auth)
self.__auth_session = Session(auth=plugin,
timeout=self.request_timeout)
return self.__auth_session
def _reduce_history_of_update_operations(self):
query = self.query.filter(TariffHistory.history_id != self.history_id,
TariffHistory.tariff_id == self.tariff_id,
TariffHistory.event == self.EVENT_UPDATE,
TariffHistory.user_id == self.user_id,
TariffHistory.date <= self.date + timedelta(seconds=1),
TariffHistory.date >= self.date - self.MAX_AGE_OF_UPDATES_TO_AUTO_COLLAPSE)
n = query.delete(False)
logbook.debug("Reduced {} history records for tariff {}", n, self.tariff)
return n
def parse_value(value: str):
logbook.debug("Parsing value '{0}'...".format(value))
if value.endswith('%'):
value = value.replace('%', '')
return '{}/10'.format(int(value) / 10)
elif value.endswith('100'):
(value, _) = value.split('/')
return '{}/10'.format(int(value) / 10)
else:
(value, _) = value.split('/')
return '{}/10'.format(float(value) * 1.0)
def new_customer(cls, email, password, creator_id, detailed_info=None, comment="New customer",
withdraw_period=None, make_prod=False, customer_type=None, promo_code=None, locale=None):
from model import Tariff, Account, CustomerHistory
customer = cls()
customer.email = email
customer.password = cls.password_hashing(password) if password else ""
customer.deleted = None
customer.created = utcnow().datetime
customer.email_confirmed = False
customer.blocked = False
customer.customer_mode = cls.CUSTOMER_PRODUCTION_MODE if make_prod else cls.CUSTOMER_TEST_MODE
customer.customer_type = customer_type if customer_type else cls.CUSTOMER_TYPE_PRIVATE_PERSON
customer.withdraw_period = withdraw_period or conf.customer.default_withdraw_period
customer.auto_withdraw_enabled = conf.payments.auto_withdraw_enable
customer.auto_withdraw_balance_limit = conf.payments.auto_withdraw_balance_limit
customer.auto_withdraw_amount = conf.payments.auto_withdraw_amount
customer.locale = locale or conf.customer.default_locale
customer.os_dashboard = conf.customer.default_openstack_dashboard
default_tariff = Tariff.get_default().first()
if not default_tariff:
default_tariff = Tariff.query.filter_by(mutable=False).first() or Tariff.query.filter_by().first()
if not default_tariff:
raise errors.TariffNotFound()
customer.tariff_id = default_tariff.tariff_id
customer.balance_limit = conf.customer.balance_limits.get(default_tariff.currency,
conf.customer.balance_limits.default)
db.session.add(customer)
db.session.flush()
customer.init_subscriptions()
template = 'customer' if make_prod else 'test_customer'
customer.quota_init(template)
customer.accounts.append(Account.create(default_tariff.currency, customer, creator_id, comment))
if promo_code is not None:
PromoCode.new_code(promo_code, customer.customer_id)
CustomerHistory.new_customer(customer, creator_id, comment)
if detailed_info:
customer.create_info(customer.customer_type, detailed_info)
auto_report_task = ScheduledTask(cls.AUTO_REPORT_TASK, customer.customer_id, customer.withdraw_period)
db.session.add(auto_report_task)
logbook.info("New customer created: {}", customer)
if not make_prod:
currency = customer.tariff.currency.upper()
initial_balance = conf.customer.test_customer.balance.get(currency)
logbook.debug("Initial balance for customer {}: {} {}", customer, initial_balance, currency)
if initial_balance:
customer.modify_balance(Decimal(initial_balance), currency, None, "Initial test balance")
return customer
def withdraw(self, delta, currency=None):
assert isinstance(delta, Decimal)
assert delta >= 0
currency = (currency or self.tariff.currency).upper()
logbook.debug("Withdraw {} {} of {}", delta, currency, self)
account = self.get_account(currency)
if account is None:
raise Exception("Customer %s doesn't have account in %s, but withdraw %s %s is required" %
(self, currency, delta, currency))
account.charge(delta)
self.check_balance(account, currency)
db.session.flush()
def updated_state_by_interpreting_new_comments(self, issue,
issue_working_state):
issue_working_state = issue_working_state.copy()
new_comments = self.get_new_comments(issue)
# Make sure we have the right label capitalisation.
issue_working_state['labels'] = [
self.get_label_by_name(l) for l in issue_working_state['labels']
]
logbook.debug(u"Examining %d new comment(s) for %s" %
(len(new_comments), issue))
for comment in new_comments:
if not comment.body:
continue
for line in comment.body.split('\n'):
line = line.strip()
# Votes look just like +<label> or -<label> where the label is the number 1 or 0.
if VOTE_REGEX.match(line):
continue
m = ADD_LABEL_REGEX.match(line)
if m:
new_label = (m.group(1) or m.group(2)).lower()
self.add_label_due_to_comment(new_label, comment,
issue_working_state)
continue
m = REMOVE_LABEL_REGEX.match(line)
if m:
remove_label = m.group(1).lower()
self.remove_label_due_to_comment(remove_label, comment,
issue_working_state)
continue
m = SET_MILESTONE_REGEX.match(line)
if m:
new_milestone = m.group(1).lower()
self.set_milestone_due_to_comment(new_milestone, comment,
issue_working_state)
continue
m = SET_ASSIGNEE_REGEX.match(line)
if m:
new_assignee = m.group(1).lower()
self.set_assignee_due_to_comment(new_assignee, comment,
issue_working_state)
continue
return issue_working_state
def send_email(email,
subject,
body,
from_addr,
cc=None,
attachments=None,
timeout=None):
""" Sends email, returns True on success
If called in test environment (conf.test == True) it just adds message to ``outbox`` list.
"""
if conf.test:
return _tests_send_email(email, subject, from_addr, body, attachments,
cc)
emails = [email] if isinstance(email, str) else list(email)
if cc:
if isinstance(cc, str):
emails.append(cc)
else:
emails.extend(cc)
test_emails = list(
filter(lambda mail: conf.test_email_server.address_pattern in mail,
emails))
if test_emails:
config = conf.test_email_server
logbook.debug("Use test server for email sending")
else:
config = conf.email_server
server_host = config.host
user = config.user
password = config.password
ssl = config.ssl
debug = config.debug
port = config.port
starttls = config.starttls
from_addr = from_addr or conf.provider.noreply
return _smtp_send_email(email,
subject,
from_addr,
body,
server_host,
port,
user,
password,
cc=cc,
attachments=attachments,
timeout=timeout,
ssl=ssl,
debug=debug,
starttls=starttls)
def _interpret(arr):
"""Adapted from libmagic.
See file-5.14/magic/Magdir/scientific
available at ftp://ftp.astron.com/pub/file/file-5.14.tar.gz
Parameters
----------
arr : ndarray
1-dimensional ndarray of dtype uint8 containing the
whole RAXIS file
Returns
-------
arr : ndarray
2-dimensional uint16 array of RAXIS detector data
"""
# Check the version field to determine the endianess
endian = '>' if arr[796:800].view('>u4') < 20 else '<'
# Width and height must be cast to at least a uint64 to
# safely multiply them. (Otherwise default numpy rules
# result in multiplication modulo 2 ** 32.)
width, height = (long(arr[768:772].view(endian + 'u4')),
long(arr[772:776].view(endian + 'u4')))
logbook.info('Interpreting as {}-endian with dimensions {}x{}'.format(
'big' if endian == '>' else 'little', width, height))
diagnostics = """
Diagnostic information:
length of the raw array (in bytes): {}
length of the raw array / 4 : {}
width: {}
heigth: {}
len / 4 - (width * height): {}
""".format(len(arr),
len(arr) / 4, width, height,
len(arr) / 4 - (width * height))
logbook.debug(diagnostics)
try:
return (arr.view(endian + 'u2')[-(width * height):].reshape(
(width, height)))
except ValueError as err:
serr = ShapeError(
"""Couldn't convert this array because of a problem interpreting
its shape. This file may be corrupt.
""" + diagnostics)
serr.original_exception = err
raise serr
