def run_usage_collection(self, end=None): # Run usage collection on all tenants present in Keystone. db.session.close() tenants = Tenant.all_active_tenants() usage = {} for tenant in tenants: try: tenant_id = tenant.tenant_id # session can be closed during next call, so we should cache tenant_id except ObjectDeletedError as e: logbook.warning( "Tenant {} was removed from db (probably during cleanup after test: ", tenant_id, e) next_run_delay = None with TenantMutex(tenant) as mutex: if mutex: logbook.debug("Processing tenant: {}", tenant_id) tenant_usage = self.collect_usage(tenant, mutex, end) usage[tenant_id] = tenant_usage db.session.commit() next_run_delay = conf.fitter.min_tenant_interval if tenant_usage else conf.fitter.tenant_interval logbook.debug( "Create mutex for tenant {} to prevent very often access to ceilometer. Delay: {}", tenant, next_run_delay) if next_run_delay and not conf.test: mutex = TenantMutex(tenant) mutex.acquire(ttl_ms=next_run_delay * 1000) db.session.close() logbook.info("Usage collection run complete.") return usage
def acquire_weights(scale_obj, sleep_time=settings.WEIGHT_MEASURE_WAIT_TIME, measure_count=settings.WEIGHT_MEASURE_COUNT): """ to ensure accuracy of the weight do a number of measures with sleep time in between :param scale_obj: BeeHiveScale object :type scale_obj: BeeHiveScale :param sleep_time: how long to wait between measures :type sleep_time: float :param measure_count: umber of measures :type measure_count: int :return: list of measures :rtype list """ logbook.info("start acquire weights") _measures = [] for count in range(measure_count + 1): weight = scale_obj.weight logbook.info("weight: {weight} kg".format(weight=weight)) _measures.append(scale_obj.weight) if count < measure_count: logbook.debug("wait for {0}s".format(sleep_time)) time.sleep(sleep_time) return _measures
def test_sanity(work_dir, source_file, disks): cmd = ['target/raiden', 'split', source_file, str(disks)] logbook.debug(cmd) subprocess.check_call(cmd) def merge(): cmd = ['target/raiden', 'merge', source_file, str(disks)] subprocess.check_call(cmd) logbook.debug(cmd) restored_file = os.path.join(work_dir, 'res__source') cmd = ['diff', source_file, restored_file] logbook.debug(cmd) subprocess.check_call(cmd) merge() @contextmanager def move_aside(disk): path = '{}_{}'.format(source_file, disk) os.rename(path, path + '~') try: yield finally: os.rename(path + '~', path) for i in xrange(disks): with move_aside(i): merge()
def doCleanups(self): """ Run cleanups for current test. This cleanups will run EVEN if setUp of test fails. Original doCleanups will be called AFTER. """ outcome = self._outcome or unittest.case._Outcome() def generic_cleanup(source): while source: function, args, kwargs = source.pop() logbook.debug('Executing cleanup: {}'.format(function)) with outcome.testPartExecutor(self): function(*(args or tuple()), **(kwargs or {})) generic_cleanup(self._cleanup_before_force_delete) for table in self._force_delete_order: if table in self._cleanup_force_delete: while self._cleanup_force_delete[table]: prefix, field = self._cleanup_force_delete[table].pop() logbook.debug('Executing cleanup: Force delete {} with prefix "{}"'.format(table, prefix)) with outcome.testPartExecutor(self): self.force_delete(table, prefix, field) generic_cleanup(self._cleanup_after_delete) generic_cleanup(self._cleanups) return outcome.success
def validate(self, value): """ This method validates given value. It doesn't catch any exception, it is a task of internal executor. :param object value: The value we have to validate. :return Validated value. """ processed_value = value for type_validator in self.validators: try: processed_value = type_validator(value) except ValidateError as exc: if self.name != self.TOKEN_NAME: name = self.name if exc.subname: name += "." + exc.subname raise BadParameter(exc, name) raise BadRequest(exc) except HTTPError as exc: raise exc except Exception as exc: logbook.info(u"Invalid parameter {}: {} ({})".format( self.name, exc, type(exc))) logbook.debug("{}", traceback.format_exc()) if hasattr(type_validator, "__class__"): validator_name = type_validator.__class__.__name__ else: validator_name = getattr(type_validator, "__name__", "<unknown type>") raise BadRequest( _(u"Invalid parameter {}. It should have type {}"), self.name, validator_name) return processed_value
async def expire_snapshots(ctx, rule): """Expire existing snapshots for the rule. """ logbook.debug('Expire existing snapshots') snapshots = await load_snapshots(ctx) snapshots = filter_snapshots_by_rule(snapshots, rule) snapshots = { s['name']: pendulum.parse(s['creationTimestamp']) for s in snapshots } to_keep = expire(snapshots, rule.deltas) logbook.info('Out of {} snapshots, we want to keep {}', len(snapshots), len(to_keep)) for snapshot_name in snapshots: if snapshot_name in to_keep: logbook.debug('Keeping snapshot {}', snapshot_name) continue if snapshot_name not in to_keep: logbook.info('Deleting snapshot {}', snapshot_name) result = await exec(ctx.gcloud.snapshots().delete( snapshot=snapshot_name, project=ctx.config['gcloud_project']).execute)
def tokens(self): """Handle auth requests, patch service catalog endpoint urls""" response = self.make_requests_request(bottle.request, urljoin(self.auth_url, 'tokens')) if response.status_code != 200 or not self.patch_service_catalog: logbook.info('Proxing tokens request to openstack without patching ({})', response.status_code) return self.make_bottle_response(response) try: parsed = response.json() for service_dict in parsed.get('access', {}).get('serviceCatalog', []): service_name = service_dict['name'] endpoint = service_dict['endpoints'][0] for item in endpoint: if item.endswith('URL'): name = service_name+'_'+item[:-3] self.service_mapping[name] = endpoint[item] # e.g. nova_public, keystone_admin endpoint[item] = self.urljoin(self.fake_auth_url, 'mock', name) + '/' dump = json.dumps(parsed) except Exception: logbook.exception('Error while patching service catalog') logbook.warning('Tokens content: {}', response.content) raise logbook.debug('service mapping is: {}', self.service_mapping) headers = self.filter_headers(response.headers) headers['Content-Length'] = len(dump) return bottle.HTTPResponse(dump, response.status_code, headers)
def main(short_break_time, long_break_time): config_path = create_config_path() poms_file = config_path / "Pomodoro.json" debug(f"config_path: {config_path} poms_file: {poms_file}") Pomodoros = [] # A single Pomodoro is 25 minutes long. pom_length = 25 pomodoro_seconds = 60 * pom_length completed_poms = 0 try: with open(poms_file, 'r') as pfile: Pomodoros = json.load(pfile) except OSError: info("No previously stored Pomodoros! Welcome!") Pom = {} try: info( "Pomodoro timer starting! Hit any key when done. Hit 'Ctrl-c' to log an interrupt." ) while True: start_time = datetime.now() Pom['start_time'] = str(start_time) info(f"Starting next Pomodoro at: {start_time}") time.sleep(pomodoro_seconds) completed_poms = completed_poms + 1 info(f"{completed_poms} pomodoros complete!") print('\a') break_time = decide_break_time(completed_poms, short_break_time, long_break_time) info(f"Well done! Take a {break_time} minute break!") time.sleep(break_time) back_to_work() except KeyboardInterrupt: end_time = datetime.now() elapsed = end_time - start_time pom_seconds = elapsed.seconds info( "Ending session. Hit 'i' if this was an interrupt. Anything else to count this pom." ) interrupt_prompt = readchar.readchar() if interrupt_prompt.lower() is not 'i': finished_one_pom = start_time + timedelta(minutes=pom_length) if datetime.now() > finished_one_pom: completed_poms = completed_poms + 1 info(f"Pom seconds: {pom_seconds} Finished poms: {completed_poms}") Pom['end_time'] = str(end_time) Pom['elapsed'] = str(elapsed) Pom['completed'] = completed_poms Pomodoros.append(Pom) with open(poms_file, 'w') as pfile: json.dump(Pomodoros, pfile)
def task_os_create_tenant_and_user(customer_id, email): # email is just for logs customer = Customer.get_by_id(customer_id) log.debug("task_os_create_tenant_and_user: {}", customer) if not customer: log.warning( "Can't find customer {} {}. Possible customer was removed by system test", customer_id, email) return info = openstack.create_tenant_and_user(email, customer_id, customer.tariff.flavors(), password=customer.os_user_password, enabled=True) log.debug("Openstack info: {}", info) # Tests can delete customer already here, therefore no updates are required if not conf.test: db.session.commit() db.session.close() customer = Customer.get_by_id(customer_id) if customer: customer.update_os_credentials(info['tenant_id'], info['tenant_name'], info['user_id'], info["username"], info['password']) send_email_os_credentials(info['email'], info['name'], info['password'], info['tenant_id'], language=customer.locale_language()) db.session.commit() else: final_delete_from_os(info['tenant_id'], info['user_id'])
def iter_api_posts(start=None): """Iterates over all posts that the api provides, starting at the post witht he given id. """ at_end = False while not at_end and start != 1: # build url for next page url = "http://pr0gramm.com/api/items/get?flags=3" if start is not None: url += "&older=%d" % start # perform api request #: :type: requests.Response logbook.debug("requesting api page {}", url) response = requests.get(url) response.raise_for_status() # parse response data = response.json() at_end = data["atEnd"] # create posts for item in data.get("items", ()): post = Post(item["id"], item["created"], item["user"], item["flags"], item["image"]) start = post.id yield post
def change_auth(self, users_auth): """Change auth to User's credentials. Use this to do something by user's privileges. """ self.__auth = { 'username': users_auth['username'], 'password': users_auth['password'], 'auth_url': users_auth.get( 'auth_url', conf.openstack.auth.auth_url), # Use this from conf here? 'tenant_id': users_auth['tenant_id'] } self.__client_nova = None self.__client_glance = None self.__client_cinder = None self.__client_neutron = None self.__client_keystone = None self.__client_ceilometer = None self.__auth_session = None log.debug("Change openstack auth to {}", users_auth["username"]) try: yield finally: self.__client_nova = None self.__client_cinder = None self.__client_neutron = None self.__client_keystone = None self.__client_ceilometer = None self.__auth_session = None self.__init__(conf.openstack)
def search_email(cls, regexp: str, prefix:str=None, timeout: int=60, dotall:bool=False): """ Search an email in mailtrap mailbox with prefix, which body matches regexp (re.search). Raise AssertionError if none matched. :param regexp regexp: regular expression to search in email body. :param str prefix: email address prefix. Current test prefix used if None. :param int timeout: searching timeout. :param bool dotall: use re.DOTALL. :return: match object. """ if prefix is None: prefix = cls.get_test_prefix() mailtrap_api = MailTrapApi.create_default() regexp = re.compile(regexp, re.DOTALL if dotall else 0) for r in cls.retries(timeout=timeout): with r: messages = mailtrap_api.get_messages(prefix=prefix) for message in messages: match = regexp.search(message['text_body']) if match: logbook.debug('Got match in: {}'.format(message['text_body'])) return match else: raise AssertionError('Emails with prefix {} matching pattern "{}" not found. {} tried.'.format( prefix, regexp.pattern, len(messages)))
def client_keystone(self): if not self.__client_keystone: log.debug("Creating keystone client") self.__client_keystone = ksclient.Client( session=self.auth_session, connect_retries=self.connect_retries, timeout=self.request_timeout) return self.__client_keystone
def get_report(self, report_id): res = self.redis.get(self.key(report_id)) if res: logbook.debug("Extract report {}. Size: {}", report_id, len(res)) else: logbook.debug("Report for {} not found. Key: {}", report_id, self.key(report_id)) return res
def merge(): cmd = ['target/raiden', 'merge', source_file, str(disks)] subprocess.check_call(cmd) logbook.debug(cmd) restored_file = os.path.join(work_dir, 'res__source') cmd = ['diff', source_file, restored_file] logbook.debug(cmd) subprocess.check_call(cmd)
def get_tenant(self, tenant_id): """Returns tenant with given tenant_id""" try: tenants = self.client_keystone.tenants.get(tenant_id) except NotFound as e: log.debug(e) return None return tenants
def _tests_send_email(email, subject, from_addr, body, attachments=None, cc=None): msg = Message(email, subject, from_addr, body, attachments or [], cc or []) logbook.debug("Send test email to {} (cc: {}) with subject {} from: {}", email, cc, subject, from_addr) outbox.append(msg)
def source_file_fixture(work_dir): path = os.path.join(work_dir, 'source') logbook.debug('Creating the source file at {}', path) with open(path, 'wb') as f: for _ in xrange(1 * 1024): f.write(bytes(random.randrange(0, 0xff))) logbook.debug('Done creating the source file') return path
def handle_rcpt(self, server, line, ctx): match = _RECIPIENT_REGEX.match(line) if not match: logbook.error("Invalid recipient line received: {!r}", line) server.send(_SYNTAX_ERROR) return recipient = match.groups()[0] logbook.debug('Will send to {!r}', recipient) ctx.recipients.append(recipient) server.send(_OK)
def _log_start(self): message = self.cstr + ' Start' if self.client.loggedin: message += ' "{}"'.format(self.client.email) message += ' {} {} ({})'.format(self.method, self.url, self.expected_status) logbook.debug(message) if len(self.kwargs) > 0: logbook.debug(self.cstr + ' Params {}'.format(json.dumps(self.kwargs)))
def cleanup(self, mailbox=None, created_after=None, search=None, prefix=None): try: messages = self.get_messages(mailbox, created_after, search, prefix) except Exception as e: logbook.error("Failed to cleanup mailtrap: {}", e) return logbook.debug("Deleting {} messages by filter {}".format(len(messages), search)) for message in messages: logbook.debug("Mailtrap deleting: {}", self.short_message_str(message)) self._delete(posixpath.join(self._mailbox_url(mailbox), str(message["id"])))
def calculate_signature(self, timestamp, method, path, parameters): timestamp = str(int(timestamp)) canonical_str = self.canonical_str(timestamp, method, path, parameters) logbook.debug("Canonical string is {}", canonical_str) signature = hmac.new(self.secret_key.encode("utf-8"), canonical_str, self.ALGORITHM).hexdigest() logbook.debug("Signature: {}", signature) return signature + timestamp
def auth_session(self): if not self.__auth_session: log.debug( "Creating auth session with args {}", {k: v for k, v in self.__auth.items() if k != 'password'}) plugin = Password(**self.__auth) self.__auth_session = Session(auth=plugin, timeout=self.request_timeout) return self.__auth_session
def _reduce_history_of_update_operations(self): query = self.query.filter(TariffHistory.history_id != self.history_id, TariffHistory.tariff_id == self.tariff_id, TariffHistory.event == self.EVENT_UPDATE, TariffHistory.user_id == self.user_id, TariffHistory.date <= self.date + timedelta(seconds=1), TariffHistory.date >= self.date - self.MAX_AGE_OF_UPDATES_TO_AUTO_COLLAPSE) n = query.delete(False) logbook.debug("Reduced {} history records for tariff {}", n, self.tariff) return n
def parse_value(value: str): logbook.debug("Parsing value '{0}'...".format(value)) if value.endswith('%'): value = value.replace('%', '') return '{}/10'.format(int(value) / 10) elif value.endswith('100'): (value, _) = value.split('/') return '{}/10'.format(int(value) / 10) else: (value, _) = value.split('/') return '{}/10'.format(float(value) * 1.0)
def new_customer(cls, email, password, creator_id, detailed_info=None, comment="New customer", withdraw_period=None, make_prod=False, customer_type=None, promo_code=None, locale=None): from model import Tariff, Account, CustomerHistory customer = cls() customer.email = email customer.password = cls.password_hashing(password) if password else "" customer.deleted = None customer.created = utcnow().datetime customer.email_confirmed = False customer.blocked = False customer.customer_mode = cls.CUSTOMER_PRODUCTION_MODE if make_prod else cls.CUSTOMER_TEST_MODE customer.customer_type = customer_type if customer_type else cls.CUSTOMER_TYPE_PRIVATE_PERSON customer.withdraw_period = withdraw_period or conf.customer.default_withdraw_period customer.auto_withdraw_enabled = conf.payments.auto_withdraw_enable customer.auto_withdraw_balance_limit = conf.payments.auto_withdraw_balance_limit customer.auto_withdraw_amount = conf.payments.auto_withdraw_amount customer.locale = locale or conf.customer.default_locale customer.os_dashboard = conf.customer.default_openstack_dashboard default_tariff = Tariff.get_default().first() if not default_tariff: default_tariff = Tariff.query.filter_by(mutable=False).first() or Tariff.query.filter_by().first() if not default_tariff: raise errors.TariffNotFound() customer.tariff_id = default_tariff.tariff_id customer.balance_limit = conf.customer.balance_limits.get(default_tariff.currency, conf.customer.balance_limits.default) db.session.add(customer) db.session.flush() customer.init_subscriptions() template = 'customer' if make_prod else 'test_customer' customer.quota_init(template) customer.accounts.append(Account.create(default_tariff.currency, customer, creator_id, comment)) if promo_code is not None: PromoCode.new_code(promo_code, customer.customer_id) CustomerHistory.new_customer(customer, creator_id, comment) if detailed_info: customer.create_info(customer.customer_type, detailed_info) auto_report_task = ScheduledTask(cls.AUTO_REPORT_TASK, customer.customer_id, customer.withdraw_period) db.session.add(auto_report_task) logbook.info("New customer created: {}", customer) if not make_prod: currency = customer.tariff.currency.upper() initial_balance = conf.customer.test_customer.balance.get(currency) logbook.debug("Initial balance for customer {}: {} {}", customer, initial_balance, currency) if initial_balance: customer.modify_balance(Decimal(initial_balance), currency, None, "Initial test balance") return customer
def withdraw(self, delta, currency=None): assert isinstance(delta, Decimal) assert delta >= 0 currency = (currency or self.tariff.currency).upper() logbook.debug("Withdraw {} {} of {}", delta, currency, self) account = self.get_account(currency) if account is None: raise Exception("Customer %s doesn't have account in %s, but withdraw %s %s is required" % (self, currency, delta, currency)) account.charge(delta) self.check_balance(account, currency) db.session.flush()
def updated_state_by_interpreting_new_comments(self, issue, issue_working_state): issue_working_state = issue_working_state.copy() new_comments = self.get_new_comments(issue) # Make sure we have the right label capitalisation. issue_working_state['labels'] = [ self.get_label_by_name(l) for l in issue_working_state['labels'] ] logbook.debug(u"Examining %d new comment(s) for %s" % (len(new_comments), issue)) for comment in new_comments: if not comment.body: continue for line in comment.body.split('\n'): line = line.strip() # Votes look just like +<label> or -<label> where the label is the number 1 or 0. if VOTE_REGEX.match(line): continue m = ADD_LABEL_REGEX.match(line) if m: new_label = (m.group(1) or m.group(2)).lower() self.add_label_due_to_comment(new_label, comment, issue_working_state) continue m = REMOVE_LABEL_REGEX.match(line) if m: remove_label = m.group(1).lower() self.remove_label_due_to_comment(remove_label, comment, issue_working_state) continue m = SET_MILESTONE_REGEX.match(line) if m: new_milestone = m.group(1).lower() self.set_milestone_due_to_comment(new_milestone, comment, issue_working_state) continue m = SET_ASSIGNEE_REGEX.match(line) if m: new_assignee = m.group(1).lower() self.set_assignee_due_to_comment(new_assignee, comment, issue_working_state) continue return issue_working_state
def send_email(email, subject, body, from_addr, cc=None, attachments=None, timeout=None): """ Sends email, returns True on success If called in test environment (conf.test == True) it just adds message to ``outbox`` list. """ if conf.test: return _tests_send_email(email, subject, from_addr, body, attachments, cc) emails = [email] if isinstance(email, str) else list(email) if cc: if isinstance(cc, str): emails.append(cc) else: emails.extend(cc) test_emails = list( filter(lambda mail: conf.test_email_server.address_pattern in mail, emails)) if test_emails: config = conf.test_email_server logbook.debug("Use test server for email sending") else: config = conf.email_server server_host = config.host user = config.user password = config.password ssl = config.ssl debug = config.debug port = config.port starttls = config.starttls from_addr = from_addr or conf.provider.noreply return _smtp_send_email(email, subject, from_addr, body, server_host, port, user, password, cc=cc, attachments=attachments, timeout=timeout, ssl=ssl, debug=debug, starttls=starttls)
def _interpret(arr): """Adapted from libmagic. See file-5.14/magic/Magdir/scientific available at ftp://ftp.astron.com/pub/file/file-5.14.tar.gz Parameters ---------- arr : ndarray 1-dimensional ndarray of dtype uint8 containing the whole RAXIS file Returns ------- arr : ndarray 2-dimensional uint16 array of RAXIS detector data """ # Check the version field to determine the endianess endian = '>' if arr[796:800].view('>u4') < 20 else '<' # Width and height must be cast to at least a uint64 to # safely multiply them. (Otherwise default numpy rules # result in multiplication modulo 2 ** 32.) width, height = (long(arr[768:772].view(endian + 'u4')), long(arr[772:776].view(endian + 'u4'))) logbook.info('Interpreting as {}-endian with dimensions {}x{}'.format( 'big' if endian == '>' else 'little', width, height)) diagnostics = """ Diagnostic information: length of the raw array (in bytes): {} length of the raw array / 4 : {} width: {} heigth: {} len / 4 - (width * height): {} """.format(len(arr), len(arr) / 4, width, height, len(arr) / 4 - (width * height)) logbook.debug(diagnostics) try: return (arr.view(endian + 'u2')[-(width * height):].reshape( (width, height))) except ValueError as err: serr = ShapeError( """Couldn't convert this array because of a problem interpreting its shape. This file may be corrupt. """ + diagnostics) serr.original_exception = err raise serr