def device(request, device_config, output_dir): devname = request.config.getoption('--device') devconfig = Loader.load(device_config) devplatform = request.config.getoption('--platform') device_info = devconfig[devname] default_caps = Loader().get_default_caps().get(devplatform) or {} default_caps.update(device_info['caps']) device_info['caps'] = default_caps device_info['port'] = request.config.getoption('--port') device_info['bp'] = request.config.getoption('--bp') device_info['host'] = request.config.getoption('--service-address') device_info['output_dir'] = output_dir if device_info['port']: ap = AppiumService(device=device_info['caps'], ports=[device_info['port'], device_info['bp']]) ap.start() procs.append(ap) device_info['caps'] = ap.device device_info['host'] = 'http://127.0.0.1:{}/wd/hub'.format(ap.port) else: if device_info['host'] and \ not device_info['host'].startswith('http') and \ not device_info['host'].endswith('/wd/hub'): device_info['host'] = 'http://{}/wd/hub'.format( device_info['host']) if devplatform == 'Android' and not device_info['caps'].get('systemPort'): device_info['caps']['systemPort'] = request.config.getoption( '--system-port') elif devplatform == 'iOS' and not device_info['caps'].get('wdaLocalPort'): device_info['caps']['wdaLocalPort'] = device_info['bp'] logger.debug('{} caps: {}'.format(devname, device_info['caps'])) return device_info
class OrdersWorker: def __init__(self): self.name = 'orders_worker' self.extractor = Extractor(self.name) self.transformer = Transformer(self.name) self.loader = Loader(self.name) self.logger = Logger(self.name) def process(self): self.logger.info('Started') orders = self.extractor.extract_orders('orders') self.logger.info(f'Extracted orders: {orders.count()}') users = self.extractor.extract_users('users') data = self.transformer.merge_users_and_orders(users, orders) self.loader.load(data) self.logger.info(f'Written records: {len(data)}') self.logger.info('Finished')
def validate_config(self, config_path, tosca_model=None): """ Validate a Toskose configuration file """ loader = Loader() config = loader.load(config_path) try: jsonschema.validate(instance=config, schema=self._config_schema) # if tosca_model is not None: # ConfigValidator._validate_nodes(config, tosca_model) except jsonschema.exceptions.ValidationError as err: raise ValidationError(err.message) except jsonschema.exceptions.SchemaError as err: logger.error(err) raise FatalError('The toskose configuration schema is corrupted. \ Validation cannot be done.')
def test_toskose_model_config_gen(data, reset_port): """ Test the auto-generation of toskose config. """ with tempfile.TemporaryDirectory() as tmp_dir: manifest_path = helpers.compute_manifest_path( tmp_dir, # also un pack the csar archive data['csar_path']) model = ToscaParser().build_model(manifest_path) docker_inputs = [ list(v['docker'].values()) for k, v in data['toskose_config_input'].items()] docker_manager_inputs = [ list(data['toskose_config_manager_input']['docker'].values())] docker_inputs += docker_manager_inputs def gen_inputs(): for entry in docker_inputs: for subentry in entry: yield subentry with mock.patch('builtins.input', side_effect=gen_inputs()): with mock.patch('getpass.getpass', return_value='password'): config_path = generate_default_config(model) cfg = Loader() config = cfg.load(config_path) if 'nodes' not in config: assert False test_data = { 'nodes': dict(data['toskose_config_input']) } assert config['nodes'] == test_data['nodes']
from app.call_duration_queue import CallDurationQueue from app.full_stat import FullStat from app.db_manager import DBManager if __name__ == '__main__': l = Loader() # Initialize queue object for in memory queue for calls and duration. queue = CallDurationQueue() # Initialize full_stat object which contains from daily_stats object. I # decide to use it because i can have call objects for several days and # thus i must have several daily_stat's objects. full_stat = FullStat() # This object manage writing data to DB. db_manager = DBManager() while True: all_files = l.load() for _obj in all_files: queue_result = queue.add_object(_obj) full_stat.make_action(queue_result) db_manager.write_data_to_db(full_stat) # Just see whats happening here, need only on dev purposes # for daily_stat in full_stat.daily_stats: # print(daily_stat.__dict__)
def generate_default_config(tosca_model, config_path=None, output_path=None): """ Generate a default toskose YAML configuration and dump it in a directory. Args: tosca_model (object): The model of a TOSCA-based application. config_path: A path containing a toskose configuration. output_path (str): The path in which the generated configuration will be dumped. """ def _autocomplete_config(config, name, is_manager=False): port = constants.gen_default_port() if is_manager: defaults = copy.deepcopy(constants.DEFAULT_MANAGER_API) else: defaults = copy.deepcopy(constants.DEFAULT_NODE_API) for key, default_value in defaults.items(): if key not in config: logger.info('Missing field [{0}] in node [{1}] \ (auto-generated)'.format(key, name)) value = config.get(key, default_value) # note: special case defaults are generated dynamically according # to the tosca model data and they cannot be store as fixed # defaults if not is_manager: # special case (auto-generation of port) # TODO fix bug: we need to check if the generated port will # not conflict with other user-defined ports of previous # containers, i.e. a user can define a port as the default # generated one if key == 'port': value = next(port) # special case (auto-generation of network alias from the # container name) if key == 'alias': value = name config[key] = value # note: no manager and no host => # the node doesn't have any api configuration # (it will not be "toskosed"). However, it needs docker # data for the deployment. if 'docker' not in config: config['docker'] = {} logger.info('Missing [docker] field in node \ [{}] (auto-generated)'.format(name)) config.update({ 'docker': generate_image_name_interactive(autocomplete_data=config['docker']) }) loader = Loader() config = dict() if config_path is None: logger.info( 'No configuration detected. Default data will be generated.') else: if not os.path.exists(config_path): raise FileNotFoundError( 'The given toskose configuration doesn\'t exists') config = loader.load(config_path) logger.info('Detected configuration [{}].'.format(config_path)) config_path = os.path.join(tosca_model.tmp_dir, constants.DEFAULT_TOSKOSE_CONFIG_FILENAME) if output_path is not None: config_path = output_path # exploiting the TOSCA model for completing missing data # nodes if 'nodes' not in config: logger.info('Missing [nodes] field (auto-generated)') config['nodes'] = dict() for container in tosca_model.containers: if not container.hosted: # no data in the toskose config for a standalone container. # skipping continue if container.name not in config['nodes'] \ or config['nodes'][container.name] is None: config['nodes'][container.name] = dict() logger.info( 'Missing node [{}] - [docker] data will be asked'.format( container.name)) _autocomplete_config(config['nodes'][container.name], container.name) # toskose-manager if 'manager' not in config: config['manager'] = dict() logger.info('Missing [manager] field (auto-generated)') _autocomplete_config(config['manager'], constants.DEFAULT_MANAGER_CONFIG_FIELD, is_manager=True) loader.dump(config, config_path, ordered=True) logger.info('{0} configuration stored in [{1}]'.format( 'A default' if config_path is None else 'Given', config_path)) return config_path