def device(request, device_config, output_dir): devname = request.config.getoption('--device') devconfig = Loader.load(device_config) devplatform = request.config.getoption('--platform') device_info = devconfig[devname] default_caps = Loader().get_default_caps().get(devplatform) or {} default_caps.update(device_info['caps']) device_info['caps'] = default_caps device_info['port'] = request.config.getoption('--port') device_info['bp'] = request.config.getoption('--bp') device_info['host'] = request.config.getoption('--service-address') device_info['output_dir'] = output_dir if device_info['port']: ap = AppiumService(device=device_info['caps'], ports=[device_info['port'], device_info['bp']]) ap.start() procs.append(ap) device_info['caps'] = ap.device device_info['host'] = 'http://127.0.0.1:{}/wd/hub'.format(ap.port) else: if device_info['host'] and \ not device_info['host'].startswith('http') and \ not device_info['host'].endswith('/wd/hub'): device_info['host'] = 'http://{}/wd/hub'.format( device_info['host']) if devplatform == 'Android' and not device_info['caps'].get('systemPort'): device_info['caps']['systemPort'] = request.config.getoption( '--system-port') elif devplatform == 'iOS' and not device_info['caps'].get('wdaLocalPort'): device_info['caps']['wdaLocalPort'] = device_info['bp'] logger.debug('{} caps: {}'.format(devname, device_info['caps'])) return device_info
def validate_config(self, config_path, tosca_model=None): """ Validate a Toskose configuration file """ loader = Loader() config = loader.load(config_path) try: jsonschema.validate(instance=config, schema=self._config_schema) # if tosca_model is not None: # ConfigValidator._validate_nodes(config, tosca_model) except jsonschema.exceptions.ValidationError as err: raise ValidationError(err.message) except jsonschema.exceptions.SchemaError as err: logger.error(err) raise FatalError('The toskose configuration schema is corrupted. \ Validation cannot be done.')
class OrdersWorker: def __init__(self): self.name = 'orders_worker' self.extractor = Extractor(self.name) self.transformer = Transformer(self.name) self.loader = Loader(self.name) self.logger = Logger(self.name) def process(self): self.logger.info('Started') orders = self.extractor.extract_orders('orders') self.logger.info(f'Extracted orders: {orders.count()}') users = self.extractor.extract_users('users') data = self.transformer.merge_users_and_orders(users, orders) self.loader.load(data) self.logger.info(f'Written records: {len(data)}') self.logger.info('Finished')
def test_toskose_model_config_gen(data, reset_port): """ Test the auto-generation of toskose config. """ with tempfile.TemporaryDirectory() as tmp_dir: manifest_path = helpers.compute_manifest_path( tmp_dir, # also un pack the csar archive data['csar_path']) model = ToscaParser().build_model(manifest_path) docker_inputs = [ list(v['docker'].values()) for k, v in data['toskose_config_input'].items()] docker_manager_inputs = [ list(data['toskose_config_manager_input']['docker'].values())] docker_inputs += docker_manager_inputs def gen_inputs(): for entry in docker_inputs: for subentry in entry: yield subentry with mock.patch('builtins.input', side_effect=gen_inputs()): with mock.patch('getpass.getpass', return_value='password'): config_path = generate_default_config(model) cfg = Loader() config = cfg.load(config_path) if 'nodes' not in config: assert False test_data = { 'nodes': dict(data['toskose_config_input']) } assert config['nodes'] == test_data['nodes']
def post(self): json_data = request.get_json() if not json_data: return form_response(code=400, message='Invalid payload') # validate and deserialize input site_loader_schema = SiteLoaderJsonSchema() try: data = site_loader_schema.load(json_data) except ValidationError as err: return form_response(code=400, message=err.messages) site_url = data.get("url") # create record to database and get id for new site site = Sites(url=site_url) site.flush() site_id = site.id # create folder for new site site_path = os.path.join(SITE_FOlDER, str(site_id), SITE_FOLDER_NAME) Path(site_path).mkdir(parents=True, exist_ok=True) # set path site.path = site_path # make commit for new record site.save_to_db() # start downloading site loader = Loader(site_url, CRAWLER_DEPTH, site_path) loader.start() # remember thread for this downloading loader_thread[site_id] = loader return form_response(code=201, data={"site_id": site_id})
from app.loader import Loader from app.call_duration_queue import CallDurationQueue from app.full_stat import FullStat from app.db_manager import DBManager if __name__ == '__main__': l = Loader() # Initialize queue object for in memory queue for calls and duration. queue = CallDurationQueue() # Initialize full_stat object which contains from daily_stats object. I # decide to use it because i can have call objects for several days and # thus i must have several daily_stat's objects. full_stat = FullStat() # This object manage writing data to DB. db_manager = DBManager() while True: all_files = l.load() for _obj in all_files: queue_result = queue.add_object(_obj) full_stat.make_action(queue_result) db_manager.write_data_to_db(full_stat) # Just see whats happening here, need only on dev purposes # for daily_stat in full_stat.daily_stats: # print(daily_stat.__dict__)
def toskose_model(tosca_model, config_path): """ Update the TOSCA model with toskose-related data of a given configuration. e.g. updated model container.envs = { ... SUPERVISORD_HTTP_PORT: xxx, SUPERVISORD_HTTP_USER: yyy, ... } """ configuration = Loader().load(config_path) tosca_model.toskose_config_path = config_path nodes_config = configuration['nodes'] for container in tosca_model.containers: if not container.hosted: logger.info('Detected a container node [{}] without sw components \ hosted on. Skipping.'.format(container.name)) # workaround: fake the toskosed image with the original one # will be used to complete the "image" field # in the final compose file container.add_artifact( ToskosedImage(container.image.name, container.image.tag)) # use the container name (defined in the TOSCA specifications) # as hostname/alias container.hostname = container.name continue supervisord_envs = { 'SUPERVISORD_{}'.format(k.upper()): v for k, v in nodes_config[container.name].items() if k != 'docker' } container.env = supervisord_envs if container.env is None else \ {**container.env, **supervisord_envs} # note: base_image and base_tag are related to the official # Toskose Docker base image used in the "toskosing" process docker_config = nodes_config[container.name]['docker'] docker_config['base_name'] = docker_config.get('base_name') docker_config['base_tag'] = docker_config.get('base_tag') # docker logic about the "toskosing" process container.add_artifact( ToskosedImage(**nodes_config[container.name]['docker'])) container.hostname = nodes_config[container.name]['alias'] # toskose-manager - container manager = Container(name='toskose-manager', is_manager=True) manager_config = configuration['manager'] # workaround # the (toskose) manager isn't "toskosed" from a given image # we need an "empty" Docker image artifact as source image. manager.add_artifact(DockerImage()) # Toskose Docker base image manager_config['docker']['base_name'] = \ manager_config['docker'].get('base_name') manager_config['docker']['base_tag'] = \ manager_config['docker'].get('base_tag') # The final "toskosed" image manager.add_artifact(ToskosedImage(**manager_config['docker'])) manager.add_port(manager_config.get('port'), manager_config.get('port')) manager.hostname = manager_config.get('alias') # toskose-manager API required envs manager.env = { 'TOSKOSE_MANAGER_PORT': manager_config['port'], 'TOSKOSE_APP_MODE': manager_config['mode'], 'SECRET_KEY': manager_config['secret_key'] } tosca_model.push(manager)
def generate_default_config(tosca_model, config_path=None, output_path=None): """ Generate a default toskose YAML configuration and dump it in a directory. Args: tosca_model (object): The model of a TOSCA-based application. config_path: A path containing a toskose configuration. output_path (str): The path in which the generated configuration will be dumped. """ def _autocomplete_config(config, name, is_manager=False): port = constants.gen_default_port() if is_manager: defaults = copy.deepcopy(constants.DEFAULT_MANAGER_API) else: defaults = copy.deepcopy(constants.DEFAULT_NODE_API) for key, default_value in defaults.items(): if key not in config: logger.info('Missing field [{0}] in node [{1}] \ (auto-generated)'.format(key, name)) value = config.get(key, default_value) # note: special case defaults are generated dynamically according # to the tosca model data and they cannot be store as fixed # defaults if not is_manager: # special case (auto-generation of port) # TODO fix bug: we need to check if the generated port will # not conflict with other user-defined ports of previous # containers, i.e. a user can define a port as the default # generated one if key == 'port': value = next(port) # special case (auto-generation of network alias from the # container name) if key == 'alias': value = name config[key] = value # note: no manager and no host => # the node doesn't have any api configuration # (it will not be "toskosed"). However, it needs docker # data for the deployment. if 'docker' not in config: config['docker'] = {} logger.info('Missing [docker] field in node \ [{}] (auto-generated)'.format(name)) config.update({ 'docker': generate_image_name_interactive(autocomplete_data=config['docker']) }) loader = Loader() config = dict() if config_path is None: logger.info( 'No configuration detected. Default data will be generated.') else: if not os.path.exists(config_path): raise FileNotFoundError( 'The given toskose configuration doesn\'t exists') config = loader.load(config_path) logger.info('Detected configuration [{}].'.format(config_path)) config_path = os.path.join(tosca_model.tmp_dir, constants.DEFAULT_TOSKOSE_CONFIG_FILENAME) if output_path is not None: config_path = output_path # exploiting the TOSCA model for completing missing data # nodes if 'nodes' not in config: logger.info('Missing [nodes] field (auto-generated)') config['nodes'] = dict() for container in tosca_model.containers: if not container.hosted: # no data in the toskose config for a standalone container. # skipping continue if container.name not in config['nodes'] \ or config['nodes'][container.name] is None: config['nodes'][container.name] = dict() logger.info( 'Missing node [{}] - [docker] data will be asked'.format( container.name)) _autocomplete_config(config['nodes'][container.name], container.name) # toskose-manager if 'manager' not in config: config['manager'] = dict() logger.info('Missing [manager] field (auto-generated)') _autocomplete_config(config['manager'], constants.DEFAULT_MANAGER_CONFIG_FIELD, is_manager=True) loader.dump(config, config_path, ordered=True) logger.info('{0} configuration stored in [{1}]'.format( 'A default' if config_path is None else 'Given', config_path)) return config_path
def test_data_load_not_exist(): with pytest.raises(ValueError): Loader().load('abcderfgh.yml')
def __init__(self): self.name = 'orders_worker' self.extractor = Extractor(self.name) self.transformer = Transformer(self.name) self.loader = Loader(self.name) self.logger = Logger(self.name)
def initializer(self, configs): self._manifest = configs['manifest'] self._config = configs['toskose_config'] self._loaded_config = Loader().load(configs['toskose_config'])