def device(request, device_config, output_dir):
devname = request.config.getoption('--device')
devconfig = Loader.load(device_config)
devplatform = request.config.getoption('--platform')
device_info = devconfig[devname]
default_caps = Loader().get_default_caps().get(devplatform) or {}
default_caps.update(device_info['caps'])
device_info['caps'] = default_caps
device_info['port'] = request.config.getoption('--port')
device_info['bp'] = request.config.getoption('--bp')
device_info['host'] = request.config.getoption('--service-address')
device_info['output_dir'] = output_dir
if device_info['port']:
ap = AppiumService(device=device_info['caps'],
ports=[device_info['port'], device_info['bp']])
ap.start()
procs.append(ap)
device_info['caps'] = ap.device
device_info['host'] = 'http://127.0.0.1:{}/wd/hub'.format(ap.port)
else:
if device_info['host'] and \
not device_info['host'].startswith('http') and \
not device_info['host'].endswith('/wd/hub'):
device_info['host'] = 'http://{}/wd/hub'.format(
device_info['host'])
if devplatform == 'Android' and not device_info['caps'].get('systemPort'):
device_info['caps']['systemPort'] = request.config.getoption(
'--system-port')
elif devplatform == 'iOS' and not device_info['caps'].get('wdaLocalPort'):
device_info['caps']['wdaLocalPort'] = device_info['bp']
logger.debug('{} caps: {}'.format(devname, device_info['caps']))
return device_info
class OrdersWorker:
def __init__(self):
self.name = 'orders_worker'
self.extractor = Extractor(self.name)
self.transformer = Transformer(self.name)
self.loader = Loader(self.name)
self.logger = Logger(self.name)
def process(self):
self.logger.info('Started')
orders = self.extractor.extract_orders('orders')
self.logger.info(f'Extracted orders: {orders.count()}')
users = self.extractor.extract_users('users')
data = self.transformer.merge_users_and_orders(users, orders)
self.loader.load(data)
self.logger.info(f'Written records: {len(data)}')
self.logger.info('Finished')
def validate_config(self, config_path, tosca_model=None):
""" Validate a Toskose configuration file """
loader = Loader()
config = loader.load(config_path)
try:
jsonschema.validate(instance=config, schema=self._config_schema)
# if tosca_model is not None:
# ConfigValidator._validate_nodes(config, tosca_model)
except jsonschema.exceptions.ValidationError as err:
raise ValidationError(err.message)
except jsonschema.exceptions.SchemaError as err:
logger.error(err)
raise FatalError('The toskose configuration schema is corrupted. \
Validation cannot be done.')
def test_toskose_model_config_gen(data, reset_port):
""" Test the auto-generation of toskose config. """
with tempfile.TemporaryDirectory() as tmp_dir:
manifest_path = helpers.compute_manifest_path(
tmp_dir, # also un pack the csar archive
data['csar_path'])
model = ToscaParser().build_model(manifest_path)
docker_inputs = [
list(v['docker'].values())
for k, v in data['toskose_config_input'].items()]
docker_manager_inputs = [
list(data['toskose_config_manager_input']['docker'].values())]
docker_inputs += docker_manager_inputs
def gen_inputs():
for entry in docker_inputs:
for subentry in entry:
yield subentry
with mock.patch('builtins.input', side_effect=gen_inputs()):
with mock.patch('getpass.getpass', return_value='password'):
config_path = generate_default_config(model)
cfg = Loader()
config = cfg.load(config_path)
if 'nodes' not in config:
assert False
test_data = {
'nodes': dict(data['toskose_config_input'])
}
assert config['nodes'] == test_data['nodes']
from app.call_duration_queue import CallDurationQueue
from app.full_stat import FullStat
from app.db_manager import DBManager
if __name__ == '__main__':
l = Loader()
# Initialize queue object for in memory queue for calls and duration.
queue = CallDurationQueue()
# Initialize full_stat object which contains from daily_stats object. I
# decide to use it because i can have call objects for several days and
# thus i must have several daily_stat's objects.
full_stat = FullStat()
# This object manage writing data to DB.
db_manager = DBManager()
while True:
all_files = l.load()
for _obj in all_files:
queue_result = queue.add_object(_obj)
full_stat.make_action(queue_result)
db_manager.write_data_to_db(full_stat)
# Just see whats happening here, need only on dev purposes
# for daily_stat in full_stat.daily_stats:
# print(daily_stat.__dict__)
def generate_default_config(tosca_model, config_path=None, output_path=None):
""" Generate a default toskose YAML configuration
and dump it in a directory.
Args:
tosca_model (object): The model of a TOSCA-based application.
config_path: A path containing a toskose configuration.
output_path (str): The path in which the generated configuration
will be dumped.
"""
def _autocomplete_config(config, name, is_manager=False):
port = constants.gen_default_port()
if is_manager:
defaults = copy.deepcopy(constants.DEFAULT_MANAGER_API)
else:
defaults = copy.deepcopy(constants.DEFAULT_NODE_API)
for key, default_value in defaults.items():
if key not in config:
logger.info('Missing field [{0}] in node [{1}] \
(auto-generated)'.format(key, name))
value = config.get(key, default_value)
# note: special case defaults are generated dynamically according
# to the tosca model data and they cannot be store as fixed
# defaults
if not is_manager:
# special case (auto-generation of port)
# TODO fix bug: we need to check if the generated port will
# not conflict with other user-defined ports of previous
# containers, i.e. a user can define a port as the default
# generated one
if key == 'port':
value = next(port)
# special case (auto-generation of network alias from the
# container name)
if key == 'alias':
value = name
config[key] = value
# note: no manager and no host =>
# the node doesn't have any api configuration
# (it will not be "toskosed"). However, it needs docker
# data for the deployment.
if 'docker' not in config:
config['docker'] = {}
logger.info('Missing [docker] field in node \
[{}] (auto-generated)'.format(name))
config.update({
'docker':
generate_image_name_interactive(autocomplete_data=config['docker'])
})
loader = Loader()
config = dict()
if config_path is None:
logger.info(
'No configuration detected. Default data will be generated.')
else:
if not os.path.exists(config_path):
raise FileNotFoundError(
'The given toskose configuration doesn\'t exists')
config = loader.load(config_path)
logger.info('Detected configuration [{}].'.format(config_path))
config_path = os.path.join(tosca_model.tmp_dir,
constants.DEFAULT_TOSKOSE_CONFIG_FILENAME)
if output_path is not None:
config_path = output_path
# exploiting the TOSCA model for completing missing data
# nodes
if 'nodes' not in config:
logger.info('Missing [nodes] field (auto-generated)')
config['nodes'] = dict()
for container in tosca_model.containers:
if not container.hosted:
# no data in the toskose config for a standalone container.
# skipping
continue
if container.name not in config['nodes'] \
or config['nodes'][container.name] is None:
config['nodes'][container.name] = dict()
logger.info(
'Missing node [{}] - [docker] data will be asked'.format(
container.name))
_autocomplete_config(config['nodes'][container.name], container.name)
# toskose-manager
if 'manager' not in config:
config['manager'] = dict()
logger.info('Missing [manager] field (auto-generated)')
_autocomplete_config(config['manager'],
constants.DEFAULT_MANAGER_CONFIG_FIELD,
is_manager=True)
loader.dump(config, config_path, ordered=True)
logger.info('{0} configuration stored in [{1}]'.format(
'A default' if config_path is None else 'Given', config_path))
return config_path