def device(request, device_config, output_dir):
devname = request.config.getoption('--device')
devconfig = Loader.load(device_config)
devplatform = request.config.getoption('--platform')
device_info = devconfig[devname]
default_caps = Loader().get_default_caps().get(devplatform) or {}
default_caps.update(device_info['caps'])
device_info['caps'] = default_caps
device_info['port'] = request.config.getoption('--port')
device_info['bp'] = request.config.getoption('--bp')
device_info['host'] = request.config.getoption('--service-address')
device_info['output_dir'] = output_dir
if device_info['port']:
ap = AppiumService(device=device_info['caps'],
ports=[device_info['port'], device_info['bp']])
ap.start()
procs.append(ap)
device_info['caps'] = ap.device
device_info['host'] = 'http://127.0.0.1:{}/wd/hub'.format(ap.port)
else:
if device_info['host'] and \
not device_info['host'].startswith('http') and \
not device_info['host'].endswith('/wd/hub'):
device_info['host'] = 'http://{}/wd/hub'.format(
device_info['host'])
if devplatform == 'Android' and not device_info['caps'].get('systemPort'):
device_info['caps']['systemPort'] = request.config.getoption(
'--system-port')
elif devplatform == 'iOS' and not device_info['caps'].get('wdaLocalPort'):
device_info['caps']['wdaLocalPort'] = device_info['bp']
logger.debug('{} caps: {}'.format(devname, device_info['caps']))
return device_info
def validate_config(self, config_path, tosca_model=None):
""" Validate a Toskose configuration file """
loader = Loader()
config = loader.load(config_path)
try:
jsonschema.validate(instance=config, schema=self._config_schema)
# if tosca_model is not None:
# ConfigValidator._validate_nodes(config, tosca_model)
except jsonschema.exceptions.ValidationError as err:
raise ValidationError(err.message)
except jsonschema.exceptions.SchemaError as err:
logger.error(err)
raise FatalError('The toskose configuration schema is corrupted. \
Validation cannot be done.')
class OrdersWorker:
def __init__(self):
self.name = 'orders_worker'
self.extractor = Extractor(self.name)
self.transformer = Transformer(self.name)
self.loader = Loader(self.name)
self.logger = Logger(self.name)
def process(self):
self.logger.info('Started')
orders = self.extractor.extract_orders('orders')
self.logger.info(f'Extracted orders: {orders.count()}')
users = self.extractor.extract_users('users')
data = self.transformer.merge_users_and_orders(users, orders)
self.loader.load(data)
self.logger.info(f'Written records: {len(data)}')
self.logger.info('Finished')
def test_toskose_model_config_gen(data, reset_port):
""" Test the auto-generation of toskose config. """
with tempfile.TemporaryDirectory() as tmp_dir:
manifest_path = helpers.compute_manifest_path(
tmp_dir, # also un pack the csar archive
data['csar_path'])
model = ToscaParser().build_model(manifest_path)
docker_inputs = [
list(v['docker'].values())
for k, v in data['toskose_config_input'].items()]
docker_manager_inputs = [
list(data['toskose_config_manager_input']['docker'].values())]
docker_inputs += docker_manager_inputs
def gen_inputs():
for entry in docker_inputs:
for subentry in entry:
yield subentry
with mock.patch('builtins.input', side_effect=gen_inputs()):
with mock.patch('getpass.getpass', return_value='password'):
config_path = generate_default_config(model)
cfg = Loader()
config = cfg.load(config_path)
if 'nodes' not in config:
assert False
test_data = {
'nodes': dict(data['toskose_config_input'])
}
assert config['nodes'] == test_data['nodes']
def post(self):
json_data = request.get_json()
if not json_data:
return form_response(code=400, message='Invalid payload')
# validate and deserialize input
site_loader_schema = SiteLoaderJsonSchema()
try:
data = site_loader_schema.load(json_data)
except ValidationError as err:
return form_response(code=400, message=err.messages)
site_url = data.get("url")
# create record to database and get id for new site
site = Sites(url=site_url)
site.flush()
site_id = site.id
# create folder for new site
site_path = os.path.join(SITE_FOlDER, str(site_id), SITE_FOLDER_NAME)
Path(site_path).mkdir(parents=True, exist_ok=True)
# set path
site.path = site_path
# make commit for new record
site.save_to_db()
# start downloading site
loader = Loader(site_url, CRAWLER_DEPTH, site_path)
loader.start()
# remember thread for this downloading
loader_thread[site_id] = loader
return form_response(code=201, data={"site_id": site_id})
from app.loader import Loader
from app.call_duration_queue import CallDurationQueue
from app.full_stat import FullStat
from app.db_manager import DBManager
if __name__ == '__main__':
l = Loader()
# Initialize queue object for in memory queue for calls and duration.
queue = CallDurationQueue()
# Initialize full_stat object which contains from daily_stats object. I
# decide to use it because i can have call objects for several days and
# thus i must have several daily_stat's objects.
full_stat = FullStat()
# This object manage writing data to DB.
db_manager = DBManager()
while True:
all_files = l.load()
for _obj in all_files:
queue_result = queue.add_object(_obj)
full_stat.make_action(queue_result)
db_manager.write_data_to_db(full_stat)
# Just see whats happening here, need only on dev purposes
# for daily_stat in full_stat.daily_stats:
# print(daily_stat.__dict__)
def toskose_model(tosca_model, config_path):
"""
Update the TOSCA model with toskose-related data of a given configuration.
e.g. updated model
container.envs = {
...
SUPERVISORD_HTTP_PORT: xxx,
SUPERVISORD_HTTP_USER: yyy,
...
}
"""
configuration = Loader().load(config_path)
tosca_model.toskose_config_path = config_path
nodes_config = configuration['nodes']
for container in tosca_model.containers:
if not container.hosted:
logger.info('Detected a container node [{}] without sw components \
hosted on. Skipping.'.format(container.name))
# workaround: fake the toskosed image with the original one
# will be used to complete the "image" field
# in the final compose file
container.add_artifact(
ToskosedImage(container.image.name, container.image.tag))
# use the container name (defined in the TOSCA specifications)
# as hostname/alias
container.hostname = container.name
continue
supervisord_envs = {
'SUPERVISORD_{}'.format(k.upper()): v
for k, v in nodes_config[container.name].items() if k != 'docker'
}
container.env = supervisord_envs if container.env is None else \
{**container.env, **supervisord_envs}
# note: base_image and base_tag are related to the official
# Toskose Docker base image used in the "toskosing" process
docker_config = nodes_config[container.name]['docker']
docker_config['base_name'] = docker_config.get('base_name')
docker_config['base_tag'] = docker_config.get('base_tag')
# docker logic about the "toskosing" process
container.add_artifact(
ToskosedImage(**nodes_config[container.name]['docker']))
container.hostname = nodes_config[container.name]['alias']
# toskose-manager - container
manager = Container(name='toskose-manager', is_manager=True)
manager_config = configuration['manager']
# workaround
# the (toskose) manager isn't "toskosed" from a given image
# we need an "empty" Docker image artifact as source image.
manager.add_artifact(DockerImage())
# Toskose Docker base image
manager_config['docker']['base_name'] = \
manager_config['docker'].get('base_name')
manager_config['docker']['base_tag'] = \
manager_config['docker'].get('base_tag')
# The final "toskosed" image
manager.add_artifact(ToskosedImage(**manager_config['docker']))
manager.add_port(manager_config.get('port'), manager_config.get('port'))
manager.hostname = manager_config.get('alias')
# toskose-manager API required envs
manager.env = {
'TOSKOSE_MANAGER_PORT': manager_config['port'],
'TOSKOSE_APP_MODE': manager_config['mode'],
'SECRET_KEY': manager_config['secret_key']
}
tosca_model.push(manager)
def generate_default_config(tosca_model, config_path=None, output_path=None):
""" Generate a default toskose YAML configuration
and dump it in a directory.
Args:
tosca_model (object): The model of a TOSCA-based application.
config_path: A path containing a toskose configuration.
output_path (str): The path in which the generated configuration
will be dumped.
"""
def _autocomplete_config(config, name, is_manager=False):
port = constants.gen_default_port()
if is_manager:
defaults = copy.deepcopy(constants.DEFAULT_MANAGER_API)
else:
defaults = copy.deepcopy(constants.DEFAULT_NODE_API)
for key, default_value in defaults.items():
if key not in config:
logger.info('Missing field [{0}] in node [{1}] \
(auto-generated)'.format(key, name))
value = config.get(key, default_value)
# note: special case defaults are generated dynamically according
# to the tosca model data and they cannot be store as fixed
# defaults
if not is_manager:
# special case (auto-generation of port)
# TODO fix bug: we need to check if the generated port will
# not conflict with other user-defined ports of previous
# containers, i.e. a user can define a port as the default
# generated one
if key == 'port':
value = next(port)
# special case (auto-generation of network alias from the
# container name)
if key == 'alias':
value = name
config[key] = value
# note: no manager and no host =>
# the node doesn't have any api configuration
# (it will not be "toskosed"). However, it needs docker
# data for the deployment.
if 'docker' not in config:
config['docker'] = {}
logger.info('Missing [docker] field in node \
[{}] (auto-generated)'.format(name))
config.update({
'docker':
generate_image_name_interactive(autocomplete_data=config['docker'])
})
loader = Loader()
config = dict()
if config_path is None:
logger.info(
'No configuration detected. Default data will be generated.')
else:
if not os.path.exists(config_path):
raise FileNotFoundError(
'The given toskose configuration doesn\'t exists')
config = loader.load(config_path)
logger.info('Detected configuration [{}].'.format(config_path))
config_path = os.path.join(tosca_model.tmp_dir,
constants.DEFAULT_TOSKOSE_CONFIG_FILENAME)
if output_path is not None:
config_path = output_path
# exploiting the TOSCA model for completing missing data
# nodes
if 'nodes' not in config:
logger.info('Missing [nodes] field (auto-generated)')
config['nodes'] = dict()
for container in tosca_model.containers:
if not container.hosted:
# no data in the toskose config for a standalone container.
# skipping
continue
if container.name not in config['nodes'] \
or config['nodes'][container.name] is None:
config['nodes'][container.name] = dict()
logger.info(
'Missing node [{}] - [docker] data will be asked'.format(
container.name))
_autocomplete_config(config['nodes'][container.name], container.name)
# toskose-manager
if 'manager' not in config:
config['manager'] = dict()
logger.info('Missing [manager] field (auto-generated)')
_autocomplete_config(config['manager'],
constants.DEFAULT_MANAGER_CONFIG_FIELD,
is_manager=True)
loader.dump(config, config_path, ordered=True)
logger.info('{0} configuration stored in [{1}]'.format(
'A default' if config_path is None else 'Given', config_path))
return config_path
def test_data_load_not_exist():
with pytest.raises(ValueError):
Loader().load('abcderfgh.yml')
def __init__(self):
self.name = 'orders_worker'
self.extractor = Extractor(self.name)
self.transformer = Transformer(self.name)
self.loader = Loader(self.name)
self.logger = Logger(self.name)
def initializer(self, configs):
self._manifest = configs['manifest']
self._config = configs['toskose_config']
self._loaded_config = Loader().load(configs['toskose_config'])