def action(self, includes: dict, variables: dict) -> any:
body = self.simple_input(variables)
method = Step.filter_predefined_keys(body) # get/put/list
oper = body[method]
conf = oper['config']
import boto3
s3_client = boto3.client('s3',
endpoint_url=conf.get('url'),
aws_access_key_id=conf['key_id'],
aws_secret_access_key=conf['secret_key'],
region_name=conf.get('region')
)
path = oper['path']
if method == 'get':
return variables, self._get_file(s3_client, path)
elif method == 'put':
content = oper.get('content')
if not content:
if 'content_resource' not in oper:
raise ValueError('No content for s3 put')
with open(join(variables['RESOURCES_DIR'], oper['content_resource']), 'r') as f:
content = f.read()
content = fill_template_str(content, variables)
return variables, self._put_file(s3_client, path, content)
elif method == 'list':
return variables, self._list_dir(conf, path)
elif method == 'delete':
return variables, self._delete(conf, path)
else:
raise AttributeError('unknown method: ' + method)
def action(self, includes: dict, variables: dict) -> any:
body = self.simple_input(variables)
method = Step.filter_predefined_keys(body) # query/<record_based>
salesforce = self._do_login(body[method]['config'])
if method == 'query':
return variables, self._query(salesforce, body['query'])
else: # record action
return variables, self.deep_convert_dict(self._record(salesforce, method, body[method]))
def action(self, includes: dict, variables: dict) -> dict or tuple:
body = self.simple_input(variables)
method = Step.filter_predefined_keys(body) # send/receive
conf = body[method]
if method == 'send':
return variables, self.send(conf, variables)
elif method == 'receive':
return variables, self.receive(conf)
else:
raise AttributeError('unknown method: ' + method)
def action(self, includes: dict, variables: dict) -> dict or tuple:
body = self.simple_input(variables)
method = Step.filter_predefined_keys(body) # read/write
step = body[method]
if method == 'read':
return variables, self._read(step)
elif method == 'write':
return variables, self._write(step)
elif method == 'delete':
return variables, self._delete(step)
else:
raise AttributeError('unknown method: ' + method)
def action(self, includes: dict, variables: dict) -> dict or tuple:
body = self.simple_input(variables)
method = Step.filter_predefined_keys(body) # run/run_status
oper = body[method]
if method == 'run':
return variables, self._run_dag(oper,
variables.get('INVENTORY_FILE'))
if method == 'run_status':
return variables, self._run_status(oper)
if method == 'get_xcom':
return variables, self._get_xcom(oper)
raise Exception('Unknown method: {}'.format(method))
def action(self, includes: dict, variables: dict) -> any:
from elasticsearch import Elasticsearch
body = self.simple_input(variables)
method = Step.filter_predefined_keys(body) # search/refresh
conf = body[method]
index = conf['index']
url = conf['url']
if isinstance(url, str):
url = [url]
es = Elasticsearch(url)
if method == 'search':
return variables, self._search(es, index, conf)
elif method == 'refresh':
return variables, self._refresh(es, index)
else:
raise AttributeError('unknown method: ' + method)
def __init__(self, **kwargs: dict) -> None:
super().__init__(**kwargs)
method = Step.filter_predefined_keys(kwargs) # produce/consume
self.method = method.lower()
conf = kwargs[method]
self.group_id = conf.get('group_id', 'catcher')
self.server = conf.get('server', '127.0.0.1:9092')
self.topic = conf['topic']
timeout = conf.get('timeout', {'seconds': 1})
self.timeout = to_seconds(timeout)
self.where = conf.get('where', None)
self.data = None
if self.method != 'consume':
self.data = conf.get('data', None)
if self.data is None:
self.file = conf['data_from_file']
def action(self, includes: dict, variables: dict) -> dict or tuple:
body = self.simple_input(variables)
# TODO determine the webdriver and check it is installed
method = Step.filter_predefined_keys(body)
step = body[method]
driver = step.get('driver')
file = step['file']
library = step.get('library', '/usr/share/java/*')
if library is not None and not isinstance(library, list):
library = [library]
my_env = os.environ.copy()
if driver is not None:
my_env['PATH'] = my_env['PATH'] + ':' + os.path.dirname(driver)
return variables, external_utils.run_cmd_simple(file,
variables,
env=my_env,
libraries=library)
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
method = Step.filter_predefined_keys(kwargs) # publish/consume
self.method = method.lower()
conf = kwargs[method]
self.config = conf['config']
self.headers = conf.get('headers', {})
self.message = None
if self.method != 'consume':
self.exchange = conf['exchange']
self.routing_key = conf['routing_key']
self.message = conf.get('data', None)
self.file = None
if self.message is None:
self.file = conf['data_from_file']
else:
self.queue = conf['queue']
def __init__(self, response_code=200, **kwargs) -> None:
super().__init__(**kwargs)
method = Step.filter_predefined_keys(kwargs) # get/post/put...
self.method = method.lower()
conf = kwargs[method]
self.url = conf['url']
self.headers = conf.get('headers', {})
self.body = None
self.verify = conf.get('verify', True)
self._should_fail = conf.get('should_fail', False)
if not self.verify:
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self.code = response_code
if self.method != 'get':
self.body = conf.get('body', None)
if self.body is None:
self.file = conf['body_from_file']
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
method = Step.filter_predefined_keys(kwargs) # get/post/put...
self.method = method.lower()
conf = kwargs[method]
self.url = conf['url']
self.headers = conf.get('headers', {})
self.verify = conf.get('verify', True)
self._should_fail = conf.get('should_fail', False)
if not self.verify:
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self.code = conf.get('response_code', 200)
self.body = conf.get('body')
self.file = conf.get('body_from_file')
self.files = conf.get('files')
self.session = conf.get('session', 'default')
self.fix_cookies = conf.get('fix_cookies', True)
self.timeout = conf.get('timeout')
def __init__(self, _get_action=None, _get_actions=None, **kwargs):
super().__init__(**kwargs)
self.type = Step.filter_predefined_keys(kwargs) # while/foreach
do = kwargs[self.type]['do']
if len(do) == 1: # just action
if isinstance(do, list): # list with single action
do = do[0]
[loop_action] = do.keys()
self.do_action = [_get_action((loop_action, do[loop_action]))]
else:
self.do_action = list(
itertools.chain.from_iterable(
[_get_actions(act) for act in do]))
self.max_cycle = kwargs[self.type].get('max_cycle')
if self.type == 'while':
if_clause = kwargs['while']['if']
if isinstance(if_clause, str):
self.if_clause = {'equals': if_clause}
else:
self.if_clause = if_clause
elif self.type == 'foreach':
self.in_var = kwargs['foreach']['in']
else:
raise ValueError('Wrong configuration for step: ' + str(kwargs))
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
method = Step.filter_predefined_keys(kwargs)
self.data = {method: kwargs[method]}