def cr_rest_request(args, rest, status_all=None, cr_next=None):
"""
sends a rest request
pipe_in: REST request (req_data)
pipe_out: response, res_body, out
"""
send_request = rest.send_request
verbose = args.verbose
print_json = args.json
while True:
pipe_in = (yield)
rest_args = get_crud(pipe_in)
loop = True
while loop:
response, res_body, out = send_request(*rest_args, verbose=verbose, print_json=print_json)
if response['status'] != '403': # 403 Forbidden
loop = False
else:
print('403 Forbidden has been returned -- will retry after 1 sec')
sleep(1)
if status_all:
try:
assert(re.match(default_status_pattern, response['status']))
status_all[0] += 1
except:
status_all[1] += 1
sleep(TRANSACTION_SLEEP)
if cr_next:
cr_next.send([response, res_body, out])
def do_pipeline(args, rest, req_data, status_all=None):
"""
Makes a pipeline and executes it
"""
def _pipe(rest, req_data):
c = None
cr = None
if not req_data:
return None
elif len(req_data) == 1:
c = req_data[0]
req_data = None
else:
c = req_data[0]
req_data = req_data[1:]
if isinstance(c, dict): # broadcast
if 'broadcast' in c:
bcr = []
for r in c['broadcast']:
bcr.append(_pipe(rest, r))
cr = cr_broadcast(*bcr)
else:
raise Exception('Unidentified coroutine in pipeline: {}'.format(c))
elif c in FILTERS:
filter_ = FILTERS[c]
cr = cr_filter(filter_, cr_next=_pipe(rest, req_data))
elif c in TRANSFORMS:
transform = TRANSFORMS[c]
cr = cr_transform(transform, cr_next=_pipe(rest, req_data))
elif c in FANCY_OUTPUTS:
fancy = FANCY_OUTPUTS[c]
cr = cr_fancy(fancy, cr_next=_pipe(rest, req_data))
elif c == 'rest_request':
cr = cr_rest_request(args, rest, status_all=status_all, cr_next=_pipe(rest, req_data))
elif c in CR_FUNCS:
cr = cr_func(CR_FUNCS[c], cr_next=_pipe(rest, req_data))
else:
raise Exception('Malformed req_data')
return cr
c = req_data[0] # Head coroutine in the pipeline
if isinstance(c, dict): # REST GET or producer
method, path, params, body = get_crud(c)
cr = None
req_data = req_data[1:]
if method:
cr = cr_rest_request(args, rest, cr_next=_pipe(rest, req_data))
cr.send(c)
cr.close()
else:
ope = list(c)[0]
if ope in PRODUCERS:
data = c[list(c)[0]]
cr = PRODUCERS[ope](cr_next=_pipe(rest, req_data))
cr.send(data)
cr.close()
else:
raise Exception("Malformed req_data")
def cr_rest_request(args, rest, status_all=None, cr_next=None):
"""
sends a rest request
pipe_in: REST request (req_data)
pipe_out: response, res_body, out
"""
send_request = rest.send_request
verbose = args.verbose
print_json = args.json
while True:
pipe_in = (yield)
rest_args = get_crud(pipe_in)
loop = True
while loop:
response, res_body, out = send_request(*rest_args,
verbose=verbose,
print_json=print_json)
if response['status'] != '403': # 403 Forbidden
loop = False
else:
print(
'403 Forbidden has been returned -- will retry after 1 sec'
)
sleep(1)
if status_all:
try:
assert (re.match(default_status_pattern, response['status']))
status_all[0] += 1
except:
status_all[1] += 1
sleep(TRANSACTION_SLEEP)
if cr_next:
cr_next.send([response, res_body, out])
def do_pipeline(args, rest, req_data, status_all=None):
"""
Makes a pipeline and executes it
"""
def _pipe(rest, req_data):
c = None
cr = None
if not req_data:
return None
elif len(req_data) == 1:
c = req_data[0]
req_data = None
else:
c = req_data[0]
req_data = req_data[1:]
if isinstance(c, dict): # broadcast
if 'broadcast' in c:
bcr = []
for r in c['broadcast']:
bcr.append(_pipe(rest, r))
cr = cr_broadcast(*bcr)
else:
raise Exception(
'Unidentified coroutine in pipeline: {}'.format(c))
elif c in FILTERS:
filter_ = FILTERS[c]
cr = cr_filter(filter_, cr_next=_pipe(rest, req_data))
elif c in TRANSFORMS:
transform = TRANSFORMS[c]
cr = cr_transform(transform, cr_next=_pipe(rest, req_data))
elif c in FANCY_OUTPUTS:
fancy = FANCY_OUTPUTS[c]
cr = cr_fancy(fancy, cr_next=_pipe(rest, req_data))
elif c == 'rest_request':
cr = cr_rest_request(args,
rest,
status_all=status_all,
cr_next=_pipe(rest, req_data))
elif c in CR_FUNCS:
cr = cr_func(CR_FUNCS[c], cr_next=_pipe(rest, req_data))
else:
raise Exception('Malformed req_data')
return cr
c = req_data[0] # Head coroutine in the pipeline
if isinstance(c, dict): # REST GET or producer
method, path, params, body = get_crud(c)
cr = None
req_data = req_data[1:]
if method:
cr = cr_rest_request(args, rest, cr_next=_pipe(rest, req_data))
cr.send(c)
cr.close()
else:
ope = list(c)[0]
if ope in PRODUCERS:
data = c[list(c)[0]]
cr = PRODUCERS[ope](cr_next=_pipe(rest, req_data))
cr.send(data)
cr.close()
else:
raise Exception("Malformed req_data")
