def listen(connector, job_name, stream, url, auth):
response = requests.get(url, auth=auth, stream=True)
for line in response.iter_lines(decode_unicode=True, chunk_size=None):
if line:
decoded_line = line.decode('utf-8')
payload = json.loads(decoded_line)
payload['oxyma.stream.hashcode'] = stream['hashcode']
Message.enqueue(
connector, 'messages', payload, 'streams',
'paprika.consumers.StreamProcessor.StreamProcessor')
logger = Logger(connector, StreamScraper())
logger.debug(job_name, json.dumps(payload))
def post_request(url, message):
logger = Logger(Message())
try:
headers = {'Content-Type': 'application/json'}
logger.debug('',
'url : ' + url + ', message : ' + json.dumps(message))
response = post(url, json.dumps(message), headers=headers)
content_type = Message.get_header(response, 'content-type')
logger.debug(
'',
'url : ' + url + ', response : ' + str(response.status_code) +
', message : ' + json.dumps(message))
logger.debug(
'', 'url : ' + url + ', content_type : ' + str(content_type) +
', message : ' + json.dumps(message))
if content_type == 'application/json':
result = json.loads(response.content)
result['status_code'] = response.status_code
else:
result = dict()
result['state'] = 'FAILED'
result['status_code'] = response.status_code
result['message'] = response.reason
result['backtrace'] = response.text
return result
except:
result = Traceback.build()
result['state'] = 'FAILED'
result['status_code'] = 400
return result