else:
transform_result = {"status_code": resp.status_code, "return_value": {"transform_succeeded": resp.ok, "message": resp.text, "status_code": resp.status_code}}
logger.debug(f'transform of entity with _id={entity.get("_id","?")}, prop_per_entity={prop_per_entity} received {transform_result} from {rendered_url}')
if endpoint == 'transform':
#if neither error code nor tolerable code abort, otherwise do the expected
if not ((transform_result.get("status_code") >= 200 and transform_result.get("status_code") < 400)
or (tolerable_status_codes_per_entity
and re.search(tolerable_status_codes_per_entity, str(transform_result.get("status_code"))))):
abort(transform_result.get("status_code"), transform_result.get("return_value"))
entity[prop_per_entity] = transform_result["return_value"]
elif endpoint == 'sink':
resp.raise_for_status()
yield json.dumps(entity)
yield "]"
# get entities from request
entities = request.get_json()
response_data_generator = generate(entities, request.endpoint)
response_data = []
if do_stream and request.endpoint != 'sink':
response_data = response_data_generator
else:
for entity in response_data_generator:
response_data.append(entity)
return Response(response=response_data, mimetype="application/json")
if __name__ == "__main__":
serve(app, port=PORT)
except Timeout as e:
logger.error(f"Timeout issue while updating ticket {ticketID}: {e}")
except ConnectionError as e:
logger.error(f"ConnectionError issue while updating ticket {ticketID}: {e}")
except Exception as e:
logger.error(f"Issue while updating ticket {ticketID} from Zendesk: {e}")
def get_items(session,unix_time_update_date):
"""Helper generator to support streaming with flask"""
url = f'{ZENURL}/incremental/tickets.json?start_time={unix_time_update_date}'
# https://developer.zendesk.com/rest_api/docs/support/tickets#list-tickets
check_items = True
while check_items:
response = session.get(url, timeout=180)
data = response.json()
end_time = data["end_time"]
url = data['next_page']
logger.debug(f"Next page url: {url}")
# logger.debug(f"Data count: {data['count']}")
for item in data['tickets']:
logger.debug(f"item: {item}")
item["_id"] = str(item["id"])
item["_updated"] = end_time
yield item
if data['count'] < 1000: # pagination used for Zendesk incremental importing. Next page will newer be none when using timestamp
check_items = False
if __name__ == "__main__":
serve(app)
if validation_resp == "Your xml file was validated :)":
files_to_send.append(file_temp)
else:
logger.error('Validation unsuccessfull! :(')
logger.info("Finished appending file to list")
file_obj.close()
schema_obj.close()
else:
files_to_send.append(file_temp)
logger.info("Finished appending file to list")
file_obj.close()
except Exception as e:
logger.error(f"Failed to get file from fileshare. Error: {e}")
conn.close()
logger.info(f"Finished appending files... ;)")
return jsonify({'files': files_to_send})
if __name__ == "__main__":
logger.info("Starting service...")
# Test connection at startup
conn = create_connection()
if not conn.connect(config.host, 445):
logger.error("Failed to authenticate with the provided credentials")
conn.close()
serve(APP)
logger.critical('Wrong type gave error: {}'.format(e))
except Exception as e:
logger.critical(
f'Error when handling entities:\n{json.dumps(req_entities)}\nError message:\n{e}'
)
# Generate the response
try:
return Response(stream_json(output), mimetype='application/json')
except BaseException as e:
return Response(
status=500,
response=f"An error occured during transform of input. Error: {e}")
if __name__ == '__main__':
variables = VariablesConfig([
'db_host', 'db_port', 'db_database', 'db_username', 'db_password',
'handler', 'query', 'LOG_LEVEL'
])
if not variables.validate():
sys.exit(-1)
databaseConnection = OracleDB(host=variables.db_host,
port=variables.db_port,
database=variables.db_database,
username=variables.db_username,
password=variables.db_password)
logger = sesam_logger(logger_name='SQL_TRANSFORM', timestamp=True, app=app)
serve(app=app)
return Response(status=500, response=json.dumps(input_data), content_type='application/json')
return Response(json.dumps(input_data), content_type='application/json')
def download_file(res: requests.Response) -> str:
"""
function that downloads a binary content of input response object and
stores into NamedTemporaryFile in file system
:param res: requests.Response instance
:return: stored file name (path)
"""
logger.debug("Chunked file download started")
with tempfile.NamedTemporaryFile(delete=False) as file:
for chunk in res.iter_content(chunk_size=config.CHUNK_SIZE):
if chunk:
file.write(chunk)
file.close()
logger.debug(f"File stored as {file.name}")
return file.name
if __name__ == "__main__":
IS_DEBUG_ENABLED = logger.isEnabledFor(logging.DEBUG)
if IS_DEBUG_ENABLED:
APP.run(debug=IS_DEBUG_ENABLED, host='0.0.0.0', port=5000)
else:
serve(APP, config={'server.max_request_body_size': 0,
'server.socket_timeout': 60})
# if we have PDF file back
if response.headers['Content-Type'] == 'application/pdf':
logging.debug(f'got PDF response')
fp = tempfile.TemporaryFile()
for chunk in response.iter_content(chunk_size=8192):
if chunk:
fp.write(chunk)
fp.flush()
def stream_and_remove_file():
yield from fp
fp.close()
return Response(stream_and_remove_file(), mimetype='application/pdf')
error = f"couldn't process MIPS response, response headers: {response.headers}"
logging.warning(error)
abort(500, error)
if __name__ == '__main__':
logging.basicConfig(level=logging.getLevelName(LOG_LEVEL))
IS_DEBUG_ENABLED = True if logging.getLogger().isEnabledFor(logging.DEBUG) else False
if IS_DEBUG_ENABLED:
APP.run(debug=IS_DEBUG_ENABLED, host='0.0.0.0', port=PORT)
else:
serve(APP, port=PORT, config={'server.thread_pool': WORKERS, 'server.max_request_body_size': 0})
exception_str = error_handling()
logging.error(exception_str)
return abort(500, exception_str)
if __name__ == '__main__':
FULL_URL_PATTERN = get_var('FULL_URL_PATTERN')
UPDATED_URL_PATTERN = get_var('UPDATED_URL_PATTERN')
UPDATED_PROPERTY = get_var('UPDATED_PROPERTY')
OFFSET_BIGGER_AND_EQUAL = get_var('OFFSET_BIGGER_AND_EQUAL')
auth_type = get_var('AUTHENTICATION')
config = json.loads(get_var('CONFIG'))
print('STARTED UP WITH:')
print('\tFULL_URL_PATTERN={}'.format(FULL_URL_PATTERN))
print('\tUPDATED_URL_PATTERN={}'.format(UPDATED_URL_PATTERN))
print('\tUPDATED_PROPERTY={}'.format(UPDATED_PROPERTY))
print('\tOFFSET_BIGGER_AND_EQUAL={}'.format(OFFSET_BIGGER_AND_EQUAL))
print('\tauth_type={}'.format(auth_type))
if not auth_type:
SYSTEM = OpenUrlSystem(config)
elif auth_type.upper() == 'OAUTH2':
SYSTEM = Oauth2System(config)
if os.environ.get('WEBFRAMEWORK', '').lower() == 'flask':
app.run(debug=True,
host='0.0.0.0',
port=int(os.environ.get('PORT', 5000)))
else:
serve(app, int(os.environ.get("PORT", 5000)))