def org_user(org):
## Validating env vars
config = VariablesConfig(required_env_vars, optional_env_vars)
if not config.validate():
sys.exit(1)
request_data = request.get_data()
json_data = json.loads(str(request_data.decode("utf-8")))
for element in json_data:
invi_username = element["username"]
if element['deleted'] == True:
data = requests.delete(
f"{config.github_base_url}/orgs/{org}/memberships/{invi_username}",
auth=(username, token))
if data.status_code == 204:
logger.info('User has been removed from organization')
if data.status_code == 403:
logger.info(
'Not allowed to remove user from organization. Status code: 403'
)
else:
logger.warning(
f'Failing with content: {data.content} and status code: {data.status_code}'
)
else:
data = requests.get(
f"{config.github_base_url}/orgs/{org}/members/{invi_username}",
auth=(username, token))
if data.status_code == 204:
logger.info('User already part of organization')
if data.status_code == 404:
logger.info('User not part of organization')
logger.info(f'Trying to add user: {invi_username}')
invi_response = requests.put(
f"{config.github_base_url}/orgs/{org}/memberships/{invi_username}",
auth=(username, token))
if invi_response.status_code == 200:
decoded_data = json.loads(
invi_response.content.decode('utf-8-sig'))
if decoded_data.get('state') == "pending":
logger.info(
f"Organization invitation sent to username: {invi_username}"
)
if decoded_data.get('state') == "active":
logger.info(
f"User with username: {invi_username}, already part of organization"
)
if invi_response.status_code == 422 or invi_response.status_code == 403:
logger.warning(
f"Organization invitation could not be sent to username: {invi_username}"
)
logger.warning(
f"Failed with error code: {invi_response.status_code}")
return jsonify({'Steve reporting': "work complete..."})
def fylke_data():
config = VariablesConfig(required_env_vars, optional_env_vars)
if not config.validate():
sys.exit(1)
logger.info(f"The geodata-connector is running")
valid_response = None
exceed_limit = True
result_offset = 0
return_object = []
result_record_count = 5000
if valid_response == None:
logger.info("Requesting access token...")
valid_response = get_token(config)
token = {'Authorization': 'Bearer ' + valid_response['token']}
if valid_response['expires'] <= 10:
logger.info("Refreshing access token...")
valid_response = get_token(config)
token = {'Authorization': 'Bearer ' + valid_response['token']}
while exceed_limit is not None:
try:
request_url = f"{config.base_url}/rest/services/Geomap_UTM33_EUREF89/GeomapMatrikkel/FeatureServer/4/query?where=fylkeid={config.fylke_id}&f=pjson&outFields={config.attributes}&returnExceededLimitFeatures=True&resultOffset={str(result_offset)}&resultRecordCount{str(result_record_count)}"
data = requests.get(request_url, headers=token)
decoded_data = json.loads(data.content.decode('utf-8-sig'))
return_object.extend(decoded_data['features'])
logger.info(
f"extending result as exceed page limit is still {exceed_limit}"
)
if not data.ok:
logger.error(
f"Unexpected response status code: {data.content}")
return f"Unexpected error : {data.content}", 500
raise
else:
try:
exceed_limit = decoded_data["exceededTransferLimit"]
except Exception:
exceed_limit = None
if exceed_limit is not None:
result_offset += int(result_record_count)
logger.info(f"Result offset is now {result_offset}")
except Exception as e:
logger.warning(
f"Service not working correctly. Failing with error : {e}")
logger.info("Returning objects...")
return Response(stream_json(return_object), mimetype='application/json')
def test_optional_variable_without_default_value():
required_vars = ["var1", "var2"]
optional_vars = ["var4"]
config = VariablesConfig(required_vars, optional_vars)
assert config.var4 == "value4"
def test_optional_variable_with_default_value():
required_vars = ["var1", "var2"]
optional_vars = [("var3", "default_value")]
config = VariablesConfig(required_vars, optional_vars)
assert config.var3 == "default_value"
def get():
## Validating env vars
config = VariablesConfig(required_env_vars, optional_env_vars)
if not config.validate():
sys.exit(1)
request_data = request.get_data()
json_data = json.loads(str(request_data.decode("utf-8")))
# Helpers
user_id = None
headers = {
"Accept": "application/json",
"Authorization": f"{password}",
"Content-type": "application/json"
}
try:
actor_id = json_data[0].get('aktorid')
data = requests.get(f"{active_users_base_url}?actor_id={actor_id}",
headers=headers)
decoded_data = json.loads(data.content.decode('utf-8-sig'))
if decoded_data.get('success') == True:
user_id = decoded_data.get('user_id')
else:
logger.info(
'Setting user_id to None, as user was not found in Dekode.')
user_id = None
except Exception as e:
logger.warning(
f"Could not get aktorid from SESAM. Failed with error: {e}")
transform_response = []
if json_data[0].get("_id"):
return_dictionary = {
"_id": f"{json_data[0].get('_id')}",
"actor_id": actor_id,
"user_id": user_id
}
transform_response.append(return_dictionary)
else:
logger.error(f"No _id provided in payload... Skipping entity")
pass
return Response(stream_json(transform_response),
mimetype='application/json')
def post():
## Validating env vars
config = VariablesConfig(required_env_vars, optional_env_vars)
if not config.validate():
sys.exit(1)
headers = {
"Accept": "application/json",
"Authorization": f"{password}",
"Content-type": "application/json"
}
request_data = request.get_data()
json_data = json.loads(str(request_data.decode("utf-8")))
for element in json_data:
function = element['properties']
del element['properties']
if '/' in function:
logger.info('trying to update user')
update_response = requests.post(f"{base_url}/{function}",
headers=headers,
data=json.dumps(element))
if update_response.status_code == 200:
logger.info(f"User has been updated!")
else:
logger.error(
f"Failed to update user, with error: {update_response.content}"
)
if function == "users":
logger.info('trying to create user')
create_response = requests.post(f"{base_url}/{function}",
headers=headers,
data=json.dumps(element))
if create_response.status_code == 201:
logger.info(f"User has been created!")
else:
logger.error(
f"Failed to create user, with error: {create_response.content}"
)
else:
logger.info('Nothing to do...')
return jsonify({'Steve reporting': "work complete..."})
import requests
from requests.exceptions import Timeout
from sesamutils import sesam_logger, VariablesConfig
from sesamutils.flask import serve
from collections import OrderedDict
app = Flask(__name__)
logger = sesam_logger('zendesk', app=app, timestamp=True)
# Default values can be given to optional environment variables by the use of tuples
required_env_vars = ["USER", "TOKEN","SUBDOMAIN"]
optional_env_vars = [("DEBUG","false"),("LOG_LEVEL", "INFO"),"DUMMY_URL"]
config = VariablesConfig(required_env_vars, optional_env_vars=optional_env_vars)
if not config.validate():
logger.error("Environment variables do not validate. Exiting system.")
os.sys.exit(1)
USER = config.USER
TOKEN = config.TOKEN
ZEN_AUTH = (USER+'/token', TOKEN)
SUBDOMAIN = config.SUBDOMAIN
DUMMY_URL = False
DEBUG = config.DEBUG in ["true","True","yes"]
if hasattr(config, 'DUMMY_URL'):
DUMMY_URL = config.DUMMY_URL
def get_data():
config = VariablesConfig(required_env_vars)
if not config.validate():
sys.exit(1)
logger.info(f"The geodata-connector is running")
request_data = request.get_data()
json_data = json.loads(str(request_data.decode("utf-8")))
valid_response = None
return_object = []
for element in json_data[0].get("payload"):
if valid_response == None:
logger.info("Requesting access token...")
valid_response = get_token(config)
token = {'Authorization': 'Bearer ' + valid_response['token']}
if valid_response['expires'] <= 10:
logger.info("Refreshing access token...")
valid_response = get_token(config)
token = {'Authorization': 'Bearer ' + valid_response['token']}
try:
## Query parameters for dynamic fetching
wkid = str(element.get("wkid"))
x = str(element.get('x_coordinate'))
y = str(element.get('y_coordinate'))
if '~f' in x or y:
x = x.strip('~f')
y = y.strip('~f')
logger.info(
f"The x, y and wkid respectively '{x}', '{y}', '{wkid}'")
if not x or not y:
logger.warning(
f"The x or y coordinates '{x}', '{y}' are not provided in the right format"
)
geometry_query = {
"x": x,
"y": y,
"spatialReference": {
"wkid": wkid
}
}
## Requesting geo data
request_url = f"{config.base_url}/rest/services/Geomap_UTM33_EUREF89/GeomapMatrikkel/MapServer/5/query?geometry={geometry_query}&geometryType=esriGeometryPoint&inSR={wkid}&spatialRel=esriSpatialRelIntersects&relationParam=&outFields=kommunenr%2Cgardsnr%2Cbruksnr&returnGeometry=false&returnTrueCurves=false&returnIdsOnly=false&returnCountOnly=false&returnZ=false&returnM=false&returnDistinctValues=false&f=pjson"
geo_data = requests.get(request_url, headers=token)
if not geo_data.ok:
logger.error(
f"Unexpected response status code: {geo_data.content}")
return f"Unexpected error : {geo_data.content}", 500
raise
try:
geo_transform = geo_data.json()['features'][0]
geo_transform["geodata"] = geo_transform.pop("attributes")
except IndexError as e:
logger.error(f"exiting with error {e}")
geo_transform = default_response
except KeyError as e:
logger.error(f"exiting with error {e}")
geo_transform = default_response
sesam_dict = dict_merger(dict(element), dict(geo_transform))
return_object.append(sesam_dict)
##
except Exception as e:
logger.warning(
f"Service not working correctly. Failing with error : {e}")
transform_response = []
if json_data[0].get("_id"):
return_dictionary = {
"_id": f"{json_data[0].get('_id')}",
"geo_response": return_object
}
transform_response.append(return_dictionary)
else:
logger.info(f"No _id provided in payload...")
return_dictionary = {"geo_response": return_object}
transform_response.append(return_dictionary)
return Response(stream_json(transform_response),
mimetype='application/json')
from sesamutils import VariablesConfig, sesam_logger
from sesamutils.flask import serve
import os
app = Flask(__name__)
# set GOOGLE_APPLICATION_CREDENTIALS envvar for GCP authentication
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'service_account_key.json'
required_env_vars = [
"GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS_CONTENT",
"QUERY_CONFIGS"
]
optional_env_vars = [("DEFAULT_PAGE_SIZE", 100)]
env_config = VariablesConfig(required_env_vars, optional_env_vars)
if not env_config.validate():
sys.exit(1)
logger = sesam_logger('google-bigquery', app=app)
env_config.QUERY_CONFIGS = json.loads(env_config.QUERY_CONFIGS)
env_config.DEFAULT_PAGE_SIZE = int(env_config.DEFAULT_PAGE_SIZE)
logger.info('started up with\n\tQUERY_CONFIGS:{}'.format(
env_config.QUERY_CONFIGS))
# write out service config from env var to known file
with open(env_config.GOOGLE_APPLICATION_CREDENTIALS, "wb") as out_file:
out_file.write(env_config.GOOGLE_APPLICATION_CREDENTIALS_CONTENT.encode())
import io
import os
import sys
import socket
from flask import Flask, abort, send_file
from smb.SMBConnection import SMBConnection
from sesamutils import VariablesConfig, sesam_logger
from sesamutils.flask import serve
APP = Flask(__name__)
logger = sesam_logger("cifs-reader", app=APP)
required_env_vars = ["username", "password", "hostname", "host", "share"]
config = VariablesConfig(required_env_vars)
if not config.validate():
sys.exit(1)
def create_connection():
return SMBConnection(config.username,
config.password,
socket.gethostname(),
config.hostname,
is_direct_tcp=True,
use_ntlm_v2=True)
@APP.route("/<path:path>", methods=['GET'])
def process_request(path):
def get_generic(txt):
method = request.method
if method == "POST" and request.is_json:
returnList = []
enteties = request.get_json()
logger.info(type(enteties))
for item in enteties:
item['Hello'] = "Hello, this is a test."
logger.info(type(item))
returnList.append(item)
return jsonify(returnList), 200, {"Content-Type": "application/json"}
else:
logger.info(f'Http method is {method}')
return "Only JSON on POST is supported.", 500, {
"Content-Type": "text/plain"
}
@app.route('/api/show/config')
def get_config():
return jsonify({'config': config})
if __name__ == "__main__":
config = VariablesConfig(required_env_vars,
optional_env_vars=optional_env_vars)
# logger.info(str(config))
# if not config.validate():
# os.sys.exit(1)
serve(app)
def test_validate_missing_variable():
required_vars = ["var1", "var3"]
config = VariablesConfig(required_vars)
assert config.validate() == False
def test_required_variables_is_set():
required_vars = ["var1", "var2"]
config = VariablesConfig(required_vars)
assert config.var1 == "value1"
assert config.var2 == "value2"
from flask import Flask, request, abort
import requests
import uuid
from sesamutils.flask import serve
from sesamutils import sesam_logger, VariablesConfig
app = Flask(__name__)
logger = sesam_logger("Slack command receiver",)
req_vars = ["NODE_JWT", "VALIDATION_TOKEN", "ENDPOINT_URL"]
config = VariablesConfig(req_vars)
if not config.validate():
logger.error("Missing required variable(s).")
headers = {
"Authorization": f"bearer {config.NODE_JWT}"
}
def valid_token(token):
return token == config.VALIDATION_TOKEN
@app.route("/command", methods=["POST"])
def command():
_id = str(uuid.uuid1())
data = dict(request.form)
data["_id"] = _id
def create_embedded_data():
logger.info(f"Test data service is ready to do your bidding..")
## Validating env vars
config = VariablesConfig(required_env_vars)
if not config.validate():
sys.exit(1)
pipe_id = request.args.get('pipe_id')
max_entities = int(request.args.get('entities'))
header = {
'Authorization': f'Bearer {config.jwt}',
"content-type": "application/json"
}
try:
sesam_config_request = requests.get(
f"{config.base_url}/pipes/{pipe_id}/config",
headers=header,
verify=False)
json_config_response = json.loads(
sesam_config_request.content.decode('utf-8-sig'))
sesam_entity_request = requests.get(
f"{config.base_url}/datasets/{pipe_id}/entities?deleted=False&history=False",
headers=header,
verify=False)
json_entity_response = json.loads(
sesam_entity_request.content.decode('utf-8-sig'))
json_schema_response = json.loads(
sesam_entity_request.content.decode('utf-8-sig'))
json_mapping_schema = cleaning_json_schema(pipe_id,
json_schema_response[:1])
flattened_entities = []
for entity in json_entity_response:
flattened_entities.append(flatten_json(entity))
json_entity_response = draw_representative_values(flattened_entities,
k=max_entities)
embedded_entities = []
for response_elements in json_entity_response:
new_entity = {}
for response_key, response_value in response_elements.items():
for schema_elements in json_mapping_schema:
if response_key in schema_elements:
new_entity[response_key] = response_value
if response_key.split(':', 1)[-1] in schema_elements:
new_entity[response_key.split(':',
1)[-1]] = response_value
if response_key not in schema_elements:
try:
for schema_key, schema_value in schema_elements.items(
):
if type(schema_value) is dict:
for nested_key, nested_value in schema_value.items(
):
if response_key.split(
':')[-1] in nested_key:
if schema_key not in new_entity:
new_entity[schema_key] = {}
new_entity[schema_key][
nested_key] = response_value
if type(nested_value) is list:
if nested_key not in new_entity[
schema_key]:
new_entity[schema_key][
nested_key] = []
for nested_dicts in nested_value:
new_entity[schema_key][
nested_key].append(
nested_dicts)
if type(nested_value) is dict:
if response_key.split(
':')[-1] in nested_key:
if nested_key not in new_entity[
schema_key]:
new_entity[schema_key][
nested_key] = {}
new_entity[schema_key][
nested_key] = nested_value
if nested_key not in new_entity[
schema_key]:
new_entity[schema_key][
nested_key] = {}
new_entity[schema_key][
nested_key] = nested_value
if type(schema_value) is list:
if schema_key not in new_entity:
new_entity[schema_key] = []
for nested_dicts in schema_value:
new_entity[schema_key].append(
nested_dicts)
except Exception:
pass
embedded_entities.append(new_entity)
new_source = {
"_id": json_config_response["_id"],
"type": json_config_response["type"],
"source": {
"type": "conditional",
"alternatives": {
"prod": json_config_response["source"],
"test": {
"type": "embedded",
"entities": embedded_entities
}
},
"condition": "$ENV(node-env)"
},
"transform": json_config_response["transform"]
}
check_response = requests.put(
f"{config.base_url}/pipes/{pipe_id}/config?force=True",
headers=header,
data=json.dumps(new_source),
verify=False)
if not check_response.ok:
return_msg = f"Unexpected error : {check_response.content}", 500
else:
return_msg = f"Your pipe with id : {pipe_id} has been updated with test data"
except Exception as e:
return_msg = f"Your pipe with id : {pipe_id} could unfortunately not be updated... I failed with the following error : {e}"
return jsonify({"Response": f"{return_msg}"})
import os
import json
import datetime
import re
from time import sleep
from sesamutils import VariablesConfig, sesam_logger
from sesamutils.flask import serve
import sys
from werkzeug.routing import Rule
app = Flask(__name__)
app.url_map.add(Rule('/', endpoint='proxy'))
required_env_vars = ["OPERATIONS", "URL_PATTERN"]
optional_env_vars = [("DELAY_DURATION_IN_SECONDS", 60), "LOG_LEVEL"]
env_config = VariablesConfig(required_env_vars,
optional_env_vars=optional_env_vars)
if not env_config.validate():
sys.exit(1)
env_config.OPERATIONS = json.loads(env_config.OPERATIONS)
env_config.DELAY_DURATION_IN_SECONDS = int(
env_config.DELAY_DURATION_IN_SECONDS)
logger = sesam_logger("delayed-rest-proxy", app=app)
logger.info(
"starting service with \n\tURL_PATTERN=%s\n\tOPERATIONS=%s, \n\tDELAY_DURATION_IN_SECONDS=%d, \n\tLOG_LEVEL=%s"
% (env_config.URL_PATTERN, env_config.OPERATIONS,
env_config.DELAY_DURATION_IN_SECONDS, logger.level))