def delete_named_document(url: str, db_name: str,
doc_name: str) -> dict or None:
"""
Delete the named document
:param url: couchDB base URL in format http://host:port/
:param db_name:name of db
:param doc_name: document name
:return: delete result or None
"""
doc_url = url + db_name + "/" + doc_name
get_resp = RestClientApis.http_get_and_check_success(doc_url)
if get_resp.http_status == HTTPStatus.OK:
rev = get_resp.json_body["_rev"]
doc_url = doc_url + "?rev=" + rev
elif get_resp.http_status == HTTPStatus.UNAUTHORIZED:
print("Delete requested but not enough permissions \n")
return None
else:
print("Error reading doc_id {}. HTTP Code {}".format(
doc_url, get_resp.http_status))
return None
del_resp = RestClientApis.http_delete_and_check_success(doc_url)
if del_resp.http_status == HTTPStatus.OK:
return del_resp.json_body
else:
print("Error deleting doc_id {}. HTTP Code {}".format(
doc_url, get_resp.http_status))
return None
def get_db_all_docs(url: str, db_name: str) -> list or None:
"""
Retrieve all docs for a DB. CouchDB returns list of documents as
chunked-encoding, therefore special reassembly is needed
:param url: CouchDB URL
:param db_name: DB name
:return: A list of JSON dicts objects
"""
# Make static analysis happy
buf = None
try:
json_dict_list = list()
all_docs = url + db_name + "/" + "_all_docs"
get_resp = RestClientApis.http_get_and_check_success(all_docs)
headers = get_resp.response_object.headers
if ("Transfer-Encoding", "chunked") in headers.items():
for buf in read_chunks(get_resp.response_object):
json_dict = json.loads(buf)
json_dict_list.append(json_dict)
return json_dict_list
else:
return None
except JSONDecodeError:
error_msg = "Error decoding response {}".format(buf)
print(error_msg)
return None
def create_opa_policy(url: str, policy):
"""
This function creates a OPA policy on the server
:param url: url address where policy is placed
:param file_path: .rego file path or a raw rego string
:return: RestReturn
"""
if os.path.isfile(policy):
with open(policy, 'r') as file:
policy_data = file.read()
policy_resp = RestClientApis.http_put_and_check_success(
url, policy_data, headers={'Content-Type': 'text/plain'})
return policy_resp
else:
# we are going to treat it as a raw string and hope for the best.
policy_resp = RestClientApis.http_put_and_check_success(
url, policy, headers={'Content-Type': 'text/plain'})
return policy_resp.success, policy_resp.message
def create_base_doc(url, json_data):
"""
This function creates a OPA policy on the server
:param url: url address where base document is placed
:param json_data: Json data for Base Document
:return: success, message
:rtype: tuple
"""
resp = RestClientApis.http_put_and_check_success(url, json_data)
return resp.success, resp.message
def get_base_doc(url: str, debug=False) -> (object):
"""
This function gets an OPA base doc on the server
:param url: url address where base document is placed
:return: RestReturn Obj
:rtype: RestReturn
"""
if debug:
url = url + DEBUG_QUERY_STRING
resp = RestClientApis.http_get_and_check_success(url)
return resp
def create_opa_base_doc(url, json_data):
"""
This function creates a OPA policy on the server
:param url: url address where base document is placed
:param json_data: Json data for Base Document
:return: Rest Return
"""
resp = RestClientApis.http_put_and_check_success(url, json_data, headers={'Content-Type': 'application/json'})
if resp.http_status != HTTPStatus.OK:
raise json.JSONDecodeError(resp.message)
return resp
def delete_base_doc(url: str, debug=False):
"""
This function deletes an OPA base doc on the server
:param url: url address where base document is placed
:return: success, message
:rtype: tuple
"""
if debug:
url = url + DEBUG_QUERY_STRING
resp = RestClientApis.http_delete_and_check_success(url)
return resp.success, resp.message
def delete_all_policies(url: str, debug=False):
"""
This function deletes all OPA base docs on the server
:param url: url address where base document is placed
:return: success, message
:rtype: tuple
"""
if debug:
url = url + DEBUG_QUERY_STRING
data = json.dumps(dict())
resp = RestClientApis.http_put_and_check_success(url, data)
return resp.success, resp.message
def execute_query(url: str, json_data: str) -> (bool, str, dict):
"""
:param url: URL for query
:param json_data: JSON in string format.
:return: success, message and json body in dict
:rtype: tuple(bool, str, dict)
"""
resp_obj = RestClientApis.http_post_and_check_success(url,
json_data,
location=False)
return resp_obj.success, resp_obj.message, resp_obj.json_body
def delete_db(url, db_name):
"""
Deletes a DB from couchDB
:param url: couchDB base URL in format http://host:port/
:param db_name: name of db to be deleted
:return: boolean
"""
del_resp = RestClientApis.http_delete_and_check_success(url + db_name)
if del_resp.success:
return 0
else:
print("Failed to delete DB {}".format(url + db_name))
return -1
def patch_base_doc(url: str, json_data: str) -> (bool, str):
"""
Patches a base document.
:param url: URL of resource to be patched
:param json_data: JSON data as string
:return: success and message
:rtype: tuple
"""
resp = RestClientApis.http_patch_and_check_success(
url,
json_data,
headers={'Content-Type': 'application/json-patch+json'})
return resp.success, resp.message
def create_named_document(url: str,
db_name: str,
doc_name: str,
document: str,
overwrite=False) -> dict or None:
"""
:param overwrite: Should we create a new revision if doc exists?
:param url: couchDB base URL in format http://host:port/
:param db_name:name of db
:param doc_name: document name
:param document: document as a json string
:return: Json body as dict or None
"""
doc_url = url + db_name + "/" + doc_name
if overwrite:
get_resp = RestClientApis.http_get_and_check_success(doc_url)
if get_resp.http_status == HTTPStatus.OK:
rev = get_resp.json_body["_rev"]
rev_json = '"_rev":"{}",'.format(rev)
document = document.replace('{', '{' + rev_json, 1)
elif get_resp.http_status == HTTPStatus.NOT_FOUND:
# If document does not exist we continue and add it
print("Overwrite requested but document does not exist \n")
elif get_resp.http_status == HTTPStatus.UNAUTHORIZED:
print("Overwrite requested but not enough permissions \n")
return None
else:
print("Error reading doc_id {}. HTTP Code {}".format(
doc_url, get_resp.http_status))
return None
put_resp = RestClientApis.http_put_and_check_success(doc_url, document)
if put_resp.http_status == HTTPStatus.CREATED:
return put_resp.json_body
else:
print("Failed to save doc_id {}, doc {}. HTTP Code: {}".format(
doc_url, document, put_resp.http_status))
return None
def execute_adhoc_query(url: str,
query_string: str = None) -> (bool, str, dict):
"""
Executed an ad-hoc query
:param query_string: Everything after the ?=
:param url: URL for query that includes the query string
:return: success, message and json body as dict
:rtype: tuple(bool, str, dict)
"""
if query_string:
enc_query_string = urllib.parse.quote_plus(query_string)
url = url + "?q=" + enc_query_string
resp_obj = RestClientApis.http_get_and_check_success(url)
return resp_obj.success, resp_obj.message, resp_obj.json_body
def create_opa_policy(url, file_path):
"""
This function creates a OPA policy on the server
:param url: url address where policy is placed
:param file_path: .rego file path
:return: Rest Return
"""
policy_resp = None
with open(file_path, 'r') as file:
policy_data = file.read()
policy_resp = RestClientApis.http_put_and_check_success(url,
policy_data, headers={'Content-Type': 'text/plain'},
params={'file': file_path})
if policy_resp.http_status != HTTPStatus.OK:
raise exceptions.HTTPError(policy_resp.message)
return policy_resp
def get_named_document(url: str, db_name: str, doc_name: str) -> dict or None:
"""
Retrieve the named document
:param url: couchDB base URL in format http://host:port/
:param db_name:name of db
:param doc_name: document name
:return: document as json dict or None
"""
doc_url = url + db_name + "/" + doc_name
get_resp = RestClientApis.http_get_and_check_success(doc_url)
if get_resp.http_status == HTTPStatus.OK:
return get_resp.json_body
else:
print("Failed to read doc_id {}. HTTP Code: {}".format(
doc_url, get_resp.http_status))
return None
def create_db(url, db_name, overwrite=False):
"""
Creates a DB in couchDB. We return success if db was created or already exists.
:param overwrite: whether to overwrite db if it exists
:param url: couchDB base URL in format http://host:port/
:param db_name: name of db to be created
:return: boolean
"""
if overwrite:
delete_db(url, db_name)
put_resp = RestClientApis.http_put_and_check_success(url + db_name, "{}")
if put_resp.http_status == HTTPStatus.CREATED or \
put_resp.http_status == HTTPStatus.PRECONDITION_FAILED:
return 0
else:
print("Failed to create DB {}".format(url + db_name))
return -1
def create_watch(url: str) -> (bool, str, OpaWatch):
"""
Creates a watch in OPA. Watches are persistent connections and changes to the watch points
are streamed back through chunked-encoding.
:param url: URL for resource to watch
:return: success, message, OpaWatch class
:rtype: tuple(bool, message, OpaWatch)
"""
orig_url = url
url = url + WATCH_QUERY_STRING
resp_obj = RestClientApis.http_get_and_check_success(url,
stream=True,
timeout=(2.0, None))
p = Process(target=process_watch_stream, args=(resp_obj.response_object, ))
p.start()
opa_watch = OpaWatch(orig_url, p, p.pid)
return resp_obj.success, resp_obj.message, opa_watch
def create_policy(self, name: str, rego_policy) -> Policy:
"""
Policy creation invokes PUT request to an OPA server and returns a policy object
:param name: name of policy; allows '/'
:type name: str
:param rego_policy: blob policy in REGO language
:type rego_policy: str or file handler
:return: policy object
:rtype: Policy
"""
def validate():
"""
validation function for name parameter
:rtype: void
"""
words = name.split('/')
valid_words = [x for x in words if x.isalpha() or '_' in x]
if len(words) != len(valid_words):
raise OPAValidationError(
'only slashes, underscores and letters are acceptable in policy name'
)
validate()
policy_url = self.base + 'policies/' + name
rego_policy = get_policy_blob(rego_policy)
rest_response = RestClientApis.http_put_and_check_success(
policy_url, rego_policy, headers={'Content-Type': 'text/plain'})
rest_json = rest_response.json_body
if rest_response.http_status == HTTPStatus.BAD_REQUEST and rest_json.get(
'code') == INVALID_REGO_ERROR:
raise OPAValidationError(
str(rest_json.get('errors', 'Invalid or Empty REGO')))
if rest_response.http_status == HTTPStatus.OK and not rest_json:
return Policy(policy_name=name,
policy_path=policy_url,
rego_contents=rego_policy)
def delete_policy(url: str, debug=False) -> (bool, str, dict):
if debug:
url = url + DEBUG_QUERY_STRING
resp = RestClientApis.http_delete_and_check_success(url)
return resp.success, resp.message, resp.json_body
os.makedirs(hackathon_data_dir)
with open("bearer.txt", "r+") as bearer_f:
bearer = bearer_f.read()
server_urls_instance = ServerUrls().get_instance()
hackaton_globals = HackathonGlobals()
hackaton_globals.data_dir = hackathon_data_dir
file_name = "test_up.txt"
src_file_full_path = os.path.join(hackaton_globals.data_dir, file_name)
magen_file = open(src_file_full_path, 'w+')
magen_file.write("this is a test")
magen_file.close()
post_json = json.loads(SIN_MAGEN_INGESTION_POST_WITH_EMPTY_DOWNLOAD_URL)
get_resp_obj = RestClientApis.http_get_and_check_success(
server_urls_instance.ingestion_server_check_url)
get_resp_json_obj = get_resp_obj.json_body
in_docker = get_resp_json_obj["response"]["docker"]
if in_docker:
# If ingestion is running inside docker, we need to use the destination volume as file url.
src_file_full_path = "/opt/data/" + file_name
post_json["asset"][0]["download_url"] = "file://" + src_file_full_path
post_resp_obj = RestClientApis.http_post_and_check_success(
server_urls_instance.ingestion_server_asset_url, json.dumps(post_json))
post_resp_json_obj = post_resp_obj.json_body
container_file_path = post_resp_json_obj["response"]["asset"][
"file_path"] + ".html"
container_file_name = file_name + ".html"
if in_docker:
# Overwrite returned docker file path with the source volume so that it has significance.
def delete_policy(url):
resp = RestClientApis.http_delete_and_check_success(url)
if resp.http_status != HTTPStatus.OK:
raise exceptions.HTTPError(resp.message)
return
def main(args):
#: setup parser -----------------------------------------------------------
parser = argparse.ArgumentParser(description='Magen IO Server',
usage=("\npython3 server.py "
"--csrf"
"--clean-init"
"--ingestion-data-dir"
"\n\nnote:\n"
"root privileges are required "))
if inside_docker():
ingestion_data_dir = os.path.join("/opt", "data")
else:
home_dir = str(Path.home())
ingestion_data_dir = os.path.join(home_dir, "magen_data", "ingestion")
parser.add_argument('--ingestion-data-dir',
default=ingestion_data_dir,
help='Set directory for data files'
'Default is %s' % ingestion_data_dir)
parser.add_argument('--clean-init',
action='store_false',
help='Clean All data when initializing'
'Default is to clean)')
parser.add_argument('--csrf',
action='store_true',
help='Enable Cross Request Forgery protection'
'Default is to not use it)')
parser.add_argument('--test',
action='store_true',
help='Run server in test mode. Used for unit tests'
'Default is to run in production mode)')
#: parse CMD arguments ----------------------------------------------------
# args = parser.parse_args()
args, _ = parser.parse_known_args(args)
""" Main Magen.io Sever """
home_dir = str(Path.home())
ingestion_globals = IngestionGlobals()
ingestion_globals.data_dir = args.ingestion_data_dir
try:
os.makedirs(ingestion_globals.data_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Starting OPA server
docker_client = docker.from_env()
# if there is no image we pull it
try:
opa_image = docker_client.images.get("openpolicyagent/opa")
except NotFound as e:
opa_image = docker_client.images.pull("openpolicyagent/opa",
tag="latest")
assert opa_image is not None
# if container is not running we will start it
try:
opa_container = docker_client.containers.get("magen_opa")
if opa_container.status == "exited" or opa_container.status == "created":
opa_container.remove()
raise NotFound("Container Exited or could not be started")
except NotFound as e:
print("OPA docker container not found or not running, starting... \n")
opa_container = docker_client.containers.run(
"openpolicyagent/opa",
command="run --server --log-level=debug",
name="magen_opa",
ports={"8181/tcp": 8181},
detach=True)
time.sleep(5)
assert opa_container.status == "running" or (
opa_container.status == "created"
and not opa_container.attrs["State"]["Error"])
ks_args = [[]]
ks_process = Process(target=ks_server.main, args=ks_args)
ks_process.start()
time.sleep(2)
serverurls = ServerUrls()
ks_check_url = serverurls.key_server_base_url + "check/"
get_resp_obj = RestClientApis.http_get_and_check_success(ks_check_url)
assert get_resp_obj.success is True
mongo_ip, mongo_port = mongo_host_port()
# We initialize at runtime everything about Mongo and its functions
# Any client of the API can change it later
db = MainDb.get_instance()
db.core_database = MongoCore.get_instance()
db.core_database.utils_strategy = MongoUtils.get_instance()
db.core_database.asset_strategy = MongoAsset.get_instance()
db.core_database.db_ip_port = '{ip}:{port}'.format(ip=mongo_ip,
port=mongo_port)
db.core_database.utils_strategy.check_db(db.core_database.db_ip_port)
db.core_database.initialize()
if args.clean_init:
success, _ = AssetDbApi.delete_all()
assert success is True
user_api.drop_user_collection()
GridFsApi.delete_all()
if args.csrf:
app.config['WTF_CSRF_ENABLED'] = True
app.config['WTF_CSRF_SECRET_KEY'] = token_hex(16)
CSRFProtect(app)
else:
app.config['WTF_CSRF_ENABLED'] = False
app.register_blueprint(main_bp)
app.register_blueprint(users_bp)
app.register_blueprint(ingestion_file_upload_bp,
url_prefix='/magen/ingestion/v2')
app.register_blueprint(ingestion_bp_v2, url_prefix='/magen/ingestion/v2')
app.register_blueprint(ingestion_bp, url_prefix='/magen/ingestion/v2')
app.run('0.0.0.0', 5005, threaded=True)
