def delete_named_document(url: str, db_name: str,
doc_name: str) -> dict or None:
"""
Delete the named document
:param url: couchDB base URL in format http://host:port/
:param db_name:name of db
:param doc_name: document name
:return: delete result or None
"""
doc_url = url + db_name + "/" + doc_name
get_resp = RestClientApis.http_get_and_check_success(doc_url)
if get_resp.http_status == HTTPStatus.OK:
rev = get_resp.json_body["_rev"]
doc_url = doc_url + "?rev=" + rev
elif get_resp.http_status == HTTPStatus.UNAUTHORIZED:
print("Delete requested but not enough permissions \n")
return None
else:
print("Error reading doc_id {}. HTTP Code {}".format(
doc_url, get_resp.http_status))
return None
del_resp = RestClientApis.http_delete_and_check_success(doc_url)
if del_resp.http_status == HTTPStatus.OK:
return del_resp.json_body
else:
print("Error deleting doc_id {}. HTTP Code {}".format(
doc_url, get_resp.http_status))
return None
def get_db_all_docs(url: str, db_name: str) -> list or None:
"""
Retrieve all docs for a DB. CouchDB returns list of documents as
chunked-encoding, therefore special reassembly is needed
:param url: CouchDB URL
:param db_name: DB name
:return: A list of JSON dicts objects
"""
# Make static analysis happy
buf = None
try:
json_dict_list = list()
all_docs = url + db_name + "/" + "_all_docs"
get_resp = RestClientApis.http_get_and_check_success(all_docs)
headers = get_resp.response_object.headers
if ("Transfer-Encoding", "chunked") in headers.items():
for buf in read_chunks(get_resp.response_object):
json_dict = json.loads(buf)
json_dict_list.append(json_dict)
return json_dict_list
else:
return None
except JSONDecodeError:
error_msg = "Error decoding response {}".format(buf)
print(error_msg)
return None
def get_base_doc(url: str, debug=False) -> (object):
"""
This function gets an OPA base doc on the server
:param url: url address where base document is placed
:return: RestReturn Obj
:rtype: RestReturn
"""
if debug:
url = url + DEBUG_QUERY_STRING
resp = RestClientApis.http_get_and_check_success(url)
return resp
def execute_adhoc_query(url: str,
query_string: str = None) -> (bool, str, dict):
"""
Executed an ad-hoc query
:param query_string: Everything after the ?=
:param url: URL for query that includes the query string
:return: success, message and json body as dict
:rtype: tuple(bool, str, dict)
"""
if query_string:
enc_query_string = urllib.parse.quote_plus(query_string)
url = url + "?q=" + enc_query_string
resp_obj = RestClientApis.http_get_and_check_success(url)
return resp_obj.success, resp_obj.message, resp_obj.json_body
def get_named_document(url: str, db_name: str, doc_name: str) -> dict or None:
"""
Retrieve the named document
:param url: couchDB base URL in format http://host:port/
:param db_name:name of db
:param doc_name: document name
:return: document as json dict or None
"""
doc_url = url + db_name + "/" + doc_name
get_resp = RestClientApis.http_get_and_check_success(doc_url)
if get_resp.http_status == HTTPStatus.OK:
return get_resp.json_body
else:
print("Failed to read doc_id {}. HTTP Code: {}".format(
doc_url, get_resp.http_status))
return None
def create_watch(url: str) -> (bool, str, OpaWatch):
"""
Creates a watch in OPA. Watches are persistent connections and changes to the watch points
are streamed back through chunked-encoding.
:param url: URL for resource to watch
:return: success, message, OpaWatch class
:rtype: tuple(bool, message, OpaWatch)
"""
orig_url = url
url = url + WATCH_QUERY_STRING
resp_obj = RestClientApis.http_get_and_check_success(url,
stream=True,
timeout=(2.0, None))
p = Process(target=process_watch_stream, args=(resp_obj.response_object, ))
p.start()
opa_watch = OpaWatch(orig_url, p, p.pid)
return resp_obj.success, resp_obj.message, opa_watch
def create_named_document(url: str,
db_name: str,
doc_name: str,
document: str,
overwrite=False) -> dict or None:
"""
:param overwrite: Should we create a new revision if doc exists?
:param url: couchDB base URL in format http://host:port/
:param db_name:name of db
:param doc_name: document name
:param document: document as a json string
:return: Json body as dict or None
"""
doc_url = url + db_name + "/" + doc_name
if overwrite:
get_resp = RestClientApis.http_get_and_check_success(doc_url)
if get_resp.http_status == HTTPStatus.OK:
rev = get_resp.json_body["_rev"]
rev_json = '"_rev":"{}",'.format(rev)
document = document.replace('{', '{' + rev_json, 1)
elif get_resp.http_status == HTTPStatus.NOT_FOUND:
# If document does not exist we continue and add it
print("Overwrite requested but document does not exist \n")
elif get_resp.http_status == HTTPStatus.UNAUTHORIZED:
print("Overwrite requested but not enough permissions \n")
return None
else:
print("Error reading doc_id {}. HTTP Code {}".format(
doc_url, get_resp.http_status))
return None
put_resp = RestClientApis.http_put_and_check_success(doc_url, document)
if put_resp.http_status == HTTPStatus.CREATED:
return put_resp.json_body
else:
print("Failed to save doc_id {}, doc {}. HTTP Code: {}".format(
doc_url, document, put_resp.http_status))
return None
os.makedirs(hackathon_data_dir)
with open("bearer.txt", "r+") as bearer_f:
bearer = bearer_f.read()
server_urls_instance = ServerUrls().get_instance()
hackaton_globals = HackathonGlobals()
hackaton_globals.data_dir = hackathon_data_dir
file_name = "test_up.txt"
src_file_full_path = os.path.join(hackaton_globals.data_dir, file_name)
magen_file = open(src_file_full_path, 'w+')
magen_file.write("this is a test")
magen_file.close()
post_json = json.loads(SIN_MAGEN_INGESTION_POST_WITH_EMPTY_DOWNLOAD_URL)
get_resp_obj = RestClientApis.http_get_and_check_success(
server_urls_instance.ingestion_server_check_url)
get_resp_json_obj = get_resp_obj.json_body
in_docker = get_resp_json_obj["response"]["docker"]
if in_docker:
# If ingestion is running inside docker, we need to use the destination volume as file url.
src_file_full_path = "/opt/data/" + file_name
post_json["asset"][0]["download_url"] = "file://" + src_file_full_path
post_resp_obj = RestClientApis.http_post_and_check_success(
server_urls_instance.ingestion_server_asset_url, json.dumps(post_json))
post_resp_json_obj = post_resp_obj.json_body
container_file_path = post_resp_json_obj["response"]["asset"][
"file_path"] + ".html"
container_file_name = file_name + ".html"
if in_docker:
# Overwrite returned docker file path with the source volume so that it has significance.
def main(args):
#: setup parser -----------------------------------------------------------
parser = argparse.ArgumentParser(description='Magen IO Server',
usage=("\npython3 server.py "
"--csrf"
"--clean-init"
"--ingestion-data-dir"
"\n\nnote:\n"
"root privileges are required "))
if inside_docker():
ingestion_data_dir = os.path.join("/opt", "data")
else:
home_dir = str(Path.home())
ingestion_data_dir = os.path.join(home_dir, "magen_data", "ingestion")
parser.add_argument('--ingestion-data-dir',
default=ingestion_data_dir,
help='Set directory for data files'
'Default is %s' % ingestion_data_dir)
parser.add_argument('--clean-init',
action='store_false',
help='Clean All data when initializing'
'Default is to clean)')
parser.add_argument('--csrf',
action='store_true',
help='Enable Cross Request Forgery protection'
'Default is to not use it)')
parser.add_argument('--test',
action='store_true',
help='Run server in test mode. Used for unit tests'
'Default is to run in production mode)')
#: parse CMD arguments ----------------------------------------------------
# args = parser.parse_args()
args, _ = parser.parse_known_args(args)
""" Main Magen.io Sever """
home_dir = str(Path.home())
ingestion_globals = IngestionGlobals()
ingestion_globals.data_dir = args.ingestion_data_dir
try:
os.makedirs(ingestion_globals.data_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Starting OPA server
docker_client = docker.from_env()
# if there is no image we pull it
try:
opa_image = docker_client.images.get("openpolicyagent/opa")
except NotFound as e:
opa_image = docker_client.images.pull("openpolicyagent/opa",
tag="latest")
assert opa_image is not None
# if container is not running we will start it
try:
opa_container = docker_client.containers.get("magen_opa")
if opa_container.status == "exited" or opa_container.status == "created":
opa_container.remove()
raise NotFound("Container Exited or could not be started")
except NotFound as e:
print("OPA docker container not found or not running, starting... \n")
opa_container = docker_client.containers.run(
"openpolicyagent/opa",
command="run --server --log-level=debug",
name="magen_opa",
ports={"8181/tcp": 8181},
detach=True)
time.sleep(5)
assert opa_container.status == "running" or (
opa_container.status == "created"
and not opa_container.attrs["State"]["Error"])
ks_args = [[]]
ks_process = Process(target=ks_server.main, args=ks_args)
ks_process.start()
time.sleep(2)
serverurls = ServerUrls()
ks_check_url = serverurls.key_server_base_url + "check/"
get_resp_obj = RestClientApis.http_get_and_check_success(ks_check_url)
assert get_resp_obj.success is True
mongo_ip, mongo_port = mongo_host_port()
# We initialize at runtime everything about Mongo and its functions
# Any client of the API can change it later
db = MainDb.get_instance()
db.core_database = MongoCore.get_instance()
db.core_database.utils_strategy = MongoUtils.get_instance()
db.core_database.asset_strategy = MongoAsset.get_instance()
db.core_database.db_ip_port = '{ip}:{port}'.format(ip=mongo_ip,
port=mongo_port)
db.core_database.utils_strategy.check_db(db.core_database.db_ip_port)
db.core_database.initialize()
if args.clean_init:
success, _ = AssetDbApi.delete_all()
assert success is True
user_api.drop_user_collection()
GridFsApi.delete_all()
if args.csrf:
app.config['WTF_CSRF_ENABLED'] = True
app.config['WTF_CSRF_SECRET_KEY'] = token_hex(16)
CSRFProtect(app)
else:
app.config['WTF_CSRF_ENABLED'] = False
app.register_blueprint(main_bp)
app.register_blueprint(users_bp)
app.register_blueprint(ingestion_file_upload_bp,
url_prefix='/magen/ingestion/v2')
app.register_blueprint(ingestion_bp_v2, url_prefix='/magen/ingestion/v2')
app.register_blueprint(ingestion_bp, url_prefix='/magen/ingestion/v2')
app.run('0.0.0.0', 5005, threaded=True)