def delete_doc_by_query(self, index: str, query: dict):
try:
res = self.client.delete_by_query(index=index, body=query)
return {"index": index, "result": res}
except Exception as e:
log_error(e)
raise
def get_status(cls, api_root, status_id):
log_debug(f'Request to Get the status of {status_id} from {api_root}')
try:
result = cls.es_client.get_doc(index=f'{api_root}-status',
doc_id=status_id)
return result['data']
except Exception as e:
log_error(e)
return EXCEPTIONS.get('StatusNotFoundException')
def get_collection(cls, api_root, collection_id):
log_debug(
f'Request to Get Collection {collection_id} from Feed: {api_root}')
try:
result = cls.es_client.get_doc(index=f'{api_root}-collections',
doc_id=collection_id).get('data')
return result
except Exception as e:
log_error(e)
return EXCEPTIONS.get('CollectionNotFoundException', {})
def update_doc(self, index: str, data: object, doc_id: str):
try:
res = self.client.update(index=index,
id=doc_id,
body={"doc": data},
refresh='wait_for')
return {"index": res['_index'], "id": res['_id'], "result": res}
except Exception as e:
log_error(e)
raise
def roots_discovery(cls):
log_debug('Request to Get Discovery Info')
try:
result = cls.es_client.get_doc(index='discovery',
doc_id='discovery')
return result['data']
except Exception as e:
log_error(e)
return EXCEPTIONS.get('DiscoveryException', {})
def delete_doc(self, index: str, doc_id: str):
try:
res = self.client.delete(index=index, id=doc_id)
return {
"index": res['_index'],
"id": res['_id'],
"result": res['result']
}
except Exception as e:
log_error(e)
raise
def get_collection_objects(cls, api_root, collection_id):
log_debug(f'Request to Get The objects of Collection: {collection_id} in the Feed Root: {api_root}')
try:
result = cls.es_client.get_doc(index=f'{api_root}-collections', doc_id=collection_id)['data']['objects']
return {
'objects': result
}
except Exception as e:
log_error(e)
return EXCEPTIONS.get('CollectionNotFoundException', {})
def get_collections(cls, api_root):
log_debug(f'Request to Get all Collections under {api_root} Root')
try:
result = cls.es_client.get_docs(
index=f'{api_root}-collections').get('data')
return {'collections': result}
except NotFoundError as e:
log_error(e)
return EXCEPTIONS.get('APIRootNotFoundException', {})
except Exception as e:
log_error(e)
return EXCEPTIONS.get('CollectionsNotFoundException', {})
def store_docs(self, index: str, data: list):
try:
def yield_bulk_data(bulk_data):
for doc in bulk_data:
yield {"_index": index, "_id": doc['id'], "_source": doc}
res = helpers.bulk(self.client, yield_bulk_data(data))
return {"result": res}
except Exception as e:
log_error(e)
raise
def get_default_root_information(cls):
log_debug('Request to Get Default API Root Information')
try:
default_api_root_url = cls.es_client.get_doc(
index='discovery', doc_id='discovery').get('data')['default']
default_api_root = urlparse(default_api_root_url)[2].partition(
'/')[2].partition('/')[0]
result = cls.es_client.get_doc(index='feeds',
doc_id=default_api_root)
return result['data']['information']
except Exception as e:
log_error(e)
return EXCEPTIONS.get('DefaultAPIRootNotFoundException')
def get_docs(self, index: str):
try:
res = self.client.search(index=index, size=10, sort='id')
results = []
for result in res['hits']['hits']:
response = {}
response.update(result['_source'])
response.update({'id': result['id']})
results.append(response)
return {
"data": results,
"total": res['hits']['total']['value'],
}
except Exception as e:
log_error(e)
raise
def post_objects(cls, cti_objects):
log_info(f'Request to Post {len(cti_objects)} Objects')
result = {}
try:
entry = cls.es_client.store_docs(
index="stix21", data=cti_objects.dict().get('objects'))
result["status"] = 'success'
result["payload"] = entry
return result
except Exception as e:
log_error(e)
result["status"] = 'fail'
result["payload"] = {
"message": "Error (E:4) while posting the object .."
}
return result
def store_doc(self,
index: str,
data: object,
doc_id=int(round(time.time() * 1000))):
try:
res = self.client.index(index=index,
id=doc_id,
body=data,
refresh='wait_for')
return {
"index": res['_index'],
"id": res['_id'],
"result": res['result']
}
except Exception as e:
log_error(e)
raise
def get_collection_manifest(cls, api_root, **query_parameters):
objects_query = None
manifest_query = None
version_range = None
added_after_range = None
size = int(query_parameters.get('limit'))
max_page_size = PAGE_SIZE
added_after = query_parameters.get('added_after')
sort_by = {'date_added': {'order': 'asc'}}
types = query_parameters.get('types')
ids = query_parameters.get('ids')
versions = query_parameters.get('versions')
spec_versions = query_parameters.get('spec_versions')
base_page = 0
next_id = 0
objects_query = None
manifest_query = None
version_range = None
added_after_range = None
log_debug(
f"Request to Get The objects Manifest of Collection: {query_parameters.get('collection_id')} "
f"in the Feed Root: {api_root}")
if query_parameters is None:
query_parameters = {}
try:
# Create a Query to filter Objects by collection id, types and spec_versions
objects_query = f"collection : {query_parameters.get('collection_id')}"
if types:
types = types.replace(",", " OR ")
objects_query = objects_query + f" AND type : ('{types}')"
if spec_versions:
spec_versions = spec_versions.replace(",", " OR ")
objects_query = objects_query + f" AND spec_version : ('{spec_versions}')"
objects_query_string = QueryString(query=objects_query,
default_operator="and")
# Create a Query to filter Manifest by collection id, object id's, versions and added after dates
manifest_query = f"collection : {query_parameters.get('collection_id')}"
if ids:
ids = ids.replace(",", " OR ")
manifest_query = manifest_query + f" AND id : ('{ids}')"
if added_after:
added_after_range = Range(
**{'date_added': {
'gt': f'{added_after}'
}})
manifests_query_string = QueryString(query=manifest_query,
default_operator="and")
# Get the intersect of both Objects and Manifest Queries
intersected_results = cls.es_client.manifest_intersect(
intersect_by='id',
objects_index=f'{api_root}-objects',
objects_query_string=objects_query_string,
manifests_index=f'{api_root}-manifest',
manifests_query_string=manifests_query_string,
added_after_range=added_after_range)
# Version and Paginate The Results
if intersected_results:
manifest_ids = ",".join(intersected_results).replace(
',', ' OR ')
query_string = QueryString(query=f"id:('{manifest_ids}')",
default_operator="AND")
pre_versioning_results = cls.es_client.scan(
index=f'{api_root}-manifest', query_string=query_string)
pre_pagination_results = Helper.fetch_objects_by_versions(
stix_objects=pre_versioning_results, versions=versions)
if -1 < size < max_page_size:
results = cls.es_client.search(
index=f'{api_root}-manifest',
query_string=query_string,
search_from=base_page,
size=size,
sort_by=sort_by)
else:
results = cls.es_client.search(
index=f'{api_root}-manifest',
query_string=query_string,
search_from=base_page,
size=max_page_size,
sort_by=sort_by)
results = {'objects': pre_pagination_results}
else:
results = {"objects": []}
return results
except Exception as e:
log_error(e)
if query_parameters.get('next'):
return EXCEPTIONS.get('NextNotFoundException', {})
else:
return EXCEPTIONS.get('CollectionNotFoundException', {})
def es_prep(self):
try:
collections = []
manifests_data = []
objects_data = []
# Prepare Discovery Data
if not self.discovery_data:
self.discovery_data = self.TAXII_DEFAULT_DISCOVERY
if not self.client.indices.exists(
self.discovery_data.get('_index')):
log_info(
f"Loading default data in {self.discovery_data.get('_index')} index..."
)
# Create The Discovery Index
self.client.indices.create(
index=self.discovery_data.get('_index'))
# Load The Discovery Data
helpers.bulk(self.client, [self.discovery_data])
# Prepare API Roots Data
if not self.roots_data:
self.roots_data = self.TAXII_DEFAULT_ROOTS
for root in self.roots_data:
if not self.client.indices.exists(root.get('_index')):
log_info(
f"Loading default data in {root.get('_index')} index..."
)
# Create The API Roots Index
self.client.indices.create(index=root.get('_index'))
# Load The API Roots Data
helpers.bulk(self.client, self.roots_data)
# Prepare Status Data
if not self.status_data:
self.status_data = self.TAXXI_DEFAULT_STATUS
for root in self.roots_data:
if not self.client.indices.exists(f"{root.get('_id')}-status"):
log_info(
f"Loading default data in status index: {root.get('_id')}-status"
)
# Create A Status Index Per Root
self.client.indices.create(
index=f"{root.get('_id')}-status")
# Load The Status Data
helpers.bulk(self.client, self.status_data)
# Prepare Collections Data
if not self.collections_data:
self.collections_data = self.TAXXI_DEFAULT_COLLECTIONS
for root in self.roots_data:
manifests_data = []
if not self.client.indices.exists(
f"{root.get('_id')}-manifest"):
log_info(
f"Loading default data in manifest index: {root.get('_id')}-manifest"
)
# Create A Manifest Index Per Root
self.client.indices.create(
index=f"{root.get('_id')}-manifest")
# Load The Manifest Data Per Collection
for collection in self.collections_data:
if root.get('_id') in collection.get('_index'):
manifests = collection['_source'].get('manifest')
if manifests:
for manifest in manifests:
manifest['collection'] = collection.get(
'_id')
manifest_default = {
"_index":
f"{root.get('_id')}-manifest",
"_source": manifest
}
manifests_data.append(manifest_default)
helpers.bulk(self.client, manifests_data)
if not self.client.indices.exists(
f"{root.get('_id')}-objects"):
log_info(
f"Loading objects data in objects index: {root.get('_id')}-objects"
)
# Create An Objects Index Per Root
self.client.indices.create(
index=f"{root.get('_id')}-objects")
# Load Objects Data Per Collection
for collection in self.collections_data:
if root.get('_id') in collection.get('_index'):
objects = collection['_source'].get('objects')
if objects:
for collection_object in objects:
collection_object[
'collection'] = collection.get('_id')
object_default = {
"_index": f"{root.get('_id')}-objects",
"_source": collection_object
}
objects_data.append(object_default)
helpers.bulk(self.client, objects_data)
for root in self.roots_data:
if not self.client.indices.exists(
f"{root.get('_id')}-collections"):
log_info(
f"Loading collections data in collections index: {root.get('_id')}-collections"
)
# Create A Collections Index Per Root
self.client.indices.create(
index=f"{root.get('_id')}-collections")
# Load Collections Per Index
for collection in self.collections_data:
collection['_source'].pop("manifest", None)
collection['_source'].pop("responses", None)
collection['_source'].pop("objects", None)
collections.append(collection)
helpers.bulk(self.client, collections)
# Create Next Index
if not self.client.indices.exists('next'):
log_info(f"Creating Next index...")
# Create The API Roots Index
self.client.indices.create('next')
except Exception as error:
log_error(error)
es_client.es_prep()
app = FastAPI(middleware=ValidationMiddleware)
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request, exc):
return JSONResponse(content={
"title": str(exc.detail),
"error_code": str(exc.status_code)
},
status_code=exc.status_code)
# TODO: Enforce Authorization
# TODO: Enable Paging
# TODO: Review Error Codes
# TODO: Review The Custom Headers
app.include_router(discovery.router)
app.include_router(collections.router)
app.include_router(objects.router)
log_info('Galaxy is running ..')
else:
log_error(
'Galaxy is not ready to start, having a problem connecting to ElasticSearch'
)
if __name__ == '__main__':
uvicorn.run(app, port=4000, host='0.0.0.0')