def list_processes():
global _process_cache
if _process_cache:
return _process_cache
added = False
result = []
dir_list = get_config("App", "extra_path")
if dir_list != "":
parts = dir_list.split(';')
for p in parts:
if p not in sys.path:
logger.info(f"adding {p} to python path")
sys.path.append(p)
for (root, dirs, files) in os.walk(p, topdown=True):
for f in files:
if f.endswith(".py"):
try:
result += load_module(root, f, p)
except Exception as e:
logger.warning(f"could not load {root}/{f} due to {e} skipping")
if added:
importlib.invalidate_caches()
_process_cache = result
logging.info(f"found {[f['name'] for f in result]}")
return result
def calculate_result(self, **kwargs):
"""
This is the entry point for a task run. Will be called by celery.
:param kwargs: arguments to the tasks.
:return:
"""
# connect to the datacube and pass that in to the users function.
# Everything should be talking to the datacube here so makes sense to pull it out and make things
# easier for the users.
result_dir = get_config("App", "result_dir")
path_prefix = path.join(result_dir, self.request.id)
os.makedirs(path_prefix, exist_ok=True)
dc = datacube.Datacube(app=self.name)
outputs = self.generate_product(dc, path_prefix,
**self.map_kwargs(**kwargs))
logging.info(f"got result of {outputs}")
self.log_query(path_prefix)
self.zip_outputs(path_prefix, outputs)
# TODO: put the results some where, send notifications etc.
output_url = self.upload_results(path_prefix)
self.ping_results(output_url)
def get_token():
payload = request.get_json()
if not payload:
abort(403, "no payload")
if not payload['name']:
abort(403, "name required")
user = payload['name']
if user in payload:
user = payload['user']
if not payload['pass']:
abort(403, "pass required")
logging.info(
f"log in request for {payload['name']} from {request.remote_addr}")
if not users.check_user(user, payload['pass'], request.remote_addr):
abort(403, "bad creds")
s = Serializer(get_config("App", "secret_key"),
expires_in=int(get_config("App", "token_duration")))
return jsonify({'token': s.dumps({'id': payload['name']})})
def validate_app_key():
"""
Get the app key parameter from a request and check that it is a valid token.
:return: Bool, True if and only if the requests app key is a valid token
"""
if 'APP_KEY' in request.args:
s = Serializer(get_config("App", "secret_key"))
try:
data = s.loads(request.args['APP_KEY'])
# TODO: look up the id and make sure its a real one.
return data['id']
except SignatureExpired:
abort(403, "Token expired")
except BadSignature:
abort(403, "Invalid token")
abort(403, "No token")
def upload_results(self, path_prefix):
source_file_path = os.path.join(path_prefix,
self.request.id + "_output.zip")
dest_file_path = os.path.join(get_config("AWS", "path_prefix"),
self.request.id + "_output.zip")
access_key = get_config("AWS", "access_key_id")
secret_key = get_config("AWS", "secret_access_key")
bucket = get_config("AWS", "bucket")
s3_tools = S3Utils(access_key, secret_key, bucket,
get_config("AWS", "end_point"),
get_config("AWS", "region"))
s3_tools.put_file(source_file_path, dest_file_path)
return dest_file_path
def ping_results(self, output_url):
result_url = get_config("App", "result_url")
if result_url:
payload = {"url": output_url, "name": ""}
print(payload)
def get_result(task_id):
validate_app_key()
result_dir = get_config("App", "result_dir")
file_name = f"{task_id}_output.zip"
target = path.join(result_dir, task_id, file_name)
return send_file(target, attachment_filename=file_name)
from flask_caching import Cache
from flask_cors import CORS
from itsdangerous import TimedJSONWebSignatureSerializer as Serializer, SignatureExpired, BadSignature
from jobtastic.cache import WrappedCache
from cubequery import get_config, users
from cubequery.packages import is_valid_task, load_task_instance, list_processes
def _to_bool(input):
return input.lower() in [
'true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'uh-huh'
]
redis_url = get_config("Redis", "url")
config = {
"DEBUG": _to_bool(get_config("App", "debug")),
"CACHE_TYPE": "redis",
"CACHE_REDIS_URL": redis_url,
}
template_dir = os.path.abspath('./webroot/templates')
static_dir = os.path.abspath('./webroot/static')
app = Flask(__name__, template_folder=template_dir, static_folder=static_dir)
app.config.from_mapping(config)
cache = WrappedCache(Cache(app))
cors = CORS(app,
origin=get_config("App", "cors_origin"),
send_wildcard=True,