def post(self,
name=None,
cpus=None,
memory=None,
network=None,
disk=None,
ssh_key=None,
user_data=None,
placed_on=None,
namespace=None,
instance_uuid=None):
global SCHEDULER
# We need to sanitise the name so its safe for DNS
name = re.sub(r'([^a-zA-Z0-9_\-])', '', name)
if not namespace:
namespace = get_jwt_identity()
# If accessing a foreign namespace, we need to be an admin
if get_jwt_identity() not in [namespace, 'system']:
return error(
401,
'only admins can create resources in a different namespace')
# The instance needs to exist in the DB before network interfaces are created
if not instance_uuid:
instance_uuid = str(uuid.uuid4())
db.add_event('instance', instance_uuid, 'uuid allocated', None,
None, None)
# Create instance object
instance = virt.from_db(instance_uuid)
if instance:
if get_jwt_identity() not in [
instance.db_entry['namespace'], 'system'
]:
LOG.info('instance(%s): instance not found, ownership test' %
instance_uuid)
return error(404, 'instance not found')
if not instance:
instance = virt.from_definition(uuid=instance_uuid,
name=name,
disks=disk,
memory_mb=memory,
vcpus=cpus,
ssh_key=ssh_key,
user_data=user_data,
owner=namespace)
if not SCHEDULER:
SCHEDULER = scheduler.Scheduler()
# Have we been placed?
if not placed_on:
candidates = SCHEDULER.place_instance(instance, network)
if len(candidates) == 0:
db.add_event('instance', instance_uuid, 'schedule', 'failed',
None, 'insufficient resources')
db.update_instance_state(instance_uuid, 'error')
return error(507, 'insufficient capacity')
placed_on = candidates[0]
db.place_instance(instance_uuid, placed_on)
db.add_event('instance', instance_uuid, 'placement', None, None,
placed_on)
else:
try:
candidates = SCHEDULER.place_instance(instance,
network,
candidates=[placed_on])
if len(candidates) == 0:
db.add_event('instance', instance_uuid, 'schedule',
'failed', None, 'insufficient resources')
db.update_instance_state(instance_uuid, 'error')
return error(507, 'insufficient capacity')
except scheduler.CandidateNodeNotFoundException as e:
return error(404, 'node not found: %s' % e)
# Have we been placed on a different node?
if not placed_on == config.parsed.get('NODE_NAME'):
body = flask_get_post_body()
body['placed_on'] = placed_on
body['instance_uuid'] = instance_uuid
body['namespace'] = namespace
token = util.get_api_token(
'http://%s:%d' % (placed_on, config.parsed.get('API_PORT')),
namespace=namespace)
r = requests.request('POST',
'http://%s:%d/instances' %
(placed_on, config.parsed.get('API_PORT')),
data=json.dumps(body),
headers={
'Authorization': token,
'User-Agent': util.get_user_agent()
})
LOG.info('Returning proxied request: %d, %s' %
(r.status_code, r.text))
resp = flask.Response(r.text, mimetype='application/json')
resp.status_code = r.status_code
return resp
# Check we can get the required IPs
nets = {}
allocations = {}
def error_with_cleanup(status_code, message):
for network_uuid in allocations:
n = net.from_db(network_uuid)
for addr, _ in allocations[network_uuid]:
with db.get_lock('sf/ipmanager/%s' % n.uuid, ttl=120) as _:
ipm = db.get_ipmanager(n.uuid)
ipm.release(addr)
db.persist_ipmanager(n.uuid, ipm.save())
return error(status_code, message)
order = 0
if network:
for netdesc in network:
if 'network_uuid' not in netdesc or not netdesc['network_uuid']:
return error_with_cleanup(404, 'network not specified')
if netdesc['network_uuid'] not in nets:
n = net.from_db(netdesc['network_uuid'])
if not n:
return error_with_cleanup(
404,
'network %s not found' % netdesc['network_uuid'])
nets[netdesc['network_uuid']] = n
n.create()
with db.get_lock('sf/ipmanager/%s' % netdesc['network_uuid'],
ttl=120) as _:
db.add_event('network', netdesc['network_uuid'],
'allocate address', None, None, instance_uuid)
allocations.setdefault(netdesc['network_uuid'], [])
ipm = db.get_ipmanager(netdesc['network_uuid'])
if 'address' not in netdesc or not netdesc['address']:
netdesc['address'] = ipm.get_random_free_address()
else:
if not ipm.reserve(netdesc['address']):
return error_with_cleanup(
409, 'address %s in use' % netdesc['address'])
db.persist_ipmanager(netdesc['network_uuid'], ipm.save())
allocations[netdesc['network_uuid']].append(
(netdesc['address'], order))
if 'model' not in netdesc or not netdesc['model']:
netdesc['model'] = 'virtio'
db.create_network_interface(str(uuid.uuid4()), netdesc,
instance_uuid, order)
order += 1
# Initialise metadata
db.persist_metadata('instance', instance_uuid, {})
# Now we can start the instance
with db.get_lock('sf/instance/%s' % instance.db_entry['uuid'],
ttl=900) as lock:
with util.RecordedOperation('ensure networks exist',
instance) as _:
for network_uuid in nets:
n = nets[network_uuid]
n.ensure_mesh()
n.update_dhcp()
with util.RecordedOperation('instance creation', instance) as _:
instance.create(lock=lock)
for iface in db.get_instance_interfaces(instance.db_entry['uuid']):
db.update_network_interface_state(iface['uuid'], 'created')
return db.get_instance(instance_uuid)
def handle_rpc_options():
response = flask.Response('', 204)
_set_cors_headers(response)
return response
def slack_slash_commands():
"""
Handles responding to slash commands for reservations.
Commands:
/crush watch <campground-tag> <DD/MM/YY> <length>
------------------------------------------------------
Registers a new watcher for a reservation. This will begin a periodic
scraping process against the recreation.gov website. When succesful we'll
send you a slack message with results.
Campgrounds are selected according to `campground-tag` you provide. The bot
will attempt to find sites within any campground that matches the tag you
provide.
To list campgrounds and their tags, use the `campgrounds` command.
/crush list
----------------------
Lists active watchers for the current user.
/crush list-all
----------------------
Lists active watchers for all users.
/crush campgrounds [tags...]
------------------
Lists known campgrounds, optionally filtered by those that match any of the
provided tags. For example, if you wish to list what the bot considers
a 'yosemite-valley' campground use `/crush campgrounds yosemite-valley`.
Syntax:
- Square brackets, as in `[param]`, denote optional parameters.
- Angle brackets, as in `<param>`, denote required parameters.
- Ellipsis, `...` following a parameter denote a space-separated list.
"""
raw_data = flask.request.get_data()
if not verify_slack_request(
flask.request.headers['X-Slack-Signature'],
flask.request.headers['X-Slack-Request-Timestamp'],
raw_data.decode('utf-8'),
):
return flask.Response(status=400)
text = flask.request.form['text']
if len(text) == 0:
return flask.jsonify({
"response_type":
"ephemeral",
# We re-use the docstring in this function as the help text.
"text":
"I need a subcommand!\n```{}```".format(
textwrap.dedent(slack_slash_commands.__doc__))
})
# Request payload mangling and subcommand delegation occurs.
parts = text.split(' ')
command = parts[0]
args = parts[1:]
if command == 'watch':
if len(args) != 3:
return flask.jsonify({
"response_type":
"ephemeral",
"text":
"Please use a format like `tuolumne DD/MM/YY <length>`."
})
campground, start, length = args
try:
date = arrow.get(start, 'DD/MM/YY')
except:
return flask.jsonify({
"response_type":
"ephemeral",
"text":
"Could not parse your date, please use a DD/MM/YY format.",
})
# Hackish workaround: 01/01/2019 successfully parses via DD/MM/YY above,
# but will subsequently get interpretted as e.g. "2020" - ignoring the
# latter two characters.
if date.format('DD/MM/YY') != start:
return flask.jsonify({
"response_type":
"ephemeral",
"text":
"Could not parse your date, please use a DD/MM/YY format.",
})
user_id = flask.request.form['user_id']
return add_watcher(user_id, campground, start, int(length))
elif command == 'list':
return slack_list_watchers(flask.request.form['user_id'])
elif command == 'list-all':
return slack_list_watchers()
elif command == 'campgrounds':
return slack_list_campgrounds(args)
elif command == 'help':
return flask.jsonify({
"response_type":
"ephemeral",
"text":
"```{}```".format(textwrap.dedent(slack_slash_commands.__doc__))
})
else:
return flask.jsonify({
"response_type": "ephemeral",
"text": "I haven't been implemented yet!",
})
def stream():
"""Handle long-lived SSE streams."""
return flask.Response(event_stream(flask.request.access_route[0]),
mimetype='text/event-stream')
def css(source_name):
source_name = urllib.unquote(str(source_name))
return flask.Response(get_css(source_name), mimetype="text/css")
def pingAPI():
return flask.Response("Pong")
return add, rm
#
# The JUNK DRAWER! Everything past this point is kind of...just random junk
#
ymd = "%Y-%m-%d"
hms = "%H:%M:%S"
ymdhms = "%Y-%m-%d %H:%M:%S"
thirty_days_ago = datetime.now() - timedelta(days=30)
# HTTP response objects
# 200s
http_200 = flask.Response(response="OK!", status=200)
HTTP_299 = flask.Response(response="This endpoint is deprecated!", status=299)
# 400s
http_400 = flask.Response(response="Bad Request!", status=400)
http_401 = flask.Response(response="Authorization required", status=401)
http_402 = flask.Response(response="This is a subscribers-only endpoint!",
status=402)
http_403 = flask.Response(response="Unauthorized!", status=403)
http_404 = flask.Response(response="Resource not found", status=404)
http_405 = flask.Response(
response="Method not allowed (did you mean to POST?)", status=405)
http_410 = flask.Response(
response="Gone. This endpoint is no longer supported! There is no\
replacement or equivalent for it.",
status=410)
def logs():
return flask.Response(event_stream(), mimetype="text/event-stream")
def trigger_update():
metadata.update()
return flask.Response(status=204)
def handle_post():
request_json = flask.request.get_data().decode('utf-8')
rpc_response = jsonrpc.JSONRPCResponseManager.handle(request_json, dispatcher)
rpc_response_json = json.dumps(rpc_response.data).encode()
response = flask.Response(rpc_response_json, 200, mimetype='application/json')
return response
def api_new_note() -> _Response:
# Basic Micropub (https://www.w3.org/TR/micropub/) query configuration support
if request.method == "GET" and request.args.get("q") == "config":
return jsonify({})
elif request.method == "GET":
abort(405)
source = None
summary = None
location = None
# Basic Micropub (https://www.w3.org/TR/micropub/) "create" support
is_micropub = False
# First, check if the Micropub specific fields are present
if (_user_api_arg("h", default=None) == "entry"
or _user_api_arg("type", default=[None])[0] == "h-entry"):
is_micropub = True
# Ensure the "create" scope is set
if "jwt_payload" not in flask.g or "create" not in flask.g.jwt_payload[
"scope"]:
abort(403)
# Handle location sent via form-data
# `geo:28.5,9.0,0.0`
location = _user_api_arg("location", default="")
if location.startswith("geo:"):
slat, slng, *_ = location[4:].split(",")
location = {
"type": ap.ActivityType.PLACE.value,
"latitude": float(slat),
"longitude": float(slng),
}
# Handle JSON microformats2 data
if _user_api_arg("type", default=None):
_logger.info(f"Micropub request: {request.json}")
try:
source = request.json["properties"]["content"][0]
except (ValueError, KeyError):
pass
# Handle HTML
if isinstance(source, dict):
source = source.get("html")
try:
summary = request.json["properties"]["name"][0]
except (ValueError, KeyError):
pass
# Try to parse the name as summary if the payload is POSTed using form-data
if summary is None:
summary = _user_api_arg("name", default=None)
# This step will also parse content from Micropub request
if source is None:
source = _user_api_arg("content", default=None)
if not source:
raise ValueError("missing content")
if summary is None:
summary = _user_api_arg("summary", default="")
if not location:
if _user_api_arg("location_lat", default=None):
lat = float(_user_api_arg("location_lat"))
lng = float(_user_api_arg("location_lng"))
loc_name = _user_api_arg("location_name", default="")
location = {
"type": ap.ActivityType.PLACE.value,
"name": loc_name,
"latitude": lat,
"longitude": lng,
}
# All the following fields are specific to the API (i.e. not Micropub related)
_reply, reply = None, None
try:
_reply = _user_api_arg("reply")
except ValueError:
pass
visibility = ap.Visibility[_user_api_arg(
"visibility", default=ap.Visibility.PUBLIC.name)]
content, tags = parse_markdown(source)
# Check for custom emojis
tags = tags + emojis.tags(content)
to: List[str] = []
cc: List[str] = []
if visibility == ap.Visibility.PUBLIC:
to = [ap.AS_PUBLIC]
cc = [ID + "/followers"]
elif visibility == ap.Visibility.UNLISTED:
to = [ID + "/followers"]
cc = [ap.AS_PUBLIC]
elif visibility == ap.Visibility.FOLLOWERS_ONLY:
to = [ID + "/followers"]
cc = []
if _reply:
reply = ap.fetch_remote_activity(_reply)
if visibility == ap.Visibility.DIRECT:
to.append(reply.attributedTo)
else:
cc.append(reply.attributedTo)
context = new_context(reply)
for tag in tags:
if tag["type"] == "Mention":
to.append(tag["href"])
raw_note = dict(
attributedTo=MY_PERSON.id,
cc=list(set(cc) - set([MY_PERSON.id])),
to=list(set(to) - set([MY_PERSON.id])),
summary=summary,
content=content,
tag=tags,
source={
"mediaType": "text/markdown",
"content": source
},
inReplyTo=reply.id if reply else None,
context=context,
)
if location:
raw_note["location"] = location
if request.files:
for f in request.files.keys():
if not request.files[f].filename:
continue
file = request.files[f]
rfilename = secure_filename(file.filename)
with BytesIO() as buf:
# bypass file.save(), because it can't save to a file-like object
copyfileobj(file.stream, buf, 16384)
oid = MEDIA_CACHE.save_upload(buf, rfilename)
mtype = mimetypes.guess_type(rfilename)[0]
raw_note["attachment"] = [{
"mediaType":
mtype,
"name":
_user_api_arg("file_description", default=rfilename),
"type":
"Document",
"url":
f"{BASE_URL}/uploads/{oid}/{rfilename}",
}]
note = ap.Note(**raw_note)
create = note.build_create()
create_id = post_to_outbox(create)
# Return a 201 with the note URL in the Location header if this was a Micropub request
if is_micropub:
resp = flask.Response("", headers={"Location": create_id})
resp.status_code = 201
return resp
return _user_api_response(activity=create_id)
def token_voice():
call = authy_api.users.request_call(current_user.authy_id, {'force': True})
if call.ok():
return flask.Response('Call request successful', status=200)
else:
return flask.Response('Call request failed', status=503)
def token_sms():
sms = authy_api.users.request_sms(current_user.authy_id, {'force': True})
if sms.ok():
return flask.Response('SMS request successful', status=200)
else:
return flask.Response('SMS request failed', status=503)
def save_fortney_result():
"""SAve the results of the Fortney grid"""
table_string = flask.request.form['data_file']
return flask.Response(table_string, mimetype="text/dat",
headers={"Content-disposition": "attachment; filename=fortney.dat"})
def serve_routes(self):
return flask.Response(json.dumps(self.routes,
cls=plotly.utils.PlotlyJSONEncoder),
mimetype='application/json')
def objects_get_common(bucket_name, object_name, revision):
# Respect the Range: header, if present.
range_header = flask.request.headers.get("range")
response_payload = revision.media
begin = 0
end = len(response_payload)
if range_header is not None:
m = re.match("bytes=([0-9]+)-([0-9]+)", range_header)
if m:
begin = int(m.group(1))
end = int(m.group(2))
response_payload = response_payload[begin : end + 1]
m = re.match("bytes=([0-9]+)-$", range_header)
if m:
begin = int(m.group(1))
response_payload = response_payload[begin:]
m = re.match("bytes=-([0-9]+)$", range_header)
if m:
last = int(m.group(1))
response_payload = response_payload[-last:]
# Process custom headers to test error conditions.
instructions = flask.request.headers.get("x-goog-testbench-instructions")
if instructions == "return-broken-stream":
def streamer():
chunk_size = 64 * 1024
for r in range(0, len(response_payload), chunk_size):
if r > 1024 * 1024:
print("\n\n###### EXIT to simulate crash\n")
sys.exit(1)
time.sleep(0.1)
chunk_end = min(r + chunk_size, len(response_payload))
yield response_payload[r:chunk_end]
length = len(response_payload)
content_range = "bytes %d-%d/%d" % (begin, end - 1, length)
headers = {
"Content-Range": content_range,
"Content-Length": length,
"x-goog-hash": revision.x_goog_hash_header(),
"x-goog-generation": revision.generation,
}
return flask.Response(streamer(), status=200, headers=headers)
if instructions == "return-corrupted-data":
response_payload = testbench_utils.corrupt_media(response_payload)
if instructions is not None and instructions.startswith(u"stall-always"):
length = len(response_payload)
content_range = "bytes %d-%d/%d" % (begin, end - 1, length)
def streamer():
chunk_size = 16 * 1024
for r in range(begin, end, chunk_size):
chunk_end = min(r + chunk_size, end)
if r == begin:
time.sleep(10)
yield response_payload[r:chunk_end]
headers = {
"Content-Range": content_range,
"x-goog-hash": revision.x_goog_hash_header(),
"x-goog-generation": revision.generation,
}
return flask.Response(streamer(), status=200, headers=headers)
if instructions == "stall-at-256KiB" and begin == 0:
length = len(response_payload)
content_range = "bytes %d-%d/%d" % (begin, end - 1, length)
def streamer():
chunk_size = 16 * 1024
for r in range(begin, end, chunk_size):
chunk_end = min(r + chunk_size, end)
if r == 256 * 1024:
time.sleep(10)
yield response_payload[r:chunk_end]
headers = {
"Content-Range": content_range,
"x-goog-hash": revision.x_goog_hash_header(),
"x-goog-generation": revision.generation,
}
return flask.Response(streamer(), status=200, headers=headers)
if instructions is not None and instructions.startswith(u"return-503-after-256K"):
length = len(response_payload)
headers = {
"Content-Range": "bytes %d-%d/%d" % (begin, end - 1, length),
"x-goog-hash": revision.x_goog_hash_header(),
"x-goog-generation": revision.generation,
}
if begin == 0:
def streamer():
chunk_size = 4 * 1024
for r in range(0, len(response_payload), chunk_size):
if r >= 256 * 1024:
print("\n\n###### EXIT to simulate crash\n")
sys.exit(1)
time.sleep(0.01)
chunk_end = min(r + chunk_size, len(response_payload))
yield response_payload[r:chunk_end]
return flask.Response(streamer(), status=200, headers=headers)
if instructions.endswith(u"/retry-1"):
print("## Return error for retry 1")
return flask.Response("Service Unavailable", status=503)
if instructions.endswith(u"/retry-2"):
print("## Return error for retry 2")
return flask.Response("Service Unavailable", status=503)
print("## Return success for %s" % instructions)
return flask.Response(response_payload, status=200, headers=headers)
response = flask.make_response(response_payload)
length = len(response_payload)
content_range = "bytes %d-%d/%d" % (begin, end - 1, length)
response.headers["Content-Range"] = content_range
response.headers["x-goog-hash"] = revision.x_goog_hash_header()
response.headers["x-goog-generation"] = revision.generation
return response
def trydocpieinfohandler():
info = {
'version_time': docpie.__timestamp__,
'version': docpie.__version__,
}
return flask.Response(json.dumps(info), mimetype='application/json')
def not_authorized():
"""Sends 401 response to bad authenticate"""
return flask.Response('Unauthorized Access!', 401)
def metadataAPI():
access_token, account_list, category_list, client_id, client_secret, refresh_token, secret_key, tmdb_api_key, token_expiry = src.config.readConfig()
tmp_metadata = src.metadata.readMetadata(category_list)
a = flask.request.args.get("a") # AUTH
c = flask.request.args.get("c") # CATEGORY
q = flask.request.args.get("q") # SEARCH-QUERY
s = flask.request.args.get("s") # SORT-ORDER
r = flask.request.args.get("r") # RANGE
id = flask.request.args.get("id") # ID
if any(a == account["auth"] for account in account_list):
if c:
tmp_metadata = [
next((i for i in tmp_metadata if i["name"] == c), None)]
if tmp_metadata:
pass
else:
return flask.Response("The category provided could not be found.", status=400)
if q:
index = 0
for category in tmp_metadata:
tmp_metadata[index]["children"] = [
item for item in category["children"] if q.lower() in item["title"].lower()]
index += 1
if s:
index = 0
for category in tmp_metadata:
if s == "alphabet-asc":
try:
tmp_metadata[index]["children"] = sorted(
category["children"], key=lambda k: k["title"])
except:
pass
elif s == "alphabet-des":
try:
tmp_metadata[index]["children"] = sorted(
category["children"], key=lambda k: k["title"], reverse=True)
except:
pass
elif s == "date-asc":
try:
tmp_metadata[index]["children"] = sorted(
category["children"], key=lambda k: tuple(map(int, k["releaseDate"].split('-'))))
except:
pass
elif s == "date-des":
try:
tmp_metadata[index]["children"] = sorted(category["children"], key=lambda k: tuple(
map(int, k["releaseDate"].split("-"))), reverse=True)
except:
pass
elif s == "popularity-asc":
try:
tmp_metadata[index]["children"] = sorted(
category["children"], key=lambda k: float(k["popularity"]))
except:
pass
elif s == "popularity-des":
try:
tmp_metadata[index]["children"] = sorted(
category["children"], key=lambda k: float(k["popularity"]), reverse=True)
except:
pass
elif s == "random":
try:
random.shuffle(tmp_metadata[index]["children"])
except:
pass
else:
return None
index += 1
if r:
index = 0
for category in tmp_metadata:
tmp_metadata[index]["children"] = eval(
"category['children']" + "[" + r + "]")
index += 1
if id:
ids = src.metadata.jsonExtract(
obj=tmp_metadata, key="id", getObj=True)
for item in ids:
if item["id"] == id:
tmp_metadata = item
return flask.jsonify(tmp_metadata)
else:
return flask.Response("The auth code provided was incorrect.", status=401)
def json_doubler():
doubled = [int(v) * 2 for v in flask.request.json]
return flask.Response(content_type='application/json',
response=json.dumps(doubled))
def render_response(self):
""" Renders an http response. """
if self.img is None:
return flask.Response(response="Image not found!", status=404)
image_file = BytesIO(self.img.read())
return flask.send_file(image_file, mimetype="image/png")
def get_looking_glass_prediction():
text = open(
'ml_enabler/tests/fixtures/looking_glass_prediction.json').read()
res = flask.Response(text, mimetype='application/json')
res.headers['Content-Length'] = len(res.get_data())
return res
def ranking_api(stat_var, place_type, place=None):
"""Returns top 100, bottom 100 or all rankings for a stats var, grouped
by place type and optionally scoped by a containing place (all is
returned if there are fewer than 100 places in the group). Each place in
the ranking has it's name returned, if available.
Optional GET args:
pc (per capita - the presence of the key enables it)
bottom (show bottom ranking instead - the presence of the key enables it)
"""
is_per_capita = flask.request.args.get('pc', False) != False
is_show_bottom = flask.request.args.get('bottom', False) != False
rank_keys = BOTTOM_KEYS_KEEP if is_show_bottom else TOP_KEYS_KEEP
delete_keys = BOTTOM_KEYS_DEL if is_show_bottom else TOP_KEYS_DEL
ranking_results = dc.get_place_ranking([stat_var], place_type, place,
is_per_capita)
if not 'payload' in ranking_results:
flask.abort(500)
payload = ranking_results['payload']
if not stat_var in ranking_results['payload']:
flask.abort(500)
# split rankAll to top/bottom if it's larger than RANK_SIZE
if 'rankAll' in payload[stat_var]:
rank_all = payload[stat_var]['rankAll']
if 'info' in rank_all and len(rank_all['info']) > RANK_SIZE:
if is_show_bottom:
payload[stat_var]['rankBottom1000'] = rank_all
else:
payload[stat_var]['rankTop1000'] = rank_all
del payload[stat_var]['rankAll']
for k in delete_keys:
try:
del payload[stat_var][k]
except KeyError:
pass # key might not exist for fewer than 1000 places
dcids = set()
for k in rank_keys:
if k in payload[stat_var] and 'info' in payload[stat_var][k]:
info = payload[stat_var][k]['info']
if is_show_bottom:
# truncate and reverse the bottom RANK_SIZE elems
info = info[-RANK_SIZE:]
else:
info = info[:RANK_SIZE]
for r in info:
dcids.add(r['placeDcid'])
payload[stat_var][k]['info'] = info
# payload[stat_var][k]['count'] = count
place_names = place_api.get_display_name('^'.join(sorted(list(dcids))),
flask.g.locale)
for k in rank_keys:
if k in payload[stat_var] and 'info' in payload[stat_var][k]:
for r in payload[stat_var][k]['info']:
r['placeName'] = place_names[r['placeDcid']]
return flask.Response(json.dumps(ranking_results),
200,
mimetype='application/json')
def return_json(data): response = flask.Response(json.dumps(data)) response.headers['Access-Control-Allow-Origin'] = '*' return response
def decorated(*args, **kwargs):
auth = request.authorization
if not auth or not check_auth(auth.username, auth.password):
return flask.Response("Access denied", 401, {"WWW-Authenticate": "Basic realm=\"Login Required\""})
return f(*args, **kwargs)
def event_stream_handler():
# Create a new Observer
observer = m.create_observer()
# Start processing the new events and send them to the client
return flask.Response(observer.get_event(), mimetype="text/event-stream")
def handle_rest(path_args, flask_request):
"""Handle /REST/ route. Query the database using get_rows or create transaction using compose_transaction."""
url_action = flask_request.path.split('/')[-1]
if url_action == 'compose':
compose = True
elif url_action == 'get':
compose = False
else:
error = 'Invalid action "%s".' % url_action
return flask.Response(error, 400, mimetype='application/json')
# Get all arguments passed via URL.
url_args = path_args.split('/')
try:
query_type = url_args.pop(0).lower()
except IndexError:
error = 'No query_type provided.'
return flask.Response(error, 400, mimetype='application/json')
# Check if message type or table name are valid.
if (compose and query_type not in API_TRANSACTIONS) or \
(not compose and query_type not in API_TABLES):
error = 'No such query type in supported queries: "%s".' % query_type
return flask.Response(error, 400, mimetype='application/json')
# Parse the additional arguments.
extra_args = flask_request.args.items()
query_data = {}
if compose:
common_args = {}
transaction_args = {}
for (key, value) in extra_args:
# Determine value type.
try:
value = int(value)
except ValueError:
try:
value = float(value)
except ValueError:
pass
# Split keys into common and transaction-specific arguments. Discard the privkey.
if key in COMMONS_ARGS:
common_args[key] = value
elif key == 'privkey':
pass
else:
transaction_args[key] = value
# Must have some additional transaction arguments.
if not len(transaction_args):
error = 'No transaction arguments provided.'
return flask.Response(error, 400, mimetype='application/json')
# Compose the transaction.
try:
query_data = compose_transaction(db, name=query_type, params=transaction_args, **common_args)
except (script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error:
error_msg = logging.warning("{} -- error composing {} transaction via API: {}".format(
str(error.__class__.__name__), query_type, str(error)))
return flask.Response(error_msg, 400, mimetype='application/json')
else:
# Need to de-generate extra_args to pass it through.
query_args = dict([item for item in extra_args])
operator = query_args.pop('op', 'AND')
# Put the data into specific dictionary format.
data_filter = [{'field': key, 'op': '==', 'value': value} for (key, value) in query_args.items()]
# Run the query.
try:
query_data = get_rows(db, table=query_type, filters=data_filter, filterop=operator)
except APIError as error:
return flask.Response(str(error), 400, mimetype='application/json')
# See which encoding to choose from.
file_format = flask_request.headers['Accept']
# JSON as default.
if file_format == 'application/json' or file_format == '*/*':
response_data = json.dumps(query_data)
elif file_format == 'application/xml':
# Add document root for XML. Note when xmltodict encounters a list, it produces separate tags for every item.
# Hence we end up with multiple query_type roots. To combat this we put it in a separate item dict.
response_data = serialize_to_xml({query_type: {'item': query_data}})
else:
error = 'Invalid file format: "%s".' % file_format
return flask.Response(error, 400, mimetype='application/json')
response = flask.Response(response_data, 200, mimetype=file_format)
return response
def all(path):
requestStartTime = datetime.datetime.now()
#print("here")
logger.info("http request "+str(flask.request.method) +"'"+str(path)+"'" +" DATA="+str(flask.request.data))
data = None
response="{}" #default is empty
responseCode = 404 # default is not found
try:
try:
#parse the incoming data
data = flask.request.data.decode('utf-8') # just the string
data = json.loads(flask.request.data.decode('utf-8'))
except:
pass
#if the path has ending /, we remove it
if path[-1]=='/':
path = path[:-1]
# server all the frontend stuff: js, css etc
if any(extension in str(path) for extension in ['.js','.css','.htm','.img','.ico','.png','.gif','.map','.svg','.wof','.ttf']):
logger.debug(" serve html "+ str(path))
if "styles.css" in path:
print("hier")
for sourceFolder in ["web","bokeh_web","bokeh_web/templates"]:
try:
obj = flask.send_from_directory(sourceFolder,path)
return obj
except Exception as exc:
logger.error("file "+ str(path)+" not found on folder"+sourceFolder)
code = 404
elif (str(path) == "_getall") and str(flask.request.method) in ["POST","GET"]:
logger.debug("execute getall")
mymodel = m.get_model_for_web()
response = json.dumps(mymodel,indent=4)# some pretty printing for debug
responseCode = 200
elif (str(path) == "_getallhashed") and str(flask.request.method) in ["POST","GET"]:
logger.debug("execute getall hashed")
mymodel = m.get_model_for_web(getHash=True)
response = json.dumps(mymodel,indent=4)# some pretty printing for debug
responseCode = 200
elif (str(path) == "_getbranch") and str(flask.request.method) in ["POST","GET"]:
logger.debug("get branch")
mymodel = m.get_branch(data)
response = json.dumps(mymodel, indent=4) # some pretty printing for debug
responseCode = 200
elif (str(path) == "_getnodes") and str(flask.request.method) in ["POST","GET"]:
logger.debug(f"getnodes")
includeLong = data["includeLongValues"]
key = data["resultKey"]
response = {}
for node in data["nodes"]:
dic=m.get_node_info(node,excludeValue=True)
if key == "browsePath":
response[m.get_browse_path(node)]=dic
response = json.dumps(response, indent=4) # some pretty printing for debug
responseCode = 200
elif (str(path) == "_getbranchpretty") and str(flask.request.method) in ["POST","GET"]:
logger.debug(f"get branch pretty {data}")
if type(data) is str:
mymodel = m.get_branch_pretty(data)
elif type(data) is dict:
ignore = []
if "ignore" in data:
ignore = data["ignore"]
mymodel= m.get_branch_pretty(data["node"],data["depth"],ignore)
response = json.dumps(mymodel, indent=4) # some pretty printing for debug
responseCode = 200
elif (str(path) == "pipelines") and str(flask.request.method) in ["GET"]:
logger.debug("execute get pipelines")
try:
pipelinesNodeIds= m.get_node('root.visualization.pipelines').get_children()
pipelines = {}
for pipeNode in pipelinesNodeIds:
pipelines[pipeNode.get_name()]= {"url":pipeNode.get_child("url").get_property("value")}
response = json.dumps(pipelines,indent=4)# some pretty printing for debug
responseCode = 200
except:
logger.error("I have no pipelines")
responseCode = 404
elif (str(path) == "modelinfo") and str(flask.request.method) in ["GET"]:
logger.debug("get modelinfo")
response = json.dumps(m.get_info(), indent=4)
responseCode = 200
elif (str(path) == "templates") and str(flask.request.method) in ["GET"]:
logger.debug(" get templates")
templates = list(m.get_templates().keys())
logger.debug(" templates are "+str(templates))
response = json.dumps(templates)
responseCode = 200
elif (str(path) == "models") and str(flask.request.method) in ["GET"]:
logger.debug(" get templates")
models = m.get_models()
response = json.dumps(models)
responseCode = 200
elif(str(path) == "_getleaves") and str(flask.request.method) in ["POST", "GET"]:
logger.debug("execute get forward")
nodes = m.get_leaves(data)
#additionally, we provide browsepaths for the forward ids of referencers
for node in nodes:
if "forwardRefs" in node:
node["forwardPaths"]=[m.get_browse_path(id) for id in node["forwardRefs"]]
if "children" in node:
for child in node["children"]:
child["forwardPaths"] = [m.get_browse_path(id) for id in child["forwardRefs"]]
response = json.dumps(nodes, indent=4)
responseCode = 200
elif(str(path) == "_delete") and str(flask.request.method) in ["POST"]:
logger.debug("delete nodes")
result = []
responseCode = 200
for nodePath in data:
delResult = m.delete_node(nodePath)
logger.debug("deleted " +nodePath + " result: "+str(delResult))
result.append(delResult)
if not delResult:
responseCode = 401
response = json.dumps(result, indent=4)
elif (str(path) == "_get") and str(flask.request.method) in ["POST", "GET"]:
logger.debug("execute get"+str(data))
nodes = []
for nodeDesc in data:
resNodes = m.get_node_with_children(nodeDesc)
print(resNodes)
nodes.append(resNodes)
response = json.dumps(nodes, indent=4) # some pretty printing for debug
logger.debug("sending"+str(len(response)))
responseCode = 200
elif (str(path) == "_getvalue") and str(flask.request.method) in ["POST", "GET"]:
logger.debug("execute getvalue")
values = []
try:
for nodeDesc in data:
value = m.get_value(nodeDesc)
if type(value) is numpy.ndarray:
value = list(value)
values.append(value) # a list of lists
response = json.dumps(values, indent=4) # some pretty printing for debug
except Exception as ex:
logger.error("problem get value")
logger.debug("sending "+str(len(response)))
responseCode = 200
elif (str(path) == "_load") and str(flask.request.method) in ["POST"]:
logger.debug("load model:" + data)
result = m.load(data)
if result:
responseCode = 200
else:
responseCode = 404
elif (str(path) == "_save") and str(flask.request.method) in ["POST"]:
logger.debug("save to model:" + data)
result = m.save(data)
if result:
responseCode = 200
else:
responseCode = 404
elif (str(path)=="_appendRow"):
logger.debug("writeRow")
for blob in data:
result = m.append_table(blob)
if not result:
responseCode = 400
break
responseCode = 200
#m.show()
elif (str(path)=="_getdata"):
logger.debug("get data")
startTime = None
endTime = None
if set(["bins","nodes"]).issubset(set(data.keys())):
#we have both bins and nodes
startTime = 0
endTime = 0
if "startTime" in data:
#try to parse it, we support iso format or epoch
if type(data["startTime"]) is str:
try:
startTime = dateutil.parser.parse(data["startTime"])
except:
logger.info("cant parse start time")
else:
startTime = data["startTime"]
if "endTime" in data:
#try to parse it, we support iso format or epoch
if type(data["endTime"]) is str:
try:
startTime = dateutil.parser.parse(data["endTime"])
except:
logger.info("cant parse end time")
else:
endTime = data["endTime"]
if "includeTimeStamps" in data:
includeTimeStamps = data["includeTimeStamps"]
else:
includeTimeStamps= None
if "includeBackGround" in data:
includeBackGround = data["includeBackGround"]
else:
includeBackGround = None
try:
result = m.get_timeseries_table(data["nodes"],startTime=startTime,endTime=endTime,noBins=int(data["bins"]),includeTimeStamps=includeTimeStamps,format="dict",includeBackGround=includeBackGround)
if type(result) != type(None):
if includeTimeStamps:
pass #XXX todo: include the timestamps converted to a certain format
#data["nodes"].append("time")
response = json.dumps(result,indent = 4)
responseCode = 200
else:
responseData = 400
except Exception as ex:
logger.error("get time series tables failed"+str(ex)+str(sys.exc_info()))
responseCode = 404
else:
responseCode = 400 # malformed
elif (str(path) == "_create"):
logger.debug("create ")
result = []
responseCode = 201
for blob in data:
#check if new or modification
id = m.get_id(blob["browsePath"])
if id:
#this is a modification
adjust = m.set_properties(blob)
result.append(id)
else:
targets = []
if "targets" in blob:
targets = blob["targets"]
del blob["targets"]
newNodeId = m.create_node_from_path(blob["browsePath"],blob)
if targets:
m.add_forward_refs(newNodeId,targets)
logger.debug('creating'+blob["browsePath"]+', result:'+str(newNodeId))
result.append(newNodeId)
if not newNodeId:
responseCode = 400
response = json.dumps(result)
responseCode = 201
elif (str(path) == "_push") and str(flask.request.method) in ["POST"]:
m.push_nodes(data)
responseCode = 201
elif (str(path) == "_createTemplate") and str(flask.request.method) in ["POST"]:
logger.debug("craete Template ")
templates = m.get_templates()
if data["type"] in templates:
m.create_template_from_path(data["browsePath"],templates[data["type"]])
responseCode = 201
else:
responseCode = 404
elif (str(path) == "setProperties"):
logger.debug("set properties ")
responseCode = 201
result = True
for blob in data:
result = result and m.set_properties(blob)
if not result:
responseCode = 400
elif (str(path) == "_references"):
logger.debug("set new references")
result = []
m.lock_model()
m.disable_observers() #avoid intermediate events
if "deleteExisting" in data and data["deleteExisting"] == True:
m.remove_forward_refs(data["parent"])
if "add" in data:
result = m.add_forward_refs(data["parent"],data["add"])
if "remove" in data:
result = m.remove_forward_refs(data["parent"], data["remove"],deleteDuplicates=True)
m.enable_observers()
m.release_model()
m.notify_observers([m.get_id(data["parent"])],"forwardRefs")
responseCode = 201
elif (str(path) == "_execute"):
logger.debug(f"execute function {data} (start the function thread)")
result =[]
launch = m.execute_function(data)
if launch==True:
responseCode = 200
elif launch == "busy":
responseCode = 429
else:
responseCode = 404
elif (str(path) == "_diffUpdate") and str(flask.request.method) in ["GET","POST"]:
if not data or data["handle"] is None:
#this is for creating a handle
curModel = m.get_model_for_web()
handle = m.create_differential_handle()
logger.debug("create new differential update, return handle:"+handle)
res = {"handle":handle,"model":curModel}
response = json.dumps(res)
responseCode = 200
else:
#we have a handle
logger.debug("get differential update with handle:"+data["handle"])
res = m.get_differential_update(data["handle"])
if res:
logger.debug("return diff update from :"+data["handle"]+"=>" + res["handle"]+"data"+json.dumps(res))
response = json.dumps(res)
responseCode = 200
else:
logger.error("requested handle does not exist"+data["handle"])
response ="requested handle does not exist"
responseCode = 404
elif (str(path)=='embedbokeh'):
# here, we must have the correct html file for rendering including css and html coloring
# as the index.html is not relevant anymore
embed = """
<!doctype html>
<body>
<link rel="stylesheet" href="templates/styles.css">
<script>{% include 'moment.min.js.js' %}</script>
<script>{% include 'moment-timezone-with-data.min.js' %} </script>
{{ script|safe }}
</body>
</html>
"""
app_url = data['url']#'http://localhost:5006/bokeh_web'
try:
if 1==0:
#this is the old style pulling the session, both work only for localhost
with pull_session(url=app_url) as session:
# customize session here
#script = server_session(session_id='12345', url=app_url)
logger.debug("bokeh pull_session done")
script= server_session(session_id=session.id, url=app_url)
logger.debug("bokeh server_session() done")
#script = server_document('http://localhost:5006/bokeh_web')
temp = render_template_string(embed, script=script)
response = temp
responseCode = 200
#return temp
else:
script = server_session(session_id=str(uuid.uuid4().hex), url=app_url)
temp = render_template_string(embed, script=script)
response = temp
responseCode = 200
except Exception as ex:
logger.error("pulling session failed"+str(ex)+str(sys.exc_info()[0]))
responseCode = 404
elif (str(path)=="dropnodes") and str(flask.request.method) in ["POST"]:
logger.debug("dropnodes")
try:
nodeIds = data["nodes"]
#check which type of widget we have to work on
if m.get_node(data["path"]).get_child("widgetType").get_value() == "timeSeriesWidget":
selectNode = m.get_node(data["path"]).get_child("selectedVariables") # pick the referencer from the widget
#also check that the dropped nodes are columns
if type(data["nodes"]) is not list:
data["nodes"]=[data["nodes"]]
newNodesIds = []
for nodeid in data["nodes"]:
if m.get_node_info(nodeid)["type"] != "column":
logger.warning("we ignore this node, is not a colum"+str(nodeid))
else:
newNodesIds.append(m.get_id(nodeid))
#the nodes are columns
logger.debug("now adding new nodes to the widget"+str(newNodesIds))
#first check, if we have this one already, if so, we don't add it again
leaves=selectNode.get_leaves_ids()
newRefs = [id for id in newNodesIds if id not in leaves]
m.add_forward_refs(selectNode.get_id(),newRefs)
responseCode = 201
except Exception as ex:
logger.error("can't add the variables to the widget"+str(ex)+str(sys.exc_info()[0]))
responseCode = 404
elif (str(path)=="move") and str(flask.request.method) in ["POST"]:
logger.debug("move")
try:
moveResult = m.move(data["nodes"],data["parent"])
responseCode = 200
except Exception as ex:
logger.error("can't move"+str(ex)+str(sys.exc_info()[0]))
responseCode = 404
elif (str(path)=="_getlayout") and str(flask.request.method) in ["POST","GET"]:
logger.debug("get layout")
try:
fileName = m.get_value(data['layoutNode'])
with open("web/customui/"+fileName) as fp:
soup = BeautifulSoup(fp, "lxml")
divs = soup.find_all(
id=re.compile("^ui-component")) # find all divs that have a string starting with "ui-component"
for div in divs:
uiinfo = json.loads(div.attrs['uiinfo'])
try:
#now find the widget by looking into the model
widgetType = m.get_node(uiinfo['component']+'.model').get_leaves()[0].get_child("widgetType").get_value()
if widgetType == "timeSeriesWidget":
#this is a bokeh time series widget, so we must do some things here
port = m.get_node(uiinfo['component']+'.settings').get_value()["port"]
#now get the host under which the client sees me:
host = flask.request.host.split(':')[0]
#script = server_session(session_id=str(uuid.uuid4().hex), url="http://localhost:"+str(port)+"/bokeh_web")
script = server_document(url="http://"+host+":"+str(port)+"/bokeh_web", relative_urls=False, resources='default',
arguments=None)
#script = script.encode('utf-8') # due to problem with "&"
scriptHtml = BeautifulSoup(script, "lxml")
scriptTag = scriptHtml.find("script")
scriptSource = scriptTag["src"]
scriptId = scriptTag["id"]
scriptinsert = scriptTag.prettify(formatter=None)
scriptTag = soup.new_tag("script",src=scriptSource,id=scriptId)
uiinfo["droppath"]=m.get_node(uiinfo['component']+'.model').get_leaves()[0].get_browse_path()
div.append(scriptTag)
div.attrs['uiinfo']=json.dumps(uiinfo)
else:
logger.error("unsupported widget type"+str(widgetType))
print(uiinfo)
except Exception as ex:
logger.error("can't get layout component" + str(uiinfo['component']))
response = str(soup)
response=response.replace('&','&') # hack: can't get beautifulsoup to suppress the conversion
logger.debug(response)
responseCode = 200
except Exception as ex:
logger.error("can't get layout" + str(ex) + str(sys.exc_info()[0]))
responseCode = 404
elif (str(path)=="_setlen") and str(flask.request.method) in ["POST"]:
logger.debug("setlen of column")
result = {}
for descriptor,newLen in data.items():
result[descriptor] = m.set_column_len(descriptor,newLen)
responseCode = 200
response = json.dumps(result)
elif ((str(path)=="_upload") and str(flask.request.method) in ["GET"]):
# Return the files from the upload folder
try:
# get the files from the upload folder
filenames = []
#listdir(mypath)
#for r, d, f in os.walk(UPLOAD_FOLDER):
for filename in os.listdir(UPLOAD_FOLDER):
fullPath=os.path.join(UPLOAD_FOLDER, filename)
if not os.path.isfile(fullPath):
continue
#for filename in f:
filenames.append({
"name": filename,
"time": os.path.getmtime(fullPath)
})
responseCode = 200
response = json.dumps(filenames)
except Exception as ex:
logger.error("can't retrieve the files from the upload folder: " + str(ex) + str(sys.exc_info()[0]))
responseCode = 404
elif ((str(path) == "_upload") and str(flask.request.method) in ["POST"]):
# Upload a file to the server
try:
# check if the post request has the file part
if 'file' not in request.files:
responseCode = 404
logger.error("File part missing in upload request")
else:
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '':
responseCode = 404
logger.error("Filename not specified")
else:
filename = secure_filename(file.filename)
file.save(os.path.join(web.config['UPLOAD_FOLDER'], filename))
responseCode = 200
response = "success"
except Exception as ex:
logger.error("Can't save the file to the upload folder: " + str(ex) + str(sys.exc_info()[0]))
responseCode = 404
else:
logger.error("CANNOT HANDLE REQUEST, is unknown"+str(path))
responseCode = 404
timeElapsed = (datetime.datetime.now() - requestStartTime).total_seconds()
logger.info("response on " + str(path) + ": " + str(responseCode) + ", len:" + str( len(response)) + " duration:"+str(timeElapsed))
return flask.Response(response, mimetype="text/html"), responseCode
except Exception as ex:
logger.error("general error " +str(sys.exc_info()[0])+".."+str(ex))
return flask.Response("",mimetype="text/html"),501
def twiml(resp):
resp = flask.Response(str(resp))
resp.headers['content-type'] = 'text/xml'
return resp
def get(self):
resp = flask.Response('Shaken Fist REST API service',
mimetype='text/plain')
resp.status_code = 200
return resp
