def update_duck_dns(domain, public_ip, opts):
token = _get(opts, 'dns.token',
_get(opts, 'dns.key', _get(opts, 'duckdns.token', None)))
if token is None:
return False
if not domain.endswith('.duckdns.org'):
return False
domain = domain.split('.')[-3]
try:
with requests.get('https://www.duckdns.org/update',
dict(domains=domain, token=token,
ip=public_ip)) as req:
if req and req.text == "OK":
return True
if req.text == 'KO':
raise JhubNginxError(
'Duck DNS refused to update -- {} token:{}'.format(
domain, token))
else:
raise JhubNginxError('Failed to contact duck DNS')
except IOError as e:
raise JhubNginxError('Failed to update duck DNS')
return False
def nginx_reload(opts):
debug('Reloading nginx config')
try:
subprocess.check_call(_get(opts, 'nginx.check_cmd'), shell=True)
subprocess.check_call(_get(opts, 'nginx.reload_cmd'), shell=True)
except FileNotFoundError as e:
raise JhubNginxError(
'Failed to reload nginx config, bad command: {}'.format(str(e)))
except subprocess.CalledProcessError as e:
raise JhubNginxError('Failed to reload nginx config: {}'.format(
str(e)))
def optimize_single_batch():
"""
optimize_single_batch
:return:
"""
print('/singlebatch')
if not SERVICE.ready():
return Response('waiting for warm-up', status=503)
# Read request content
# Source: https://cloud.google.com/tasks/docs/creating-appengine-handlers
req = json.loads(request.get_data(as_text=True))
# warehouse_uuid = str(_get(req, '_meta.warehouse.uuid'))
warehouse_tag = str(_get(req, '_meta.warehouse.tag'))
if warehouse_tag.lower() == 'demo':
warehouse_tag = 'DADC'
# Get task ID
task_id = _get(req, '_meta.taskId')
if task_id is None:
print('Error: Task ID is invalid: ' + str(task_id))
return json.dumps({'error': 'Task ID is invalid: ' + str(task_id)})
if not TASKS.has_task(task_id):
print('Error: No task with task ID: ' + task_id)
return json.dumps({'error': 'No task with task ID: ' + task_id})
try:
# Read payload and update tasks table
data = TASKS.get_payload(task_id)
TASKS.set_status(task_id, 'running')
handler = Handler(data, SERVICE.wh_dict(), warehouse_tag)
# INSERT SOLVER HERE e.g. SingleBatchOptimizer(handler.solve_dict)
# client_response, _, bq_optim = handler.gen_response(handler.solve_dict, warehouse_uuid)
client_response, _, bq_optim = handler.gen_response(handler.solve_dict)
BQ.batching(bq_optim)
# Update tasks table
TASKS.set_result(task_id, client_response)
TASKS.set_status(task_id, 'done')
# Return results
return client_response
except Exception as exc:
# If optimization failed, put error message in the results field and set status to failed
TASKS.set_result(task_id, str(exc))
TASKS.set_status(task_id, 'failed')
# Return empty string
print('Error: ' + str(exc))
return json.dumps({'error': str(exc)})
def update_dns(domain, public_ip, opts):
if domain.endswith('.duckdns.org'):
return update_duck_dns(domain, public_ip, opts)
if libcloud is not None and _get(opts, 'dns.type') is not None:
return update_dns_libcloud(domain, public_ip, opts)
return False
def run_certbot(num_tries):
debug('Running certbot for {}'.format(domain))
if standalone:
cmd = ('certbot certonly'
' --standalone'
' --text --agree-tos --no-eff-email'
' --email {email}'
' --domains {domain}').format(email=email,
domain=domain).split()
else:
webroot = Path(_get(opts, 'letsencrypt.webroot'))
if not webroot.exists():
debug('Creating webroot directory: {}'.format(webroot))
webroot.mkdir(parents=True)
cmd = ('certbot certonly'
' --webroot -w {webroot}'
' --text --agree-tos --no-eff-email'
' --email {email}'
' --domains {domain}').format(email=email,
webroot=webroot,
domain=domain).split()
while num_tries > 0:
num_tries -= 1
try:
out = subprocess.check_output(cmd).decode('utf-8')
debug(out)
return True
except FileNotFoundError as e:
raise JhubNginxError('certbot is not installed')
except subprocess.CalledProcessError as e:
if num_tries > 0:
debug('Will re-try in one minute')
time.sleep(60)
raise JhubNginxError('certbot reported an error')
return False
def optimize_pick_route():
"""
Pickroute request is opened, passed on to the handler that
prepares a dict (called solve_dict) that is then passed on to the
optimizer. After optimization a client_response is returned and
BigQuery jsons are sent to worker.
:return: client_response: the response the client (WMS) gets.
"""
print('/pickroute')
if not SERVICE.ready():
return Response('waiting for warm-up', status=503)
# Read request content
req = json.loads(request.get_data(as_text=True))
warehouse_tag = str(_get(req, '_meta.warehouse.tag'))
if warehouse_tag.lower() == 'demo':
warehouse_tag = 'DADC'
# Perform optimization
handler = Handler(req, SERVICE.wh_dict(), warehouse_tag)
if len(handler.solve_dict['coords_path_bef_sol']) <= 6:
TrivialInstanceSolver(handler.solve_dict)
# brute force search (but still very fast since there are very few nodes in this case)
else:
pass
# INSERT SOLVER with input = handler.solve_dict
client_response, bq_hist, bq_optim = handler.gen_response(
handler.solve_dict)
# HENCE this returns all finished jsons as in Legacy. As much as this as possible of this
# functionality should be moved to api
# Send data to BigQuery
BQ.pro(bq_hist)
BQ.pro(bq_optim)
# Return results
return client_response
def add_or_check_vhost(domain,
hub_ip='127.0.0.1',
hub_port='8000',
skip_dns_check=False,
standalone=False,
dns_wait_timeout=5 * 60,
min_dns_wait=60,
opts=None):
opts = utils.default_opts(opts)
vhost_cfg_file = domain_config_path(domain, opts)
public_ip = None if skip_dns_check else utils.public_ip()
email = _get(opts, 'letsencrypt.email', None)
def run_certbot(num_tries):
debug('Running certbot for {}'.format(domain))
if standalone:
cmd = ('certbot certonly'
' --standalone'
' --text --agree-tos --no-eff-email'
' --email {email}'
' --domains {domain}').format(email=email,
domain=domain).split()
else:
webroot = Path(_get(opts, 'letsencrypt.webroot'))
if not webroot.exists():
debug('Creating webroot directory: {}'.format(webroot))
webroot.mkdir(parents=True)
cmd = ('certbot certonly'
' --webroot -w {webroot}'
' --text --agree-tos --no-eff-email'
' --email {email}'
' --domains {domain}').format(email=email,
webroot=webroot,
domain=domain).split()
while num_tries > 0:
num_tries -= 1
try:
out = subprocess.check_output(cmd).decode('utf-8')
debug(out)
return True
except FileNotFoundError as e:
raise JhubNginxError('certbot is not installed')
except subprocess.CalledProcessError as e:
if num_tries > 0:
debug('Will re-try in one minute')
time.sleep(60)
raise JhubNginxError('certbot reported an error')
return False
def gen_config(**kwargs):
txt = render_vhost(domain,
opts,
hub_port=hub_port,
hub_ip=hub_ip,
**kwargs)
if not vhost_cfg_file.parent.exists():
debug('Missing folder: {}, creating'.format(vhost_cfg_file.parent))
vhost_cfg_file.parent.mkdir(parents=True)
return utils.write_if_different(str(vhost_cfg_file), txt)
def attempt_cleanup():
debug('Cleaning up {}'.format(vhost_cfg_file))
try:
os.remove(str(vhost_cfg_file))
nginx_reload(opts)
except JhubNginxError as e:
debug('Ooops failure within a failure: {}'.format(str(e)))
except OSError as e:
debug('Ooops failure within a failure: {}'.format(str(e)))
def have_ssl_files():
ssl_root = Path(_get(opts, 'nginx.ssl_root')) / domain
privkey = ssl_root / "privkey.pem"
fullchain = ssl_root / "fullchain.pem"
return privkey.exists() and fullchain.exists()
def obtain_ssl():
if email is None:
raise JhubNginxError("Can't request SSL without an E-mail address")
if standalone:
return run_certbot(2)
debug(' writing temp vhost config')
gen_config(nossl=True)
try:
nginx_reload(opts)
run_certbot(2)
except JhubNginxError as e:
attempt_cleanup()
raise e
def add_ssl_vhost():
updated = gen_config()
if updated:
debug('Updated vhost config {}'.format(vhost_cfg_file))
if standalone:
return
try:
nginx_reload(opts)
except JhubNginxError as e:
attempt_cleanup()
raise e
else:
debug('No changes were required {}'.format(vhost_cfg_file))
def on_dns_update(domain, ip):
if min_dns_wait:
debug('Waiting for {} seconds after updating DNS'.format(
min_dns_wait))
time.sleep(min_dns_wait)
def cbk(t):
debug("Still waiting for DNS to update")
if dns_wait(domain, ip, dns_wait_timeout, cbk=cbk,
use_dig=True) is False:
warn(
'Requested DNS record update, but failed to observe the change, will continue anyway'
)
if vhost_cfg_file.exists():
if not skip_dns_check:
try:
check_dns(domain, public_ip, opts, message=debug)
except JhubNginxError as e:
warn(
'Virtual host config already exists but DNS check/update failed:\n {}'
.format(str(e)))
add_ssl_vhost() # Make sure content is up to date
else:
if not skip_dns_check:
check_dns(domain,
public_ip,
opts,
on_update=on_dns_update,
message=debug)
if have_ssl_files():
debug('Found SSL files, no need to run certbot')
else:
debug('Obtaining SSL for {}'.format(domain))
obtain_ssl()
add_ssl_vhost()
return True
def domain_config_path(domain, opts):
return Path(_get(opts, 'nginx.sites')) / (domain + '.conf')
def ssl_cert_file():
ssl_root = Path(_get(opts, 'nginx.ssl_root')) / domain
return ssl_root / "cert.pem"
def have_ssl_files():
ssl_root = Path(_get(opts, 'nginx.ssl_root')) / domain
privkey = ssl_root / "privkey.pem"
fullchain = ssl_root / "fullchain.pem"
return privkey.exists() and fullchain.exists()
