def update_duck_dns(domain, public_ip, opts): token = _get(opts, 'dns.token', _get(opts, 'dns.key', _get(opts, 'duckdns.token', None))) if token is None: return False if not domain.endswith('.duckdns.org'): return False domain = domain.split('.')[-3] try: with requests.get('https://www.duckdns.org/update', dict(domains=domain, token=token, ip=public_ip)) as req: if req and req.text == "OK": return True if req.text == 'KO': raise JhubNginxError( 'Duck DNS refused to update -- {} token:{}'.format( domain, token)) else: raise JhubNginxError('Failed to contact duck DNS') except IOError as e: raise JhubNginxError('Failed to update duck DNS') return False
def nginx_reload(opts): debug('Reloading nginx config') try: subprocess.check_call(_get(opts, 'nginx.check_cmd'), shell=True) subprocess.check_call(_get(opts, 'nginx.reload_cmd'), shell=True) except FileNotFoundError as e: raise JhubNginxError( 'Failed to reload nginx config, bad command: {}'.format(str(e))) except subprocess.CalledProcessError as e: raise JhubNginxError('Failed to reload nginx config: {}'.format( str(e)))
def optimize_single_batch(): """ optimize_single_batch :return: """ print('/singlebatch') if not SERVICE.ready(): return Response('waiting for warm-up', status=503) # Read request content # Source: https://cloud.google.com/tasks/docs/creating-appengine-handlers req = json.loads(request.get_data(as_text=True)) # warehouse_uuid = str(_get(req, '_meta.warehouse.uuid')) warehouse_tag = str(_get(req, '_meta.warehouse.tag')) if warehouse_tag.lower() == 'demo': warehouse_tag = 'DADC' # Get task ID task_id = _get(req, '_meta.taskId') if task_id is None: print('Error: Task ID is invalid: ' + str(task_id)) return json.dumps({'error': 'Task ID is invalid: ' + str(task_id)}) if not TASKS.has_task(task_id): print('Error: No task with task ID: ' + task_id) return json.dumps({'error': 'No task with task ID: ' + task_id}) try: # Read payload and update tasks table data = TASKS.get_payload(task_id) TASKS.set_status(task_id, 'running') handler = Handler(data, SERVICE.wh_dict(), warehouse_tag) # INSERT SOLVER HERE e.g. SingleBatchOptimizer(handler.solve_dict) # client_response, _, bq_optim = handler.gen_response(handler.solve_dict, warehouse_uuid) client_response, _, bq_optim = handler.gen_response(handler.solve_dict) BQ.batching(bq_optim) # Update tasks table TASKS.set_result(task_id, client_response) TASKS.set_status(task_id, 'done') # Return results return client_response except Exception as exc: # If optimization failed, put error message in the results field and set status to failed TASKS.set_result(task_id, str(exc)) TASKS.set_status(task_id, 'failed') # Return empty string print('Error: ' + str(exc)) return json.dumps({'error': str(exc)})
def update_dns(domain, public_ip, opts): if domain.endswith('.duckdns.org'): return update_duck_dns(domain, public_ip, opts) if libcloud is not None and _get(opts, 'dns.type') is not None: return update_dns_libcloud(domain, public_ip, opts) return False
def run_certbot(num_tries): debug('Running certbot for {}'.format(domain)) if standalone: cmd = ('certbot certonly' ' --standalone' ' --text --agree-tos --no-eff-email' ' --email {email}' ' --domains {domain}').format(email=email, domain=domain).split() else: webroot = Path(_get(opts, 'letsencrypt.webroot')) if not webroot.exists(): debug('Creating webroot directory: {}'.format(webroot)) webroot.mkdir(parents=True) cmd = ('certbot certonly' ' --webroot -w {webroot}' ' --text --agree-tos --no-eff-email' ' --email {email}' ' --domains {domain}').format(email=email, webroot=webroot, domain=domain).split() while num_tries > 0: num_tries -= 1 try: out = subprocess.check_output(cmd).decode('utf-8') debug(out) return True except FileNotFoundError as e: raise JhubNginxError('certbot is not installed') except subprocess.CalledProcessError as e: if num_tries > 0: debug('Will re-try in one minute') time.sleep(60) raise JhubNginxError('certbot reported an error') return False
def optimize_pick_route(): """ Pickroute request is opened, passed on to the handler that prepares a dict (called solve_dict) that is then passed on to the optimizer. After optimization a client_response is returned and BigQuery jsons are sent to worker. :return: client_response: the response the client (WMS) gets. """ print('/pickroute') if not SERVICE.ready(): return Response('waiting for warm-up', status=503) # Read request content req = json.loads(request.get_data(as_text=True)) warehouse_tag = str(_get(req, '_meta.warehouse.tag')) if warehouse_tag.lower() == 'demo': warehouse_tag = 'DADC' # Perform optimization handler = Handler(req, SERVICE.wh_dict(), warehouse_tag) if len(handler.solve_dict['coords_path_bef_sol']) <= 6: TrivialInstanceSolver(handler.solve_dict) # brute force search (but still very fast since there are very few nodes in this case) else: pass # INSERT SOLVER with input = handler.solve_dict client_response, bq_hist, bq_optim = handler.gen_response( handler.solve_dict) # HENCE this returns all finished jsons as in Legacy. As much as this as possible of this # functionality should be moved to api # Send data to BigQuery BQ.pro(bq_hist) BQ.pro(bq_optim) # Return results return client_response
def add_or_check_vhost(domain, hub_ip='127.0.0.1', hub_port='8000', skip_dns_check=False, standalone=False, dns_wait_timeout=5 * 60, min_dns_wait=60, opts=None): opts = utils.default_opts(opts) vhost_cfg_file = domain_config_path(domain, opts) public_ip = None if skip_dns_check else utils.public_ip() email = _get(opts, 'letsencrypt.email', None) def run_certbot(num_tries): debug('Running certbot for {}'.format(domain)) if standalone: cmd = ('certbot certonly' ' --standalone' ' --text --agree-tos --no-eff-email' ' --email {email}' ' --domains {domain}').format(email=email, domain=domain).split() else: webroot = Path(_get(opts, 'letsencrypt.webroot')) if not webroot.exists(): debug('Creating webroot directory: {}'.format(webroot)) webroot.mkdir(parents=True) cmd = ('certbot certonly' ' --webroot -w {webroot}' ' --text --agree-tos --no-eff-email' ' --email {email}' ' --domains {domain}').format(email=email, webroot=webroot, domain=domain).split() while num_tries > 0: num_tries -= 1 try: out = subprocess.check_output(cmd).decode('utf-8') debug(out) return True except FileNotFoundError as e: raise JhubNginxError('certbot is not installed') except subprocess.CalledProcessError as e: if num_tries > 0: debug('Will re-try in one minute') time.sleep(60) raise JhubNginxError('certbot reported an error') return False def gen_config(**kwargs): txt = render_vhost(domain, opts, hub_port=hub_port, hub_ip=hub_ip, **kwargs) if not vhost_cfg_file.parent.exists(): debug('Missing folder: {}, creating'.format(vhost_cfg_file.parent)) vhost_cfg_file.parent.mkdir(parents=True) return utils.write_if_different(str(vhost_cfg_file), txt) def attempt_cleanup(): debug('Cleaning up {}'.format(vhost_cfg_file)) try: os.remove(str(vhost_cfg_file)) nginx_reload(opts) except JhubNginxError as e: debug('Ooops failure within a failure: {}'.format(str(e))) except OSError as e: debug('Ooops failure within a failure: {}'.format(str(e))) def have_ssl_files(): ssl_root = Path(_get(opts, 'nginx.ssl_root')) / domain privkey = ssl_root / "privkey.pem" fullchain = ssl_root / "fullchain.pem" return privkey.exists() and fullchain.exists() def obtain_ssl(): if email is None: raise JhubNginxError("Can't request SSL without an E-mail address") if standalone: return run_certbot(2) debug(' writing temp vhost config') gen_config(nossl=True) try: nginx_reload(opts) run_certbot(2) except JhubNginxError as e: attempt_cleanup() raise e def add_ssl_vhost(): updated = gen_config() if updated: debug('Updated vhost config {}'.format(vhost_cfg_file)) if standalone: return try: nginx_reload(opts) except JhubNginxError as e: attempt_cleanup() raise e else: debug('No changes were required {}'.format(vhost_cfg_file)) def on_dns_update(domain, ip): if min_dns_wait: debug('Waiting for {} seconds after updating DNS'.format( min_dns_wait)) time.sleep(min_dns_wait) def cbk(t): debug("Still waiting for DNS to update") if dns_wait(domain, ip, dns_wait_timeout, cbk=cbk, use_dig=True) is False: warn( 'Requested DNS record update, but failed to observe the change, will continue anyway' ) if vhost_cfg_file.exists(): if not skip_dns_check: try: check_dns(domain, public_ip, opts, message=debug) except JhubNginxError as e: warn( 'Virtual host config already exists but DNS check/update failed:\n {}' .format(str(e))) add_ssl_vhost() # Make sure content is up to date else: if not skip_dns_check: check_dns(domain, public_ip, opts, on_update=on_dns_update, message=debug) if have_ssl_files(): debug('Found SSL files, no need to run certbot') else: debug('Obtaining SSL for {}'.format(domain)) obtain_ssl() add_ssl_vhost() return True
def domain_config_path(domain, opts): return Path(_get(opts, 'nginx.sites')) / (domain + '.conf')
def ssl_cert_file(): ssl_root = Path(_get(opts, 'nginx.ssl_root')) / domain return ssl_root / "cert.pem"
def have_ssl_files(): ssl_root = Path(_get(opts, 'nginx.ssl_root')) / domain privkey = ssl_root / "privkey.pem" fullchain = ssl_root / "fullchain.pem" return privkey.exists() and fullchain.exists()