def post(self, log_id):
log = Log.query.get_or_raise(log_id)
data = request.get_json()
file = data.get('file')
if log.mode in (Mode.REPO_MIRROR, Mode.REPO_ROOT):
file = file.format(
LOG_REPO=os.path.join(current_app.dm.config.config_dir, defaults.LOG_SENDER_REPO,
clean_string(log.source_server.name)))
if data.get('compress', False):
data_log = zlib.decompress(base64.b64decode(data.get('data').encode('ascii')))
else:
data_log = base64.b64decode(data.get('data').encode('ascii'))
if not os.path.exists(os.path.dirname(file)):
try:
os.makedirs(os.path.dirname(file))
except PermissionError:
raise errors.GenericError(f"Permission denied creating '{os.path.dirname(file)}'", 500)
try:
with open(file, 'ab') as fh:
fh.write(data_log)
except Exception as e:
raise errors.GenericError(f"{e}", 500)
return {'offset': os.path.getsize(file)}
def set_software_server(soft, server, path, recalculate_data=False):
file = os.path.join(path, soft.filename)
if not os.path.exists(file):
raise errors.FileNotFound(file)
if soft.size != os.path.getsize(file):
return errors.GenericError(f"file is not of specified size",
file=file,
size=soft.size)
if soft.checksum != md5(file):
return errors.GenericError(f"checksum error on file", file=file)
return SoftwareServerAssociation(software=soft, server=server, path=path)
def file_sync(file_id):
if get_jwt_identity() == '00000000-0000-0000-0000-000000000001':
data = request.get_json()
file = File.query.get(file_id)
if file is None and not data.get('force', False):
raise errors.EntityNotFound("File", file_id)
file = data.get('file')
content = zlib.decompress(base64.b64decode(data.get('data').encode('ascii')))
_logger.debug(f"received file sync {file}.")
try:
if not os.path.exists(os.path.dirname(file)):
os.makedirs(os.path.dirname(file))
with open(file, 'wb') as fh:
fh.write(content)
except Exception as e:
raise errors.GenericError(f"Error while trying to create/write file: {e}", 500)
return {}, 204
else:
raise errors.UserForbiddenError
def launch_orchestration(orchestration_id):
data = request.get_json()
if orchestration_id:
orchestration = Orchestration.query.get_or_raise(orchestration_id)
else:
iden = (data.get('orchestration'), )
columns = ('orchestration', )
query = Orchestration.query.filter_by(name=data.get('orchestration'))
if 'version' in data:
iden += (data.get('version'), )
columns += ('version', )
query = query.filter_by(version=data.get('version'))
query = query.order_by(Orchestration.version.desc())
if query.count() <= 1:
orchestration = query.one_or_none()
else:
orchestration = query.first()
if not orchestration:
raise errors.EntityNotFound('Orchestration', iden, columns)
if not orchestration.steps:
return errors.GenericError(
'orchestration does not have steps to execute',
orchestration_id=orchestration_id)
params = data.get('params') or {}
hosts = data.get('hosts', Server.get_current().id)
a = set(orchestration.target)
if not isinstance(hosts, dict):
hosts = dict(all=hosts)
b = set(hosts.keys())
c = a - b
if len(c) > 0:
raise errors.TargetUnspecified(c)
c = b - a
if len(c) > 0:
raise errors.TargetNotNeeded(c)
not_found = normalize_hosts(hosts)
if not_found:
raise errors.ServerNormalizationError(not_found)
for target, target_hosts in hosts.items():
if len(target_hosts) == 0:
raise errors.EmptyTarget(target)
# check param entries
# rest = orchestration.user_parameters - set(params.keys())
# if rest:
# rest = list(rest)
# rest.sort()
# return {'error': f"Parameter(s) not specified: {', '.join(rest)}"}, 404
execution_id = str(uuid.uuid4())
executor_id = get_jwt_identity()
vc = Context(params,
dict(execution_id=None,
root_orch_execution_id=execution_id,
orch_execution_id=execution_id,
executor_id=executor_id),
vault=Vault.get_variables_from(executor_id,
scope=data.get(
'scope', 'global')))
if not data.get('skip_validation', False):
validate_input_chain(
orchestration, {
'input': set(params.keys()),
'env': set(vc.env.keys()),
'vault': set(vc.vault.keys())
})
if request.get_json().get('background', True):
future = executor.submit(deploy_orchestration,
orchestration=orchestration.id,
var_context=vc,
hosts=hosts,
timeout=data.get('timeout', None))
try:
future.result(5)
except concurrent.futures.TimeoutError:
return {'execution_id': execution_id}, 202
except Exception as e:
current_app.logger.exception(
f"Exception got when executing orchestration {orchestration}")
raise
else:
try:
deploy_orchestration(orchestration=orchestration,
var_context=vc,
hosts=hosts,
timeout=data.get('timeout', None))
except Exception as e:
current_app.logger.exception(
f"Exception got when executing orchestration {orchestration}")
raise
return OrchExecution.query.get(execution_id).to_json(
add_step_exec=True,
human=check_param_in_uri('human'),
split_lines=True), 200
def join():
global fetched_catalog
if get_jwt_identity() == '00000000-0000-0000-0000-000000000004':
js = request.get_json()
current_app.logger.debug(
f"New server wanting to join: {json.dumps(js, indent=2)}")
if db.session.query(Server).filter_by(
id=js.get('id', None)).count() > 0:
raise errors.DuplicatedId(js.get('id', None))
if db.session.query(Server).filter_by(
name=js.get('name', None)).count() > 0:
raise errors.AlreadyExists('name', js.get('name', None))
s = Server.from_json(js)
s.created_on = get_now()
external_ip = ipaddress.ip_address(request.remote_addr)
if not external_ip.is_loopback and external_ip not in [
gate.ip for gate in s.gates
]:
for port in set([gate.port for gate in s.gates]):
s.add_new_gate(external_ip, port, hidden=True)
certfile = current_app.dm.config.http_conf.get('certfile', None)
keyfile = current_app.dm.config.http_conf.get('keyfile', None)
if keyfile and os.path.exists(keyfile):
with open(keyfile, 'rb') as fh:
keyfile_content = fh.read()
else:
raise errors.FileNotFound(keyfile)
if certfile and os.path.exists(certfile):
with open(certfile, 'rb') as fh:
certfile_content = fh.read()
else:
raise errors.FileNotFound(certfile)
data = {
'keyfile': base64.b64encode(keyfile_content).decode(),
'certfile': base64.b64encode(certfile_content).decode()
}
data.update(Dimension=g.dimension.to_json())
data.update(me=str(Server.get_current().id))
with _lock:
if fetched_catalog[1] is None or fetched_catalog[0] < get_now(
) - dt.timedelta(minutes=1):
c = fetch_catalog()
fetched_catalog = (get_now(), c)
else:
c = fetched_catalog[1]
data.update(catalog=c)
if _lock_delete.acquire(False):
try:
delete_old_temp_servers()
finally:
_lock_delete.release()
server_data = s.to_json(add_gates=True)
servers_to_be_created.update({s.id: server_data})
del s
return data, 200
else:
raise errors.GenericError('Invalid token', status_code=400)
def raise_error():
raise errors.GenericError('error content', some='payload')
def change_destinations(file: File, destinations: t.List, action: str = None):
"""
:param file: file to change its destinations
:param destinations:
:param action:
:return:
"""
current = set([d.dst_server_id for d in file.destinations])
new = set([d.get('dst_server_id') for d in destinations])
new_fsas = []
if action == 'add':
to_remove = []
to_modify = []
already_there = current.intersection(new)
if already_there:
raise errors.InvalidValue("destination servers already exist",
destinations=[{
'id':
ident,
'name':
Server.query.get(ident).name
} for ident in already_there])
to_add = new
elif action == 'delete':
to_remove = new
not_there = new - current
if not_there:
raise errors.InvalidValue("destination servers do not exist",
destinations=[{
'id':
ident,
'name':
Server.query.get(ident).name
} for ident in not_there])
to_modify = []
to_add = []
else:
to_remove = current - new
to_modify = new.intersection(current)
to_add = new - current
for dst_server_id in to_add:
s = Server.query.get_or_raise(dst_server_id)
if s._me:
raise errors.InvalidValue(
"Destination cannot be the same as the source server",
source_server={
'id': s.id,
'name': s.name
})
dest = [
d for d in destinations if d['dst_server_id'] == dst_server_id
][0]
fsa = FileServerAssociation(file=file,
destination_server=s,
dest_folder=dest.get('dest_folder', None))
db.session.add(fsa)
new_fsas.append(fsa)
for dst_server_id in to_modify:
s = Server.query.get_or_raise(dst_server_id)
curr_dest = [
d for d in file.destinations if d.dst_server_id == dst_server_id
][0]
in_dest = [
d for d in destinations if d['dst_server_id'] == dst_server_id
][0]
if curr_dest.dest_folder != in_dest.get('dest_folder', None):
curr_dest.dest_folder = in_dest.get('dest_folder', None)
for dst_server_id in to_remove:
s = Server.query.get_or_raise(dst_server_id)
dest = [
d for d in file.destinations if d.dst_server_id == dst_server_id
]
if dest:
db.session.delete(dest[0])
else:
raise errors.GenericError("File does not have this destination",
status_code=404,
file_id=file.id,
destination={
'id': dst_server_id,
'name': s.name
})
return new_fsas