def request(path, query=None, body=None, method='get', config=None):
query = query or {}
if isinstance(query, dict):
query = urlencode(query)
if '?' in path:
path += '&' + query
else:
path += '?' + query
config = read_home_config() if config is None else config
if method == 'get' and body is not None:
method = 'post'
ssh_stream = create_ssh_stream(config)
stdin, stdout, stderr = ssh_stream.exec_command('api ' + method + ' ' + simplejson.dumps(path))
if body is not None:
input = six.b(simplejson.dumps(body))
stdin.write(input)
stdin.flush()
stdin.channel.shutdown_write()
stdout = drain_stream(stdout)
stderr = drain_stream(stderr)
if len(stderr) > 0:
if hasattr(stderr, 'decode'):
stderr = stderr.decode('utf-8')
raise ApiError('Could not request api: ' + config['host'] + path, stderr)
return stdout
def connect(self, channel):
"""
In the write-thread we detect that no connection is living anymore and try always again.
Up to the 3 connection try, we report to user. We keep trying but in silence.
Also, when more than 10 connection tries are detected, we delay extra 15 seconds.
"""
if self.connection_tries > 10:
time.sleep(10)
if self.in_connecting[channel]:
return False
self.in_connecting[channel] = True
self.logger.debug('[%s] Wanna connect ...' % (channel, ))
try:
if self.is_connected(channel) or self.online is False:
if self.is_connected(channel):
self.logger.debug('[%s] Already connected' % (channel, ))
if self.online is False:
self.logger.debug('[%s] self.online=False' % (channel, ))
return True
self.channel_lock[channel].acquire()
self.connected[channel] = None
self.registered[channel] = None
self.ssh_stream[channel] = False
self.ssh_channel[channel] = False
messages = None
stderrdata = ''
try:
if not self.ssh_stream[channel]:
self.logger.debug('[%s] Open ssh connection' % (channel, ))
self.ssh_stream[channel] = create_ssh_stream(self.config, exit_on_failure=False)
self.logger.debug('[%s] open channel' % (channel, ))
self.ssh_channel[channel] = self.ssh_stream[channel].get_transport().open_session()
self.ssh_channel[channel].exec_command('stream')
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
self.connected[channel] = False
self.registered[channel] = False
self.logger.debug('[%s] connection failed: %s' % (channel, str(e)))
return False
finally:
self.channel_lock[channel].release()
if self.ssh_channel[channel]:
messages = self.wait_for_at_least_one_message(channel)
if not messages:
stderrdata = self.ssh_channel[channel].recv_stderr().decode("utf-8").strip()
self.connected[channel] = False
self.registered[channel] = False
else:
self.logger.debug('[%s] opened and received %d messages' % (channel, len(messages)))
self.connected[channel] = True
self.registered[channel] = self.on_connect(self.was_connected_once[channel], channel)
self.connected_since[channel] = time.time()
if channel == '' and self.registered[channel] and self.was_connected_once[channel]:
self.logger.info("Successfully reconnected.")
if not self.registered[channel]:
# make sure to close channel and connection first
try:
self.ssh_channel[channel] and self.ssh_channel[channel].close()
except: pass
try:
self.ssh_stream[channel] and self.ssh_stream[channel].close()
except: pass
self.logger.debug("[%s] Client: registration failed. stderrdata: %s" % (channel, stderrdata))
self.connected[channel] = False
try:
self.logger.debug('[%s] Client: ssh_tream close due to registration failure' % (channel, ))
self.ssh_stream[channel].close()
except (KeyboardInterrupt, SystemExit):
raise
self.connection_tries += 1
if not self.was_connected_once[channel] and self.go_offline_on_first_failed_attempt:
# initial try needs to be online, otherwise we go offline
self.go_offline()
if stderrdata:
if 'Connection refused' not in stderrdata and 'Permission denied' not in stderrdata:
self.logger.error(stderrdata)
if 'Permission denied' in stderrdata:
if self.connection_tries < 3:
self.logger.warning("Access denied. Did you setup your SSH public key correctly "
"and saved it in your AETROS Trainer user account?")
self.close()
sys.exit(1)
self.connection_error(channel, "Connection error during connecting to %s: %s" % (self.host, str(stderrdata)))
else:
self.was_connected_once[channel] = True
except Exception as error:
self.connection_error(channel, error)
finally:
self.in_connecting[channel] = False
return self.is_connected(channel)
def main(self, args):
import aetros.const
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog=aetros.const.__prog__ + ' authenticate',
description=
'Authenticates the machine with a new pair of SSH keys with a user account.'
)
parsed_args = parser.parse_args(args)
home_config = read_home_config()
host = home_config['host']
installed_key = get_ssh_key_for_host(host)
key_exists_and_valid = False
if installed_key:
try:
create_ssh_stream(home_config, exit_on_failure=False)
key_exists_and_valid = True
except Exception:
pass
if key_exists_and_valid:
choice = six.moves.input(
"You have already configured a valid SSH (ssk_key: " +
installed_key + ") "
"for " + host +
".\nWant to create a new key? (y/N): ").lower()
if choice != 'y' and choice != 'yes':
print("Aborted.")
sys.exit(1)
ssh_key = paramiko.RSAKey.generate(4096)
ssh_key_private = ssh_key.key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.TraditionalOpenSSL,
serialization.NoEncryption()).decode()
ssh_key_public = 'rsa ' + ssh_key.get_base64()
fingerprint = hashlib.md5(ssh_key.__str__()).hexdigest()
fingerprint = ':'.join(
a + b for a, b in zip(fingerprint[::2], fingerprint[1::2]))
token = api.http_request('machine-token', None, {
'host': socket.getfqdn(),
'key': ssh_key_public
})
print(
"Open following link and login to confirm this machine's SSH key in your account."
)
print("Public Key Fingerprint: MD5:" + fingerprint)
print("\n https://" + host + "/confirm-machine/" + token)
print("\nWaiting for confirmation ...")
while True:
time.sleep(3)
response = api.http_request('machine-token/authorized?id=' + token,
method='post')
if response['status'] == 'confirmed':
print(
"\n" + response['username'] +
' confirmed the public key. Test with "aetros id" or "ssh git@'
+ host + '".')
private_key_path = os.path.expanduser('~/.ssh/aetros_' +
response['username'] +
'_rsa')
public_key_path = os.path.expanduser('~/.ssh/aetros_' +
response['username'] +
'_rsa.pub')
if not os.path.exists(os.path.dirname(private_key_path)):
os.makedirs(os.path.dirname(private_key_path))
with open(private_key_path, 'w') as f:
f.write(ssh_key_private)
with open(public_key_path, 'w') as f:
f.write(ssh_key_public)
os.chmod(private_key_path, 0o600)
os.chmod(public_key_path, 0o600)
ssh_config_path = os.path.expanduser('~/.ssh/config')
if not os.path.exists(os.path.dirname(ssh_config_path)):
os.makedirs(os.path.dirname(ssh_config_path))
host_section = 'host ' + host + '\n'
identity_section = ' IdentityFile ~/.ssh/aetros_' + response[
'username'] + '_rsa\n'
if os.path.exists(ssh_config_path):
import re
regex = re.compile(r"^host\s+" + re.escape(host) + '\s*',
re.IGNORECASE | re.MULTILINE)
with open(ssh_config_path, 'r+') as f:
config = f.read()
if regex.match(config):
config = regex.sub(host_section + identity_section,
config, 1)
else:
config = host_section + identity_section + config
f.seek(0)
f.write(config)
else:
with open(ssh_config_path, 'w') as f:
f.write(host_section + identity_section)
print("Private key " + private_key_path +
" installed in ~/.ssh/config for " + host + ".\n")
user = api.user()
print("Key installed of account %s (%s)." %
(user['username'], user['name']))
sys.exit(0)
if response['status'] == 'expired':
print("Token expired.")
sys.exit(1)
def diff_objects(self, latest_commit_sha):
"""
Push all changes to origin, based on objects, not on commits.
Important: Call this push after every new commit, or we lose commits.
"""
base = ['git', '--bare', '--git-dir', self.git_path]
object_shas = []
summary = {'commits': [], 'trees': [], 'files': []}
def read_parents_and_tree_from(commit):
if commit in self.synced_object_shas or commit in object_shas:
# this commit has already been synced or read
return None, None
self.synced_object_shas[commit] = True
summary['commits'].append(commit)
object_shas.append(commit)
object_content = subprocess.check_output(base + ['cat-file', '-p', commit]).decode('utf-8').strip()
parents = []
tree = ''
for line in object_content.splitlines():
if line.startswith('tree '):
tree = line[len('tree '):]
if line.startswith('parent '):
parents.append(line[len('parent '):])
return parents, tree
def collect_files_from_tree(tree):
if tree in self.synced_object_shas or tree in object_shas:
# we have exactly this tree already synced or read, meaning all its objects as well
return
self.synced_object_shas[tree] = True
summary['trees'].append(tree)
object_shas.append(tree)
object_content = subprocess.check_output(base + ['ls-tree', '-r', '-t', tree]).decode('utf-8').strip()
for line in object_content.splitlines():
exploded = line.split(' ')
if len(exploded) < 3:
sys.stderr.write("Error: Wrong line format of ls-tree for %s: %s\n" % (tree, line,))
sys.exit(1)
object_to_add = str(exploded[2][:40])
path = str(exploded[2][41:])
if object_to_add in self.synced_object_shas or object_to_add in object_shas:
# have it already in the list or already synced
continue
object_shas.append(object_to_add)
self.synced_object_shas[object_to_add] = True
summary['files'].append([object_to_add, path])
commits_to_check = [latest_commit_sha]
while len(commits_to_check):
sha = commits_to_check.pop(0)
parents, tree = read_parents_and_tree_from(sha)
if parents:
for parent in parents:
if parent not in commits_to_check:
commits_to_check.append(parent)
if tree:
collect_files_from_tree(tree)
is_debug2() and self.logger.debug("shas_to_check %d: %s " % (len(object_shas), str(object_shas),))
if not object_shas:
return [], summary
try:
is_debug2() and self.logger.debug("Do git-cat-file-check.sh")
ssh_stream = create_ssh_stream(read_home_config(), exit_on_failure=False)
channel = ssh_stream.get_transport().open_session()
channel.exec_command('git-cat-file-check.sh "%s"' % (self.model_name + '.git',))
channel.sendall('\n'.join(object_shas))
channel.shutdown_write()
def readall(c):
content = b''
while True:
try:
chunk = c.recv(1024)
if chunk == b'':
break
content += chunk
except (KeyboardInterrupt, SystemExit):
return
return content
missing_objects = readall(channel).decode('utf-8').splitlines()
channel.close()
ssh_stream.close()
# make sure we have in summary only SHAs we actually will sync
for stype in six.iterkeys(summary):
ids = summary[stype][:]
for sha in ids:
if stype == 'files':
if sha[0] not in missing_objects:
summary[stype].remove(sha)
else:
if sha not in missing_objects:
summary[stype].remove(sha)
return missing_objects, summary
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
self.logger.error("Failed to generate diff_objects: %s" % (str(e),))
for sha in object_shas:
if sha in self.synced_object_shas:
del self.synced_object_shas[sha]
return None, None
def connect(self, channel):
"""
In the write-thread we detect that no connection is living anymore and try always again.
Up to the 3 connection try, we report to user. We keep trying but in silence.
Also, when more than 10 connection tries are detected, we delay extra 15 seconds.
"""
if self.connection_tries > 10:
time.sleep(10)
if self.in_connecting[channel]:
return False
self.in_connecting[channel] = True
self.logger.debug('[%s] Wanna connect ...' % (channel, ))
try:
if self.is_connected(channel) or self.online is False:
if self.is_connected(channel):
self.logger.debug('[%s] Already connected' % (channel, ))
if self.online is False:
self.logger.debug('[%s] self.online=False' % (channel, ))
return True
self.channel_lock[channel].acquire()
self.connected[channel] = None
self.registered[channel] = None
self.ssh_stream[channel] = False
self.ssh_channel[channel] = False
messages = None
stderrdata = ''
try:
if not self.ssh_stream[channel]:
self.logger.debug('[%s] Open ssh connection' % (channel, ))
self.ssh_stream[channel] = create_ssh_stream(
self.config, exit_on_failure=False)
self.logger.debug('[%s] open channel' % (channel, ))
self.ssh_channel[channel] = self.ssh_stream[
channel].get_transport().open_session()
self.ssh_channel[channel].exec_command('stream')
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
self.connected[channel] = False
self.registered[channel] = False
self.logger.debug('[%s] connection failed: %s' %
(channel, str(e)))
return False
finally:
self.channel_lock[channel].release()
if self.ssh_channel[channel]:
messages = self.wait_for_at_least_one_message(channel)
if not messages:
stderrdata = self.ssh_channel[channel].recv_stderr().decode(
"utf-8").strip()
self.connected[channel] = False
self.registered[channel] = False
else:
self.logger.debug('[%s] opened and received %d messages' %
(channel, len(messages)))
self.connected[channel] = True
self.registered[channel] = self.on_connect(
self.was_connected_once[channel], channel)
self.connected_since[channel] = time.time()
if channel == '' and self.registered[
channel] and self.was_connected_once[channel]:
self.logger.info("Successfully reconnected.")
if not self.registered[channel]:
# make sure to close channel and connection first
try:
self.ssh_channel[channel] and self.ssh_channel[
channel].close()
except:
pass
try:
self.ssh_stream[channel] and self.ssh_stream[
channel].close()
except:
pass
self.logger.debug(
"[%s] Client: registration failed. stderrdata: %s" %
(channel, stderrdata))
self.connected[channel] = False
try:
self.logger.debug(
'[%s] Client: ssh_tream close due to registration failure'
% (channel, ))
self.ssh_stream[channel].close()
except (KeyboardInterrupt, SystemExit):
raise
self.connection_tries += 1
if not self.was_connected_once[
channel] and self.go_offline_on_first_failed_attempt:
# initial try needs to be online, otherwise we go offline
self.go_offline()
if stderrdata:
if 'Connection refused' not in stderrdata and 'Permission denied' not in stderrdata:
self.logger.error(stderrdata)
if 'Permission denied' in stderrdata:
if self.connection_tries < 3:
self.logger.warning(
"Access denied. Did you setup your SSH public key correctly "
"and saved it in your AETROS Trainer user account?"
)
self.close()
sys.exit(1)
self.connection_error(
channel, "Connection error during connecting to %s: %s" %
(self.host, str(stderrdata)))
else:
self.was_connected_once[channel] = True
except Exception as error:
self.connection_error(channel, error)
finally:
self.in_connecting[channel] = False
return self.is_connected(channel)
def main(self, args):
import aetros.const
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
prog=aetros.const.__prog__ + ' authenticate',
description='Authenticates the machine with a new pair of SSH keys with a user account.')
parsed_args = parser.parse_args(args)
home_config = read_home_config()
host = home_config['host']
installed_key = get_ssh_key_for_host(host)
key_exists_and_valid = False
if installed_key:
try:
create_ssh_stream(home_config, exit_on_failure=False)
key_exists_and_valid = True
except Exception: pass
if key_exists_and_valid:
choice = six.moves.input("You have already configured a valid SSH (ssk_key: "+installed_key+") "
"for "+host+".\nWant to create a new key? The old won't be removed. (y/N): ").lower()
if choice != 'y' and choice != 'yes':
print("Aborted.")
sys.exit(1)
ssh_key = paramiko.RSAKey.generate(4096)
ssh_key_private = ssh_key.key.private_bytes(
serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption()
).decode()
ssh_key_public = 'rsa ' + ssh_key.get_base64()
string_key = ssh_key.__str__()
if not isinstance(string_key, six.binary_type):
string_key = string_key.encode('utf-8')
md5 = hashlib.md5(string_key)
fingerprint = md5.hexdigest()
fingerprint = ':'.join(a + b for a, b in zip(fingerprint[::2], fingerprint[1::2]))
try:
token = api.http_request('machine-token', None, {
'host': socket.getfqdn(),
'key': ssh_key_public
})
except requests.exceptions.SSLError:
sys.exit(1)
print("Open following link and login to confirm this machine's SSH key in your account.")
print("Public Key Fingerprint: MD5:" + fingerprint)
print("\n " + home_config['url'] + "/confirm-machine/" + token)
print("\nWaiting for confirmation ...")
key_prefix = home_config['host'] + '_'
while True:
time.sleep(3)
response = api.http_request('machine-token/authorized?id=' + token, method='post')
if response['status'] == 'confirmed':
print("\n" + response['username'] + ' confirmed the public key. Test with "aetros id" or "ssh git@'+host+'".')
private_key_path = os.path.expanduser('~/.ssh/' + key_prefix + response['username']+'_rsa')
public_key_path = os.path.expanduser('~/.ssh/' + key_prefix + response['username']+'_rsa.pub')
if not os.path.exists(os.path.dirname(private_key_path)):
os.makedirs(os.path.dirname(private_key_path))
with open(private_key_path, 'w') as f:
f.write(ssh_key_private)
with open(public_key_path, 'w') as f:
f.write(ssh_key_public)
os.chmod(private_key_path, 0o600)
os.chmod(public_key_path, 0o600)
ssh_config_path = os.path.expanduser('~/.ssh/config')
if not os.path.exists(os.path.dirname(ssh_config_path)):
os.makedirs(os.path.dirname(ssh_config_path))
host_section = 'host '+host+'\n'
identity_section = ' IdentityFile ~/.ssh/' + key_prefix + response['username']+'_rsa\n'
if os.path.exists(ssh_config_path):
import re
regex = re.compile(r"^host\s+" + re.escape(host)+'\s*', re.IGNORECASE | re.MULTILINE)
with open(ssh_config_path, 'r+') as f:
config = f.read()
if regex.match(config):
config = regex.sub(host_section + identity_section, config, 1)
else:
config = host_section + identity_section + config
f.seek(0)
f.write(config)
else:
with open(ssh_config_path, 'w') as f:
f.write(host_section + identity_section)
print("Private key " + private_key_path + " installed in ~/.ssh/config for "+host+".\n")
user = api.user()
print("Key installed of account %s (%s)." % (user['username'], user['name']))
sys.exit(0)
if response['status'] == 'expired':
print("Token expired.")
sys.exit(1)