def __init__(self, cam_num, ssh):
self.cam_num = cam_num
self.last_frame = np.zeros((1,1))
self.ssh = ssh
self.scp = SCPClient(self.ssh.get_transport(), socket_timeout=None)
def _startRemoteJob(self):
try:
cmd = self._getMpiCommand()
remoteConfig = self.jobParams[PARAM_REMOTE_JOB_CONFIG]
remoteWorkingDir = remoteConfig[u"repo"]
remoteHost = remoteConfig[u"host"]
remoteUser = remoteConfig[u"user"]
remotePass = remoteConfig[u"pass"]
if self.jobParams[PARAM_SIM_MAX_RUNNING_TIME] > 0:
maxRuntime = datetime.timedelta(
minutes=self.jobParams[PARAM_SIM_MAX_RUNNING_TIME])
else:
maxRuntime = u"99-00:00:00"
template = remoteConfig[u"template"]
template = template.replace(u"{PARAM_SIM_MAX_RUNNING_TIME}",
unicode(maxRuntime))
template = template.replace(
u"{PARAM_SIM_NODES}", unicode(self.jobParams[PARAM_SIM_NODES]))
template = template.replace(u"{PARAM_REMOTE_WORKING_DIR}",
unicode(remoteWorkingDir))
template = template.replace(u"{PARAM_JOB_COMMAND}", cmd)
self.logger.debug(u"Remote job started")
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'w') as tmp:
# do stuff with temp file
tmp.write(template)
from paramiko import SSHClient
from scp import SCPClient
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(hostname=remoteHost,
username=remoteUser,
password=remotePass)
with SCPClient(ssh.get_transport()) as scp:
scp.put(path,
'{0}/remoteSbatchJob.sh'.format(remoteWorkingDir))
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
'sbatch {0}/remoteSbatchJob.sh'.format(remoteWorkingDir))
self.remoteJobId = int(
re.search("[0-9]*$",
ssh_stdout.readlines()[0]).group(0))
while True:
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(
'scontrol show jobid -dd {0}'.format(self.remoteJobId))
line = ssh_stdout.readlines()[3]
jobStatus = re.search(u"JobState=(\w*)", line).group(1)
if jobStatus == "PENDING":
self.status = STATUS_WAITING
elif jobStatus == "RUNNING":
self.status = STATUS_RUNNING
elif jobStatus == "CANCELLED":
self.status = STATUS_ERROR
break
elif jobStatus == "ERROR":
self.status = STATUS_ERROR
break
elif jobStatus == "COMPLETED":
self.status = STATUS_FINISHED
scp.get(
'{0}/{1}'.format(remoteWorkingDir, self.outputFile),
self.outputFile)
break
self.sigJobStatusChanged.emit(self.status)
time.sleep(1)
self.sigJobStatusChanged.emit(self.status)
except Exception as e:
self.logger.error(u"Exception: {0}".format(e))
self.status = STATUS_ERROR
self.sigJobStatusChanged.emit(self.status)
finally:
self.sigFinishJob.emit()
def copy_to_host(host, remote_file, local_file, is_root=False):
"""Copy file to remote host
Function first copies file to remote user directory and then moving.
Final move depends to right to directory.
"""
_lpath, lfilename = os.path.split(local_file)
rpath, rfilename = os.path.split(remote_file)
local_filepath = os.path.join('./', lfilename)
if len(rfilename) < 1:
rfilename = lfilename
remote_file = os.path.join(rpath, rfilename)
if not os.path.isfile(local_file):
logger.error("File to copy does not exist",
file=local_file,
module=COMMAND_MODULE_CUSTOM)
return False
ssh_run = SshRun(host.hostname, host.ssh_hostname, local_file,
helpers.get_function_name(), is_root)
if not ssh_run.create_ssh_channel():
return False
success = True
if not CREDS.DRY_RUN:
retries = 0
success = False
while retries < MAX_SCP_RETRIES:
# Copies file to remote users directory
with SCPClient(ssh_run.ssh.get_transport()) as scp:
try:
scp.put(local_file, lfilename)
success = True
break
except Exception as e:
if _error_check(e.message, remote_file, host.hostname,
"copy_to_host"):
return
retries += 1
time.sleep(2)
if not success:
logger.error("Problem using scp",
hostname=host.hostname,
local_file=local_file,
module=COMMAND_MODULE_CUSTOM)
return False
if local_filepath != remote_file:
# Only copy file if not the same location
ssh_run.add_cmd("mkdir -p %s" % rpath)
ssh_run.add_cmd("mv %s %s" % (lfilename, remote_file))
if is_root:
# Make sure root permission are set if needed
ssh_run.add_cmd("chown root:root %s" % remote_file)
ssh_run.add_cmd("restorecon %s" % remote_file)
results = ssh_run.run()
_log_rc(results,
ssh_run.function_name,
hostname=host.hostname,
remote_file=remote_file,
local_file=local_file,
outputs=results['outputs'],
module=COMMAND_MODULE_BUILTIN)
return results['rc'] < 1
import paramiko
from scp import SCPClient
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect('10.1.10.200',
port=22,
username='******',
password='******',
look_for_keys=False,
allow_agent=False)
scp = SCPClient(ssh_client.get_transport())
#scp.put('10.1.10.200-2019-10-9.txt', '/var/tmp/newfile.txt')
#Copy a directory
#scp.put('directory1', recursive=True, remote_path='/var/tmp')
scp.get('/var/tmp/directory1/10.1.10.200-2019-10-9.txt', 'copiedfile.txt')
scp.close()
def Update(sip, S_md5, App, path):
ssh = paramiko.SSHClient()
key = paramiko.RSAKey.from_private_key_file(key_file)
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for ip in sip:
ip = ip.strip()
Redis.lpush(Key, '-' * 60)
Redis.lpush(Key, 'publish ' + path + ' to ' + ip + '\n')
try:
ssh.connect(ip, 22, username, pkey=key, timeout=15)
cmd = "mkdir -p {0}".format(web_path)
ssh.exec_command(cmd)
scp = SCPClient(ssh.get_transport())
scp.put('%s%s.zip' % (web_path, App), web_path)
cmd = '/usr/bin/md5sum %s/%s.zip' % (web_path, App)
stdin, stdout, stderr = ssh.exec_command(cmd)
R_md5 = stdout.read().split()[0]
if R_md5 == S_md5:
Redis.lpush(Key, '%s md5 verify --->pass!' % App)
Redis.lpush(Key, 'unzip start ......')
cmds = [
'cd %s && /usr/bin/unzip -qo %s.zip && /bin/rm -f %s.zip' %
(web_path, App, App),
'[ -e %s ] && echo ok' % tag_path
]
for cmd in cmds:
stdin, stdout, stderr = ssh.exec_command(cmd)
result_zip = stdout.read().strip()
if result_zip == 'ok':
Redis.lpush(Key, '%s --->unzip success!' % App)
Redis.lpush(Key, 'backup start ......')
cmds = ('/bin/mkdir -p %s' % bak_path,
'[ -e %s/%s ] && /bin/rm -rf %s/%s' %
(bak_path, App, bak_path, App),
'[ -e %s%s ] && /bin/mv %s%s %s/%s' %
(web_path, App, web_path, App, bak_path, App))
for cmd in cmds:
stdin, stdout, stderr = ssh.exec_command(cmd)
result_backup = stderr.read()
if result_backup:
Redis.lpush(Key, result_backup)
Redis.lpush(Key, '%s --->backup fail!' % App)
else:
Redis.lpush(Key,
'%s --->backup success!' % App)
Redis.lpush(Key, 'publish start ......')
cmd = '/bin/mv %s %s%s' % (tag_path, web_path, App)
stdin, stdout, stderr = ssh.exec_command(cmd)
result_rsync = stderr.read()
if result_rsync:
Redis.lpush(Key, result_rsync)
Redis.lpush(Key,
'%s --->publish fail!' % App)
else:
Redis.lpush(
Key, '%s --->publish success!' % App)
ssh.close()
Redis.lpush(Key, '-' * 60)
else:
Redis.lpush(Key, '%s --->unzip fail!' % App)
else:
Redis.lpush(Key, 'S_MD5:%r' % S_md5)
Redis.lpush(Key, 'R_MD5:%r' % R_md5)
Redis.lpush(Key, ' verify %s on %s fail!!!' % (App, ip))
except Exception as e:
Redis.lpush(Key, 'Update:{0}'.format(e))
ssh.close()
sys.exit()
def __init__(self,
file_path,
timer,
onehot=False,
reward_scaler=1e6,
op_age=min,
save_ll=False,
remote=False,
with_age=True,
ip_address=None,
h_size=2):
super(LLVMEnv, self).__init__()
self.file_path = file_path
self.reward_scaler = reward_scaler
self.timer = timer
self.extremum = op_age
self.onehot = onehot
self.MAX_AGE = 10
self.save_ll = save_ll
self.with_age = with_age
self.ssh_con = False
self.scp_con = False
self.ip_address = None
self.h_size = h_size # size of the history
# configure ssh connection to raspberryPi
if remote:
self.ip_address = ip_address
self.ssh_con = paramiko.SSHClient()
self.ssh_con.load_system_host_keys()
self.ssh_con.connect(self.ip_address,
username="******",
password="******")
self.scp_con = SCPClient(self.ssh_con.get_transport())
# instructions handle by the environment
self.instr_to_opcode = {
"call": 1,
"tail": 1,
"store": 2,
"load": 3,
"fadd": 4,
"fsub": 5,
"fmul": 6,
"fdiv": 7,
"bitcast": 8,
"fpext": 9,
"sitofp": 10,
"fptrunc": 11,
"other": 12
}
# self.instr_to_opcode = {
# "alloca": 1,
# "alloca": 1,
# "store": 2,
# "load": 3,
# "fpext": 4,
# "fmul": 5,
# "fdiv": 6,
# "sitofp": 7,
# "other": 8
# }
self.max_step = 50000
self.curr_step = 0
# Define action and state spaces
# 2 actions: choose instruction or not.
self.action_space = spaces.Discrete(2)
if not self.onehot:
if self.with_age:
self.observation_space = spaces.Box(
low=np.array([0, 0, 0, 0, 1]),
high=np.array([12, 12, 12, self.MAX_AGE, 4]),
dtype=int)
else:
self.observation_space = spaces.Box(low=np.array([0, 0, 0, 0]),
high=np.array(
[12, 12, 12, 4]),
dtype=int)
else:
if self.with_age:
low = np.concatenate(
(np.array([0] * (self.h_size + 1) *
len(self.instr_to_opcode)),
np.array([0] * (self.MAX_AGE + 1)), np.array([0] * 4)))
high = np.concatenate(
(np.array([1] * (self.h_size + 1) *
len(self.instr_to_opcode)),
np.array([1] * (self.MAX_AGE + 1)), np.array([1] * 4)))
else:
low = np.concatenate(
(np.array([0] * (self.h_size + 1) *
(len(self.instr_to_opcode))), np.array([0] * 4)))
high = np.concatenate(
(np.array([1] * (self.h_size + 1) *
(len(self.instr_to_opcode))), np.array([1] * 4)))
self.observation_space = spaces.Box(low=low, high=high, dtype=int)
self.selected_instruction = None # current selected instruction
self.schedulable_instructions = None
# last scheduled instructions's opcode
self.history = collections.deque([0] * (self.h_size), self.h_size)
self.llvm = LLVMController(self.file_path)
print("timing original file")
self.original_time = self.llvm.run(self.timer,
self.curr_step,
False,
self.ssh_con,
self.scp_con,
self.ip_address,
time_original=True)
print(f"original program time is: {self.original_time}s")
ssh.load_system_host_keys()
ssh.connect('sundog.uchicago.edu',username='******')
loading_string = ''
while True:
files = []
for ext in extensions:
files.extend(glob.glob(input_dir + ext))
for f in files:
f = f.replace('\\','/') # To handle windows paths.
t = os.path.getmtime(f)
if t >= call_time:
loading_string = '.'
# SCPCLient takes a paramiko transport and progress callback as its arguments.
# Might want to look into a keep alive if I only want to ssh once and not every call.
with SCPClient(ssh.get_transport(), progress=progress) as scp:
#ready file
f_new = output_dir + f.split('/')[-1]
try:
print('Removing %s'%f)
os.remove(f)
except Exception as e:
print('Failed deleting %s.'%f)
print(e)
#met file
try:
f = f.replace('.ready','.met')
f_new = f_new.replace('.ready','.met')
print('Transferring met file modified since last call')
scp.put(f, f_new)
print('\n')
def getDataFromRemote(self, sshClient, ori, dest):
scp = SCPClient(sshClient.getConnection().get_transport())
scp.get(ori, recursive=True, local_path=dest)
scp.close()
def connect_transfer_client(self):
print("Connected to server", self.host)
self.scp_client = SCPClient(self.client.get_transport(),
progress=self.progress)
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(hostname,
username='******',
password='******',
timeout=120)
for index in range(IMAGES_PER_MACHINE):
print("Limpiando foto %d: " % index)
ssh.exec_command("rm -f /tmp/out.xwd")
ssh.exec_command("rm -f /tmp/img.jpg")
print("Tomando Foto %d: " % index)
ssh.exec_command("xwd -out /tmp/out.xwd -root -display :0.0 ",
timeout=50)
print("transformando foto: %d: " % index)
ssh.exec_command("convert -quality 8% /tmp/out.xwd /tmp/img.jpg",
timeout=50)
print("Copiando imagen ... ")
fname = "img_%d_%s.jpg" % (
index, str(datetime.now().strftime("%Y%m%d-%H_%M_%S")))
filepath = "%s/%s" % (maquina, fname)
try:
with SCPClient(ssh.get_transport(), socket_timeout=30) as scp:
scp.get('/tmp/img.jpg', filepath)
except Exception as e:
print("error al copiar la imagen %d" % index, e)
ssh.close()
except Exception as e:
print("error en la conexion", e)
def putDataInRemote(self, sshClient, ori, dest):
scp = SCPClient(sshClient.getConnection().get_transport())
scp.put(ori, dest, recursive=True)
scp.close()
def install_apache():
# Read apche hosts metadata from the json file
with open('hosts.json', 'r') as fp:
data = json.load(fp)
for hosts in data['apache_hosts']:
print hosts
name = hosts['cname']
user = hosts['username']
passwd = hosts['password']
diskt = hosts['disk_threshold']
override = hosts['override_port']
install = hosts['install_pkg']
port = hosts['web_port']
documentroot = hosts['documentroot']
logging.debug("%s, %s, %s, %,s", name, user, passwd, install)
# Command to install apache related packages
ssh = SshConnect()
ssh.connection(name, user, passwd)
ssh1 = ssh.connection(name, user, passwd)
scp = SCPClient(ssh1)
scp.put('scripts/find_deleted.sh', '/tmp/find_deleted.sh')
output = ssh.run_command('sh -c /tmp/find_deleted.sh')
command = "df -h / | tail -n +2 | awk '{print $5}'"
disk = ssh.run_command(command)
logging.info("Disk usage is %s", disk)
if int(disk.split('%')[0]) < int(diskt):
command = "apt-get -y install " + install
output = ssh.run_command(command)
if output:
PATH = os.getcwd()
TEMPLATE_ENVIRONMENT = Environment(
autoescape=False,
loader=FileSystemLoader(os.path.join(PATH, 'templates')),
trim_blocks=False)
# Replace the following context from the template files
context = {"DocumentRoot": documentroot, "ports": port}
for files in os.listdir("templates"):
logging.info("file name is %s", files)
# Create new files from the template files after rendering
create_file(files, context)
indexdoc = documentroot + "/index.php"
ssh1 = ssh.connection(name, user, passwd)
scp = SCPClient(ssh1)
# Copy the newly generated files
scp.put('ports.conf', '/etc/apache2/ports.conf')
scp.put('apache2.conf', '/etc/apache2/apache2.conf')
scp.put('app.conf', '/etc/apache2/sites-available/app.conf')
scp.put('resolv.conf', '/etc/resolv.conf')
command = 'mkdir -p ' + documentroot
output = ssh.run_command(command)
scp.put('index.php', indexdoc)
output = ssh.run_command(
'a2dissite 000-default && a2ensite app')
scp.put('scripts/fix_iptables.sh', '/tmp/fix_iptables.sh')
command = "/tmp/fix_iptables.sh " + port
logging.info("running command %s", command)
output = ssh.run_command(command)
output = ssh.run_command("service apache2 stop")
# We can implement override port here similar to disk threshold
# (Not implemented correctly here)
if override:
command = "lsof -i :" + port + \
" | tail -n +2 | awk '{print $2}'| xargs kill -9"
logging.info("running command %s", command)
output = ssh.run_command(command)
output = ssh.run_command('service apache2 start')
hostconfig = hosts['cname'] + ":" + hosts['web_port']
vhost.append(hostconfig)
ssh1.close()
else:
file.write(
name +
"::: installation failed please check output below \n")
file.write(name + "::" + output + "\n")
else:
logging.exception(
"Disk Usgae is above threshold.Please delete unwanted data and try agian"
)
exit("Installation failed on host %s", name)
print "vhost fils is ", vhost
return vhost
def Download(self):
# Start by setting a DL state
self.inst['state'] = 9 # Downloading
self.Save_Instance()
# Do we have a temp folder locally we can store the files?
savedir = download_dir + '/' + self.inst['uuid']
if not os.path.exists(savedir):
os.makedirs(savedir, exist_ok=True)
self.logger.debug("Download: Made save dir of " + savedir)
# Are we in good, or error state? If error, DL error files
our_dls = ast.literal_eval(self.inst['downloads'])
if self.inst['backend_build_state'] == "failed":
our_dls = ['/var/log/cloud-init-output.log', '/var/log/syslog']
# Before we start the downloads, open up an SSH session
# SSH and check status
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
# Loop so we can monitor from this function, because f**k adding jank
# to backend.py
loop_counter = 0
while True:
# Attempt our connection
try:
ssh.connect(self.inst['ip'], ssh_port, ssh_user, timeout=3)
except Exception as e:
# Did we except? Make sure it's a known message we can see
msgs = [
"Unable to connect", "No existing session",
"Authentication failed"
]
for error in msgs:
if error in str(e):
ssh.close() # Cleanup
time.sleep(1) # zzz to prevent racing
continue
# if we are here, we hit an unknown error. Have we been here
# too many times?
if loop_counter > 10:
ssh.close() # Cleanup
self.inst['state'] = 50
self.Save_Instance()
raise Exception(
"Error SSHing into droplet repeatedly, last error of "
+ str(e))
# Count up
loop_counter += 1
continue
else:
# We connected, carry on to SCP
for dl in ast.literal_eval(self.inst['downloads']):
with SCPClient(ssh.get_transport(),
socket_timeout=30) as scp:
self.logger.debug(
"Download: Downloading file/folder of " + dl)
try:
scp.get(dl, savedir, recursive=True)
except Exception as e:
self.inst['state'] = 50
self.Save_Instance()
raise Exception(
"Error with SCP Download, recieved an error of: "
+ str(e))
finally:
# This should not be needed with the "with"
# statement, but play it safe
scp.close()
finally:
ssh.close()
# If we are here, break out of the loop
break
# We done here
return True
def scp_file_here(filename, name, user, password):
print('{0} - SCP copy file here from router {1}'.format(
_get_time(), filename))
ssh = createSSHClient(name, 22, user, password)
with SCPClient(ssh.get_transport()) as scp:
scp.get(filename)
def transfer_file(self,
hostname=None,
username=None,
password=None,
allow_agent=False,
look_for_keys=False,
ssh_config=None):
"""Transfer the file to the remote device over SCP.
Note:
If any arguments are omitted, the corresponding attributes
of the ``self.device`` will be used.
Args:
hostname (str): OPTIONAL - The name or
IP address of the remote device.
proto (str): OPTIONAL - Protocol to be used
for transfer - 'scp' or 'sftp'
username (str): OPTIONAL - The SSH username
for the remote device.
password (str): OPTIONAL - The SSH password
for the remote device.
Raises:
FileTransferError: if an error occurs during the file transfer.
FileHashMismatchError: if the source and
destination hashes don't match.
FileNotReadableError: if the local file doesn't exist or isn't readable.
FileNotEnoughSpaceError: if there isn't enough space on the device.
FileRemoteDirDoesNotExist: if the remote directory doesn't exist.
"""
hostname = hostname or self.device.host
username = username or self.device.username
password = password or self.device.password
ssh_config = ssh_config or self.device.ssh_config
key_filename = None
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if ssh_config is not None:
sshconfig = paramiko.SSHConfig()
sshconfig.parse(open(ssh_config))
cfg = sshconfig.lookup('dev')
key_filename = cfg['identityfile'][0]
ssh.connect(hostname=hostname,
username=username,
password=password,
port=self.port,
allow_agent=allow_agent,
look_for_keys=look_for_keys,
key_filename=key_filename)
if self.proto == "scp":
scp = SCPClient(ssh.get_transport())
try:
scp.get(self.src, self.dst)
except:
raise FileTransferError
scp.close()
elif self.proto == "sftp":
sftp = ssh.open_sftp()
sftp.sshclient = ssh
try:
sftp.get(self.src.strip('flash:'), self.dst)
except:
raise FileTransferError
sftp.close()
else:
raise FileTransferUnsupportedProtocol(self.proto)
src_hash = self._get_local_md5()
dst_hash = self._get_remote_md5()
if src_hash != dst_hash:
raise FileHashMismatchError(self.src, self.dst, src_hash, dst_hash)
def _download_mlcnet_file(self, remote_file, download_to):
scp = SCPClient(self.mlcnet_ssh.get_transport())
scp.get(remote_file, download_to)
scp.close()
def deploy_orchestrator(self):
"""
Deploy Cloudify Manager.
network, security group, fip, VM creation
"""
# network creation
start_time = time.time()
self.__logger.info("Creating keypair ...")
kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
keypair_settings = KeypairConfig(name='cloudify_ims_kp',
private_filepath=kp_file)
keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings)
keypair_creator.create()
self.created_object.append(keypair_creator)
self.__logger.info("Creating full network ...")
subnet_settings = SubnetConfig(name='cloudify_ims_subnet',
cidr='10.67.79.0/24')
network_settings = NetworkConfig(name='cloudify_ims_network',
subnet_settings=[subnet_settings])
network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
network_creator.create()
self.created_object.append(network_creator)
ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
router_creator = OpenStackRouter(
self.snaps_creds,
RouterConfig(name='cloudify_ims_router',
external_gateway=ext_net_name,
internal_subnets=[subnet_settings.name]))
router_creator.create()
self.created_object.append(router_creator)
# security group creation
self.__logger.info("Creating security group for cloudify manager vm")
sg_rules = list()
sg_rules.append(
SecurityGroupRuleConfig(sec_grp_name="sg-cloudify-manager",
direction=Direction.ingress,
protocol=Protocol.tcp,
port_range_min=1,
port_range_max=65535))
sg_rules.append(
SecurityGroupRuleConfig(sec_grp_name="sg-cloudify-manager",
direction=Direction.ingress,
protocol=Protocol.udp,
port_range_min=1,
port_range_max=65535))
securit_group_creator = OpenStackSecurityGroup(
self.snaps_creds,
SecurityGroupConfig(name="sg-cloudify-manager",
rule_settings=sg_rules))
securit_group_creator.create()
self.created_object.append(securit_group_creator)
# orchestrator VM flavor
self.__logger.info("Get or create flavor for cloudify manager vm ...")
flavor_settings = FlavorConfig(
name=self.orchestrator['requirements']['flavor']['name'],
ram=self.orchestrator['requirements']['flavor']['ram_min'],
disk=50,
vcpus=2)
flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
flavor_creator.create()
self.created_object.append(flavor_creator)
image_settings = ImageConfig(
name=self.orchestrator['requirements']['os_image'],
image_user='******',
exists=True)
port_settings = PortConfig(name='cloudify_manager_port',
network_name=network_settings.name)
manager_settings = VmInstanceConfig(
name='cloudify_manager',
flavor=flavor_settings.name,
port_settings=[port_settings],
security_group_names=[securit_group_creator.sec_grp_settings.name],
floating_ip_settings=[
FloatingIpConfig(
name='cloudify_manager_fip',
port_name=port_settings.name,
router_name=router_creator.router_settings.name)
])
manager_creator = OpenStackVmInstance(self.snaps_creds,
manager_settings, image_settings,
keypair_settings)
self.__logger.info("Creating cloudify manager VM")
manager_creator.create()
self.created_object.append(manager_creator)
public_auth_url = keystone_utils.get_endpoint(self.snaps_creds,
'identity')
self.__logger.info("Set creds for cloudify manager")
cfy_creds = dict(keystone_username=self.snaps_creds.username,
keystone_password=self.snaps_creds.password,
keystone_tenant_name=self.snaps_creds.project_name,
keystone_url=public_auth_url)
cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip,
username='******',
password='******',
tenant='default_tenant')
self.orchestrator['object'] = cfy_client
self.__logger.info("Attemps running status of the Manager")
cfy_status = None
retry = 10
while str(cfy_status) != 'running' and retry:
try:
cfy_status = cfy_client.manager.get_status()['status']
self.__logger.debug("The current manager status is %s",
cfy_status)
except Exception: # pylint: disable=broad-except
self.__logger.warning("Cloudify Manager isn't " +
"up and running. Retrying ...")
retry = retry - 1
time.sleep(30)
if str(cfy_status) == 'running':
self.__logger.info("Cloudify Manager is up and running")
else:
raise Exception("Cloudify Manager isn't up and running")
self.__logger.info("Put OpenStack creds in manager")
secrets_list = cfy_client.secrets.list()
for k, val in cfy_creds.iteritems():
if not any(d.get('key', None) == k for d in secrets_list):
cfy_client.secrets.create(k, val)
else:
cfy_client.secrets.update(k, val)
duration = time.time() - start_time
self.__logger.info("Put private keypair in manager")
if manager_creator.vm_ssh_active(block=True):
ssh = manager_creator.ssh_client()
scp = SCPClient(ssh.get_transport(), socket_timeout=15.0)
scp.put(kp_file, '~/')
cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
run_blocking_ssh_command(ssh, cmd)
cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
run_blocking_ssh_command(ssh, cmd)
cmd = "sudo yum install -y gcc python-devel"
run_blocking_ssh_command(
ssh, cmd, "Unable to install packages \
on manager")
self.details['orchestrator'].update(status='PASS', duration=duration)
self.vnf['inputs'].update(
dict(external_network_name=ext_net_name,
network_name=network_settings.name))
self.result = 1 / 3 * 100
return True
def upload(self, file, remote_folder):
scp = SCPClient(self.ssh.get_transport())
scp.put(f'{file}', recursive=True, remote_path=remote_folder)
scp.close()
'sudo pip install --upgrade google-api-python-client',
'sudo pip install beaker', 'sudo pip install peewee',
'sudo apt-get install unzip'
]
for n, com in enumerate(commands):
stdin, stdout, stderr = client.exec_command(com)
if n == 1:
# time.sleep(3)
stdin.write('y\n')
time.sleep(100)
else:
time.sleep(20)
print "executed '" + com + "'"
if __name__ == "__main__":
scp = SCPClient(client.get_transport())
print "Uploading Webserver"
scp.put(file_name)
time.sleep(5)
stdin, stdout, stderr = client.exec_command("unzip " + file_name)
print "Setting up server"
time.sleep(10)
stdin, stdout, stderr = client.exec_command("cd " + file_name[:-4])
time.sleep(1)
print "Executing"
stdin, stdout, stderr = client.exec_command(
"screen -d -m sudo python server.py")
time.sleep(2)
# print stdout.next()
print "connection %s" % ("ok" if connection else "failed")
def deploy(path: str = './', hostname: str = "ev3dev", username: str = "robot", password: str = "maker",
execute_file: Optional[str] = None, executable: List[str] = ('*.py',),
exclude_path: str = "./.ignore", print_console: bool = True,
redirect_stdout: bool = True, redirect_stderr: bool = True, redirect_stdin: bool = False) -> None:
"""
Send code to Ev3
:param path: The Directory to send (default is current directory).
:param hostname: The ssh hostname (default is 'ev3dev')
:param username: The ssh username (default is 'robot')
:param password: The ssh password (default is 'maker')
:param execute_file: A file to run on the ev3 when finished. 'None' to disable.
Note: this file must be marked as executable.
:param executable: A list of patterns of files that should be marked as executable (default is ['*.py']).
:param exclude_path: The file containing the list of files to ignore (default is '.ignore').
:param print_console: Should we print info to the console?
:param redirect_stdout: Should we redirect stdout form ev3 to console?
:param redirect_stderr: Should we redirect stderr form ev3 to console?
:param redirect_stdin: Should we redirect console input to ev3 stdin?
This is disabled by default as it cannot terminate without reading from stdin.
"""
# Get / Set working directory
if print_console:
print("CD", path)
os.chdir(path)
working_dir = os.getcwd()
dir_name = os.path.basename(working_dir)
exclude = read_exclude(exclude_path)
# Set up ssh
if print_console:
print("Starting ssh ...")
ssh = SSHClient()
ssh.load_system_host_keys()
if print_console:
print("Connecting to", F"{username}@{hostname} ...")
ssh.connect(hostname=hostname, username=username, password=password)
with SCPClient(ssh.get_transport()) as scp:
for subdir, dirs, files in os.walk('.'): # for every file in current working directory:
for filename in files:
filepath = subdir + '/' + filename # get full file path (relative to working directory)
if not match(filepath, exclude): # if the file path does not match any of the excluded patterns:
if print_console:
print("Sending", Path(filepath), "... ", end='')
# create the directory if it does not exist
ssh.exec_command('mkdir -p ' + path_join('~', dir_name, subdir).as_posix())
# copy files using scp
scp.put(str(path_join(working_dir, filepath)), path_join('~', dir_name, filepath).as_posix())
if print_console:
print("Sent")
if match(filepath, executable): # if file path matches any of the executable patterns:
# mark as executable
if print_console:
print(path_join('~', dir_name, filepath).as_posix(), "marked as executable.")
ssh.exec_command('chmod u+x ' + path_join('~', dir_name, filepath).as_posix())
else:
if print_console:
print('Excluding', Path(filepath), '.')
if execute_file:
if print_console:
print(F'\nExecuting {execute_file} ...\n')
# execute the file.
stdin, stdout, stderr = ssh.exec_command(path_join('~', dir_name, execute_file).as_posix(), get_pty=True)
# create the redirecting threads
if redirect_stdout:
out = threading.Thread(target=redirect_stdout_handler, args=(stdout,))
if redirect_stderr:
err = threading.Thread(target=redirect_stderr_handler, args=(stderr,))
if redirect_stdin:
child_pid = os.fork()
if child_pid == 0:
redirect_stdin_handler(stdin)
os.kill(os.getpid(), SIGTERM)
# sin = threading.Thread(target=redirect_stdin_handler, args=(stdin,))
# start them
if redirect_stdout:
out.start()
if redirect_stderr:
err.start()
# if redirect_stdin:
# sin.start()
# wait for them to terminate
if redirect_stdout:
out.join()
if redirect_stderr:
err.join()
if redirect_stdin:
global run_stdin
# tell redirect_stdin_handler to exit without sending data to stdin
run_stdin = False
# sys.stdin.close()
# sin.join()
os.kill(child_pid, SIGTERM)
if print_console:
print('\nFinished.')
def main(username, password, test=False, force=False):
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect('nova.cs.tau.ac.il', username=username, password=password)
scp = SCPClient(ssh.get_transport(), progress = progress)
# copy benchmark unbound receptor and ligand to nova:
all_complex_codes = next(os.walk('../data'))[1]
if test:
all_complex_codes = all_complex_codes[:1]
with open('./copied_successfully.json', 'r') as f:
copied_successfully = json.load(f)
if force:
copied_successfully = []
for complex_code in all_complex_codes:
if len(complex_code) != 4:
continue
print("Copying complex: %s" % complex_code)
try:
ssh.exec_command('mkdir ~/data/' + complex_code + '/')
ssh.exec_command('mkdir ~/data/' + complex_code + '/benchmark/')
ssh.exec_command('mkdir ~/data/' + complex_code + '/patch_dock/')
scp.put('../data/' + complex_code + '/benchmark/' + complex_code + '_l_u.pdb', '~/data/' + complex_code + '/benchmark/')
scp.put('../data/' + complex_code + '/benchmark/' + complex_code + '_r_u.pdb', '~/data/' + complex_code + '/benchmark/')
scp.put("../data/{0}/patch_dock/{0}.patch_dock_output".format(complex_code), '~/data/' + complex_code + '/patch_dock/')
copied_successfully.append(complex_code)
with open('./copied_successfully.json', 'w') as f:
json.dump(copied_successfully, f)
except:
print("ERROR with: %s" % complex_code)
# run patch dock on each benchmark receptor and ligand which doesn't have an ouput file yet
with open('./patch_dock_ran_successfully.json', 'r') as f:
patch_dock_ran_successfully = json.load(f)
if force:
patch_dock_ran_successfully = []
for complex_code in all_complex_codes:
if len(complex_code) != 4 or complex_code in patch_dock_ran_successfully:
continue
print("Running PatchDock on: %s" % complex_code)
try:
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('cd ~/PatchDock; sh run_patch_dock_on_one_complex.sh ' + complex_code)
print(ssh_stdout.read(), ssh_stderr.read())
if ssh_stderr.read() == b'':
patch_dock_ran_successfully.append(complex_code)
with open('./patch_dock_ran_successfully.json', 'w') as f:
json.dump(patch_dock_ran_successfully, f)
except Exception as e:
print(e.message)
print("ERROR with: %s" % complex_code)
print("DONE")
scp.close()
def put(self, files, remote_path=b'.', recursive=False):
client = self._get_client()
with SCPClient(client.get_transport()) as scp:
scp.put(files, remote_path, recursive)
def connect(self):
""" Establish a connection to the device.
Purpose: This method is used to make a connection to the junos
| device. The internal property conn_type is what
| determines the type of connection we make to the device.
| - 'paramiko' is used for operational commands (to allow
| pipes in commands)
| - 'scp' is used for copying files
| - 'shell' is used for to send shell commands
| - 'root' is used when logging into the device as root, and
| wanting to send operational commands
| - 'ncclient' is used for the rest (commit, compare_config,
| commit_check)
@returns: None
@rtype: None
"""
if self.conn_type == 'paramiko':
self._session = paramiko.SSHClient()
# These two lines set the paramiko logging to Critical to
# remove extra messages from being sent to the user output.
logger = logging.Logger.manager.getLogger('paramiko.transport')
logger.setLevel(logging.CRITICAL)
self._session.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._session.connect(hostname=self.host,
username=self.username,
password=self.password,
port=self.port,
timeout=self.connect_timeout)
if self.conn_type == 'scp':
self._scp_session = paramiko.SSHClient()
logger = logging.Logger.manager.getLogger('paramiko.transport')
logger.setLevel(logging.CRITICAL)
self._scp_session.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._scp_session.connect(hostname=self.host,
username=self.username,
password=self.password,
port=self.port,
timeout=self.connect_timeout)
self._scp = SCPClient(self._scp_session.get_transport())
elif self.conn_type == "ncclient":
self._session = manager.connect(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
timeout=self.connect_timeout,
device_params={'name': 'junos'},
hostkey_verify=False
)
elif self.conn_type == 'shell':
if not self._session:
self.conn_type = 'paramiko'
self.connect()
self.conn_type = 'shell'
if not self._shell:
self._shell = self._session.invoke_shell()
time.sleep(2)
if self.username != 'root' and not self._in_cli:
self._in_cli = True
if not self.cli_to_shell():
self._shell.recv(9999)
elif self.conn_type == 'root':
# open the shell if necessary, and move into CLI
if not self._shell:
self._shell = self._session.invoke_shell()
time.sleep(2)
if not self.shell_to_cli():
self._shell.recv(9999)
self._update_timeout(self.session_timeout)
def get(self, remote_path, local_path='/tmp/', recursive=True):
client = self._get_client()
with SCPClient(client.get_transport()) as scp:
scp.get(remote_path, local_path, recursive)
cur_time = int(time.time())
while cur_time < timeout:
if IsOnline(sync_ip):
break
cur_time = int(time.time())
if not IsOnline(sync_ip):
print("Failed to wake %s" % sync_ip)
time.sleep(60)
continue
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(sync_ip, username=username)
scp = SCPClient(ssh.get_transport(), progress=progress)
sync(monitor_dir, sync_dir, scp)
scp.close()
ssh.close()
# if there are, pause main queue and sync them to NAS
# resume when sync complete
time.sleep(60)
def cpfiles(self):
with SCPClient(ssh.get_transport()) as scp:
scp.put('a.csv', 'b.csv')
scp.get('23.txt')
with tarfile.open(
f"/tmp/packetframedeploy/{node['name']}.tar.gz",
"w:gz") as tar:
output_file = f"/tmp/packetframedeploy/{node['name']}/"
tar.add(output_file, arcname=os.path.basename(output_file))
try:
ssh.connect(node["management_ip"],
username="******",
port=34553,
key_filename=configuration["ssh-key"])
except (TimeoutError, NoValidConnectionsError):
print(node["name"] + " unable to connect.")
else:
with SCPClient(ssh.get_transport()) as scp:
scp.put(
f"/tmp/packetframedeploy/{node['name']}.tar.gz",
"/etc/bind/deploy.tar.gz")
stdin, stdout, stderr = ssh.exec_command(
"tar -xvzf /etc/bind/deploy.tar.gz -C /etc/bind/ ; rm /etc/bind/deploy.tar.gz ; bash /root/cleanup-zones.sh ; rndc reload"
)
for line in stdout:
print(line.strip())
if line in stderr:
print(line.strip())
# Clean up the tmp files
shutil.rmtree("/tmp/packetframedeploy/")
def _publish_code(package_type, d_files, package_name, ssh, ip, ssh_port, app_port, package_path, package_md5,execute):
#上线操作前进行数据备份
_console_out(channel, Msg_Key,"ip:%s ssh_port:%s app_port:%s --->start deploy %s......" % (ip, ssh_port, app_port,project))
if execute == 'publish':
try:
cmd = "[ -e %s%s ] && echo ok" % (web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = str(stdout.read().strip(),encoding='utf8')
if result == 'ok':
if package_name.endswith('.war'):
cmd = "\cp -rf %s%s %s%s" % (web_path, package_name, bak_path,package_name)
else:
cmd = "/usr/bin/rsync -av --delete %s%s/ %s%s/" % (web_path, package_name, bak_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number,"Error:%s"%result)
return "ip:%s ssh_port:%s app_port:%s --->backup Fail !" % (ip, ssh_port, app_port)
except Exception as e:
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->backup Fail !" % (ip, ssh_port, app_port)
scp = SCPClient(ssh.get_transport())
#增量包部署
if package_type == 'part':
try:
scp.put("%s/"%d_files, "%s%s/" % (web_path, package_name), recursive=True)
except Exception as e:
# 传输错误重试3次
for i in range(3):
time.sleep(3)
try:
scp.put("%s/" % d_files, "%s%s/" % (web_path, package_name), recursive=True)
except:
if i >=2:
break
continue
else:
cmd = "chown %s:%s -R %s%s/" % (service_user, service_user, web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port, web_path, package_name)
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
else:
cmd = "chown %s:%s -R %s%s/" %(service_user,service_user,web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port,web_path, package_name)
#整包部署
if package_type == 'full':
try:
d_zip = "%s%s" % (web_path, package_path.split('/')[-1])
try:
scp.put(package_path, d_zip)
except Exception as e:
# 传输错误重试3次
for i in range(3):
time.sleep(3)
try:
scp.put(package_path, d_zip)
except:
if i >= 2:
break
continue
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->transfers Fail !" % (ip, ssh_port, app_port)
cmd = '/usr/bin/md5sum %s' % d_zip
stdin, stdout, stderr = ssh.exec_command(cmd)
R_md5 = str(stdout.read().split()[0],encoding='utf8')
if R_md5 == package_md5:
package_zip = package_path.split('/')[-1]
cmd = "cd %s && /usr/bin/unzip -qo %s && /bin/rm -f %s && [ -e %s ] && echo ok" % (web_path,package_zip,package_zip,package_zip.replace('.zip', ''))
stdin, stdout, stderr = ssh.exec_command(cmd)
result_zip = str(stdout.read().strip(),encoding='utf8')
result = stderr.read()
if result_zip == 'ok':
cmd = "cd %s && /bin/rm -rf %s{,.war} &&/bin/mv %s %s" % (web_path,project,package_zip.replace('.zip', ''), package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number,"Error:%s"%result)
return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
else:
cmd = "chown %s:%s %s%s" % (service_user, service_user, web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port, web_path, package_name)
else:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s app_port:%s --->unzip Fail !" % (ip, ssh_port, app_port)
else:
return "ip:%s ssh_port:%s app_port:%s --->md5 Fail !" % (ip, ssh_port, app_port)
except Exception as e:
if 'old-style' not in str(e):
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
from paramiko import SSHClient
from scp import SCPClient
import tarfile, os, shutil, time
overall_time = time.time()
start_time = time.time()
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect('34.72.152.18', username="******")
print("ssh connection took: ", time.time() - start_time)
start_time = time.time()
# SCPCLient takes a paramiko transport as an argument
scp = SCPClient(ssh.get_transport())
scp.get("/home/marekjachym99/lavalanche/weather.tar", "../data")
scp.close()
print("scp copying took: ", time.time() - start_time)
# extract .tar file
start_time = time.time()
archive = tarfile.open("../data/weather.tar", "r")
archive.extractall("../data/")
archive.close()
if os.path.isdir("../data/weather_data"):
shutil.rmtree("../data/weather_data")
shutil.move("../data/home/marekjachym99/lavalanche/weather_data", "../data/")
os.remove("../data/weather.tar")
shutil.rmtree("../data/home")
print("file handling took: ", time.time() - start_time)
print("whole process took: ", time.time() - overall_time)
def ltm(IP, ACC, PASS):
# 找出錯誤log並儲存到 \IP_ERR_LOG中
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(IP, username=ACC, password=PASS) # 以paramiko進行ssh連線
_, stdout, _ = client.exec_command(
"cd /var/log; ls") # 列出 /var/log目錄下所有檔案
log = ""
lst = [
line.replace('\n', '') for line in stdout.readlines()
if line[:3] == "ltm"
] # 尋找ltm log並儲存到lst中
scp = SCPClient(client.get_transport()) # 準備以scp協定傳輸檔案
for line in lst:
scp.get("/var/log/" + line,
PATH + "\\" + IP + "_log\\" + line) # 取得/var/log中的ltm log
if line[-2:] == "gz": # 如果該log為壓縮擋 則解壓縮再讀取
with gzip.open(IP + "_log\\" + line, "rb") as f_in:
log += f_in.read().decode() # log字串增加讀取到的檔案內容
else: # 如果該log不是壓縮擋 則直接讀取
with open(IP + "_log\\" + line, "rb") as f_in:
log += f_in.read().decode() # log字串增加讀取到的檔案內容
except Exception as e:
print(e)
return
# == HA state change
P = re.compile("\n.*HA unit.*\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef:
ef.writelines(res)
P = re.compile("\n.*No failover status messages received for.*\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef:
ef.writelines(res)
P = re.compile("\n.*Active\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef:
ef.writelines(res)
P = re.compile("\n.*Offline\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef:
ef.writelines(res)
P = re.compile("\n.*Standby\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef:
ef.writelines(res)
# == VS state change
P = re.compile("\n.*Virtual Address .*GREEN to RED.*\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\VS_ERR.log", "w", newline='') as ef:
ef.writelines(res)
# == Pool
P = re.compile("\n.*Pool.*GREEN to RED.*\n")
res = re.findall(P, log)
if len(res) != 0:
if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG")
with open(IP + "_ERR_LOG\\Pool_ERR.log", "w", newline='') as ef:
ef.writelines(res)
