def test_it_can_write_to_a_file(self):
content = "This is some file content"
sh("save", self.test_root, "test_file.txt", content)
with open(join(self.test_root, "test_file.txt"), 'rb') as file:
self.assertEqual(content, file.read())
def main(self):
data = requests.head(self.targetURL)
#print(data.headers)
headerDict = ast.literal_eval(str(data.headers))
filesize = headerDict['Content-Length']
# print 'Filesize is %s' % filesize
startPos = 0
partSize = int(filesize) / self.numWorkers
endPos = partSize
# print 'End Position is %s' % endPos
threads = []
for i in range(0, self.numWorkers):
endPos = partSize * (i + 1)
if (i + 1) == self.numWorkers: # Is last iteration?
endPos = filesize
# print '%i> Start: %s End: %s' % (i, startPos, endPos)
t = threading.Thread(target=self.downloadSection,
args=(i, startPos, endPos))
t.start()
threads.append(t)
startPos = int(endPos) + 1
# endPos = partSize * (i + 1)
map(lambda t: t.join(), threads)
print('combining all pieces...')
sh('./combine.sh')
fileName = self.targetURL.split('/')[-1]
mv('movie.mkv', fileName)
print('Moving file to video storage')
mv(fileName, '/media/storage/videos/')
return 0
def compile( self, source_dir, build_dir, install_dir ):
package_source_dir = os.path.join( source_dir, self.dirname )
assert( os.path.exists( package_source_dir ) )
package_build_dir = os.path.join( build_dir, self.dirname )
sh.cd( os.path.join( package_source_dir, 'scripts/Resources' ) )
sh.sh( './copyresources.sh' )
# the install target doesn't copy the stuff that copyresources.sh puts in place
sh.cp( '-v', os.path.join( package_source_dir, 'bin/Release/Readme.txt' ), os.path.join( install_dir, 'Readme.meshy.txt' ) )
sh.cp( '-v', '-r', os.path.join( package_source_dir, 'bin/Release_Linux/Resources/' ), install_dir )
sh.mkdir( '-p', package_build_dir )
sh.cd( package_build_dir )
if ( platform.system() == 'Darwin' ):
sh.cmake(
'-G', 'Xcode',
'-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
package_source_dir,
_out = sys.stdout )
sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
else:
sh.cmake(
'-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
package_source_dir,
_out = sys.stdout )
sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
sh.make.install( _out = sys.stdout )
def test_it_knows_how_big_a_file_is(self):
content = "Test file content"
sh("save", self.test_root, "test_file.txt", content)
size = sh("stat -s", self.test_root, "test_file.txt")
self.assertEquals(len(content), size)
def test_it_can_list_a_dir_with_things(self):
sh("mkdir", self.test_root, "test1")
sh("mkdir", self.test_root, "test2")
content = sh("ls", self.test_root)
self.assertTrue(len(content) == 2)
self.assertIn("test1", content)
self.assertIn("test2", content)
def test_throws_exception_when_file_exists_as_parent(self):
test_dir = join(self.test_root, "test_dir")
open(test_dir, "wb").close()
with self.assertRaises(SHException):
sh("mkdir -p", test_dir, "target_dir")
self.assertFalse(exists(join(test_dir, "target_dir")))
def test_it_can_remove_a_file(self):
test_file = join(self.test_root, "test_file.bin")
open(test_file, "wb")
self.assertTrue(exists(join(self.test_root, "test_file.bin")))
sh("rm", test_file)
self.assertFalse(exists(test_file))
def test_it_can_expand_several_environment_variables(self):
os.environ['USER'] = "******"
os.environ['OTHER'] = "OtherThing"
path = sh('join', '/home/$USER/Desktop/$OTHER')
self.assertEqual(path, '/home/TestUser/Desktop/OtherThing')
def dupsh(sock = 'ebp', os = None):
"""Args: [sock (imm/reg) = ebp]
Duplicates sock to stdin, stdout and stderr and spawns a shell."""
if os in ['freebsd', 'linux']:
return dup(sock), sh()
else:
bug('OS was neither linux nor freebsd')
def execute(self, name, command):
"""executes the command on the named host"""
if name in ["localhost"]:
r = '\n'.join(sh.sh("-c", command).split()[-1:])
else:
r = '\n'.join(sh.ssh(name, command).split()[-1:])
return r
def get_real_url(self, url):
ret = None
m = self.proto_fmt.search(url)
if m:
proto = m.group(1).strip().lower()
if proto == 'file':
value = m.group(2).strip()
exists = False
i = 0
while i < 2:
if not os.path.exists(value):
value = os.path.join(prog_home, value)
i += 1
continue
else:
exists = True
break
if exists :
ret = []
try:
for line in sh.sh(value): ## exec the script
line = self.remove_newline(line)
if line: ret.append( line )
except Exception, e:
cblog("get_real_url: exception" + repr(e))
print e
else:
ret = url
def execute(self, name, command):
"""executes the command on the named host"""
if name in ["localhost"]:
r = '\n'.join(sh.sh("-c", command).split()[-1:])
else:
r = '\n'.join(sh.ssh(name, command).split()[-1:])
return r
def dupsh(sock='ebp', os=None):
"""Args: [sock (imm/reg) = ebp]
Duplicates sock to stdin, stdout and stderr and spawns a shell."""
if os in ['freebsd', 'linux']:
return dup(sock), sh()
else:
bug('OS was neither linux nor freebsd')
def check_symbol(self, env, sofile, symbol):
nm = env.get('NM', 'nm')
syms = sh.sh('-c', "{} -gp {} | cut -d' ' -f3".format(
nm, sofile), _env=env).splitlines()
if symbol in syms:
return True
print('{} missing symbol {}; rebuilding'.format(sofile, symbol))
return False
def check_symbol(self, env, sofile, symbol):
nm = env.get('NM', 'nm')
syms = sh.sh('-c', "{} -gp {} | cut -d' ' -f3".format(
nm, sofile), _env=env).splitlines()
if symbol in syms:
return True
print('{} missing symbol {}; rebuilding'.format(sofile, symbol))
return False
def test_it_can_remove_a_file_recursively(self):
test_dir = join("a", "b", "c", "d")
sh("mkdir -p", self.test_root, test_dir)
self.assertTrue(sh("exists", self.test_root, test_dir))
sh("cd", self.test_root)
sh("rm -r", ".", test_dir)
self.assertFalse(sh("exists", self.test_root, test_dir))
def test_it_can_remove_a_single_empty_directory(self):
target = "target"
sh("mkdir", self.test_root, target)
self.assertTrue(sh("isdir", self.test_root, target))
self.assertTrue(sh("exists", self.test_root, target))
sh("rm", self.test_root, target)
self.assertFalse(sh("exists", self.test_root, target))
def test_it_can_list_the_cwd(self):
sh("mkdir", self.test_root, "test1")
sh("mkdir", self.test_root, "test2")
sh("cd", self.test_root)
content = sh("ls")
self.assertTrue(len(content) == 2)
self.assertIn("test1", content)
self.assertIn("test2", content)
def compile(self, source_dir, build_dir, install_dir):
package_source_dir = os.path.join(source_dir, self.dirname)
assert (os.path.exists(package_source_dir))
package_build_dir = os.path.join(build_dir, self.dirname)
sh.cd(os.path.join(package_source_dir, 'scripts/Resources'))
sh.sh('./copyresources.sh')
# the install target doesn't copy the stuff that copyresources.sh puts in place
sh.cp('-v', os.path.join(package_source_dir, 'bin/Release/Readme.txt'),
os.path.join(install_dir, 'Readme.meshy.txt'))
sh.cp('-v', '-r',
os.path.join(package_source_dir, 'bin/Release_Linux/Resources/'),
install_dir)
sh.mkdir('-p', package_build_dir)
sh.cd(package_build_dir)
if (platform.system() == 'Darwin'):
sh.cmake('-G',
'Xcode',
'-D',
'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D',
'CMAKE_MODULE_PATH=%s' %
os.path.join(install_dir, 'CMake'),
package_source_dir,
_out=sys.stdout)
sh.xcodebuild('-configuration', 'Release', _out=sys.stdout)
else:
sh.cmake('-D',
'CMAKE_INSTALL_PREFIX=%s' % install_dir,
'-D',
'CMAKE_MODULE_PATH=%s' %
os.path.join(install_dir, 'lib/OGRE/cmake'),
package_source_dir,
_out=sys.stdout)
sh.make('-j4', 'VERBOSE=1', _out=sys.stdout)
sh.make.install(_out=sys.stdout)
def main(args=None):
parser = gitboots.argparse_setup(globals())
parser.add_argument("-e", metavar="COMMAND", default="rm -f", help="execute command on each found lock")
opts = parser.parse_args(args)
path = git_dir()
for lock in sh.find(path, "-type", "f", "-name", "index.lock"):
lock = lock.strip()
if opts.e != "rm -f":
if "{}" in opts.e:
cmd = opts.e.replace("{}", lock)
else:
cmd = "{cmd} {lock}".format(cmd=opts.e, lock=lock)
logger.info(cmd)
if sh.which("sh"):
sh.sh("-c", cmd)
elif sh.which("cmd.exe"):
sh.cmd("\c", cmd)
else:
raise ValueError("Do not know how to run this command: %s" % cmd)
else:
logger.info("rm -f %s", lock)
os.remove(lock)
def _get_routes(self):
routes = []
for iface in self.model.db:
match = self._cmd_regex.match(iface)
if not match:
routes.append(iface)
continue
try:
with open("{}/iface_cmd.sh".format(self._path_root), "w") as f:
f.write(match.group(1))
_iface = sh.sh("{}/iface_cmd.sh".format(self._path_root))
routes.append(str(_iface).rstrip())
except Exception as e:
_logger.debug(e)
return routes
def upload(self):
"""upload converted image to openstack
Returns:
bool: True is success, or False for failed"""
from sh import sh
size_name, size = get_size(self.convert_name)
# if not all True
if not all([size_name, size]):
return False
openrc_path = os.path.abspath(os.path.curdir)
source_cmd = "source " + os.path.join(openrc_path, "openrc")
# logger.debug("use glancn command to upload image.")
cmd = source_cmd + ";glance image-create --name %s --min-disk %s --disk-format raw --container-format bare --file %s --progress"
# pass vmname, min-disk, file path
cmd = cmd % (self.vmname + "-" + size_name, int(size),
self.convert_name)
try:
logger.info("excute command: %s" % cmd)
logger.info("convert vm: %s" % self.vmname)
proc = sh("-c", cmd, _iter=True)
logger.info("glance upload command start.")
except Exception as e:
logger.critical("upload image failed.")
logger.critical("excute command failed: %s " % cmd)
logger.critical(e)
return False
for line in proc:
logger.debug(line)
ret = not proc.exit_code
if ret:
if not os.path.exists(".vm"):
os.mkdir(".vm")
logger.info("upload completed.....")
open(".vm/" + self.vmname + ".upload", "a").close()
logger.info("Remove the converted image file")
# remove orignal vmdk file
if VmwareVM.remove(self.convert_name):
logger.warning("remove converted image file success.")
else:
logger.warning("remove converted image file failed.")
return ret
def virt_v2v_convert(self, out="./"):
""""convert the vm to raw format with virt-v2v
Args:
out(str): path for output, default ./
Returns:
bool: True is success, or False for failed"""
from sh import sh
vm = self.down_name
cmd = "export LIBGUESTFS_BACKEND=direct; virt-v2v -v -i disk %s -of raw -o local -os %s "
cmd = cmd % (vm, out)
try:
logger.info("excute command: %s" % cmd)
logger.info("convert vm: %s" % self.vmname)
proc = sh("-c", cmd, _iter=True)
logger.info("virt-v2v convert command start.")
except Exception as e:
logger.critical("convert image failed.")
logger.critical("excute command failed: %s " % cmd)
logger.critical(e)
return False
for line in proc:
logger.debug(line)
# 0 mean success
ret = not proc.exit_code
if ret:
if not os.path.exists(".vm"):
os.mkdir(".vm")
logger.info("convert completed.....")
open(".vm/" + self.vmname + ".convert", "a").close()
logger.debug("try remove the original vmdk file")
# remove orignal vmdk file
if VmwareVM.remove(vm):
logger.info("remove orignal vmdk file success.")
else:
logger.warning("remove orignal vmdk file failed.")
return ret
def push(name):
# a terrible CIDR check
if not any([request.remote_addr.startswith(x) for x in ALLOWED_RANGES]):
return jsonify({'status': 'access denied'}), 401
site = mongo.db.sites.find_one({'_id': name})
if site == None:
return jsonify({'status': 'invalid repo spec'}), 404
rev = str(git('rev-parse', 'origin/master', _cwd=site['path']))
sudo.git.fetch(_cwd=site['path'])
after_rev = str(git('rev-parse', 'origin/master', _cwd=site['path']))
if rev != after_rev:
git_output, deploy, restart, err = [''] * 4
try:
git_output = str(sudo.git.pull(_cwd=site['path'])).strip()
deploy = str(sh('./deploy.sh', _cwd=site['path'])).strip()
restart = str(sudo('service', name, 'restart'))
except ErrorReturnCode as e:
err = str(e)
print('weird error', err)
output = {
'site': name,
'git_revs': [rev, after_rev],
'git_output': git_output,
'deploy': deploy,
'restart': restart,
'err': err,
}
mongo.db.deploys.insert(output)
output['_id'] = str(output['_id'])
return jsonify(output), 201
return jsonify({
'status': 'same',
'rev': rev or '...',
}), 200
def push(name):
# a terrible CIDR check
if not any([request.remote_addr.startswith(x) for x in ALLOWED_RANGES]):
return jsonify({'status': 'access denied'}), 401
site = mongo.db.sites.find_one({'_id': name})
if site == None:
return jsonify({'status': 'invalid repo spec'}), 404
rev = str(git('rev-parse', 'origin/master', _cwd=site['path']))
sudo.git.fetch(_cwd=site['path'])
after_rev = str(git('rev-parse', 'origin/master', _cwd=site['path']))
if rev != after_rev:
git_output, deploy, restart, err = [''] * 4
try:
git_output = str(sudo.git.pull(_cwd=site['path'])).strip()
deploy = str(sh('./deploy.sh', _cwd=site['path'])).strip()
restart = str(sudo('service', name, 'restart'))
except ErrorReturnCode as e:
err = str(e)
print('weird error', err)
output = {
'site': name,
'git_revs': [rev, after_rev],
'git_output': git_output,
'deploy': deploy,
'restart': restart,
'err': err,
}
mongo.db.deploys.insert(output)
output['_id'] = str(output['_id'])
return jsonify(output), 201
return jsonify({
'status': 'same',
'rev': rev or '...',
}), 200
def test_it_converts_all_wslashes_to_uslashes(self):
path = sh("join", "c:\\my_dir\\my_sub")
self.assertEqual(path, "c:/my_dir/my_sub")
def status():
try:
return sh("git status --porcelain 2> /dev/null")
except ValueError:
return None
for variant_file in os.listdir('simulator/configs'):
log.info("Building {}".format(variant_file))
variant = os.path.basename(variant_file)
build_variant_dir = os.path.join(BUILD_DIR, variant)
mkdir(build_variant_dir)
cp('-r', PRISTINE, build_variant_dir)
with sh.pushd(os.path.join(build_variant_dir, 'simulator')):
tup('init')
tup('generate', 'build.sh', '--config',
os.path.join('configs', variant))
try:
errbuf = io.StringIO()
sh.sh('-x', 'build.sh', _err=errbuf)
except:
errbuf.seek(0)
for l in errbuf.readlines():
print(l, end='')
raise
sh.test('-x', 'simulator')
log.info("Built {}".format(variant_file))
any_fail = False
sim = sh.Command('./simulator')
for test in glob.iglob('tests/**/*.bin', recursive=True):
log.info("\t\ttest: %s", test)
try:
def general_command_run(self, command):
result = sh.sh(command, _bg=True)
result_status = "Success" if result.exit_code() == 0 else "Fail"
return {"status": result_status, "result": result}
#!/usr/bin/python
import json
import base64
import sh
sh.sh("./make-zip.sh")
open('resources/communication-with-storage/request/payload.bin','wb').write(
json.dumps(
dict( __nt_name="ExportToKramerius",
uuid='e65d9072-2c9b-11e5-99fd-b8763f0a3d61',
b64_data=base64.encodestring(open("/tmp/e65d9072-2c9b-11e5-99fd-b8763f0a3d61.zip","rb").read())
)
)
)
def test_it_joins_deep_paths_with_backslashes(self):
path = sh("join", "c:\\my_file", "a\\b\\c\\d")
self.assertEqual("c:/my_file/a/b/c/d", path)
image = "654874881458.dkr.ecr.us-east-1.amazonaws.com/hoopla/pgbouncer"
# Copy configuration from the default to the project folder.
# hoopla-deployment/pgbouncer/ENVIRONMENT/BUILD/*
# ex: hoopla-deployment/pgbouncer/staging/default/*
pgbouncer = 'hoopla-deployment/pgbouncer'
for environment in os.listdir(pgbouncer):
environment_folder = "{}/{}".format(pgbouncer, environment)
for build in os.listdir(environment_folder):
build_folder = '{}/{}'.format(environment_folder, build)
files_copied = []
for configuration_file in os.listdir(build_folder):
configuration_file_path = '{}/{}'.format(build_folder, configuration_file)
shutil.copyfile(configuration_file_path, configuration_file)
files_copied.append(configuration_file)
# Configuration had been copied, build and publish docker image
full_image_name = "{}:{}.{}".format(image, environment, build)
logging.info("Creating and publishing image {}".format(full_image_name))
sh.sh("-c", "sudo docker build -t {} .".format(full_image_name), _out=log_sh)
sh.sh("-c", "sudo docker push {}".format(full_image_name))
logging.info("Deleting temporary configuration files")
for file_copied in files_copied:
os.remove(file_copied)
logging.info("Done!")
def test_it_doesnt_butcher_paths(self):
raw_path = "c:/users/jcsoko~1/appdata/local/temp/tmpkfex8c/test_file.bin"
path = sh("join", raw_path)
self.assertEqual(raw_path, path)
def test_short_option(self):
from sh import sh
s1 = sh(c="echo test").strip()
s2 = "test"
self.assertEqual(s1, s2)
def branch():
"""The current git branch."""
try:
return sh("git symbolic-ref --short HEAD 2> /dev/null").strip()
except ValueError:
return None
def get_current_arch():
return sh(
"-c",
f". {PROJECT_DIR / 'shared-build-context/shared/scripts/functions.sh; normalize_to_docker_arch'}",
).strip()
def handle(self, *args, **options):
session = requests.session()
session.verify = False # internal network, self-signed cert
session.headers['User-Agent'] = 'python-requests/ConfigMaster'
# global lock - remove after Repository integration is done
global_lock = locking.FileLock("/run/configmaster-esxi-backup")
global_lock.acquire()
for name, uri in settings.ESXI_BACKUP:
self.stdout.write("Acquiring lock for %s..." % name)
run_lock = locking.FileLock("/run/esxi-%s" % name)
tempdir = tempfile.mkdtemp('esxi')
self.stdout.write("Lock acquired, running... (%s)" % tempdir)
try:
run_lock.acquire(non_blocking=True)
except IOError:
self.stderr.write(
"A run is already in progress")
return
try:
resp = session.get(uri, timeout=5)
except IOError as e:
self.stderr.write(
"Config backup failure for %s: %s" % (name, str(e)))
continue
if resp.status_code != 200:
self.stderr.write(
"Config backup failed for %s: %r" % (name, resp.content))
continue
stage_1 = os.path.join(tempdir, 'stage1.tgz')
with open(stage_1, 'wb') as f:
f.write(resp.content)
os.chdir(tempdir)
try:
stage1_ret = sh.aunpack('stage1.tgz')
except sh.ErrorReturnCode:
self.stderr.write(
"Config backup failed for %s: failed to unpack stage1"
% name)
continue
if not "\nstate.tgz" in stage1_ret.stdout or not os.path.exists(
'stage1'):
self.stderr.write(
"Config backup failed for %s: invalid stage1" % name)
continue
os.chdir('stage1')
try:
stage2_ret = sh.aunpack('state.tgz')
except sh.ErrorReturnCode:
self.stderr.write(
"Config backup failed for %s: failed to unpack state.tgz"
% name)
continue
if not os.path.exists('local.tgz'):
self.stderr.write(
"Config backup failed for %s: invalid state.tgz, "
"local.tgz missing" % name)
continue
try:
stage3_ret = sh.aunpack('local.tgz')
except sh.ErrorReturnCode:
self.stderr.write(
"Config backup failed for %s: "
"failed to unpack local.tgz" % name)
continue
if not os.path.exists('etc'):
self.stderr.write(
"Config backup failed for %s: "
"invalid local.tgz, etc/ missing" % name)
continue
for path in settings.ESXI_FILE_BLACKLIST:
if os.path.exists(path):
os.unlink(path)
repo_dir = os.path.join(settings.ESXI_BACKUP_REPO, name)
if not os.path.exists(repo_dir):
os.mkdir(repo_dir)
sh.sh('-c', '/usr/bin/find -type d -exec chmod 750 {} \;'.split())
sh.sh('-c', '/usr/bin/find -type d -exec chmod 640 {} \;'.split())
for root, dirs, files in os.walk(repo_dir):
for d in dirs:
os.chmod(os.path.join(root,d), 0o750)
for f in files:
os.chmod(os.path.join(root,f), 0o640)
sh.cp('-r', '.', repo_dir+'/')
os.chdir(repo_dir)
try:
sh.git.add('-u', '.')
sh.git.add('.')
if NetworkDeviceConfigBackupHandler._git_commit(
"ESXi config change (%s)" % name):
sh.git.push()
except sh.ErrorReturnCode as e:
self.stderr.write(
"Git commit or push failed: " + str(e))
continue
sh.cp(
os.path.join(tempdir, 'stage1.tgz'),
os.path.join(settings.ESXI_BACKUP_REPO_RAW, '%s.tgz' % name)
)
os.chdir(settings.ESXI_BACKUP_REPO_RAW)
try:
sh.git.add('-u', '.')
sh.git.add('.')
if NetworkDeviceConfigBackupHandler._git_commit(
"ESXi raw config change (%s)" % name):
sh.git.push()
except sh.ErrorReturnCode as e:
self.stderr.write(
"Git commit or push failed: " + str(e))
continue
shutil.rmtree(tempdir)
run_lock.release()
for path in (settings.ESXI_BACKUP_REPO_RAW, settings.ESXI_BACKUP_REPO):
os.chdir(path)
sh.git.push()
global_lock.release()
self.stdout.write("Run completed.")
def test_it_joins_windows_paths(self):
path = sh("join", "c:/my_dir", "my_file.txt")
self.assertEqual(path, "c:/my_dir/my_file.txt")
que = subprocess.Popen(cmd,stdout=subprocess.PIPE,shell=True).communicate()[0]
queue = json.loads(que)
queue = queue['QueueUrl']
cmd = "aws sqs receive-message --queue-url '"+queue+"'"
done = "aws sqs delete-message --queue-url '"+queue+"' --receipt-handle "
if queue:
report(env+queue)
failed_receive_message = 0
while maxfails == 0 or ( maxfails > 0 and failed_receive_message < maxfails ):
if maxfails > 0:
failed_receive_message += 1
mes = subprocess.Popen(cmd,stdout=subprocess.PIPE,shell=True).communicate()[0]
if not mes:
continue
messages = json.loads(mes)
for message in messages['Messages']:
body = urllib.unquote_plus(message['Body']).replace('\n',' ').replace('\r','')
rcpt = "'"+message['ReceiptHandle']+"'"
try:
sh.sh('-c',env+body,_tty_in=True,_out=job+'/out.txt',_err=job+'/err.txt')
subprocess.call(done+rcpt,shell=True)
failed_receive_message = 0
except Exception as e:
report(str(e) + ': cannot service '+body)
if maxfails < 0:
break
if failed_receive_message > 0:
report('giving up... nothing received after '+str(failed_receive_message)+' trys')
def execPort(self, code): sh(self.__cmd, self.__timeout) return ERoutingResult.STOP_OK
def setup_canopy(ip_address):
ssh = digitalocean.get_ssh("root", ip_address, process_out(ip_address))
# log("creating primary virtual environment")
# ssh("/root/python/bin/python3 -m venv /root/canopy")
# ssh(
# "cat > /root/runinenv",
# stdin=textwrap.dedent(
# """\
# #!/usr/bin/env bash
# VENV=$1
# . ${VENV}/bin/activate
# shift 1
# exec "$@"
# deactivate"""
# ),
# )
ssh("chmod", "+x", "/root/runinenv")
log("installing canopy")
ssh("/root/runinenv", "/root/canopy", "pip", "install", "canopy-platform")
return
# if self.env_dir.exists():
# return
# log("creating primary virtual environment")
get_python_sh()("-m", "venv", self.env_dir)
sh.echo(
textwrap.dedent("""\
#!/usr/bin/env bash
VENV=$1
. ${VENV}/bin/activate
shift 1
exec "$@"
deactivate"""),
_out=f"{self.home_dir}/runinenv",
)
sh.chmod("+x", self.home_dir / "runinenv")
# log("installing SQLite")
# log(" ", "downloading")
# sh.wget("https://www.sqlite.org/src/tarball/sqlite.tar.gz", _cwd=self.src_dir)
# log(" ", "extracting")
# sh.tar("xf", "sqlite.tar.gz", _cwd=self.src_dir)
# sqlite_dir = self.src_dir / "sqlite"
# log(" ", "configuring")
# sh.bash("./configure", _cwd=sqlite_dir)
# sh.make("sqlite3.c", _cwd=sqlite_dir)
# sh.git("clone", "https://github.com/coleifer/pysqlite3", _cwd=self.src_dir)
# pysqlite_dir = self.src_dir / "pysqlite3"
# sh.cp(sqlite_dir / "sqlite3.c", ".", _cwd=pysqlite_dir)
# sh.cp(sqlite_dir / "sqlite3.h", ".", _cwd=pysqlite_dir)
# sh.sh(
# self.home_dir / "runinenv",
# self.env_dir,
# "python",
# "setup.py",
# "build_static",
# _cwd=pysqlite_dir,
# )
# sh.sh(
# self.home_dir / "runinenv",
# self.env_dir,
# "python",
# "setup.py",
# "install",
# _cwd=pysqlite_dir,
# )
log("installing Gaea")
sh.sh(
self.home_dir / "runinenv",
self.env_dir,
"pip",
"install",
"libgaea",
)
log("installing Poetry")
get_python_sh()(
sh.wget(
"https://raw.githubusercontent.com/python-poetry/poetry"
"/master/install-poetry.py",
"-q",
"-O",
"-",
),
"-",
)
for variant_file in os.listdir('simulator/configs'):
log.info("Building {}".format(variant_file))
variant = os.path.basename(variant_file)
build_variant_dir = os.path.join(BUILD_DIR, variant)
mkdir(build_variant_dir)
cp('-r', PRISTINE, build_variant_dir)
with sh.pushd(os.path.join(build_variant_dir, 'simulator')):
tup('init')
tup('generate', 'build.sh', '--config', os.path.join('configs', variant))
try:
errbuf = io.StringIO()
sh.sh('-x', 'build.sh', _err=errbuf)
except:
errbuf.seek(0)
for l in errbuf.readlines():
print(l, end='')
raise
sh.test('-x', 'simulator')
log.info("Built {}".format(variant_file))
any_fail = False
sim = sh.Command('./simulator')
for test in glob.iglob('tests/**/*.bin', recursive=True):
log.info("\t\ttest: %s", test)
try:
import sh
sh.cd ("ext/deps/")
sh.wget ("http://download.nanomsg.org/nanomsg-0.3-beta.tar.gz")
sh.tar ("xzf", "nanomsg-0.3-beta.tar.gz")
sh.cd ("nanomsg-0.3-beta")
sh.sh ("configure")
sh.make ()
def run(input_file: KGTKFiles,
output_file: KGTKFiles,
columns: typing.Optional[typing.List[str]] = None,
reverse: bool = False,
extra: typing.Optional[str] = None,
verbose: bool = False) -> int:
import os
from pathlib import Path
import sh # type: ignore
import sys
from kgtk.io.kgtkreader import KgtkReader
input_path: Path = KGTKArgumentParser.get_input_file(input_file)
output_path: Path = KGTKArgumentParser.get_output_file(output_file)
error_file = sys.stderr
try:
header_read_fd: int
header_write_fd: int
header_read_fd, header_write_fd = os.pipe()
os.set_inheritable(header_write_fd, True)
if verbose:
print("header pipe: read_fd=%d write_fd=%d" %
(header_read_fd, header_write_fd),
file=error_file,
flush=True)
sortopt_read_fd: int
sortopt_write_fd: int
sortopt_read_fd, sortopt_write_fd = os.pipe()
os.set_inheritable(sortopt_read_fd, True)
if verbose:
print("sort options pipe: read_fd=%d write_fd=%d" %
(sortopt_read_fd, sortopt_write_fd),
file=error_file,
flush=True)
cmd: str = ""
if str(input_path) != "-":
# Feed the named file into the data processing pipeline,
# otherwise read from standard input.
cmd += "cat " + repr(str(input_path)) + " | "
cmd += " { IFS= read -r header ; " # Read the header line
cmd += " { printf \"%s\\n\" \"$header\" >&" + str(
header_write_fd) + " ; } ; " # Send the header to Python
cmd += " printf \"%s\\n\" \"$header\" ; " # Send the header to standard output (which may be redirected to a file, below).
cmd += " IFS= read -u " + str(
sortopt_read_fd
) + " -r options ; " # Read the sort command options from Python.
cmd += " LC_ALL=C sort -t '\t' $options ; } " # Sort the remaining input lines using the options read from Python.
if str(output_path) != "-":
# Feed the sorted output to the named file. Otherwise, the sorted
# output goes to standard output without passing through Python.
cmd += " > " + repr(str(output_path))
if verbose:
print("sort command: %s" % cmd, file=error_file, flush=True)
# Sh version 1.13 or greater is required for _pass_fds.
proc = sh.sh("-c",
cmd,
_in=sys.stdin,
_out=sys.stdout,
_err=sys.stderr,
_bg=True,
_pass_fds={header_write_fd, sortopt_read_fd})
if verbose:
print("Reading the KGTK input file header line with KgtkReader",
file=error_file,
flush=True)
kr: KgtkReader = KgtkReader.open(Path("<%d" % header_read_fd))
if verbose:
print("KGTK header: %s" % " ".join(kr.column_names),
file=error_file,
flush=True)
sort_options: str = ""
if reverse:
sort_options += " --reverse"
if extra is not None and len(extra) > 0:
sort_options += " " + extra
sort_idx: int
if columns is not None and len(columns) > 0:
# Process the list of column names, including splitting
# comma-separated lists of column names.
column_name: str
for column_name in columns:
column_name_2: str
for column_name_2 in column_name.split(","):
column_name_2 = column_name_2.strip()
if len(column_name_2) == 0:
continue
if column_name_2.isdigit():
sort_idx = int(column_name_2)
if sort_idx > len(kr.column_names):
proc.kill_group()
raise KGTKException(
"Invalid column number %d (max %d)." %
(sort_idx, len(kr.column_names)))
else:
if column_name_2 not in kr.column_names:
proc.kill_group()
raise KGTKException("Unknown column_name %s" %
column_name_2)
sort_idx = kr.column_name_map[column_name_2] + 1
sort_options += " -k %d,%d" % (sort_idx, sort_idx)
else:
if kr.is_node_file:
sort_idx = kr.id_column_idx + 1
sort_options += " -k %d,%d" % (sort_idx, sort_idx)
elif kr.is_edge_file:
if kr.id_column_idx >= 0:
sort_idx = kr.id_column_idx + 1
sort_options += " -k %d,%d" % (sort_idx, sort_idx)
sort_idx = kr.node1_column_idx + 1
sort_options += " -k %d,%d" % (sort_idx, sort_idx)
sort_idx = kr.label_column_idx + 1
sort_options += " -k %d,%d" % (sort_idx, sort_idx)
sort_idx = kr.node2_column_idx + 1
sort_options += " -k %d,%d" % (sort_idx, sort_idx)
else:
proc.kill_group()
raise KGTKException(
"Unknown KGTK file mode, please specify the sorting columns."
)
if verbose:
print("sort options: %s" % sort_options,
file=error_file,
flush=True)
kr.close() # We are done with the KgtkReader now.
# Send the sort options back to the data processing pipeline.
with open(sortopt_write_fd, "w") as options_file:
options_file.write(sort_options + "\n")
proc.wait()
# Clean up the file descriptors, just in case.
#
# Note: Should close these when we raise an exception, too.
try:
os.close(header_read_fd)
except os.error:
pass
try:
os.close(header_write_fd)
except os.error:
pass
try:
os.close(sortopt_read_fd)
except os.error:
pass
try:
os.close(sortopt_write_fd)
except os.error:
pass
return 0
except Exception as e:
#import traceback
#traceback.print_tb(sys.exc_info()[2], 10)
raise KGTKException('INTERNAL ERROR: ' + type(e).__module__ + '.' +
str(e) + '\n')
def main():
nupdated = 0
if os.path.exists("wip"):
print(
"In-progress commits detected: wip/ exists. Please resolve manually."
)
sys.exit(1)
override_commit = None
if "--override" in sys.argv:
override_commit = sys.argv[sys.argv.index("--override") + 1]
if "-m" in sys.argv:
override_msg = sys.argv[sys.argv.index("-m") + 1]
for repo, proj in PROJECTS.items():
osc = proj["cmd"]
commit = override_commit or proj["version-tag"]
if "--fetch-changes" in sys.argv:
f, filename = tempfile.mkstemp('.txt', text=True)
tf = os.fdopen(f, "w")
fetch_changelog(osc, proj["version-tag"], tf)
tf.close()
print(open(filename).read())
os.remove(filename)
sys.exit(0)
try:
tarball = osc.api(
"-X", "GET",
"/source/{}/{}/update-tarball.sh".format(repo, PACKAGE))
m = re.search('ROOK_REV="(.+)"', tarball.stdout.decode('utf-8'))
if m and m.group(1) == commit:
continue
except sh.ErrorReturnCode as err:
print("Error code {}, skipping {}/{}...".format(
err.exit_code, repo, PACKAGE))
continue
print("Updating {}/{} to version {}...".format(repo, PACKAGE, commit))
curr = os.getcwd()
try:
wip = os.path.join(curr, "wip")
try:
os.mkdir(wip)
except os.error:
pass
os.chdir(wip)
osc("bco", repo, PACKAGE)
os.chdir(os.path.join(wip, BRANCHBASE.format(repo), PACKAGE))
if proj["version-tag"] == commit:
update_changelog(osc, proj["version-tag"])
else:
update_changelog_with_message(
osc, override_msg or "Update to commit {}".format(commit))
update_tarball(commit)
for toremove in glob.glob('./rook-*.xz'):
print("Deleting {}...".format(toremove))
os.remove(toremove)
print(sh.sh("./update-tarball.sh"))
print(osc.ar())
print(osc.commit("-m", "Update to version {}:".format(commit)))
nupdated = nupdated + 1
finally:
os.chdir(curr)
if nupdated > 0:
print("{} updated projects now in:\nwip".format(nupdated))
def test_joining_a_dot_path(self):
path = sh("join", "./abc", "def")
self.assertEqual(path, "./abc/def")
