def WriteRootCaCerts(client):
"""Write the internal root CA certs to a file."""
logging.debug('WriteRootCaCerts')
managed_installs_dir = munkicommon.pref('ManagedInstallDir')
certs_dir = os.path.join(managed_installs_dir, 'certs')
cert_file_path = os.path.join(certs_dir, 'ca.pem')
if not os.path.isdir(certs_dir):
os.makedirs(certs_dir)
tmpfile = tempfile.NamedTemporaryFile(
dir=os.path.dirname(os.path.realpath(cert_file_path)))
tmpfile.write(client.GetSystemRootCACertChain())
logging.debug('WriteRootCaCerts: writing to tmp %s', tmpfile.name)
try:
os.unlink(cert_file_path)
except OSError:
pass
try:
os.link(tmpfile.name, cert_file_path)
except OSError, e:
tmpfile.close()
raise client.Error('Error writing root CA certs: %s' % str(e))
def revert(self):
if self.__state != self.__STATE['APPLIED']:
raise Error("Attempt to revert configuration from state %s" % self.__state)
for child in self.__children:
child.revert()
log("Reverting changes to %s configuration" % self.__path)
# Remove existing new configuration
if os.access(self.__newpath, os.F_OK):
os.unlink(self.__newpath)
# Revert new configuration.
if os.access(self.__path, os.F_OK):
os.link(self.__path, self.__newpath)
os.unlink(self.__path)
# Revert to old configuration.
if os.access(self.__oldpath, os.F_OK):
os.link(self.__oldpath, self.__path)
os.unlink(self.__oldpath)
# Leave .*.xapi-new as an aid to debugging.
self.__state = self.__STATE['REVERTED']
def acquire(self, timeout=None):
try:
open(self.unique_name, "wb").close()
except IOError:
raise LockFailed("failed to create %s" % self.unique_name)
end_time = time.time()
if timeout is not None and timeout > 0:
end_time += timeout
while True:
# Try and create a hard link to it.
try:
os.link(self.unique_name, self.lock_file)
except OSError:
# Link creation failed. Maybe we've double-locked?
nlinks = os.stat(self.unique_name).st_nlink
if nlinks == 2:
# The original link plus the one I created == 2. We're
# good to go.
return
else:
# Otherwise the lock creation failed.
if timeout is not None and time.time() > end_time:
os.unlink(self.unique_name)
if timeout > 0:
raise LockTimeout
else:
raise AlreadyLocked
time.sleep(timeout is not None and timeout/10 or 0.1)
else:
# Link creation succeeded. We're good to go.
return
def _copy_game_file(self, src, dst):
ifs = self.open(src, prefer_path=True)
if not ifs:
raise NotFoundError("Could not find file for {0}".format(src))
if os.path.exists(dst):
print("removing existing file", dst)
os.remove(dst)
if isinstance(ifs, str):
# we got a direct path
try:
os.link(ifs, dst)
return
except Exception:
# couldn't link file, normal on non-unix systems and also
# if the files are on different file systems
pass
shutil.copyfile(ifs, dst)
else:
dst_partial = dst + ".partial"
with open(dst_partial, "wb") as ofs:
# ifs_sha1 = hashlib.sha1()
while True:
# noinspection PyUnresolvedReferences
data = ifs.read()
if not data:
break
# ifs_sha1.update(data)
ofs.write(data)
print("rename file from", dst_partial, "to", dst)
os.rename(dst_partial, dst)
def apply(self):
if self.__state != self.__STATE['NOT-APPLIED']:
raise Error("Attempt to apply configuration from state %s" % self.__state)
for child in self.__children:
child.apply()
log("Applying changes to %s configuration" % self.__path)
# Remove previous backup.
if os.access(self.__oldpath, os.F_OK):
os.unlink(self.__oldpath)
# Save current configuration.
if os.access(self.__path, os.F_OK):
os.link(self.__path, self.__oldpath)
os.unlink(self.__path)
# Apply new configuration.
assert(os.path.exists(self.__newpath))
os.link(self.__newpath, self.__path)
# Remove temporary file.
os.unlink(self.__newpath)
self.__state = self.__STATE['APPLIED']
def test_clean_stale_with_threads_active(self):
"""verify locks for multiple threads are cleaned up """
# create sentinels for four threads in our process, and a 'dead'
# process
sentinel1 = self._create_sentinel(self.hostname, self.pid, 'Default-1')
sentinel2 = self._create_sentinel(self.hostname, self.pid, 'Default-2')
sentinel3 = self._create_sentinel(self.hostname, self.pid, 'Default-3')
sentinel4 = self._create_sentinel(self.hostname, self.pid, 'Default-4')
sentinel5 = self._create_sentinel(self.hostname, self.dead_pid,
'Default-1')
os.link(sentinel1, self.lock_file)
utils.cleanup_file_locks()
self.assertTrue(os.path.exists(sentinel1))
self.assertTrue(os.path.exists(sentinel2))
self.assertTrue(os.path.exists(sentinel3))
self.assertTrue(os.path.exists(sentinel4))
self.assertTrue(os.path.exists(self.lock_file))
self.assertFalse(os.path.exists(sentinel5))
os.unlink(sentinel1)
os.unlink(sentinel2)
os.unlink(sentinel3)
os.unlink(sentinel4)
os.unlink(self.lock_file)
def hardLink(self, source, target):
try:
os.link(source, target)
return True
except OSError as (t, e):
qDebug("FileSystem.hardLink - %s: '%s' -> '%s'" % (e, source, target))
return False
def tst_link(mnt_dir):
name1 = pjoin(mnt_dir, name_generator())
name2 = pjoin(mnt_dir, name_generator())
shutil.copyfile(TEST_FILE, name1)
assert filecmp.cmp(name1, TEST_FILE, False)
fstat1 = os.lstat(name1)
assert fstat1.st_nlink == 1
os.link(name1, name2)
fstat1 = os.lstat(name1)
fstat2 = os.lstat(name2)
for attr in ('st_mode', 'st_dev', 'st_uid', 'st_gid',
'st_size', 'st_atime', 'st_mtime', 'st_ctime'):
assert getattr(fstat1, attr) == getattr(fstat2, attr)
assert os.path.basename(name2) in os.listdir(mnt_dir)
assert filecmp.cmp(name1, name2, False)
os.unlink(name2)
assert os.path.basename(name2) not in os.listdir(mnt_dir)
with pytest.raises(FileNotFoundError):
os.lstat(name2)
os.unlink(name1)
def fly_CopterAVC(viewerip=None, map=False):
'''fly ArduCopter in SIL for AVC2013 mission
'''
global homeloc
if TARGET != 'sitl':
util.build_SIL('ArduCopter', target=TARGET)
home = "%f,%f,%u,%u" % (AVCHOME.lat, AVCHOME.lng, AVCHOME.alt, AVCHOME.heading)
sil = util.start_SIL('ArduCopter', wipe=True, model='heli', home=home, speedup=speedup_default)
mavproxy = util.start_MAVProxy_SIL('ArduCopter', options='--sitl=127.0.0.1:5501 --out=127.0.0.1:19550')
mavproxy.expect('Received [0-9]+ parameters')
# setup test parameters
mavproxy.send("param load %s/Helicopter.parm\n" % testdir)
mavproxy.expect('Loaded [0-9]+ parameters')
# reboot with new parameters
util.pexpect_close(mavproxy)
util.pexpect_close(sil)
sil = util.start_SIL('ArduCopter', model='heli', home=home, speedup=speedup_default)
options = '--sitl=127.0.0.1:5501 --out=127.0.0.1:19550 --streamrate=5'
if viewerip:
options += ' --out=%s:14550' % viewerip
if map:
options += ' --map'
mavproxy = util.start_MAVProxy_SIL('ArduCopter', options=options)
mavproxy.expect('Telemetry log: (\S+)')
logfile = mavproxy.match.group(1)
print("LOGFILE %s" % logfile)
buildlog = util.reltopdir("../buildlogs/CopterAVC-test.tlog")
print("buildlog=%s" % buildlog)
if os.path.exists(buildlog):
os.unlink(buildlog)
try:
os.link(logfile, buildlog)
except Exception:
pass
# the received parameters can come before or after the ready to fly message
mavproxy.expect(['Received [0-9]+ parameters', 'Ready to FLY'])
mavproxy.expect(['Received [0-9]+ parameters', 'Ready to FLY'])
util.expect_setup_callback(mavproxy, expect_callback)
expect_list_clear()
expect_list_extend([sil, mavproxy])
if map:
mavproxy.send('map icon 40.072467969730496 -105.2314389590174\n')
mavproxy.send('map icon 40.072600990533829 -105.23146100342274\n')
# get a mavlink connection going
try:
mav = mavutil.mavlink_connection('127.0.0.1:19550', robust_parsing=True)
except Exception, msg:
print("Failed to start mavlink connection on 127.0.0.1:19550" % msg)
raise
def test_dont_copy_file_onto_link_to_itself(self):
# bug 851123.
os.mkdir(TESTFN)
src = os.path.join(TESTFN, 'cheese')
dst = os.path.join(TESTFN, 'shop')
try:
f = open(src, 'w')
f.write('cheddar')
f.close()
os.link(src, dst)
self.assertRaises(shutil.Error, shutil.copyfile, src, dst)
self.assertEqual(open(src,'r').read(), 'cheddar')
os.remove(dst)
# Using `src` here would mean we end up with a symlink pointing
# to TESTFN/TESTFN/cheese, while it should point at
# TESTFN/cheese.
os.symlink('cheese', dst)
self.assertRaises(shutil.Error, shutil.copyfile, src, dst)
self.assertEqual(open(src,'r').read(), 'cheddar')
os.remove(dst)
finally:
try:
shutil.rmtree(TESTFN)
except OSError:
pass
def link(src, dst):
if os.name == 'nt':
import ctypes
if ctypes.windll.kernel32.CreateHardLinkW(unicode(dst), unicode(src), 0) == 0: raise ctypes.WinError()
else:
os.link(src, dst)
def import_files(cls, from_dir, to_dir, unlink_after):
"""
Import audio files from from_dir into to_dir
"""
audio_files = cls.get_filelist(from_dir)
for audio_file in audio_files:
from_path = audio_file.filename
to_path = os.path.join(to_dir, audio_file.destination)
makedirs(os.path.split(to_path)[0])
try:
if on_same_partition(from_path, os.path.dirname(to_path)):
# Hardlink here because we may not have write permission on the
# source, so a move would fail, but we still want it put into the library.
# It also saves space if we're not deleting the file after
os.link(from_path, to_path)
else:
shutil.copy(from_path, to_path)
except OSError as e:
if e.errno == errno.EEXIST:
print "Warning: Target file already exists: %s. Skipping" % to_path
else:
raise
if unlink_after:
try:
os.unlink(from_path)
except OSError as e:
print "Warning: Caught error trying to unlink %s: %s. Ignoring" % (from_path, str(e))
pass
def Main(argv):
# Check number of parameters
if len(argv) != 3:
print("The function requires two parameters to be passed in.")
return
# Check parameters
dirA = argv[1]
if not os.path.isdir(dirA):
print("The first parameter should be an existing directory.")
return
dirB = argv[2]
if not os.path.isdir(dirB):
print("The second parameter should be an existing directory.")
return
# Build a dictionary with key-value pair { file name - True }
nameDict = { }
for fileName in os.listdir(dirB):
filePath = os.path.join(dirB, fileName)
if os.path.isfile(filePath):
nameDict[fileName] = True
for fileName in os.listdir(dirA):
filePathA = os.path.join(dirA, fileName)
filePathB = os.path.join(dirB, fileName)
if not nameDict.get(fileName, False):
os.link(filePathA, filePathB)
elif os.path.getmtime(filePathA) > os.path.getmtime(filePathB):
os.remove(filePathB)
os.link(filePathA, filePathB)
def extract_item(self, item, restore_attrs=True, dry_run=False):
if dry_run:
if b'chunks' in item:
for _ in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True):
pass
return
dest = self.cwd
if item[b'path'].startswith('/') or item[b'path'].startswith('..'):
raise Exception('Path should be relative and local')
path = os.path.join(dest, item[b'path'])
# Attempt to remove existing files, ignore errors on failure
try:
st = os.lstat(path)
if stat.S_ISDIR(st.st_mode):
os.rmdir(path)
else:
os.unlink(path)
except OSError:
pass
mode = item[b'mode']
if stat.S_ISDIR(mode):
if not os.path.exists(path):
os.makedirs(path)
if restore_attrs:
self.restore_attrs(path, item)
elif stat.S_ISREG(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# Hard link?
if b'source' in item:
source = os.path.join(dest, item[b'source'])
if os.path.exists(path):
os.unlink(path)
os.link(source, path)
else:
with open(path, 'wb') as fd:
ids = [c[0] for c in item[b'chunks']]
for data in self.pipeline.fetch_many(ids, is_preloaded=True):
fd.write(data)
fd.flush()
self.restore_attrs(path, item, fd=fd.fileno())
elif stat.S_ISFIFO(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
os.mkfifo(path)
self.restore_attrs(path, item)
elif stat.S_ISLNK(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
source = item[b'source']
if os.path.exists(path):
os.unlink(path)
os.symlink(source, path)
self.restore_attrs(path, item, symlink=True)
elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
os.mknod(path, item[b'mode'], item[b'rdev'])
self.restore_attrs(path, item)
else:
raise Exception('Unknown archive item type %r' % item[b'mode'])
def copy(src, dst, hardlink=False, keep_symlink=True):
assert not P.isdir(src), 'Source path must not be a dir'
assert not P.isdir(dst), 'Destination path must not be a dir'
if keep_symlink and P.islink(src):
assert not P.isabs(readlink(src)), 'Cannot copy symlink that points to an absolute path (%s)' % src
logger.debug('%8s %s -> %s' % ('symlink', src, dst))
if P.exists(dst):
assert readlink(dst) == readlink(src), 'Refusing to retarget already-exported symlink %s' % dst
else:
symlink(readlink(src), dst)
return
if P.exists(dst):
assert hash_file(src) == hash_file(dst), 'Refusing to overwrite already exported dst %s' % dst
else:
if hardlink:
try:
link(src, dst)
logger.debug('%8s %s -> %s' % ('hardlink', src, dst))
return
except OSError, o:
if o.errno != errno.EXDEV: # Invalid cross-device link, not an error, fall back to copy
raise
logger.debug('%8s %s -> %s' % ('copy', src, dst))
shutil.copy2(src, dst)
def download(self, image_href, image_file):
"""Downloads image to specified location.
:param image_href: Image reference.
:param image_file: File object to write data to.
:raises: exception.ImageRefValidationFailed if source image file
doesn't exist.
:raises: exception.ImageDownloadFailed if exceptions were raised while
writing to file or creating hard link.
"""
source_image_path = self.validate_href(image_href)
dest_image_path = image_file.name
local_device = os.stat(dest_image_path).st_dev
try:
# We should have read and write access to source file to create
# hard link to it.
if (local_device == os.stat(source_image_path).st_dev and
os.access(source_image_path, os.R_OK | os.W_OK)):
image_file.close()
os.remove(dest_image_path)
os.link(source_image_path, dest_image_path)
else:
filesize = os.path.getsize(source_image_path)
with open(source_image_path, 'rb') as input_img:
sendfile.sendfile(image_file.fileno(), input_img.fileno(),
0, filesize)
except Exception as e:
raise exception.ImageDownloadFailed(image_href=image_href,
reason=e)
def paste_hardlink(self):
for f in self.copy_buffer:
try:
new_name = next_available_filename(f.basename)
link(f.path, join(getcwd(), new_name))
except Exception as x:
self.notify(x)
def rename_no_overwrite(source_path, dest_path):
# link will fail with EEXIST if there is already something at dest_path.
os.link(source_path, dest_path)
try:
os.unlink(source_path)
except EnvironmentError:
reraise(UnableToUnlinkReplacementError)
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# 2600-01-01 > 2**64 ns
os.utime('input/empty', (19880895600, 19880895600))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File owner
os.chown('input/file1', 100, 200)
# File mode
os.chmod('input/file1', 0o7755)
os.chmod('input/dir2', 0o555)
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled():
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
def hard_link_dir(self, path, destination, existing):
if not os.path.exists(destination):
try:
os.makedirs(destination)
except OSError as e:
# Raised when it already exists, but are there other cases?
log.debug('Failed to create destination dir %s: %s', destination, e)
# 'recursively' traverse and hard link
working_dir = os.getcwd()
os.chdir(path) # change working dir to make dir joins easier
for root, dirs, files in os.walk('.'):
dst_dir = os.path.abspath(os.path.join(destination, root))
for d in dirs:
try:
os.mkdir(d)
except OSError as e:
# Raised when it already exists, but are there other cases?
log.debug('Failed to create subdir %s: %s', d, e)
for f in files:
src_file = os.path.join(root, f)
dst_file = os.path.join(dst_dir, f)
log.debug('Hardlinking %s to %s', src_file, dst_file)
try:
os.link(src_file, dst_file)
except OSError as e:
log.debug('Failed to create hardlink for file %s: %s', f, e)
if existing == 'fail':
raise # reraise to fail the entry in the calling function
os.chdir(working_dir)
def test_subfiles(self):
p = lambda x: os.path.join(self.tmp, x)
def create(name, text):
a = open(p(name), 'w')
a.write(text)
a.flush()
return a
def check(name, text):
path = next(notified)
self.assertEqual(path, p(name))
with open(path) as f:
self.assertEqual(f.read(), text)
a = create('first', 'blah')
a.write('...')
notified = inotify.subfiles(self.tmp)
check('first', 'blah')
os.link(p(a.name), p('a hard link')) # ignored
b = create('other', 'hello')
b.close()
check('other', 'hello')
c = create('last', '!!!')
a.close()
check('first', 'blah...')
os.rename(p(a.name), p(b.name))
check('other', 'blah...')
c.close()
check('last', '!!!')
def my_link(src, dest):
dest = destpath(src, dest)
try:
os.link(src, dest)
except OSError, e:
e.filename = src + " -> " + dest
raise
def opensocket(self):
sockname = self.options.sockname
tempname = "%s.%d" % (sockname, os.getpid())
self.unlink_quietly(tempname)
while 1:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
sock.bind(tempname)
os.chmod(tempname, 0o700)
try:
os.link(tempname, sockname)
break
except os.error:
# Lock contention, or stale socket.
self.checkopen()
# Stale socket -- delete, sleep, and try again.
msg = "Unlinking stale socket %s; sleep 1" % sockname
sys.stderr.write(msg + "\n")
sys.stderr.flush() # just in case
self.logger.warn(msg)
self.unlink_quietly(sockname)
sock.close()
time.sleep(1)
continue
finally:
self.unlink_quietly(tempname)
sock.listen(1)
sock.setblocking(0)
self.mastersocket = sock
def unify(src, dst):
""" Unify destination and source """
# NOTE: at this point it is assumed files are unifiable
# get a temp file name
dir = os.path.split(src)[0]
tmp_handle, tmp_path = tempfile.mkstemp(dir=dir)
os.close(tmp_handle)
# rename the destination, in case we need to back out
os.rename(dst, tmp_path)
# link source to destination
try:
os.link(src, dst)
except:
# back out
print "Could not link %s -> %s, backing out" % (src, dst)
try:
if os.path.exists(dst):
os.unlink(dst)
os.rename(tmp_path, dst)
except:
print "Could not back out!!! the destination file is still there as", tmp_file
raise exceptions.OSError
# done, remove the temp file
os.unlink(tmp_path)
def render_task(arg):
"""
This is the worker task run on a sub-process,
it needs TARG and j2env set properly (done inside render())
"""
fn, root = arg
src = join(root, fn)
dst = normpath(join("..", TARG, src))
lvl = root.count(os.sep)
#log("processing/f: %s" % src, nl=False)
if fn.endswith(".html"):
# we ignore html files starting with "_" (e.g. language specific templates)
# except the ones in doc, which might be __init__.html
if fn.startswith("_") and fn != "__init__.html":
return
# assume it's a template and process it
tmpl = j2env.get_template(src)
c = fn.rsplit(".", 1)[0].split("-", 1)[0]
content = tmpl.render(level=lvl, filename=fn, category=c)
with open(dst, "wb") as output:
output.write(content.encode("utf-8"))
output.write(b"\n")
elif islink(src):
# we have a symlink, copy it
# log("SYMLINK/files %s" % src)
if islink(dst):
os.remove(dst)
os.symlink(os.readlink(src), dst)
else:
# all other files, hardlink them
# log("hardlink %s -> %s" % (src, dst))
if exists(dst):
os.remove(dst)
os.link(src, dst)
def dump(self,outPath=None,overwrite=False,saveBackup=True):
"""Write the file back to the disk.
If outPath is given, the file is saved under that name.
If no outPath is given, and overwirte is True, the file is
saved under the same name and path that it was loaded from;
if save backup is true a backup copy (.bak) is made first.
"""
if type(outPath) == StringType:
if os.path.exists(outPath) and saveBackup:
bakName = outPath+'.bak'
if os.path.exists(bakName):
os.unlink(bakName)
os.link(outPath,bakName)
os.unlink(outPath)
file(outPath,'w').write(str(self))
else:
if overwrite:
outPath=self.rawFile.name
if not self.rawFile.closed:
self.rawFile.close()
self.dump(outPath=outPath,overwrite=overwrite,saveBackup=saveBackup)
else:
warnings.warn ("dump: must specify outPath or set overwrite to True")
def copyfile(src, dst, overwrite=False, link=False):
"""
Copy a file from `src` to `dst`; return `True` if the copy was
actually made. If `overwrite` is `False` (default), an existing
destination entry is left unchanged and `False` is returned.
If `link` is `True`, an attempt at hard-linking is done first;
failing that, we copy the source file onto the destination
one. Permission bits and modification times are copied as well.
If `dst` is a directory, a file with the same basename as `src` is
created (or overwritten) in the directory specified.
"""
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
if os.path.exists(dst) and not overwrite:
return False
if samefile(src, dst):
return False
try:
dstdir = os.path.dirname(dst)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
if link:
try:
os.link(src, dst)
except OSError, ex:
# retry with normal copy
shutil.copy2(src, dst)
else:
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regual_file('empty', size=0)
self.create_regual_file('file1', size=1024 * 80)
# Directory
self.create_regual_file('dir2/file2', size=1024 * 80)
# File owner
os.chown('input/file1', 100, 200)
# File mode
os.chmod('input/file1', 0o7755)
os.chmod('input/dir2', 0o555)
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
if xattr.is_enabled():
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
def linux_fixup_databasedir():
''' Under Linux move database from /var/neubot to /var/lib/neubot '''
# Explanation: /var/lib/neubot is FHS, /var/neubot isn't
if os.name != 'posix':
return
if not sys.platform.startswith('linux'):
return
if os.getuid() != 0:
return
if not os.path.isfile('/var/neubot/database.sqlite3'):
return
if os.path.exists('/var/lib/neubot/database.sqlite3'):
return
logging.debug('database_xxx: /var/neubot -> /var/lib/neubot...')
# Lazy import
from neubot import utils_posix
#
# Here we create the new link as root, and we assume that
# the caller will fix permissions afterwards. This should
# happen as long as we are invoked before the database
# function that checks database path.
#
utils_posix.mkdir_idempotent('/var/lib/neubot')
os.link('/var/neubot/database.sqlite3', '/var/lib/neubot/database.sqlite3')
os.unlink('/var/neubot/database.sqlite3')
logging.debug('database_xxx: /var/neubot -> /var/lib/neubot... done')
def __init__(self):
self._uuid = str(uuid.uuid4())
self.hadoop_cmd_file = tmp_path + "." + self._uuid
abs_dir = os.path.dirname(os.path.abspath(__file__))
self.mock_hadoop_client_path = os.path.join(abs_dir, "mock_hadoop_client.py_" + self._uuid)
# os.path.abspath(__file__) could be mock_hadoop_client.pyc
os.link(os.path.join(abs_dir, "mock_hadoop_client.py"), self.mock_hadoop_client_path)
('=f', 'v2', [-1, (161, -1), 0, 0], 1.0, 'ricker1', 2.0),
]
# read mesh from disk
fieldio += [
('=R', 'x1', [0, 0, 1, 0], 'x.bin'),
('=R', 'x2', [0, 0, 1, 0], 'y.bin'),
]
# specify output
for c in '12':
fieldio += [
('=w', 'u' + c, [-1, -1, 1, 0], 'source-u%s.bin' % c),
('=w', 'u' + c, [1, 0, 1, 0], 'canyon-u%s.bin' % c),
('=w', 'u' + c, [(2, 158), 1, 1, 0], 'flank-u%s.bin' % c),
('=w', 'v' + c, [0, 0, 1, (1, -1, 10)], 'snap-v%s.bin' % c),
('=w', 'u' + c, [0, 0, 1, (1, -1, 10)], 'snap-u%s.bin' % c),
]
# continue if command line
if __name__ == '__main__':
# stage job, copy mesh files, and run job
rundir = os.path.join('run', 'sim')
job = cst.sord.stage(locals())
for f in 'x.bin', 'y.bin':
a = os.path.join('run', 'mesh', f)
b = os.path.join('run', 'sim', f)
os.link(a, b)
cst.sord.run(job)
def hardlink_scons(self, src, dst, ver):
try:
os.unlink(dst)
except OSError:
pass
os.link(ver, dst)
def link(self, target, name):
return os.link(self._full_path(target), self._full_path(name))
def main():
module = AnsibleModule(
argument_spec=dict(
state=dict(choices=[
'file', 'directory', 'link', 'hard', 'touch', 'absent'
],
default=None),
path=dict(aliases=['dest', 'name'], required=True, type='path'),
original_basename=dict(
required=False), # Internal use only, for recursive ops
recurse=dict(default=False, type='bool'),
force=dict(required=False, default=False, type='bool'),
follow=dict(required=False, default=False, type='bool'),
diff_peek=dict(
default=None
), # Internal use only, for internal checks in the action plugins
validate=dict(
required=False,
default=None), # Internal use only, for template and copy
src=dict(required=False, default=None, type='path'),
),
add_file_common_args=True,
supports_check_mode=True)
params = module.params
state = params['state']
recurse = params['recurse']
force = params['force']
diff_peek = params['diff_peek']
src = params['src']
b_src = to_bytes(src, errors='surrogate_or_strict')
follow = params['follow']
# modify source as we later reload and pass, specially relevant when used by other modules.
path = params['path']
b_path = to_bytes(path, errors='surrogate_or_strict')
# short-circuit for diff_peek
if diff_peek is not None:
appears_binary = False
try:
f = open(b_path, 'rb')
head = f.read(8192)
f.close()
if b("\x00") in head:
appears_binary = True
except:
pass
module.exit_json(path=path,
changed=False,
appears_binary=appears_binary)
prev_state = get_state(b_path)
# state should default to file, but since that creates many conflicts,
# default to 'current' when it exists.
if state is None:
if prev_state != 'absent':
state = prev_state
elif recurse:
state = 'directory'
else:
state = 'file'
# source is both the source of a symlink or an informational passing of the src for a template module
# or copy module, even if this module never uses it, it is needed to key off some things
if src is None:
if state in ('link', 'hard'):
if follow and state == 'link':
# use the current target of the link as the source
src = to_native(os.path.realpath(b_path), errors='strict')
b_src = to_bytes(os.path.realpath(b_path), errors='strict')
else:
module.fail_json(
msg='src and dest are required for creating links')
# original_basename is used by other modules that depend on file.
if state not in ("link", "absent") and os.path.isdir(b_path):
basename = None
if params['original_basename']:
basename = params['original_basename']
elif src is not None:
basename = os.path.basename(src)
if basename:
params['path'] = path = os.path.join(path, basename)
b_path = to_bytes(path, errors='surrogate_or_strict')
prev_state = get_state(b_path)
# make sure the target path is a directory when we're doing a recursive operation
if recurse and state != 'directory':
module.fail_json(path=path,
msg="recurse option requires state to be 'directory'")
file_args = module.load_file_common_arguments(params)
changed = False
diff = {
'before': {
'path': path
},
'after': {
'path': path
},
}
state_change = False
if prev_state != state:
diff['before']['state'] = prev_state
diff['after']['state'] = state
state_change = True
if state == 'absent':
if state_change:
if not module.check_mode:
if prev_state == 'directory':
try:
shutil.rmtree(b_path, ignore_errors=False)
except Exception as e:
module.fail_json(msg="rmtree failed: %s" %
to_native(e))
else:
try:
os.unlink(b_path)
except Exception as e:
module.fail_json(path=path,
msg="unlinking failed: %s " %
to_native(e))
module.exit_json(path=path, changed=True, diff=diff)
else:
module.exit_json(path=path, changed=False)
elif state == 'file':
if state_change:
if follow and prev_state == 'link':
# follow symlink and operate on original
b_path = os.path.realpath(b_path)
path = to_native(b_path, errors='strict')
prev_state = get_state(b_path)
file_args['path'] = path
if prev_state not in ('file', 'hard'):
# file is not absent and any other state is a conflict
module.fail_json(path=path,
msg='file (%s) is %s, cannot continue' %
(path, prev_state))
changed = module.set_fs_attributes_if_different(file_args,
changed,
diff,
expand=False)
module.exit_json(path=path, changed=changed, diff=diff)
elif state == 'directory':
if follow and prev_state == 'link':
b_path = os.path.realpath(b_path)
path = to_native(b_path, errors='strict')
prev_state = get_state(b_path)
if prev_state == 'absent':
if module.check_mode:
module.exit_json(changed=True, diff=diff)
changed = True
curpath = ''
try:
# Split the path so we can apply filesystem attributes recursively
# from the root (/) directory for absolute paths or the base path
# of a relative path. We can then walk the appropriate directory
# path to apply attributes.
for dirname in path.strip('/').split('/'):
curpath = '/'.join([curpath, dirname])
# Remove leading slash if we're creating a relative path
if not os.path.isabs(path):
curpath = curpath.lstrip('/')
b_curpath = to_bytes(curpath, errors='surrogate_or_strict')
if not os.path.exists(b_curpath):
try:
os.mkdir(b_curpath)
except OSError as ex:
# Possibly something else created the dir since the os.path.exists
# check above. As long as it's a dir, we don't need to error out.
if not (ex.errno == errno.EEXIST
and os.path.isdir(b_curpath)):
raise
tmp_file_args = file_args.copy()
tmp_file_args['path'] = curpath
changed = module.set_fs_attributes_if_different(
tmp_file_args, changed, diff, expand=False)
except Exception as e:
module.fail_json(
path=path,
msg='There was an issue creating %s as requested: %s' %
(curpath, to_native(e)))
# We already know prev_state is not 'absent', therefore it exists in some form.
elif prev_state != 'directory':
module.fail_json(path=path,
msg='%s already exists as a %s' %
(path, prev_state))
changed = module.set_fs_attributes_if_different(file_args,
changed,
diff,
expand=False)
if recurse:
changed |= recursive_set_attributes(
module,
to_bytes(file_args['path'], errors='surrogate_or_strict'),
follow, file_args)
module.exit_json(path=path, changed=changed, diff=diff)
elif state in ('link', 'hard'):
if not os.path.islink(b_path) and os.path.isdir(b_path):
relpath = path
else:
b_relpath = os.path.dirname(b_path)
relpath = to_native(b_relpath, errors='strict')
absrc = os.path.join(relpath, src)
b_absrc = to_bytes(absrc, errors='surrogate_or_strict')
if not force and not os.path.exists(b_absrc):
module.fail_json(
path=path,
src=src,
msg=
'src file does not exist, use "force=yes" if you really want to create the link: %s'
% absrc)
if state == 'hard':
if not os.path.isabs(b_src):
module.fail_json(msg="absolute paths are required")
elif prev_state == 'directory':
if not force:
module.fail_json(
path=path,
msg='refusing to convert between %s and %s for %s' %
(prev_state, state, path))
elif os.listdir(b_path):
# refuse to replace a directory that has files in it
module.fail_json(
path=path,
msg='the directory %s is not empty, refusing to convert it'
% path)
elif prev_state in ('file', 'hard') and not force:
module.fail_json(
path=path,
msg='refusing to convert between %s and %s for %s' %
(prev_state, state, path))
if prev_state == 'absent':
changed = True
elif prev_state == 'link':
b_old_src = os.readlink(b_path)
if b_old_src != b_src:
diff['before']['src'] = to_native(b_old_src, errors='strict')
diff['after']['src'] = src
changed = True
elif prev_state == 'hard':
if not (state == 'hard'
and os.stat(b_path).st_ino == os.stat(b_src).st_ino):
changed = True
if not force:
module.fail_json(
dest=path,
src=src,
msg=
'Cannot link, different hard link exists at destination'
)
elif prev_state == 'file':
changed = True
if not force:
module.fail_json(dest=path,
src=src,
msg='Cannot link, %s exists at destination' %
prev_state)
elif prev_state == 'directory':
changed = True
if os.path.exists(b_path):
if state == 'hard' and os.stat(b_path).st_ino == os.stat(
b_src).st_ino:
module.exit_json(path=path, changed=False)
elif not force:
module.fail_json(
dest=path,
src=src,
msg=
'Cannot link, different hard link exists at destination'
)
else:
module.fail_json(dest=path,
src=src,
msg='unexpected position reached')
if changed and not module.check_mode:
if prev_state != 'absent':
# try to replace atomically
b_tmppath = to_bytes(os.path.sep).join([
os.path.dirname(b_path),
to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))
])
try:
if prev_state == 'directory' and state == 'link':
os.rmdir(b_path)
elif prev_state == 'directory' and state == 'hard':
if os.path.exists(b_path):
os.remove(b_path)
if state == 'hard':
os.link(b_src, b_tmppath)
else:
os.symlink(b_src, b_tmppath)
os.rename(b_tmppath, b_path)
except OSError as e:
if os.path.exists(b_tmppath):
os.unlink(b_tmppath)
module.fail_json(path=path,
msg='Error while replacing: %s' %
to_native(e, nonstring='simplerepr'))
else:
try:
if state == 'hard':
os.link(b_src, b_path)
else:
os.symlink(b_src, b_path)
except OSError as e:
module.fail_json(path=path,
msg='Error while linking: %s' %
to_native(e, nonstring='simplerepr'))
if module.check_mode and not os.path.exists(b_path):
module.exit_json(dest=path, src=src, changed=changed, diff=diff)
changed = module.set_fs_attributes_if_different(file_args,
changed,
diff,
expand=False)
module.exit_json(dest=path, src=src, changed=changed, diff=diff)
elif state == 'touch':
if not module.check_mode:
if prev_state == 'absent':
try:
open(b_path, 'wb').close()
except OSError as e:
module.fail_json(path=path,
msg='Error, could not touch target: %s' %
to_native(e, nonstring='simplerepr'))
elif prev_state in ('file', 'directory', 'hard'):
try:
os.utime(b_path, None)
except OSError as e:
module.fail_json(
path=path,
msg='Error while touching existing target: %s' %
to_native(e, nonstring='simplerepr'))
else:
module.fail_json(
msg=
'Cannot touch other than files, directories, and hardlinks (%s is %s)'
% (path, prev_state))
try:
module.set_fs_attributes_if_different(file_args,
True,
diff,
expand=False)
except SystemExit as e:
if e.code:
# We take this to mean that fail_json() was called from
# somewhere in basic.py
if prev_state == 'absent':
# If we just created the file we can safely remove it
os.remove(b_path)
raise e
module.exit_json(dest=path, changed=True, diff=diff)
module.fail_json(path=path, msg='unexpected position reached')
def link(src, dst):
if os.name == 'nt':
import ctypes
if ctypes.windll.kernel32.CreateHardLinkW(six.text_type(dst), six.text_type(src), 0) == 0: raise ctypes.WinError()
else:
os.link(src, dst)
def do_sillydiff(dirname, oldfile, newfile, outfile):
today = time.strftime("%d.%m.%Y")
try:
oldfile = open(os.path.join(dirname, oldfile), "r")
line = oldfile.readline()
line = line.rstrip()
except IOError:
logger.warn("Warning, old file did not exist, assuming first run ever")
os.link(os.path.join(dirname, newfile), os.path.join(dirname, outfile))
return
old_dict = dict()
while line:
key = line[0:12]
value = old_dict.get(key, list())
value.append(line[13:])
old_dict[key] = value
line = oldfile.readline()
line = line.rstrip()
oldfile.close()
out = AtomicFileWriter(os.path.join(dirname, outfile), 'w')
newin = open(os.path.join(dirname, newfile))
for newline in newin:
newline = newline.rstrip()
pnr = newline[0:12]
data = newline[13:]
if pnr in old_dict:
if data not in old_dict[pnr]:
# Some change, want to update with new values.
out.write(newline + "\n")
else:
old_dict[pnr].remove(data)
# If nothing else is left, delete the key from the dictionary
if not old_dict[pnr]:
del old_dict[pnr]
else:
# completely new entry, output unconditionally
out.write(newline + "\n")
# Now, there is one problem left: we cannot output the old data blindly,
# as people's names might have changed. So, we force *every* old record to
# the current names in Cerebrum. This may result in the exactly same
# record being output twice, but it should be fine.
person = Factory.get("Person")(db)
const = Factory.get("Constants")(db)
logger.debug("%d old records left", len(old_dict))
for leftpnr in old_dict:
# FIXME: it is unsafe to assume that this will succeed
first, last = fnr2names(person, const, leftpnr[:-1])
if not (first and last):
logger.warn(
"No name information for %s is available. %d "
"entry(ies) will be skipped", leftpnr[:-1],
len(old_dict[leftpnr]))
continue
for entry in old_dict[leftpnr]:
vals = entry.split(";")
vals[2] = first
vals[3] = last
vals[13] = today
vals[17] = ""
out.write("%s;%s\n" % (leftpnr, ";".join(vals)))
out.close()
newin.close()
def __main__():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option(
'-p',
'--num-threads',
dest='num_threads',
help='Use this many threads to align reads. The default is 1.')
parser.add_option('-C',
'--color-space',
dest='color_space',
action='store_true',
help='This indicates color-space data')
parser.add_option('-J',
'--junctions-output',
dest='junctions_output_file',
help='Junctions output file; formate is BED.')
parser.add_option('-H',
'--hits-output',
dest='accepted_hits_output_file',
help='Accepted hits output file; formate is BAM.')
parser.add_option('', '--own-file', dest='own_file', help='')
parser.add_option(
'-D',
'--indexes-path',
dest='index_path',
help='Indexes directory; location of .ebwt and .fa files.')
parser.add_option(
'-r',
'--mate-inner-dist',
dest='mate_inner_dist',
help='This is the expected (mean) inner distance between mate pairs. \
For, example, for paired end runs with fragments selected at 300bp, \
where each end is 50bp, you should set -r to be 200. There is no default, \
and this parameter is required for paired end runs.'
)
parser.add_option(
'',
'--mate-std-dev',
dest='mate_std_dev',
help=
'Standard deviation of distribution on inner distances between male pairs.'
)
parser.add_option('-n',
'--transcriptome-mismatches',
dest='transcriptome_mismatches')
parser.add_option('',
'--genome-read-mismatches',
dest='genome_read_mismatches')
parser.add_option('', '--read-mismatches', dest='read_mismatches')
parser.add_option('', '--bowtie-n', action="store_true", dest='bowtie_n')
parser.add_option(
'-a',
'--min-anchor-length',
dest='min_anchor_length',
help=
'The "anchor length". TopHat will report junctions spanned by reads with at least this many bases on each side of the junction.'
)
parser.add_option(
'-m',
'--splice-mismatches',
dest='splice_mismatches',
help=
'The maximum number of mismatches that can appear in the anchor region of a spliced alignment.'
)
parser.add_option(
'-i',
'--min-intron-length',
dest='min_intron_length',
help=
'The minimum intron length. TopHat will ignore donor/acceptor pairs closer than this many bases apart.'
)
parser.add_option(
'-I',
'--max-intron-length',
dest='max_intron_length',
help=
'The maximum intron length. When searching for junctions ab initio, TopHat will ignore donor/acceptor pairs farther than this many bases apart, except when such a pair is supported by a split segment alignment of a long read.'
)
parser.add_option('-g',
'--max_multihits',
dest='max_multihits',
help='Maximum number of alignments to be allowed')
parser.add_option(
'',
'--seg-mismatches',
dest='seg_mismatches',
help=
'Number of mismatches allowed in each segment alignment for reads mapped independently'
)
parser.add_option('',
'--seg-length',
dest='seg_length',
help='Minimum length of read segments')
parser.add_option(
'',
'--library-type',
dest='library_type',
help=
'TopHat will treat the reads as strand specific. Every read alignment will have an XS attribute tag. Consider supplying library type options below to select the correct RNA-seq protocol.'
)
parser.add_option(
'',
'--allow-indels',
action="store_true",
help=
'Allow indel search. Indel search is disabled by default.(Not used since version 1.3.0)'
)
parser.add_option('',
'--max-insertion-length',
dest='max_insertion_length',
help='The maximum insertion length. The default is 3.')
parser.add_option('',
'--max-deletion-length',
dest='max_deletion_length',
help='The maximum deletion length. The default is 3.')
# Options for supplying own junctions
parser.add_option(
'-G',
'--GTF',
dest='gene_model_annotations',
help='Supply TopHat with a list of gene model annotations. \
TopHat will use the exon records in this file to build \
a set of known splice junctions for each gene, and will \
attempt to align reads to these junctions even if they \
would not normally be covered by the initial mapping.'
)
parser.add_option(
'-j',
'--raw-juncs',
dest='raw_juncs',
help='Supply TopHat with a list of raw junctions. Junctions are \
specified one per line, in a tab-delimited format. Records \
look like: <chrom> <left> <right> <+/-> left and right are \
zero-based coordinates, and specify the last character of the \
left sequenced to be spliced to the first character of the right \
sequence, inclusive.'
)
parser.add_option('',
'--no-novel-juncs',
action="store_true",
dest='no_novel_juncs',
help="Only look for junctions indicated in the \
supplied GFF file. (ignored without -G)"
)
parser.add_option(
'',
'--no-novel-indels',
action="store_true",
dest='no_novel_indels',
help="Skip indel search. Indel search is enabled by default.")
# Types of search.
parser.add_option(
'',
'--microexon-search',
action="store_true",
dest='microexon_search',
help=
'With this option, the pipeline will attempt to find alignments incident to microexons. Works only for reads 50bp or longer.'
)
parser.add_option(
'',
'--coverage-search',
action="store_true",
dest='coverage_search',
help=
'Enables the coverage based search for junctions. Use when coverage search is disabled by default (such as for reads 75bp or longer), for maximum sensitivity.'
)
parser.add_option('',
'--no-coverage-search',
action="store_false",
dest='coverage_search')
parser.add_option(
'',
'--min-segment-intron',
dest='min_segment_intron',
help=
'Minimum intron length that may be found during split-segment search')
parser.add_option(
'',
'--max-segment-intron',
dest='max_segment_intron',
help=
'Maximum intron length that may be found during split-segment search')
parser.add_option(
'',
'--min-coverage-intron',
dest='min_coverage_intron',
help='Minimum intron length that may be found during coverage search')
parser.add_option(
'',
'--max-coverage-intron',
dest='max_coverage_intron',
help='Maximum intron length that may be found during coverage search')
# Fusion search options.
parser.add_option('',
'--fusion-search',
action='store_true',
dest='fusion_search')
parser.add_option('',
'--fusion-anchor-length',
dest='fusion_anchor_length')
parser.add_option('', '--fusion-min-dist', dest='fusion_min_dist')
parser.add_option('',
'--fusion-read-mismatches',
dest='fusion_read_mismatches')
parser.add_option('', '--fusion-multireads', dest='fusion_multireads')
parser.add_option('', '--fusion-multipairs', dest='fusion_multipairs')
parser.add_option('',
'--fusion-ignore-chromosomes',
dest='fusion_ignore_chromosomes')
# Wrapper options.
parser.add_option(
'-1',
'--input1',
dest='input1',
help='The (forward or single-end) reads file in Sanger FASTQ format')
parser.add_option('-2',
'--input2',
dest='input2',
help='The reverse reads file in Sanger FASTQ format')
parser.add_option('', '--single-paired', dest='single_paired', help='')
parser.add_option('', '--settings', dest='settings', help='')
(options, args) = parser.parse_args()
# Color or base space
space = ''
if options.color_space:
space = '-C'
# Creat bowtie index if necessary.
tmp_index_dir = tempfile.mkdtemp()
if options.own_file:
index_path = os.path.join(
tmp_index_dir,
'.'.join(os.path.split(options.own_file)[1].split('.')[:-1]))
try:
os.link(options.own_file, index_path + '.fa')
except:
# Tophat prefers (but doesn't require) fasta file to be in same directory, with .fa extension
pass
cmd_index = 'bowtie-build %s -f %s %s' % (space, options.own_file,
index_path)
try:
tmp = tempfile.NamedTemporaryFile(dir=tmp_index_dir).name
tmp_stderr = open(tmp, 'wb')
proc = subprocess.Popen(args=cmd_index,
shell=True,
cwd=tmp_index_dir,
stderr=tmp_stderr.fileno())
returncode = proc.wait()
tmp_stderr.close()
# get stderr, allowing for case where it's very large
tmp_stderr = open(tmp, 'rb')
stderr = ''
buffsize = 1048576
try:
while True:
stderr += tmp_stderr.read(buffsize)
if not stderr or len(stderr) % buffsize != 0:
break
except OverflowError:
pass
tmp_stderr.close()
if returncode != 0:
raise Exception, stderr
except Exception, e:
if os.path.exists(tmp_index_dir):
shutil.rmtree(tmp_index_dir)
stop_err('Error indexing reference sequence\n' + str(e))
def ensure_hardlink(path, src, follow, force, timestamps):
b_path = to_bytes(path, errors='surrogate_or_strict')
b_src = to_bytes(src, errors='surrogate_or_strict')
prev_state = get_state(b_path)
file_args = module.load_file_common_arguments(module.params)
mtime = get_timestamp_for_time(timestamps['modification_time'], timestamps['modification_time_format'])
atime = get_timestamp_for_time(timestamps['access_time'], timestamps['access_time_format'])
# src is the source of a hardlink. We require it if we are creating a new hardlink.
# We require path in the argument_spec so we know it is present at this point.
if src is None:
raise AssibleModuleError(results={'msg': 'src is required for creating new hardlinks'})
if not os.path.exists(b_src):
raise AssibleModuleError(results={'msg': 'src does not exist', 'dest': path, 'src': src})
diff = initial_diff(path, 'hard', prev_state)
changed = False
if prev_state == 'absent':
changed = True
elif prev_state == 'link':
b_old_src = os.readlink(b_path)
if b_old_src != b_src:
diff['before']['src'] = to_native(b_old_src, errors='strict')
diff['after']['src'] = src
changed = True
elif prev_state == 'hard':
if not os.stat(b_path).st_ino == os.stat(b_src).st_ino:
changed = True
if not force:
raise AssibleModuleError(results={'msg': 'Cannot link, different hard link exists at destination',
'dest': path, 'src': src})
elif prev_state == 'file':
changed = True
if not force:
raise AssibleModuleError(results={'msg': 'Cannot link, %s exists at destination' % prev_state,
'dest': path, 'src': src})
elif prev_state == 'directory':
changed = True
if os.path.exists(b_path):
if os.stat(b_path).st_ino == os.stat(b_src).st_ino:
return {'path': path, 'changed': False}
elif not force:
raise AssibleModuleError(results={'msg': 'Cannot link: different hard link exists at destination',
'dest': path, 'src': src})
else:
raise AssibleModuleError(results={'msg': 'unexpected position reached', 'dest': path, 'src': src})
if changed and not module.check_mode:
if prev_state != 'absent':
# try to replace atomically
b_tmppath = to_bytes(os.path.sep).join(
[os.path.dirname(b_path), to_bytes(".%s.%s.tmp" % (os.getpid(), time.time()))]
)
try:
if prev_state == 'directory':
if os.path.exists(b_path):
try:
os.unlink(b_path)
except OSError as e:
if e.errno != errno.ENOENT: # It may already have been removed
raise
os.link(b_src, b_tmppath)
os.rename(b_tmppath, b_path)
except OSError as e:
if os.path.exists(b_tmppath):
os.unlink(b_tmppath)
raise AssibleModuleError(results={'msg': 'Error while replacing: %s'
% to_native(e, nonstring='simplerepr'),
'path': path})
else:
try:
os.link(b_src, b_path)
except OSError as e:
raise AssibleModuleError(results={'msg': 'Error while linking: %s'
% to_native(e, nonstring='simplerepr'),
'path': path})
if module.check_mode and not os.path.exists(b_path):
return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
changed = module.set_fs_attributes_if_different(file_args, changed, diff, expand=False)
changed |= update_timestamp_for_file(file_args['path'], mtime, atime, diff)
return {'dest': path, 'src': src, 'changed': changed, 'diff': diff}
def drive_APMrover2(viewerip=None, map=False):
'''drive APMrover2 in SIL
you can pass viewerip as an IP address to optionally send fg and
mavproxy packets too for local viewing of the mission in real time
'''
global homeloc
options = '--sitl=127.0.0.1:5501 --out=127.0.0.1:19550 --streamrate=10'
if viewerip:
options += " --out=%s:14550" % viewerip
if map:
options += ' --map'
sim_cmd = util.reltopdir(
'Tools/autotest/pysim/sim_wrapper.py'
) + ' --frame=rover --rate=200 --speedup=100 --home=%f,%f,%u,%u' % (
HOME.lat, HOME.lng, HOME.alt, HOME.heading)
sil = util.start_SIL('APMrover2', wipe=True)
mavproxy = util.start_MAVProxy_SIL('APMrover2', options=options)
runsim = pexpect.spawn(sim_cmd, logfile=sys.stdout, timeout=10)
runsim.delaybeforesend = 0
runsim.expect('Starting at lat')
print("WAITING FOR PARAMETERS")
mavproxy.expect('Received [0-9]+ parameters')
# setup test parameters
mavproxy.send("param load %s/Rover.parm\n" % testdir)
mavproxy.expect('Loaded [0-9]+ parameters')
# restart with new parms
util.pexpect_close(mavproxy)
util.pexpect_close(sil)
util.pexpect_close(runsim)
sil = util.start_SIL('APMrover2')
mavproxy = util.start_MAVProxy_SIL('APMrover2', options=options)
mavproxy.expect('Logging to (\S+)')
logfile = mavproxy.match.group(1)
print("LOGFILE %s" % logfile)
sim_cmd = util.reltopdir(
'Tools/autotest/pysim/sim_wrapper.py'
) + ' --frame=rover --rate=200 --speedup=100 --home=%f,%f,%u,%u' % (
HOME.lat, HOME.lng, HOME.alt, HOME.heading)
runsim = pexpect.spawn(sim_cmd, logfile=sys.stdout, timeout=10)
runsim.delaybeforesend = 0
util.pexpect_autoclose(runsim)
runsim.expect('Starting at lat')
buildlog = util.reltopdir("../buildlogs/APMrover2-test.tlog")
print("buildlog=%s" % buildlog)
if os.path.exists(buildlog):
os.unlink(buildlog)
try:
os.link(logfile, buildlog)
except Exception:
pass
mavproxy.expect('Received [0-9]+ parameters')
util.expect_setup_callback(mavproxy, expect_callback)
expect_list_clear()
expect_list_extend([runsim, sil, mavproxy])
print("Started simulator")
# get a mavlink connection going
try:
mav = mavutil.mavlink_connection('127.0.0.1:19550',
robust_parsing=True)
except Exception, msg:
print("Failed to start mavlink connection on 127.0.0.1:19550" % msg)
raise
def link(self, newpath):
""" Create a hard link at 'newpath', pointing to this file. """
os.link(self, newpath)
def make_deb(distro, build_os, arch, spec, srcdir):
# I can't remember the details anymore, but the initscript/upstart
# job files' names must match the package name in some way; and
# see also the --name flag to dh_installinit in the generated
# debian/rules file.
suffix = spec.suffix()
sdir = setupdir(distro, build_os, arch, spec)
if re.search("debian", distro.name()):
os.unlink(sdir + "debian/mongod.upstart")
os.link(
sdir + "debian/mongod.service", sdir +
"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix))
os.unlink(sdir + "debian/init.d")
elif re.search("ubuntu", distro.name()):
os.unlink(sdir + "debian/init.d")
if build_os in ("ubuntu1204", "ubuntu1404", "ubuntu1410"):
os.link(
sdir + "debian/mongod.upstart",
sdir + "debian/%s%s-server.mongod.upstart" %
(distro.pkgbase(), suffix))
os.unlink(sdir + "debian/mongod.service")
else:
os.link(
sdir + "debian/mongod.service",
sdir + "debian/%s%s-server.mongod.service" %
(distro.pkgbase(), suffix))
os.unlink(sdir + "debian/mongod.upstart")
else:
raise Exception("unknown debianoid flavor: not debian or ubuntu?")
# Rewrite the control and rules files
write_debian_changelog(sdir + "debian/changelog", spec, srcdir)
distro_arch = distro.archname(arch)
sysassert([
"cp", "-v",
srcdir + "debian/%s%s.control" % (distro.pkgbase(), suffix),
sdir + "debian/control"
])
sysassert([
"cp", "-v", srcdir + "debian/%s%s.rules" % (distro.pkgbase(), suffix),
sdir + "debian/rules"
])
# old non-server-package postinst will be hanging around for old versions
#
if os.path.exists(sdir + "debian/postinst"):
os.unlink(sdir + "debian/postinst")
# copy our postinst files
#
sysassert([
"sh", "-c",
"cp -v \"%sdebian/\"*.postinst \"%sdebian/\"" % (srcdir, sdir)
])
# Do the packaging.
oldcwd = os.getcwd()
try:
os.chdir(sdir)
sysassert(["dpkg-buildpackage", "-uc", "-us", "-a" + distro_arch])
finally:
os.chdir(oldcwd)
r = distro.repodir(arch, build_os, spec)
ensure_dir(r)
# FIXME: see if shutil.copyfile or something can do this without
# much pain.
sysassert(["sh", "-c", "cp -v \"%s/../\"*.deb \"%s\"" % (sdir, r)])
return r
def Install(**kwargs):
"""
This does the grunt-work of actually doing the install.
The possible arguments are:
- config Object containing configuration. This is where the download URL and
package directories will be specified.
- interactive Whether or not to be interactive. If true (default), then
bsd.Dialog will be used to for status and error messages.
- disks An array of Disk objects to install to. If set, then the disks
will be partitioned and erased. If NOT set, then the installer
will create a new boot environment on the existing freenas-boot pool.
- efi Boolean indicating whether or not to use EFI (default is False).
- upgrade_from An unimported ZFSPool object to install to. This must be set
when upgrading, and when creating a new BE on an existing pool.
- upgrade Boolean indicating whether or not to upgrade. Requires upgrade_from to
be valid.
- data_dir A string indicating the location of the /data. Normally this will just
be "/data", but if installing from something other than the ISO, it will
be necessary to specify it.
- password A string indicating the root password. Ignored for upgrades; may be None
(indicating no password, not recommended).
- partitions An array of Partition objects (see Utils). Note that the OS
partition will always be installed last.
- post_install An array of callable objects, which will be called after installation,
as func(mount_point=/path, **kwargs). MUST BE AN ARRAY.
- package_handler Call-back for the start of each package. Arguments are
(index [int], name [string], packages [array of package names])
be installed.
- progress_handler Call-back after each file/directory is installed. Arguments are **kwargs,
will [currently] be either done=True (indicating the package is installed),
or (total=int [number of objects], index=int [current index], name=string
[name of object that was just installed]).
- manifest A manifest object. Must be set.
- package_directory A path where the package files are located. The package files must
already be located in this directory.
- trampoline A boolean indicating whether the post-install scripts should be run
on reboot (True, default) or during the install (False).
"""
LogIt("Install({})".format(kwargs))
orig_kwargs = kwargs.copy()
config = kwargs.get("config", Configuration.SystemConfiguration())
interactive = kwargs.get("interactive", True)
disks = kwargs.get("disks", [])
efi = kwargs.get("efi", False)
upgrade_pool = kwargs.get("upgrade_from", None)
upgrade = kwargs.get("upgrade", False)
data_dir = kwargs.get("data_dir", "/data")
password = kwargs.get("password", None)
extra_partitions = kwargs.get("partitions", [])
post_install = kwargs.get("post_install", [])
package_notifier = kwargs.get("package_handler", None)
progress_notifier = kwargs.get("progress_handler", None)
manifest = kwargs.get("manifest", None)
trampoline = kwargs.get("trampoline", True)
# The default is based on ISO layout
package_dir = kwargs.get("package_directory", "/.mount/{}/Packages".format(Project()))
if type(post_install) != list:
post_install = [post_install]
if not manifest:
if interactive:
try:
Dialog.MessageBox(Title(),
"No manifest specified for the installation",
height=7, width=45).run()
except:
pass
raise InstallationError("No manifest specified for the installation")
config.SetPackageDir(package_dir)
mount_point = tempfile.mkdtemp()
# Quick sanity check
if upgrade and upgrade_pool is None:
if interactive:
Dialog.MessageBox(Title(), "\nNo pool to upgrade from",
height=7, width=30).run()
raise InstallationError("Upgrade selected but not previous boot pool selected")
if disks is None and upgrade_pool is None:
if interactive:
Dialog.MessageBox(Title(), "\nNo disks or previous pool selected",
height=10, width=30).run()
raise InstallationError("No disks or previous boot pool selected")
if IsTruenas():
# We use a 16g swap partition in TrueNAS.
# Note that this is only used if the disks are being formatted.
extra_partitions.append(Partition(type="swap", index="3", size=16*1024*1024*1024))
def make_tn_swap(mount_point=None, **kwargs):
# This uses the previously-defined variables, not kwargs
if disks and mount_point:
try:
RunCommand("/sbin/gmirror", "label", "-b", "prefer",
["{}p3".format(disk.name) for disk in disks])
with open(os.path.join(mount_point, "data/fstab.swap"), "w") as swaptab:
print("/dev/mirror/swap.eli\tnone\tswap\tsw\t0\t0", file=swaptab)
except RunCommandException as e:
LogIt("Could not create mirrored swap: {}".format(str(e)))
post_install.append(make_tn_swap)
# First step is to see if we're upgrading.
# If so, we want to copy files from the active BE to
# a location in /tmp, so we can copy them back later.
# This will import, and then export, the freenas-boot pool.
if upgrade_pool and upgrade:
upgrade_dir = SaveConfiguration(interactive=interactive,
pool=upgrade_pool)
else:
upgrade_dir = None
# Second step is to see if we're formatting drives.
# If so, we first will destroy the freenas-boot pool;
# after that, we will partition the drives. How we partition
# depends on the boot method -- efi or bios. We set the
# BE name to "default" and create the freenas-boot pool, and
# then the grub dataset.
#
# If we're NOT formatting the drive, we set the pool name
# to time.strftime("default-%Y%m%d-%H%M%S")
LogIt("disks = {}".format(disks))
if disks:
# This means we're formatting
# We need to know what size and types to make the partitions.
# If we're using EFI, then we need a 100mbyte msdosfs partition;
# otherwise a 512k bios-boot. If we have any extra partitions,
# we'll take those into account as well. For the big freebsd-zfs
# partition, we'll take the minimum of the remaining space,
# rounded down to the nearest gbyte.
gByte = 1024 * 1024 * 1024
if efi:
# 100mbytes for efi partition
used = 100 * 1024 * 1024
boot_part = Partition(type="efi",
index=1,
size=used)
else:
# BIOS partition gets 512kbytes
used = 512 * 1024
boot_part = Partition(type="bios-boot",
index=1,
size=used)
partitions = [boot_part]
# For now, we always make the freenas-boot partition index 2, and place
# it at the end of the disk.
next_index = 3
for part in (extra_partitions or []):
# We will ignore the index given here.
part.index = next_index
used += part.size
LogIt("Additional partition {}".format(part))
partitions.append(part)
next_index += 1
# At this point, used is the sum of the partitions, in bytes.
# This isn't really correct - we should be rounding the size up
# to the blocksize of the disk. But partitioning behaves strangely
# sometimes with flash drives. As a result, when we do the actual
# partitioning, we use the smart-size (e.g., 1G), which rounds down.
min_size = 0
for disk in disks:
# If the remaining space is too small, this installation won't work well.
size = disk.size
size = size - used
if size < gByte:
if size < 0:
fspace = "no free space after the other partitions"
else:
fspace = "free space is {}, minimum is 1Gbyte".format(SmartSize(size))
name = disk.name
LogIt("Disk {} is too small {}".format(name, fspace))
ssize = SmartSize(disk.size)
if interactive:
Dialog.MessageBox(Title(),
"Disk {} is too small ({})".format(name, ssize),
height=10, width=25).run()
raise InstallationException("Disk {} is too small ({})".format(name, ssize))
if (size < min_size) or (not min_size):
min_size = size
if min_size == 0:
if interactive:
Dialog.MessageBox(Title(),
"Unable to find the size of any of the selected disks",
height=15, weidth=60).run()
raise InstallationError("Unable to find disk size")
# Round min_size down to a gbyte
part_size = int(min_size / gByte) * gByte
os_part = Partition(type="freebsd-zfs",
index=2,
size=part_size,
os=True)
LogIt("OS partition {}".format(os_part))
partitions.append(os_part)
# We need to destroy any existing freenas-boot pool.
# To do that, we may first need to import the pool.
if upgrade_pool is None:
try:
old_pools = list(zfs.find_import(name="freenas-boot"))
except libzfs.ZFSException as e:
LogIt("Got ZFS error {} while trying to import freenas-boot for destruction".format(str(e)))
old_pools = []
else:
old_pools = [upgrade_pool]
# We'll be destroying it, so..
upgrade_pool = None
for pool in old_pools:
try:
dead_pool = zfs.import_pool(pool, "freenas-boot", {})
if dead_pool is None:
dead_pool = zfs.get("freenas-boot")
zfs.destroy("freenas-boot")
except libzfs.ZFSException as e:
LogIt("Trying to destroy a freenas-boot pool got error {}".format(str(e)))
try:
freenas_boot = FormatDisks(disks, partitions, interactive)
except BaseException as e:
LogIt("FormatDisks got exception {}".format(str(e)))
raise
bename = "freenas-boot/ROOT/default"
else:
# We need to import the pool (we exported it above if upgrade_pool)
try:
if upgrade_pool:
freenas_boot = zfs.import_pool(upgrade_pool, "freenas-boot", {})
else:
freenas_boot = None
pools = list(zfs.find_import(name="freenas-boot"))
if len(pools) > 1:
raise InstallationError("There are multiple unimported freenas-boot pools")
if len(pools) == 1:
freenas_boot = zfs.import_pool(upgrade_pool, "freenas-boot", {})
if freenas_boot is None:
freenas_boot = zfs.get("freenas-boot")
except libzfs.ZFSException as e:
LogIt("Got ZFS error {} while trying to import pool".format(str(e)))
if interactive:
Dialog.MessageBox("Error importing boot pool",
"The {} Installer was unable to import the boot pool:\n\n\t{}".format(Project(), str(e)),
height=25, width=60).run()
raise InstallationError("Unable to import boot pool")
bename = time.strftime("freenas-boot/ROOT/default-%Y%m%d-%H%M%S")
# Next, we create the dataset, and mount it, and then mount
# the grub dataset.
# We also mount a devfs and tmpfs in the new environment.
LogIt("BE name is {}".format(bename))
try:
freenas_boot.create(bename, fsopts={
"mountpoint" : "legacy",
"sync" : "disabled",
})
except libzfs.ZFSException as e:
LogIt("Could not create BE {}: {}".format(bename, str(e)))
if interactive:
Dialog.MessageBox(Title(),
"An error occurred creatint the installation boot environment\n" +
"\n\t{}".format(str(e)),
height=25, width=60).run()
raise InstallationError("Could not create BE {}: {}".format(bename, str(e)))
MountFilesystems(bename, mount_point)
# After this, any exceptions need to have the filesystems unmounted
try:
# If upgrading, copy the stashed files back
if upgrade_dir:
RestoreConfiguration(save_path=upgrade_dir,
interactive=interactive,
destination=mount_point)
else:
if os.path.exists(data_dir):
try:
copytree(data_dir, "{}/data".format(mount_point),
progress_callback=lambda src, dst: LogIt("Copying {} -> {}".format(src, dst)))
except:
pass
#
# We should also handle some FN9 stuff
# In this case, we want the newer database file, for migration purposes
# XXX -- this is a problem when installing from FreeBSD
for dbfile in ["freenas-v1.db", "factory-v1.db"]:
if os.path.exists("/data/{}".format(dbfile)):
copytree("/data/{}".format(dbfile), "{}/data/{}".format(mount_point, dbfile))
# After that, we do the installlation.
# This involves mounting the new BE,
# and then running the install code on it.
installer = Installer.Installer(manifest=manifest,
root=mount_point,
config=config)
if installer.GetPackages() is not True:
LogIt("Installer.GetPackages() failed")
raise InstallationError("Unable to load packages")
# This should only be true for the ISO installer.
installer.trampoline = trampoline
start_time = time.time()
try:
installer.InstallPackages(progressFunc=progress_notifier,
handler=package_notifier)
except BaseException as e:
LogIt("InstallPackaages got exception {}".format(str(e)))
raise InstallationError("Could not install packages")
# Packages installed!
if interactive:
try:
status = Dialog.MessageBox(Title(), "Preparing new boot environment",
height=5, width=35, wait=False)
status.clear()
status.run()
except:
pass
for f in ["{}/conf/default/etc/fstab".format(mount_point),
"{}/conf/base/etc/fstab".format(mount_point)
]:
try:
os.remove(f)
except:
LogIt("Unable to remove {} -- ignoring".format(f))
try:
with open("{}/etc/fstab".format(mount_point), "w") as fstab:
print("freenas-boot/grub\t/boot/grub\tzfs\trw,noatime\t1\t0", file=fstab)
except OSError as e:
LogIt("Unable to create fstab: {}".format(str(e)))
raise InstallationError("Unable to create filesystem table")
try:
os.link("{}/etc/fstab".format(mount_point),
"{}/conf/base/etc/fstab".format(mount_point))
except OSError as e:
LogIt("Unable to link /etc/fstab to /conf/base/etc/fstab: {}".format(str(e)))
# Here, I should change module_path in boot/loader.conf, and get rid of the kernel line
try:
lines = []
boot_config = "{}/boot/loader.conf".format(mount_point)
with open(boot_config, "r") as bootfile:
for line in bootfile:
line = line.rstrip()
if line.startswith("module_path="):
lines.append('module_path="/boot/kernel;/boot/modules;/usr/local/modules"')
elif line.startswith("kernel="):
lines.append('kernel="kernel"')
else:
lines.append(line)
with open(boot_config, "w") as bootfile:
for line in lines:
print(line, file=bootfile)
except BaseException as e:
LogIt("While modifying loader.conf, got exception {}".format(str(e)))
# Otherwise I'll ignore it, I think
# This is to support Xen
try:
hvm = RunCommand("/usr/local/sbin/dmidecode", "-s", "system-product-name",
chroot=mount_point)
if hvm == "HVM domU":
with open(os.path.join(mount_point, "boot", "loader.conf.local"), "a") as f:
print('hint.hpet.0.clock="0"', file=f)
except BaseException as e:
LogIt("Got an exception trying to set XEN boot loader hint: {}".format(str(e)))
# Now I have to mount a tmpfs on var
try:
LogIt("Mounting tmpfs on var")
bsd.nmount(source="tmpfs",
fspath=os.path.join(mount_point, "var"),
fstype="tmpfs")
except BaseException as e:
LogIt("Got exception {} while trying to mount {}/var: {}".format(mount_point, str(e)))
raise InstallationError("Unable to mount temporary space in newly-created BE")
# Now we need to populate a data structure
mtree_command = ["/usr/sbin/mtree", "-deUf" ]
if os.path.exists("/usr/sbin/mtree"):
mtree_command.append("{}/etc/mtree/BSD.var.dist".format(mount_point))
mtree_command.extend(["-p", "{}/var".format(mount_point)])
chroot=None
else:
mtree_command.extend(["/etc/mtree/BSD.var.dist", "-p", "/var"])
chroot=mount_point
try:
RunCommand(*mtree_command,
chroot=chroot)
except RunCommandException as e:
LogIt("{} (chroot={}) failed: {}".format(mtree_command, chroot, str(e)))
raise InstallationError("Unable to prepare new boot environment")
try:
# Now we need to install grub
# We do this even if we didn't format the disks.
# But if we didn't format the disks, we need to use the same type
# of boot loader.
if interactive:
try:
status = Dialog.MessageBox(Title(), "Installing boot loader",
height=5, width=35, wait=False)
status.clear()
status.run()
except:
pass
# We've just repartitioned, so rescan geom
geom.scan()
# Set the boot dataset
freenas_boot.properties["bootfs"].value = bename
LogIt("Set bootfs to {}".format(bename))
# This is EXTREMELY ANNOYING.
# I'd like to use libzfs to set the property here, but
# I have to do it in the chrooted environment, because otherwise
# zfs returns an error and doesn't set it.
#freenas_boot.properties["cachefile"].value = "/boot/zfs/rpool.cache"
try:
RunCommand("/sbin/zpool",
"set", "cachefile=/boot/zfs/rpool.cache",
"freenas-boot",
chroot=mount_point)
except RunCommandException as e:
LogIt("Got exception {} while trying to set cachefile".format(str(e)))
raise InstallationException("Could not set cachefile on boot pool")
LogIt("Set cachefile to /boot/zfs/rpool.cache")
# We need to set the serial port stuff in the database before running grub,
# because it'll use that in the configuration file it generates.
try:
SaveSerialSettings(mount_point)
except:
raise InstallationError("Could not save serial console settings")
try:
# All boot pool disks are partitioned using the same type.
# Or the darkness rises and squit once again rule the earth.
# (It's happened.)
use_efi = Utils.BootPartitionType(freenas_boot.disks[0]) == "efi"
InstallGrub(chroot=mount_point,
disks=freenas_boot.disks,
bename=bename, efi=use_efi)
except RunCommandException as e:
LogIt("Command {} failed: {} (code {})".format(e.command, e.message, e.code))
raise InstallationError("Boot loader installation failure")
except BaseException as e:
LogIt("InstallGrub got exception {}".format(str(e)))
raise
if interactive:
try:
status = Dialog.MessageBox(Title(), "Finalizing installation",
height=5, width=35, wait=False)
status.clear()
status.run()
except BaseException as e:
LogIt("Finalizing got exception {}".format(str(e)))
# This is FN9 specific
with open("{}/data/first-boot".format(mount_point), "wb"):
pass
if upgrade:
for sentinel in ["/data/cd-upgrade", "/data/need-update"]:
with open(mount_point + sentinel, "wb") as f:
pass
elif password is not None:
if interactive:
try:
status = Dialog.MessageBox(Title(), "\nSetting root password",
height=7, width=35, wait=False)
status.clear()
status.run()
except:
pass
try:
RunCommand("/etc/netcli", "reset_root_pw", password,
chroot=mount_point)
except RunCommandException as e:
LogIt("Setting root password: {}".format(str(e)))
raise InstallationError("Unable to set root password")
except BaseException as e:
LogIt("Got exception {} during configuration".format(str(e)))
if interactive:
try:
Dialog.MessageBox(Title(),
"Error during configuration",
height=7, width=35).run()
except:
pass
raise
# Let's turn sync back to default for the dataset
try:
ds = zfs.get_dataset(bename)
except libzfs.ZFSException as e:
LogIt("Got ZFS error {} while trying to get {} dataset".format(str(e), bename))
raise InstallationError("Could not fid newly-created BE {}".format(bename))
try:
ds.properties["sync"].inherit()
except BaseException as e:
LogIt("Unable to set sync on {} to inherit: {}".format(bename, str(e)))
# That's all I'm going to do for now
# We save the manifest
manifest.Save(mount_point)
# Okay! Now if there are any post-install functions, we call them
for fp in post_install:
fp(mount_point=mount_point, **kwargs)
# And we're done!
end_time = time.time()
except InstallationError as e:
# This is the outer try block -- it needs to ensure mountpoints are
# cleaned up
LogIt("Outer block got error {}".format(str(e)))
if interactive:
try:
Dialog.MessageBox("{} Installation Error".format(Project()),
e.message,
height=25, width=50).run()
except:
pass
raise
except BaseException as e:
LogIt("Outer block got base exception {}".format(str(e)))
raise
finally:
if package_dir is None:
LogIt("Removing downloaded packages directory {}".format(cache_dir))
shutil.rmtree(cache_dir, ignore_errors=True)
UnmountFilesystems(mount_point)
LogIt("Exporting freenas-boot at end of installation")
try:
zfs.export_pool(freenas_boot)
except libzfs.ZFSException as e:
LogIt("Could not export freenas boot: {}".format(str(e)))
raise
if interactive:
total_time = int(end_time - start_time)
Dialog.MessageBox(Title(),
"The {} installer has finished the installation in {} seconds".format(Project(), total_time),
height=8, width=40).run()
def os_func():
'''
操作系统模块
该模块下的方法,对各个版本的兼容不明确,须谨慎使用.
测试版本: Python:3.6.1 Windows:Windows10,64bit
'''
# === 系统 ===
strs = os.name # 当前系统: Linux'posix' / Windows'nt' / 'ce' / 'java'
strs = os.sep # 分隔符 \\ (windows:\\ linux:/)
strs = os.pathsep # path变量分隔符 ; (windows:; linux::)
strs = os.linesep # 换行分隔符 \r\n (windows:/r/n linux:\n)
dics = os.environ # 查看系统环境变量(字典)
strs = os.getenv("Path", default=-1) # 读取环境变量, 没有返回None
os.putenv("Path", "C:\\python") # 添加环境变量 (windows无效)
os.unsetenv("Path") # 删除环境变量 (windows不可用)
strs = os.getlogin() # 当前登录的用户名
num = os.getpid() # 当前进程PID
num = os.system("cmd") # 执行操作系统命令, 返回0/1(0执行正确;1执行错误)
strs = os.popen("dir").read() # 执行系统命令,并去读结果
tups = os.times(
) # 当前进程时间(user用户 / system系统 / children_user子用户(windews=0) / children_system子系统(windews=0) / elapsed过去时间)
bytes = os.urandom(10) # n字节用以加密的随机字符
num = os.cpu_count() # CUP数量
# === 进程 ===
os.abort() # 结束进程
# execl(file, *args) / execle / execlp / execlpe / execv / execve / execvp / execvpe // 运行新程序替换当前进程
os.execl(r"C:\python", 'python', 'hello.py', 'i') # (windows执行失败)
os._exit(0) # 退出
os.kill(
8480, signal.SIGTERM
) # (系统) 终止进程(需要导入:signal) SIGINT (终止进程) / SIGTERM (终止进程) / SIGKILL (终止进程) / SIGALRM (闹钟信号)
# === 文件 / 文件夹 ===
strs = os.getcwd() # 当前路径
bytes = os.getcwdb() # 当前路径
os.chdir(r"C:\Windows") # 切换路径
strs = os.curdir # 当前目录 .
strs = os.pardir # 上级目录 ..
strs = os.sep # 路径分隔符 ('/' or '\\')
bytes = os.fsencode(r"C:\c.obj") # (编码) 文件路径字符串转为bytes类型 => b'C:\\c.obj'
strs = os.fsdecode(b"C:\c.obj") # (解码) 文件路径转为strs类型 => 'C:\\c.obj'
# chmod(path, mode, *, dir_fd=None, follow_symlinks=True)
os.chmod(r"C:\python\hello.py", 777) # 修改模式
os.link("file.txt", "file.txt.link") # 创建硬链接
os.symlink("file.txt", "file.txt.link") # 创建软链接 (windows执行失败)
lists = os.listdir() # 所有文件和文件夹(列表) ".""..""D:"
# lstat(path, *, dir_fd=None)
tups = os.lstat(r"C:\c.obj") # 状态信息列表
boolean = os.access(
r"C:\c.obj", os.F_OK
) # (文件/文件夹) 权限测试 (mode: F_OK存在? / R_OK可读? / W_OK可写? / X_OK可执行?)
# scandir(path='.') // DirEntry迭代器, 文件目录
lists = os.scandir()
# st_atime (访问时间) / st_ctime (修改时间) / st_mtime (模式修改时间) / st_size (大小(字节bytes))
# st_uid (用户ID) / st_gid (用户组ID)
# st_ino (inode) / st_mode (模式) / st_dev (设备) / st_nlink (硬链接)
# count = cpu_count() # (系统) CPU线程数(非核数)
tups = os.stat(".") # 获取状态信息, 返回stat_result对象
# utime(path, times=None, *, ns=None, dir_fd=None, follow_symlinks=True) // 更新文件的访问和修改时间
num = os.utime(r"C:\c.obj")
# walk(top, topdown=True, onerror=None, followlinks=False) // 根目录(top)遍历目录树,返回迭代器 (dirpath, dirnames[], filenames[]).
root, dirnames, filenames = os.walk(r"C:\python")
os.removedirs(r"c:\python") # 删除多个文件夹 (windews删除多个文件夹失败,单个成功)
# mkdir(path, mode=0o777, *, dir_fd=None) // 创建单个目录, 文件夹已存在,抛 FileExistsError 异常
os.mkdir("test")
# makedirs(name, mode=0o777, exist_ok=False) // 创建(多个)文件夹
os.makedirs(r"./t1/t2/t3")
os.rmdir("test") # 删除单个目录
# mknod(path, mode=0o600, device=0, *, dir_fd=None) // 创建空文件 (windows不可用)
os.mknod("test.txt")
# remove(path, *, dir_fd=None)
os.remove("test.txt") # 删除文件
# rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None)
os.rename("text.txt", "file.txt") # 重命名
os.renames("text.txt", "file.txt")
# replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None) // 重命名, 存在则替换
os.replace("text.txt", "file.txt")
tups = os.stat(r"text.txt") # 文件属性
# === 文件读写 ===
# open(path, flags, mode=0o777, *, dir_fd=None) // 打开文件 fd:文件描述符
fd = os.open(r"C:\c.obj", os.O_RDWR | os.O_CREAT)
readfd, writefd = os.pipe() # 打开管道,返回读取,写入 (windows失败)
# fdopen(fd, *args, **kwargs) // 打开 (windews写入失败,读取""字符)
f = os.fdopen(readfd)
os.read(fd, 150) # 读取
os.write(fd, "String".encode("utf-8")) # 写入
os.fsync(fd) # 强行写入
os.ftruncate(fd, 100) # 裁剪文件
bytes = os.lseek(
fd, 10,
os.SEEK_SET) # 设置指针 SEEK_SET(0开始) SEEK_CUR(1当前位置) SEEK_END(2末尾)
fd_temp = os.dup(fd) # 副本
boolean = os.isatty(fd) # 是否是tty设备
stat = os.fstat(fd) # 状态信息
strs = os.device_encoding(fd) # 返回终端字符编码,非终端None
os.close(fd) # 关闭
os.closerange(fd, fd) # 关闭并忽略错误
# === DirEntry ===
for dir in os.scandir():
strs = dir.name # 文件名
strs = dir.path # 完整路径名
num = dir.inode() # inode编号
boolean = dir.is_dir() # 是否是文件夹
boolean = dir.is_file() # 是否是文件
boolean = dir.is_symlink() # 是否是链接
tups = dir.stat() # 状态信息的stat_result对象
def make_rpm(distro, build_os, arch, spec, srcdir):
# Create the specfile.
suffix = spec.suffix()
sdir = setupdir(distro, build_os, arch, spec)
specfile = srcdir + "rpm/mongodb%s.spec" % suffix
init_spec = specfile.replace(".spec", "-init.spec")
# The Debian directory is here for the manpages so we we need to remove the service file
# from it so that RPM packages don't end up with the Debian file.
os.unlink(sdir + "debian/mongod.service")
# Swap out systemd files, different systemd spec files, and init scripts as needed based on
# underlying os version. Arranged so that new distros moving forward automatically use
# systemd. Note: the SUSE init packages use a different init script than then other RPM
# distros.
#
if distro.name() == "suse" and distro.repo_os_version(build_os) in ("10",
"11"):
os.unlink(sdir + "rpm/init.d-mongod")
os.link(sdir + "rpm/init.d-mongod.suse", sdir + "rpm/init.d-mongod")
os.unlink(specfile)
os.link(init_spec, specfile)
elif distro.name() == "redhat" and distro.repo_os_version(build_os) in (
"5", "6"):
os.unlink(specfile)
os.link(init_spec, specfile)
elif distro.name() == "amazon":
os.unlink(specfile)
os.link(init_spec, specfile)
topdir = ensure_dir('%s/rpmbuild/%s/' % (os.getcwd(), build_os))
for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]:
ensure_dir("%s/%s/" % (topdir, subdir))
distro_arch = distro.archname(arch)
# RPM tools take these macro files that define variables in
# RPMland. Unfortunately, there's no way to tell RPM tools to use
# a given file *in addition* to the files that it would already
# load, so we have to figure out what it would normally load,
# augment that list, and tell RPM to use the augmented list. To
# figure out what macrofiles ordinarily get loaded, older RPM
# versions had a parameter called "macrofiles" that could be
# extracted from "rpm --showrc". But newer RPM versions don't
# have this. To tell RPM what macros to use, older versions of
# RPM have a --macros option that doesn't work; on these versions,
# you can put a "macrofiles" parameter into an rpmrc file. But
# that "macrofiles" setting doesn't do anything for newer RPM
# versions, where you have to use the --macros flag instead. And
# all of this is to let us do our work with some guarantee that
# we're not clobbering anything that doesn't belong to us.
#
# On RHEL systems, --rcfile will generally be used and
# --macros will be used in Ubuntu.
#
macrofiles = [
l for l in backtick(["rpm", "--showrc"]).split("\n")
if l.startswith("macrofiles")
]
flags = []
macropath = os.getcwd() + "/macros"
write_rpm_macros_file(macropath, topdir, distro.release_dist(build_os))
if len(macrofiles) > 0:
macrofiles = macrofiles[0] + ":" + macropath
rcfile = os.getcwd() + "/rpmrc"
write_rpmrc_file(rcfile, macrofiles)
flags = ["--rcfile", rcfile]
else:
# This hard-coded hooey came from some box running RPM
# 4.4.2.3. It may not work over time, but RPM isn't sanely
# configurable.
flags = [
"--macros",
"/usr/lib/rpm/macros:/usr/lib/rpm/%s-linux/macros:/usr/lib/rpm/suse/macros:/etc/rpm/macros.*:/etc/rpm/macros:/etc/rpm/%s-linux/macros:~/.rpmmacros:%s"
% (distro_arch, distro_arch, macropath)
]
# Put the specfile and the tar'd up binaries and stuff in
# place.
#
# The version of rpm and rpm tools in RHEL 5.5 can't interpolate the
# %{dynamic_version} macro, so do it manually
with open(specfile, "r") as spec_source:
with open(topdir + "SPECS/" + os.path.basename(specfile),
"w") as spec_dest:
for line in spec_source:
line = line.replace('%{dynamic_version}',
spec.pversion(distro))
line = line.replace('%{dynamic_release}', spec.prelease())
spec_dest.write(line)
oldcwd = os.getcwd()
os.chdir(sdir + "/../")
try:
sysassert([
"tar", "-cpzf", topdir + "SOURCES/mongodb%s-%s.tar.gz" %
(suffix, spec.pversion(distro)),
os.path.basename(os.path.dirname(sdir))
])
finally:
os.chdir(oldcwd)
# Do the build.
flags.extend([
"-D", "dynamic_version " + spec.pversion(distro), "-D",
"dynamic_release " + spec.prelease(), "-D", "_topdir " + topdir
])
# Versions of RPM after 4.4 ignore our BuildRoot tag so we need to
# specify it on the command line args to rpmbuild
if ((distro.name() == "suse" and distro.repo_os_version(build_os) == "15")
or (distro.name() == "redhat"
and distro.repo_os_version(build_os) == "8")):
flags.extend([
"--buildroot",
os.path.join(topdir, "BUILDROOT"),
])
sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags +
["%s/SPECS/mongodb%s.spec" % (topdir, suffix)])
repo_dir = distro.repodir(arch, build_os, spec)
ensure_dir(repo_dir)
# FIXME: see if some combination of shutil.copy<hoohah> and glob
# can do this without shelling out.
sysassert([
"sh", "-c",
"cp -v \"%s/RPMS/%s/\"*.rpm \"%s\"" % (topdir, distro_arch, repo_dir)
])
return repo_dir
def fq2fq(f_in, f_in_type, f_out, f_out_type, link='symbolic', tmp_dir=None):
"""
It converts qualities in a FASTQ file.
f_in_type ('sanger','solexa','illumina','auto-detect')
f_out_type ('sanger','solexa','illumina','illumina-1.5')
link ('soft','hard','copy')
tmp_dir temporary directory
"""
if f_in_type.lower() == 'auto-detect':
# detect the input FASTQ format type
f_in_type = detect_fastq_format(f_in)
print >> sys.stderr, "Auto-detect found " + f_in_type.upper(
) + " FASTQ format!"
fit = 'fastq-' + f_in_type
fot = ''
if f_out_type == 'illumina-1.5':
fot = 'fastq-illumina'
else:
fot = 'fastq-' + f_out_type
if fit == fot and fit != '-':
# input type is same as output type
if os.path.isfile(f_out) or os.path.islink(f_out):
os.remove(f_out)
if link == 'soft':
if os.path.islink(f_in):
linkto = os.readlink(f_in)
os.symlink(linkto, f_out)
else:
os.symlink(f_in, f_out)
elif link == 'hard':
linkto = f_in
if os.path.islink(f_in):
linkto = os.readlink(f_in)
try:
os.link(linkto, f_out)
except OSError as er:
print >> sys.stderr, "WARNING: Cannot do hard links ('%s' and '%s')!" % (
linkto, f_out)
shutil.copyfile(linkto, f_out)
# if er.errno == errno.EXDEV:
# # they are on different partitions
# # [Errno 18] Invalid cross-device link
# shutil.copyfile(linkto,f_out)
# else:
# print >>sys.stderr,"ERROR: Cannot do hard links ('%s' and '%s')!" % (linkto,f_out)
# print >>sys.stderr,er
# sys.exit(1)
elif options.link == 'copy':
shutil.copyfile(f_in, f_out)
else:
print >> sys.stderr, "ERROR: unknown operation of linking!", link
sys.exit(1)
else:
if hasattr(Bio.SeqIO, 'convert'):
fin = f_in
if f_in == "-":
fin = sys.stdin
elif f_in.lower().endswith('.gz'):
fid = gzip.open(f_in, 'r')
else:
fin = open(f_in, 'r')
fout = f_out
if f_out == "-":
fout = sys.stdout
elif f_out.lower().endswith('.gz'):
fout = gzip.open(f_out, 'r')
else:
fout = open(f_out, 'w')
counts = Bio.SeqIO.convert(f_in, fit, f_out, fot)
fin.close()
fout.close()
else:
print >> sys.stderr, "Bio.SeqIO.convert() not supported!"
print >> sys.stderr, "Trying to go around it!"
if f_in == '-' or f_out == '-':
print >> sys.stderr, "ERROR: BioPython library from Python is tool old! Please, upgrade it!"
sys.exit(1)
input_handle = open(f_in, "rU")
output_handle = open(f_out, "w")
sequences = Bio.SeqIO.parse(input_handle, fit)
counts = Bio.SeqIO.write(sequences, output_handle, fot)
output_handle.close()
input_handle.close()
if f_out_type == 'illumina-1.5':
ftemp = give_me_temp_filename(tmp_dir)
fi = open(f_out, 'rb')
fo = open(ftemp, 'wb')
size_buffer = 10**8
i = 0
while True:
lines = fi.readlines(size_buffer)
if not lines:
break
lines = [
line.replace('@', 'B').replace('A', 'B') if
((i + j + 1) % 4 == 0) else line
for (j, line) in enumerate(lines)
]
i = i + len(lines)
fo.writelines(lines)
fi.close()
fo.close()
shutil.move(ftemp, f_out)
print >> sys.stderr, "Converted %i records" % counts
def _hardlink(src, dst):
try:
os.link(src, dst)
except:
_copy(src, dst)
output.append("-" + prev[old].key)
old = old + 1
topfile.write(version.to_line(len(output)) + "\n")
for entry in output:
topfile.write(entry + "\n")
prev = msgs
topfile.close()
# For paranoia at the moment we're keeping a few revisions. This
# could almost certainly be done more elegantly, and it would be nice
# to do versioning in subdirectories.
try:
os.unlink(topfile_name + ".older")
except FileNotFoundError as e:
pass
try:
os.link(topfile_name + ".old", topfile_name + ".older")
except FileNotFoundError as e:
pass
else:
os.unlink(topfile_name + ".old")
try:
os.link(topfile_name, topfile_name + ".old")
except FileNotFoundError as e:
pass
else:
os.unlink(topfile_name)
os.link(topfile_name + ".new", topfile_name)
os.unlink(topfile_name + ".new")
exit(0)
def handle_event(self, event, filetype):
# pylint: disable=redefined-variable-type
rsa_public_key = None
site = event.get("site")
if not site:
site = self.find_site_for_file(event["full_path"])
encryption_key_id = self.config["backup_sites"][site][
"encryption_key_id"]
if encryption_key_id:
rsa_public_key = self.config["backup_sites"][site][
"encryption_keys"][encryption_key_id]["public"]
compressed_blob = None
if event.get("compress_to_memory"):
output_obj = BytesIO()
compressed_filepath = None
else:
compressed_filepath = self.get_compressed_file_path(
site, filetype, event["full_path"])
output_obj = NamedTemporaryFile(
dir=os.path.dirname(compressed_filepath),
prefix=os.path.basename(compressed_filepath),
suffix=".tmp-compress")
input_obj = event.get("input_data")
if not input_obj:
input_obj = open(event["full_path"], "rb")
with output_obj, input_obj:
if filetype == "xlog":
wal.verify_wal(wal_name=os.path.basename(event["full_path"]),
fileobj=input_obj)
original_file_size, compressed_file_size = rohmufile.write_file(
input_obj=input_obj,
output_obj=output_obj,
compression_algorithm=self.config["compression"]["algorithm"],
compression_level=self.config["compression"]["level"],
rsa_public_key=rsa_public_key,
log_func=self.log.info,
)
if compressed_filepath:
os.link(output_obj.name, compressed_filepath)
else:
compressed_blob = output_obj.getvalue()
if event.get("delete_file_after_compression", True):
os.unlink(event["full_path"])
metadata = event.get("metadata", {})
metadata.update({
"pg-version":
self.config["backup_sites"][site].get("pg_version"),
"compression-algorithm":
self.config["compression"]["algorithm"],
"compression-level":
self.config["compression"]["level"],
"original-file-size":
original_file_size,
})
if encryption_key_id:
metadata.update({"encryption-key-id": encryption_key_id})
if compressed_filepath:
metadata_path = compressed_filepath + ".metadata"
write_json_file(metadata_path, metadata)
self.set_state_defaults_for_site(site)
self.state[site][filetype]["original_data"] += original_file_size
self.state[site][filetype]["compressed_data"] += compressed_file_size
self.state[site][filetype]["count"] += 1
if original_file_size:
size_ratio = compressed_file_size / original_file_size
self.stats.gauge("pghoard.compressed_size_ratio",
size_ratio,
tags={
"algorithm":
self.config["compression"]["algorithm"],
"site":
site,
"type":
filetype,
})
transfer_object = {
"callback_queue": event.get("callback_queue"),
"file_size": compressed_file_size,
"filetype": filetype,
"metadata": metadata,
"opaque": event.get("opaque"),
"site": site,
"type": "UPLOAD",
}
if compressed_filepath:
transfer_object["local_path"] = compressed_filepath
else:
transfer_object["blob"] = compressed_blob
transfer_object["local_path"] = event["full_path"]
self.transfer_queue.put(transfer_object)
return True
def make_rpm(distro, build_os, arch, spec, srcdir): # pylint: disable=too-many-locals
"""Create the RPM specfile."""
suffix = spec.suffix()
sdir = setupdir(distro, build_os, arch, spec)
specfile = srcdir + "rpm/mongodb%s.spec" % suffix
init_spec = specfile.replace(".spec", "-init.spec")
# The Debian directory is here for the manpages so we we need to remove the service file
# from it so that RPM packages don't end up with the Debian file.
os.unlink(sdir + "debian/mongod.service")
# Swap out systemd files, different systemd spec files, and init scripts as needed based on
# underlying os version. Arranged so that new distros moving forward automatically use
# systemd. Note: the SUSE init packages use a different init script than then other RPM
# distros.
#
if distro.name() == "suse" and distro.repo_os_version(build_os) in ("10", "11"):
os.unlink(sdir + "rpm/init.d-mongod")
os.link(sdir + "rpm/init.d-mongod.suse", sdir + "rpm/init.d-mongod")
os.unlink(specfile)
os.link(init_spec, specfile)
elif distro.name() == "redhat" and distro.repo_os_version(build_os) in ("5", "6"):
os.unlink(specfile)
os.link(init_spec, specfile)
elif distro.name() == "amazon":
os.unlink(specfile)
os.link(init_spec, specfile)
topdir = ensure_dir('%s/rpmbuild/%s/' % (os.getcwd(), build_os))
for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]:
ensure_dir("%s/%s/" % (topdir, subdir))
distro_arch = distro.archname(arch)
# The version of rpm and rpm tools in RHEL 5.5 can't interpolate the
# %{dynamic_version} macro, so do it manually
with open(specfile, "r") as spec_source:
with open(topdir + "SPECS/" + os.path.basename(specfile), "w") as spec_dest:
for line in spec_source:
line = line.replace('%{dynamic_version}', spec.pversion(distro))
line = line.replace('%{dynamic_release}', spec.prelease())
spec_dest.write(line)
oldcwd = os.getcwd()
os.chdir(sdir + "/../")
try:
sysassert([
"tar", "-cpzf",
topdir + "SOURCES/mongodb%s-%s.tar.gz" % (suffix, spec.pversion(distro)),
os.path.basename(os.path.dirname(sdir))
])
finally:
os.chdir(oldcwd)
# Do the build.
flags = [
"-D",
f"_topdir {topdir}",
"-D",
f"dist .{distro.release_dist(build_os)}",
"-D",
"_use_internal_dependency_generator 0",
"-D",
f"dynamic_version {spec.pversion(distro)}",
"-D",
f"dynamic_release {spec.prelease()}",
]
# Versions of RPM after 4.4 ignore our BuildRoot tag so we need to
# specify it on the command line args to rpmbuild
if ((distro.name() == "suse" and distro.repo_os_version(build_os) == "15")
or (distro.name() == "redhat" and distro.repo_os_version(build_os) == "8")):
flags.extend([
"--buildroot",
os.path.join(topdir, "BUILDROOT"),
])
sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags +
["%s/SPECS/mongodb%s.spec" % (topdir, suffix)])
repo_dir = distro.repodir(arch, build_os, spec)
ensure_dir(repo_dir)
# FIXME: see if some combination of shutil.copy<hoohah> and glob
# can do this without shelling out.
sysassert(["sh", "-c", "cp -v \"%s/RPMS/%s/\"*.rpm \"%s\"" % (topdir, distro_arch, repo_dir)])
return repo_dir
def run_local_tar_basebackup(self):
pgdata = self.config["backup_sites"][self.site]["pg_data_directory"]
if not os.path.isdir(pgdata):
raise errors.InvalidConfigurationError(
"pg_data_directory {!r} does not exist".format(pgdata))
temp_basebackup_dir, compressed_basebackup = self.get_paths_for_backup(
self.basebackup_path)
compression_algorithm = self.config["compression"]["algorithm"]
compression_level = self.config["compression"]["level"]
rsa_public_key = None
encryption_key_id = self.config["backup_sites"][
self.site]["encryption_key_id"]
if encryption_key_id:
rsa_public_key = self.config["backup_sites"][
self.site]["encryption_keys"][encryption_key_id]["public"]
self.log.debug("Connecting to database to start backup process")
connection_string = connection_string_using_pgpass(
self.connection_info)
with psycopg2.connect(connection_string) as db_conn:
cursor = db_conn.cursor()
if self.pg_version_server >= 90600:
# We'll always use the the non-exclusive backup mode on 9.6 and newer
cursor.execute("SELECT pg_start_backup(%s, false, false)",
[BASEBACKUP_NAME])
backup_label = None
backup_mode = "non-exclusive"
else:
# On older versions, first check if we're in recovery, and find out the version of a possibly
# installed pgespresso extension. We use pgespresso's backup control functions when they're
# available, and require them in case we're running on a replica. We also make sure the
# extension version is 1.2 or newer to prevent crashing when using tablespaces.
cursor.execute(
"SELECT pg_is_in_recovery(), "
" (SELECT extversion FROM pg_extension WHERE extname = 'pgespresso')"
)
in_recovery, pgespresso_version = cursor.fetchone()
if in_recovery and (not pgespresso_version
or pgespresso_version < "1.2"):
raise errors.InvalidConfigurationError(
"pgespresso version 1.2 or higher must be installed "
"to take `local-tar` backups from a replica")
if pgespresso_version and pgespresso_version >= "1.2":
cursor.execute("SELECT pgespresso_start_backup(%s, false)",
[BASEBACKUP_NAME])
backup_label = cursor.fetchone()[0]
backup_mode = "pgespresso"
else:
cursor.execute("SELECT pg_start_backup(%s)",
[BASEBACKUP_NAME])
with open(os.path.join(pgdata, "backup_label"), "r") as fp:
backup_label = fp.read()
backup_mode = "legacy"
backup_stopped = False
try:
# Look up tablespaces and resolve their current filesystem locations
cursor.execute(
"SELECT oid, spcname FROM pg_tablespace WHERE spcname NOT IN ('pg_default', 'pg_global')"
)
tablespaces = {
spcname: {
"path":
os.readlink(os.path.join(pgdata, "pg_tblspc",
str(oid))),
"oid":
oid,
}
for oid, spcname in cursor.fetchall()
}
db_conn.commit()
self.log.info("Starting to backup %r to %r", pgdata,
compressed_basebackup)
start_time = time.monotonic()
with NamedTemporaryFile(
dir=temp_basebackup_dir,
prefix="data.",
suffix=".tmp-compress") as raw_output_obj:
with rohmufile.file_writer(
fileobj=raw_output_obj,
compression_algorithm=compression_algorithm,
compression_level=compression_level,
rsa_public_key=rsa_public_key) as output_obj:
with tarfile.TarFile(fileobj=output_obj,
mode="w") as output_tar:
self.write_init_entries_to_tar(
pgdata=pgdata,
tablespaces=tablespaces,
tar=output_tar)
files = self.find_files_to_backup(
pgdata=pgdata,
tablespaces=tablespaces) # NOTE: generator
self.write_files_to_tar(files=files,
tar=output_tar)
self.write_pg_control_to_tar(pgdata=pgdata,
tar=output_tar)
# Call the stop backup functions now to get backup label for 9.6+ non-exclusive backups
if backup_mode == "non-exclusive":
cursor.execute(
"SELECT labelfile FROM pg_stop_backup(false)"
)
backup_label = cursor.fetchone()[0]
elif backup_mode == "pgespresso":
cursor.execute(
"SELECT pgespresso_stop_backup(%s)",
[backup_label])
else:
cursor.execute("SELECT pg_stop_backup()")
db_conn.commit()
backup_stopped = True
backup_label_data = backup_label.encode("utf-8")
self.write_backup_label_to_tar(
tar=output_tar, backup_label=backup_label_data)
input_size = output_obj.tell()
os.link(raw_output_obj.name, compressed_basebackup)
result_size = raw_output_obj.tell()
rohmufile.log_compression_result(
elapsed=time.monotonic() - start_time,
encrypted=True if rsa_public_key else False,
log_func=self.log.info,
original_size=input_size,
result_size=result_size,
source_name=pgdata,
)
finally:
db_conn.rollback()
if not backup_stopped:
if backup_mode == "non-exclusive":
cursor.execute("SELECT pg_stop_backup(false)")
elif backup_mode == "pgespresso":
cursor.execute("SELECT pgespresso_stop_backup(%s)",
[backup_label])
else:
cursor.execute("SELECT pg_stop_backup()")
db_conn.commit()
start_wal_segment, backup_start_time = self.parse_backup_label(
backup_label_data)
metadata = {
"compression-algorithm": compression_algorithm,
"encryption-key-id": encryption_key_id,
"format": "pghoard-bb-v1",
"original-file-size": input_size,
"pg-version": self.pg_version_server,
"start-time": backup_start_time,
"start-wal-segment": start_wal_segment,
}
for spcname, spcinfo in tablespaces.items():
metadata["tablespace-name-{}".format(spcinfo["oid"])] = spcname
metadata["tablespace-path-{}".format(
spcinfo["oid"])] = spcinfo["path"]
self.transfer_queue.put({
"callback_queue": self.callback_queue,
"file_size": result_size,
"filetype": "basebackup",
"local_path": compressed_basebackup,
"metadata": metadata,
"site": self.site,
"type": "UPLOAD",
})
except Exception, e:
error(extract_dents, 'Warn', 'DIR Fail: %s' % e)
if 'dent' in inode:
for dnode in inode['dent']:
extract_dents(ubifs, inodes, dnode, dent_path, perms)
elif dent_node.type == UBIFS_ITYPE_REG:
try:
if inode['ino'].nlink > 1:
if 'hlink' not in inode:
inode['hlink'] = dent_path
buf = _process_reg_file(ubifs, inode, dent_path)
_write_reg_file(dent_path, buf)
else:
os.link(inode['hlink'], dent_path)
log(extract_dents,
'Make Link: %s > %s' % (dent_path, inode['hlink']))
else:
buf = _process_reg_file(ubifs, inode, dent_path)
_write_reg_file(dent_path, buf)
if perms:
_set_file_perms(dent_path, inode)
except Exception, e:
error(extract_dents, 'Warn', 'FILE Fail: %s' % e)
elif dent_node.type == UBIFS_ITYPE_LNK:
try:
# probably will need to decompress ino data if > UBIFS_MIN_COMPR_LEN
def _update_index(self, story, delete=False):
index = "%s/topics/.INDEX" % self.path
lk = open(index + ".lck", "w")
fcntl.flock(lk, 2)
oi = open(index, "r")
if oi is not None:
# new index
ni = open(index + ".new", "w")
t = story.get("topic_id")
s = story.get("id")
d = story.get("date")
if delete is True:
# null record entry
r = None
else:
# record entry
r = ":".join([
d,
t,
s,
",".join(story.get("tags")),
story.get("udate")
]) + "\n"
# iterate current index
for l in oi:
tr = l.replace("\n", "").split(":")
# if not already saved and this record
# is older, store here and destroy
if r is not None and d > tr[0]:
ni.write(r)
r = None
# store this record if it's not this story
if t != tr[1] or s != tr[2]:
ni.write(l)
# not yet stored? do it now
if r is not None:
ni.write(r)
# now swap
try:
os.unlink(index + ".old")
except:
pass
os.link(index, index + ".old")
os.rename(index + ".new", index)
# finally close and release lock
oi.close()
ni.close()
else:
# no index; create it
l = []
# loop al stories
for t in self.topics():
for s in self.stories(t):
story = self.story(t, s)
r = [
story.get("date") or ("0" * 14),
t,
s,
",".join(story.get("tags")),
story.get("udate")
]
l.append(":".join(r))
# reverse order
l.sort(reverse=True)
with open(index, "w") as ni:
for r in l:
ni.write(r + "\n")
lk.close()
def tar_one_file(self,
*,
temp_dir,
chunk_path,
files_to_backup,
callback_queue,
filetype="basebackup_chunk",
extra_metadata=None):
start_time = time.monotonic()
site_config = self.config["backup_sites"][self.site]
encryption_key_id = site_config["encryption_key_id"]
if encryption_key_id:
rsa_public_key = site_config["encryption_keys"][encryption_key_id][
"public"]
else:
rsa_public_key = None
with NamedTemporaryFile(dir=temp_dir,
prefix=os.path.basename(chunk_path),
suffix=".tmp") as raw_output_obj:
# pylint: disable=bad-continuation
with rohmufile.file_writer(
compression_algorithm=self.config["compression"]
["algorithm"],
compression_level=self.config["compression"]["level"],
compression_threads=site_config[
"basebackup_compression_threads"],
rsa_public_key=rsa_public_key,
fileobj=raw_output_obj) as output_obj:
with tarfile.TarFile(fileobj=output_obj,
mode="w") as output_tar:
self.write_files_to_tar(files=files_to_backup,
tar=output_tar)
input_size = output_obj.tell()
result_size = raw_output_obj.tell()
# Make the file persist over the with-block with this hardlink
os.link(raw_output_obj.name, chunk_path)
rohmufile.log_compression_result(
encrypted=bool(encryption_key_id),
elapsed=time.monotonic() - start_time,
original_size=input_size,
result_size=result_size,
source_name="$PGDATA files ({})".format(len(files_to_backup)),
log_func=self.log.info,
)
size_ratio = result_size / input_size
self.metrics.gauge("pghoard.compressed_size_ratio",
size_ratio,
tags={
"algorithm":
self.config["compression"]["algorithm"],
"site": self.site,
"type": "basebackup",
})
metadata = {
"compression-algorithm": self.config["compression"]["algorithm"],
"encryption-key-id": encryption_key_id,
"format": "pghoard-bb-v2",
"original-file-size": input_size,
"host": socket.gethostname(),
}
if extra_metadata:
metadata.update(extra_metadata)
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": result_size,
"filetype": filetype,
"local_path": chunk_path,
"metadata": metadata,
"site": self.site,
"type": "UPLOAD",
})
# Get the name of the chunk and the name of the parent directory (ie backup "name")
chunk_name = "/".join(chunk_path.split("/")[-2:])
return chunk_name, input_size, result_size
def link(space, src, dst):
"Create a hard link to a file."
try:
os.link(src, dst)
except OSError as e:
raise wrap_oserror(space, e)
def drive_APMrover2(binary,
viewerip=None,
use_map=False,
valgrind=False,
gdb=False,
frame=None,
params=None,
gdbserver=False,
speedup=10):
"""Drive APMrover2 in SITL.
you can pass viewerip as an IP address to optionally send fg and
mavproxy packets too for local viewing of the mission in real time
"""
global homeloc
if frame is None:
frame = 'rover'
options = '--sitl=127.0.0.1:5501 --out=127.0.0.1:19550 --streamrate=10'
if viewerip:
options += " --out=%s:14550" % viewerip
if use_map:
options += ' --map'
home = "%f,%f,%u,%u" % (HOME.lat, HOME.lng, HOME.alt, HOME.heading)
sitl = util.start_SITL(binary,
wipe=True,
model=frame,
home=home,
speedup=speedup)
mavproxy = util.start_MAVProxy_SITL('APMrover2', options=options)
progress("WAITING FOR PARAMETERS")
mavproxy.expect('Received [0-9]+ parameters')
# setup test parameters
if params is None:
params = vinfo.options["APMrover2"]["frames"][frame][
"default_params_filename"]
if not isinstance(params, list):
params = [params]
for x in params:
mavproxy.send("param load %s\n" % os.path.join(testdir, x))
mavproxy.expect('Loaded [0-9]+ parameters')
set_parameter(mavproxy, 'LOG_REPLAY', 1)
set_parameter(mavproxy, 'LOG_DISARMED', 1)
# restart with new parms
util.pexpect_close(mavproxy)
util.pexpect_close(sitl)
sitl = util.start_SITL(binary,
model='rover',
home=home,
speedup=speedup,
valgrind=valgrind,
gdb=gdb,
gdbserver=gdbserver)
mavproxy = util.start_MAVProxy_SITL('APMrover2', options=options)
mavproxy.expect('Telemetry log: (\S+)')
logfile = mavproxy.match.group(1)
progress("LOGFILE %s" % logfile)
buildlog = util.reltopdir("../buildlogs/APMrover2-test.tlog")
progress("buildlog=%s" % buildlog)
if os.path.exists(buildlog):
os.unlink(buildlog)
try:
os.link(logfile, buildlog)
except Exception:
pass
mavproxy.expect('Received [0-9]+ parameters')
util.expect_setup_callback(mavproxy, expect_callback)
expect_list_clear()
expect_list_extend([sitl, mavproxy])
progress("Started simulator")
# get a mavlink connection going
try:
mav = mavutil.mavlink_connection('127.0.0.1:19550',
robust_parsing=True)
except Exception as msg:
progress("Failed to start mavlink connection on 127.0.0.1:19550" % msg)
raise
mav.message_hooks.append(message_hook)
mav.idle_hooks.append(idle_hook)
failed = False
e = 'None'
try:
progress("Waiting for a heartbeat with mavlink protocol %s" %
mav.WIRE_PROTOCOL_VERSION)
mav.wait_heartbeat()
progress("Setting up RC parameters")
setup_rc(mavproxy)
progress("Waiting for GPS fix")
mav.wait_gps_fix()
homeloc = mav.location()
progress("Home location: %s" % homeloc)
if not arm_rover(mavproxy, mav):
progress("Failed to ARM")
failed = True
if not drive_mission(mavproxy, mav, os.path.join(
testdir, "rover1.txt")):
progress("Failed mission")
failed = True
if not drive_brake(mavproxy, mav):
progress("Failed brake")
failed = True
if not disarm_rover(mavproxy, mav):
progress("Failed to DISARM")
failed = True
if not log_download(mavproxy, mav,
util.reltopdir("../buildlogs/APMrover2-log.bin")):
progress("Failed log download")
failed = True
# if not drive_left_circuit(mavproxy, mav):
# progress("Failed left circuit")
# failed = True
# if not drive_RTL(mavproxy, mav):
# progress("Failed RTL")
# failed = True
# do not move this to be the first test. MAVProxy's dedupe
# function may bite you.
progress("Getting banner")
if not do_get_banner(mavproxy, mav):
progress("FAILED: get banner")
failed = True
progress("Getting autopilot capabilities")
if not do_get_autopilot_capabilities(mavproxy, mav):
progress("FAILED: get capabilities")
failed = True
progress("Setting mode via MAV_COMMAND_DO_SET_MODE")
if not do_set_mode_via_command_long(mavproxy, mav):
failed = True
except pexpect.TIMEOUT as e:
progress("Failed with timeout")
failed = True
mav.close()
util.pexpect_close(mavproxy)
util.pexpect_close(sitl)
valgrind_log = util.valgrind_log_filepath(binary=binary, model='rover')
if os.path.exists(valgrind_log):
os.chmod(valgrind_log, 0o644)
shutil.copy(valgrind_log,
util.reltopdir("../buildlogs/APMrover2-valgrind.log"))
if failed:
progress("FAILED: %s" % e)
return False
return True
def start(ds, **kwargs):
# Read the user input from the POST
global HOST_IP
urllib3.disable_warnings()
yaml = read_yaml(WORKDIR + '/INPUT.yaml')
REC_ISO_IMAGE_NAME = yaml['iso_primary']
REC_PROVISIONING_ISO_NAME = yaml['iso_secondary']
INPUT_YAML_URL = yaml['input_yaml']
HOST_IP = yaml['rc_host']
CLOUDNAME = 'CL-'+POD.POD
ISO = '%s/images/install-%s.iso' % (RI_DIR, POD.POD)
BOOTISO = '%s/images/bootcd-%s.iso' % (RI_DIR, POD.POD)
USERCONF = '%s/user-configs/%s/user_config.yaml' % (RI_DIR, CLOUDNAME)
print('-----------------------------------------------------------------------------------------------')
print(' POD is '+POD.POD)
print(' CLOUDNAME is '+CLOUDNAME)
print(' WORKDIR is '+WORKDIR)
print(' HOST_IP is '+HOST_IP)
print(' EXTERNALROOT is '+EXTERNALROOT)
print(' REC_ISO_IMAGE_NAME is '+REC_ISO_IMAGE_NAME)
print('REC_PROVISIONING_ISO_NAME is '+REC_PROVISIONING_ISO_NAME)
print(' INPUT_YAML_URL is '+INPUT_YAML_URL)
print(' ISO is '+ISO)
print(' BOOTISO is '+BOOTISO)
print(' USERCONF is '+USERCONF)
print('-----------------------------------------------------------------------------------------------')
# Setup RI_DIR
initialize_RI(CLOUDNAME)
# Fetch the three files into WORKDIR
fetchURL(REC_ISO_IMAGE_NAME, WORKDIR + '/install.iso');
fetchURL(REC_PROVISIONING_ISO_NAME, WORKDIR + '/bootcd.iso');
fetchURL(INPUT_YAML_URL, WORKDIR + '/user_config.yaml');
# Link files to RI_DIR with unique names
os.link(WORKDIR + '/install.iso', ISO)
os.link(WORKDIR + '/bootcd.iso', BOOTISO)
os.link(WORKDIR + '/user_config.yaml', USERCONF)
PWFILE = '%s/user-configs/%s/admin_passwd' % (RI_DIR, CLOUDNAME)
with open(PWFILE, "w") as f:
f.write(ADMIN_PASSWD + '\n')
# Start the remote_installer
client = docker.from_env()
namefilt = { 'name': RI_NAME }
ri = client.containers.list(filters=namefilt)
if len(ri) == 0:
print(RI_NAME + ' is not running.')
c = start_RI(client)
else:
print(RI_NAME + ' is running.')
c = ri[0]
# Send request to remote_installer
id = send_request(HOST_IP, CLOUDNAME, ISO, BOOTISO)
# Wait up to WAIT_TIME minutes for completion
if wait_for_completion(HOST_IP, id, WAIT_TIME):
print('Installation failed after %d minutes.' % (WAIT_TIME))
sys.exit(1)
# Remove the ISOs?
if REMOVE_ISO:
for iso in (WORKDIR + '/install.iso', ISO, WORKDIR + '/bootcd.iso', BOOTISO):
os.unlink(iso)
# Done!
print('Installation complete!')
# sys.exit(0) Don't exit as this will cause the task to fail!
return 'Complete.'
def install(self, spec, prefix):
quux_cc = '''#include "quux.h"
#include "garply/garply.h"
#include "quux_version.h"
#include <iostream>
#include <stdexcept>
const int Quux::version_major = quux_version_major;
const int Quux::version_minor = quux_version_minor;
Quux::Quux() {}
int
Quux::get_version() const
{
return 10 * version_major + version_minor;
}
int
Quux::quuxify() const
{
int quux_version = get_version();
std::cout << "Quux::quuxify version " << quux_version
<< " invoked" <<std::endl;
std::cout << "Quux config directory is %s" <<std::endl;
Garply garply;
int garply_version = garply.garplinate();
if (garply_version != quux_version) {
throw std::runtime_error(
"Quux found an incompatible version of Garply.");
}
return quux_version;
}
'''
quux_h = '''#ifndef QUUX_H_
class Quux
{
private:
static const int version_major;
static const int version_minor;
public:
Quux();
int get_version() const;
int quuxify() const;
};
#endif // QUUX_H_
'''
quuxifier_cc = '''
#include "quux.h"
#include <iostream>
int
main()
{
Quux quux;
quux.quuxify();
return 0;
}
'''
quux_version_h = '''const int quux_version_major = %s;
const int quux_version_minor = %s;
'''
mkdirp('%s/quux' % prefix.include)
mkdirp('%s/quux' % self.stage.source_path)
with open('%s/quux_version.h' % self.stage.source_path, 'w') as f:
f.write(quux_version_h % (self.version[0], self.version[1:]))
with open('%s/quux/quux.cc' % self.stage.source_path, 'w') as f:
f.write(quux_cc % (prefix.config))
with open('%s/quux/quux.h' % self.stage.source_path, 'w') as f:
f.write(quux_h)
with open('%s/quux/quuxifier.cc' % self.stage.source_path, 'w') as f:
f.write(quuxifier_cc)
gpp = which('/usr/bin/g++')
if sys.platform == 'darwin':
gpp = which('/usr/bin/clang++')
gpp('-Dquux_EXPORTS', '-I%s' % self.stage.source_path,
'-I%s' % spec['garply'].prefix.include, '-O2', '-g', '-DNDEBUG',
'-fPIC', '-o', 'quux.cc.o', '-c', 'quux/quux.cc')
gpp('-Dquux_EXPORTS', '-I%s' % self.stage.source_path,
'-I%s' % spec['garply'].prefix.include, '-O2', '-g', '-DNDEBUG',
'-fPIC', '-o', 'quuxifier.cc.o', '-c', 'quux/quuxifier.cc')
if sys.platform == 'darwin':
gpp('-fPIC', '-O2', '-g', '-DNDEBUG', '-dynamiclib',
'-Wl,-headerpad_max_install_names', '-o', 'libquux.dylib',
'-install_name', '@rpath/libcorge.dylib', 'quux.cc.o',
'-Wl,-rpath,%s' % prefix.lib64,
'-Wl,-rpath,%s' % spec['garply'].prefix.lib64,
'%s/libgarply.dylib' % spec['garply'].prefix.lib64)
gpp('-O2', '-g', '-DNDEBUG', 'quuxifier.cc.o', '-o', 'quuxifier',
'-Wl,-rpath,%s' % prefix.lib64,
'-Wl,-rpath,%s' % spec['garply'].prefix.lib64, 'libquux.dylib',
'%s/libgarply.dylib' % spec['garply'].prefix.lib64)
mkdirp(prefix.lib64)
copy('libquux.dylib', '%s/libquux.dylib' % prefix.lib64)
os.link('%s/libquux.dylib' % prefix.lib64,
'%s/libquux.dylib.3.0' % prefix.lib64)
else:
gpp(
'-fPIC', '-O2', '-g', '-DNDEBUG', '-shared',
'-Wl,-soname,libquux.so', '-o', 'libquux.so', 'quux.cc.o',
'-Wl,-rpath,%s:%s::::' %
(prefix.lib64, spec['garply'].prefix.lib64),
'%s/libgarply.so' % spec['garply'].prefix.lib64)
gpp(
'-O2', '-g', '-DNDEBUG', '-rdynamic', 'quuxifier.cc.o', '-o',
'quuxifier', '-Wl,-rpath,%s:%s::::' %
(prefix.lib64, spec['garply'].prefix.lib64), 'libquux.so',
'%s/libgarply.so' % spec['garply'].prefix.lib64)
mkdirp(prefix.lib64)
copy('libquux.so', '%s/libquux.so' % prefix.lib64)
os.link('%s/libquux.so' % prefix.lib64,
'%s/libquux.so.3.0' % prefix.lib64)
copy('quuxifier', '%s/quuxifier' % prefix.lib64)
copy('%s/quux/quux.h' % self.stage.source_path,
'%s/quux/quux.h' % prefix.include)
mkdirp(prefix.bin)
copy('quux_version.h', '%s/quux_version.h' % prefix.bin)
os.symlink('%s/quuxifier' % prefix.lib64, '%s/quuxifier' % prefix.bin)
os.symlink('%s/garplinator' % spec['garply'].prefix.lib64,
'%s/garplinator' % prefix.bin)
def basebackup_compression_pipe(self, proc, basebackup_path):
rsa_public_key = None
encryption_key_id = self.config["backup_sites"][
self.site]["encryption_key_id"]
if encryption_key_id:
rsa_public_key = self.config["backup_sites"][
self.site]["encryption_keys"][encryption_key_id]["public"]
compression_algorithm = self.config["compression"]["algorithm"]
compression_level = self.config["compression"]["level"]
self.log.debug("Compressing basebackup directly to file: %r",
basebackup_path)
set_stream_nonblocking(proc.stderr)
metadata = {
"compression-algorithm": compression_algorithm,
"encryption-key-id": encryption_key_id,
"host": socket.gethostname(),
}
with NamedTemporaryFile(prefix=basebackup_path,
suffix=".tmp-compress") as output_obj:
def extract_header_func(input_data):
# backup_label should always be first in the tar ball
if input_data[0:12].startswith(b"backup_label"):
# skip the 512 byte tar header to get to the actual backup label content
start_wal_segment, start_time = self.parse_backup_label(
input_data[512:1024])
metadata.update({
"start-wal-segment": start_wal_segment,
"start-time": start_time
})
def progress_callback():
stderr_data = proc.stderr.read()
if stderr_data:
self.latest_activity = datetime.datetime.utcnow()
self.log.debug("pg_basebackup stderr: %r", stderr_data)
original_input_size, compressed_file_size = rohmufile.write_file(
input_obj=proc.stdout,
output_obj=output_obj,
compression_algorithm=compression_algorithm,
compression_level=compression_level,
rsa_public_key=rsa_public_key,
progress_callback=progress_callback,
log_func=self.log.info,
header_func=extract_header_func)
os.link(output_obj.name, basebackup_path)
if original_input_size:
size_ratio = compressed_file_size / original_input_size
self.metrics.gauge("pghoard.compressed_size_ratio",
size_ratio,
tags={
"algorithm": compression_algorithm,
"site": self.site,
"type": "basebackup",
})
return original_input_size, compressed_file_size, metadata
tc = trpc.Client(host, port, user, password)
torrent = tc.get_torrents(hash)[0]
for each in torrent.files().values():
if each['completed'] == each['size']:
if each['name'].endswith(
('.rar', '.avi', '.mkv',
'.mp4')) and 'sample' not in each['name'].lower(
) and '/subs' not in each['name'].lower():
if each['name'].endswith('.rar'):
file = os.path.basename(each['name'])
if 'part' not in each['name'] or 'part01' in each['name']:
print file
print dir + each['name']
print output
subprocess.call([
'7z', 'x', dir + each['name'], '-aos',
'-o' + output
])
print 'Successfully extracted {}'.format(dir +
each['name'])
if 'tv_shows' in type:
autoProcessTV.processEpisode(output)
else:
file = os.path.basename(each['name'])
os.link(dir + each['name'], output + file)
if 'tv_shows' in type:
autoProcessTV.processEpisode(output, file)
except KeyError, e:
print "Environment Variables not supplied - is this being called from Transmission?"
sys.exit()
#dst_dir = os.path.join(dst, platform, year, month, day, pair_folder)
dst_dir = os.path.join(dst, platform, year, month, day)
if not os.path.isdir(dst_dir):
if not args.dryrun:
os.makedirs(dst_dir)
for ifp in glob.glob(
os.path.join(raster.srcdir, raster.stripid) + "*"):
ofp = os.path.join(dst_dir, os.path.basename(ifp))
if os.path.isfile(ofp) and args.overwrite:
logger.debug("Copying {} to {}".format(ifp, ofp))
if not args.dryrun:
os.remove(ofp)
if args.try_link:
os.link(ifp, ofp)
else:
shutil.copy2(ifp, ofp)
elif not os.path.isfile(ofp):
logger.debug("Copying {} to {}".format(ifp, ofp))
if not args.dryrun:
if args.try_link:
os.link(ifp, ofp)
else:
shutil.copy2(ifp, ofp)
else:
logger.warning("Cannot copy {} to {}".format(ifp, ofp))
logger.info('Done')
