def __init__(self, config, timeout):
# config
self.host = config["host"]
self.username = config["username"]
self.python = config["python"]
self.port = config["port"]
self.interval = config["interval"]
self.custom = config["custom"]
self.telegraf = config["telegraf"]
self.comment = config["comment"]
self.config = AgentConfig(config)
# connection
self.session = None
self.ssh = SecuredShell(self.host, self.port, self.username, timeout)
self.incoming_queue = Queue.Queue()
self.buffer = ""
self.reader = MonitoringReader(self.incoming_queue)
handle, cfg_path = tempfile.mkstemp(".cfg", "agent_")
os.close(handle)
self.path = {
# Destination path on remote host
"AGENT_REMOTE_FOLDER": "/tmp/",
# Source path on tank
"AGENT_LOCAL_FOLDER": os.path.dirname(__file__) + "/agent",
"TELEGRAF_REMOTE_PATH": "/tmp/telegraf",
"TELEGRAF_LOCAL_PATH": self.telegraf,
}
def test_create_volume_from_image_exception(self):
"""Verify that create volume from image, the volume status is
'downloading'."""
dst_fd, dst_path = tempfile.mkstemp()
os.close(dst_fd)
self.stubs.Set(self.volume.driver, 'local_path', lambda x: dst_path)
image_id = 'aaaaaaaa-0000-0000-0000-000000000000'
# creating volume testdata
volume_id = 1
db.volume_create(self.context,
{'id': volume_id,
'updated_at': datetime.datetime(1, 1, 1, 1, 1, 1),
'display_description': 'Test Desc',
'size': 20,
'status': 'creating',
'host': 'dummy'})
self.assertRaises(exception.ImageNotFound,
self.volume.create_volume,
self.context,
volume_id, None, None, None,
None,
image_id)
volume = db.volume_get(self.context, volume_id)
self.assertEqual(volume['status'], "error")
# cleanup
db.volume_destroy(self.context, volume_id)
os.unlink(dst_path)
def onConnect():
# if keyAgent and options['agent']:
# cc = protocol.ClientCreator(reactor, SSHAgentForwardingLocal, conn)
# cc.connectUNIX(os.environ['SSH_AUTH_SOCK'])
if hasattr(conn.transport, 'sendIgnore'):
_KeepAlive(conn)
if options.localForwards:
for localPort, hostport in options.localForwards:
s = reactor.listenTCP(localPort,
forwarding.SSHListenForwardingFactory(conn,
hostport,
SSHListenClientForwardingChannel))
conn.localForwards.append(s)
if options.remoteForwards:
for remotePort, hostport in options.remoteForwards:
log.msg('asking for remote forwarding for %s:%s' %
(remotePort, hostport))
conn.requestRemoteForwarding(remotePort, hostport)
reactor.addSystemEventTrigger('before', 'shutdown', beforeShutdown)
if not options['noshell'] or options['agent']:
conn.openChannel(SSHSession())
if options['fork']:
if os.fork():
os._exit(0)
os.setsid()
for i in range(3):
try:
os.close(i)
except OSError as e:
import errno
if e.errno != errno.EBADF:
raise
def close_safe(fd):
'''Close a file descriptor ignoring any exception it generates'''
#pylint: disable-msg=W0704
try:
os.close(fd)
except OSError:
pass
def print_xcf(self, filename_or_obj, *args, **kwargs):
"Writes the figure to a GIMP XCF image file"
# If filename_or_obj is a file-like object we need a temporary file for
# GIMP's output too...
if is_string(filename_or_obj):
out_temp_handle, out_temp_name = None, filename_or_obj
else:
out_temp_handle, out_temp_name = tempfile.mkstemp(suffix='.xcf')
try:
# Create a temporary file and write the "layer" to it as a PNG
in_temp_handle, in_temp_name = tempfile.mkstemp(suffix='.png')
try:
FigureCanvasAgg.print_png(self, in_temp_name, *args, **kwargs)
run_gimp_script(
SINGLE_LAYER_SCRIPT.format(
input=quote_string(in_temp_name),
output=quote_string(out_temp_name)))
finally:
os.close(in_temp_handle)
os.unlink(in_temp_name)
finally:
if out_temp_handle:
os.close(out_temp_handle)
# If we wrote the XCF to a temporary file, write its content to
# the file-like object we were given (the copy is chunked as
# XCF files can get pretty big)
with open(out_temp_name, 'rb') as source:
for chunk in iter(lambda: source.read(131072), ''):
filename_or_obj.write(chunk)
os.unlink(out_temp_name)
def config_control(control_path):
"""
Edit the control file to adapt the current environment.
Replace CLIENTIP with guestip, and replace SERVERIP with hostip.
:return: Path of a temp file which contains the result of replacing.
"""
pattern2repl_dict = {r'CLIENTIP': vm.get_address(),
r'SERVERIP': utils_net.get_host_ip_address(params)}
control_file = open(control_path)
lines = control_file.readlines()
control_file.close()
for pattern, repl in pattern2repl_dict.items():
for index in range(len(lines)):
line = lines[index]
lines[index] = re.sub(pattern, repl, line)
fd, temp_control_path = tempfile.mkstemp(prefix="control",
dir=data_dir.get_tmp_dir())
os.close(fd)
temp_control = open(temp_control_path, "w")
temp_control.writelines(lines)
temp_control.close()
return temp_control_path
def daemonize( errfile ):
"""
Detach process and become a daemon.
"""
pid = os.fork()
if pid:
os._exit(0)
os.setsid()
signal.signal(signal.SIGHUP, signal.SIG_IGN)
os.umask(0)
pid = os.fork()
if pid:
os._exit(0)
os.chdir("/")
for fd in range(0,20):
try:
os.close(fd)
except OSError:
pass
sys.stdin = open("/dev/null","r")
sys.stdout = open("/dev/null","w")
sys.stderr = ErrorLog( errfile )
def get_img_id(doc, image, bpc, nr_channels, cs, p = default_cfg()):
spec = doc.image_definition()
spec.dimensions(*p.img_dim)
spec.bits_per_component(bpc)
spec.dpi(*p.dpi)
spec.color_space(cs)
spec.format(jagpdf.IMAGE_FORMAT_NATIVE)
img_data = g_img_cache.img_data(image, bpc, nr_channels, *p.img_dim)
if p.from_file:
handle, tmp_file = tempfile.mkstemp()
img_data.tofile(open(tmp_file,'wb'))
os.close(handle)
g_temp_files.add(tmp_file)
if p.from_file:
spec.file_name(tmp_file)
else:
spec.data(img_data)
desc = "%d levels per channel (%d %s)" % (2**bpc, bpc, bpc > 1 and "bits" or "bit")
if p.spec_fn:
desc2 = p.spec_fn(spec)
if desc2:
desc = desc2
id_ = doc.image_load(spec)
testlib.must_throw(doc.image_load, spec) # cannot load the same spec twice
return id_, desc
def _write_file_chunks(self, chunk_map, file_extension=""):
"""
Given a mapping of chunks, write their contents to a temporary file,
returning the path to that file.
Returned file path should be manually removed by the user.
:param chunk_map: Mapping of integer index to file-like chunk
:type chunk_map: dict of (int, StringIO)
:param file_extension: String extension to suffix the temporary file
with
:type file_extension: str
:raises OSError: OS problems creating temporary file or writing it out.
:return: Path to temporary combined file
:rtype: str
"""
# Make sure write dir exists...
if not os.path.isdir(self.working_dir):
safe_create_dir(self.working_dir)
tmp_fd, tmp_path = tempfile.mkstemp(file_extension, dir=self.working_dir)
self.log.debug("Combining chunks into temporary file: %s", tmp_path)
# tmp_file = os.fdopen(tmp_fd, 'wb')
tmp_file = open(tmp_path, "wb")
for idx, chunk in sorted(chunk_map.items(), key=lambda p: p[0]):
data = chunk.read()
tmp_file.write(data)
tmp_file.close() # apparently also closes file descriptor?
os.close(tmp_fd)
return tmp_path
def save_data(data, output_filename):
"""
Save data to file.
If the file already exists, the function will not save the data and return
1
Parameters
----------
data : str
String containing the data you wish to write out to a file
output_filename : str
Path (full or relative) to the file you will save the data into.
Returns
-------
out : int
Return 0 if the data was saved successfully. Return 1 if the file
already exists.
Hint
----
Check out the os module for determining whether a file exists already.
"""
if not os.path.exists(output_filename): # output_file doesn't currently exist
fd = os.open(output_filename,os.O_WRONLY|os.O_CREAT)
os.write(fd,data)
os.close(fd)
return 0
else:
return 1
def create(obj, dummy_eng):
#FIXME change share tmp directory
from invenio.config import CFG_TMPSHAREDDIR
from invenio.legacy.bibsched.bibtask import task_low_level_submission, \
bibtask_allocate_sequenceid
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.seal()
tmp_file_fd, tmp_file_path = mkstemp(
prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
suffix='.xml',
dir=CFG_TMPSHAREDDIR,
)
os.write(tmp_file_fd, sip.package)
os.close(tmp_file_fd)
# Trick to have access to task_sequence_id in subsequent tasks.
d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()
task_id = task_low_level_submission(
'bibupload', 'webdeposit',
'-r' if 'recid' in sip.metadata else '-i', tmp_file_path,
'-I', str(d.workflow_object.task_sequence_id)
)
sip.task_ids.append(task_id)
d.update()
def processFileSize(fileName):
global rpmDict
sizeDict = rpmDict['fileSizes']
fd = os.open(fileName, os.O_RDONLY)
header = transactionSet.hdrFromFdno(fd)
os.close(fd)
files = header.fiFromHeader()
for fileObj in files:
fileName = fileObj[FILEOBJ_FILENAME]
size = roundUpTo(fileObj[FILEOBJ_SIZE], BLOCK_SIZE)
mode = fileObj[FILEOBJ_MODE]
if stat.S_ISDIR(mode):
if fileName not in sizeDict:
sizeDict[fileName] = BLOCK_SIZE
elif stat.S_ISREG(mode) or stat.S_ISLNK(mode):
dirName = os.path.dirname(fileName)
if dirName not in sizeDict:
sizeDict[dirName] = BLOCK_SIZE
if stat.S_ISLNK(mode):
sizeDict[dirName] += BLOCK_SIZE
else:
sizeDict[dirName] += size
else:
print "Didn't know how to handle %s" % fileName
def setUp(self):
self.packager = packager_app.PackagerApp()
self.input = os.path.join(
test_env.SRC_DIR, 'media', 'test', 'data', 'bear-1280x720.mp4')
self.tmpdir = tempfile.mkdtemp()
fd, self.output = tempfile.mkstemp(dir=self.tmpdir)
os.close(fd)
def install_nvm(options):
print '---------- Installing NVM ---------'
check_run_quick('sudo chmod 775 /usr/local')
check_run_quick('sudo mkdir -m 777 -p /usr/local/node /usr/local/nvm')
result = check_fetch(
'https://raw.githubusercontent.com/creationix/nvm/{nvm_version}/install.sh'
.format(nvm_version=NVM_VERSION))
fd, temp = tempfile.mkstemp()
os.write(fd, result.content)
os.close(fd)
try:
run_and_monitor(
'bash -c "NVM_DIR=/usr/local/nvm source {temp}"'.format(temp=temp))
finally:
os.remove(temp)
# curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.26.0/install.sh | NVM_DIR=/usr/local/nvm bash
check_run_and_monitor('sudo bash -c "cat > /etc/profile.d/nvm.sh"',
input=__NVM_SCRIPT)
print '---------- Installing Node {version} ---------'.format(
version=NODE_VERSION)
run_and_monitor('bash -c "source /etc/profile.d/nvm.sh'
'; nvm install {version}'
'; nvm alias default {version}"'
.format(version=NODE_VERSION))
def WriteToTemporaryFile(data):
(fd, fname) = tempfile.mkstemp()
os.close(fd)
tmp_file = open(fname, "w")
tmp_file.write(data)
tmp_file.close()
return fname
def Close(self):
"""
The physical file is automatically deleted after being closed.
"""
super(TemporaryFile, self).Close()
os.close(self.__fd)
os.remove(self.__tmp_path)
def saved_fd(fd):
new_fd = os.dup(fd)
try:
yield
finally:
os.dup2(new_fd, fd)
os.close(new_fd)
def __call__(self, *args): ##, **kw): kw unused
import hotshot, hotshot.stats, os, tempfile ##, time already imported
f, filename = tempfile.mkstemp()
os.close(f)
prof = hotshot.Profile(filename)
stime = time.time()
result = prof.runcall(self.func, *args)
stime = time.time() - stime
prof.close()
import cStringIO
out = cStringIO.StringIO()
stats = hotshot.stats.load(filename)
stats.stream = out
stats.strip_dirs()
stats.sort_stats('time', 'calls')
stats.print_stats(40)
stats.print_callers()
x = '\n\ntook '+ str(stime) + ' seconds\n'
x += out.getvalue()
# remove the tempfile
try:
os.remove(filename)
except IOError:
pass
return result, x
def dumpScreen(utilityPath):
"""dumps a screenshot of the entire screen to a directory specified by
the MOZ_UPLOAD_DIR environment variable"""
# Need to figure out which OS-dependent tool to use
if mozinfo.isUnix:
utility = [os.path.join(utilityPath, "screentopng")]
utilityname = "screentopng"
elif mozinfo.isMac:
utility = ['/usr/sbin/screencapture', '-C', '-x', '-t', 'png']
utilityname = "screencapture"
elif mozinfo.isWin:
utility = [os.path.join(utilityPath, "screenshot.exe")]
utilityname = "screenshot"
# Get dir where to write the screenshot file
parent_dir = os.environ.get('MOZ_UPLOAD_DIR', None)
if not parent_dir:
log.info('Failed to retrieve MOZ_UPLOAD_DIR env var')
return
# Run the capture
try:
tmpfd, imgfilename = tempfile.mkstemp(prefix='mozilla-test-fail-screenshot_', suffix='.png', dir=parent_dir)
os.close(tmpfd)
returncode = subprocess.call(utility + [imgfilename])
printstatus(returncode, utilityname)
except OSError, err:
log.info("Failed to start %s for screenshot: %s",
utility[0], err.strerror)
return
def __exit__(self, exe_type, exe_val, tb):
sys.stdout.flush()
sys.stderr.flush()
os.dup2(self.old_out, 1)
os.dup2(self.old_err, 2)
os.close(self.old_out)
os.close(self.old_err)
self.temp.seek(0)
# dump everything if an exception was thrown
if exe_type is not None:
for line in self.temp:
sys.stderr.write(line)
return False
# if no exception is thrown only dump important lines
for line in self.temp:
if self._should_veto(line):
continue
accept = self._should_accept(line)
if self.re is not None:
re_found = self.re.search(line)
else:
re_found = False
if accept or re_found:
sys.stderr.write(line)
def _load_master_key(self):
"""Load the master key from file, or create one if not available."""
# TODO(jamielennox): This is but one way that a key file could be
# stored. This can be pluggable later for storing/fetching keys from
# better locations.
mkey = None
try:
with open(CONF.crypto.master_key_file, 'r') as f:
mkey = base64.b64decode(f.read())
except IOError as e:
if e.errno == errno.ENOENT:
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
mkey = self.crypto.new_key(self.KEY_SIZE)
f = None
try:
f = os.open(CONF.crypto.master_key_file, flags, 0o600)
os.write(f, base64.b64encode(mkey))
except Exception as x:
_logger.warn('Failed to read master key initially: %s', e)
_logger.warn('Failed to create new master key: %s', x)
raise x
finally:
if f:
os.close(f)
else:
# the file could be unreadable due to bad permissions
# so just pop up whatever error comes
raise e
return mkey
def getTerminalSize():
def ioctl_GWINSZ(fd):
try:
import fcntl, termios, struct, os
cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
except:
return None
return cr
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
if not cr:
import os
try:
fd = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_GWINSZ(fd)
os.close(fd)
except:
pass
if cr:
return int(cr[1]), int(cr[0])
try:
h, w = os.popen("stty size", "r").read().split()
return int(w), int(h)
except:
pass
return 80, 25
def __deserialize_file(self, response):
"""
Saves response body into a file in a temporary folder,
using the filename from the `Content-Disposition` header if provided.
:param response: RESTResponse.
:return: file path.
"""
config = Configuration()
fd, path = tempfile.mkstemp(dir=config.temp_folder_path)
os.close(fd)
os.remove(path)
content_disposition = response.getheader("Content-Disposition")
if content_disposition:
filename = re.\
search(r'filename=[\'"]?([^\'"\s]+)[\'"]?', content_disposition).\
group(1)
path = os.path.join(os.path.dirname(path), filename)
with open(path, "w") as f:
f.write(response.data)
return path
def closerange(fd_low, fd_high):
# Iterate through and close all file descriptors.
for fd in range(fd_low, fd_high):
try:
os.close(fd)
except OSError: # ERROR, fd wasn't open to begin with (ignored)
pass
def sort( self, fields, offset = 0 ):
"""sort file according to criteria."""
if self.mIsOpened:
reopen = 1
self.close()
else:
reopen = 0
sort_criteria = []
for field in fields:
id, modifier = self.mMapKey2Field[ field ]
field_id = 1 + id + offset
sort_criteria.append( "-k%i,%i%s" % (field_id, field_id, modifier) )
outfile, filename_temp = tempfile.mkstemp()
os.close(outfile)
## empty fields are a problem with sort, which by default uses whitespace to non-whitespace
## transitions as field separate.
## make \t explicit field separator.
statement = "sort -t'\t' %s %s > %s" % (string.join( sort_criteria, " " ), self.mFilename, filename_temp)
exit_code = os.system(statement)
if exit_code:
raise "error while sorting, statement =%s" % statement
os.system( "mv %s %s" % (filename_temp, self.mFilename))
if reopen: self.open()
def create_signed_cert(self, ou, san="IP:127.0.0.1,IP:::1,DNS:localhost"):
print_ok("generating {0}.key, {0}.crt, {0}.p12".format(ou))
fd, openssl_config = mkstemp(dir='.')
os.write(fd, "extendedKeyUsage=clientAuth,serverAuth\n".encode('utf-8'))
os.write(fd, "subjectAltName = {0}".format(san).encode('utf-8'))
call("openssl genrsa -out {0}.key 1024".format(ou),
shell=True, stderr=FNULL)
call(
"openssl req -new -key {0}.key -out {0}.csr -subj /C=US/ST=CA/O=ghostunnel/OU={0}".format(ou),
shell=True,
stderr=FNULL)
call("chmod 600 {0}.key".format(ou), shell=True)
call(
"openssl x509 -req -in {0}.csr -CA {1}.crt -CAkey {1}.key -CAcreateserial -out {0}_temp.crt -days 5 -extfile {2}".format(
ou,
self.name,
openssl_config),
shell=True,
stderr=FNULL)
call(
"openssl pkcs12 -export -out {0}_temp.p12 -in {0}_temp.crt -inkey {0}.key -password pass:"******"{0}_temp.crt".format(ou), "{0}.crt".format(ou))
os.rename("{0}_temp.p12".format(ou), "{0}.p12".format(ou))
os.close(fd)
os.remove(openssl_config)
self.leaf_certs.append(ou)
def install_rust(self):
"""Download and run the rustup installer."""
import errno
import stat
import tempfile
platform = rust.platform()
url = rust.rustup_url(platform)
checksum = rust.rustup_hash(platform)
if not url or not checksum:
print('ERROR: Could not download installer.')
sys.exit(1)
print('Downloading rustup-init... ', end='')
fd, rustup_init = tempfile.mkstemp(prefix=os.path.basename(url))
os.close(fd)
try:
self.http_download_and_save(url, rustup_init, checksum)
mode = os.stat(rustup_init).st_mode
os.chmod(rustup_init, mode | stat.S_IRWXU)
print('Ok')
print('Running rustup-init...')
subprocess.check_call([rustup_init, '-y',
'--default-toolchain', 'stable',
'--default-host', platform,
])
cargo_home, cargo_bin = self.cargo_home()
self.print_rust_path_advice(RUST_INSTALL_COMPLETE,
cargo_home, cargo_bin)
finally:
try:
os.remove(rustup_init)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def verboseCallback(self, what, amount, total, mydata, wibble):
""" This method provides a verbose callback.
@param what: callback type
@param amount: bytes processed
@param total: total bytes
@param mydata: package key (hdr, path)
@param wibble: user data
"""
if wibble: print wibble
if what == rpm.RPMCALLBACK_TRANS_START:
pass
elif what == rpm.RPMCALLBACK_INST_OPEN_FILE:
hdr, path = mydata
print "Installing %s\r" % (hdr["name"])
fdno = os.open(path, os.O_RDONLY)
nvr = '%s-%s-%s' % ( hdr['name'], hdr['version'], hdr['release'] )
self.fdnos[nvr] = fdno
return fdno
elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE:
hdr, path = mydata
nvr = '%s-%s-%s' % ( hdr['name'], hdr['version'], hdr['release'] )
os.close(self.fdnos[nvr])
elif what == rpm.RPMCALLBACK_INST_PROGRESS:
hdr, path = mydata
print "%s: %.5s%% done\r" % (hdr["name"], (float(amount) / total) * 100)
return None
def make_tmp_file(self, content):
fd, path = tempfile.mkstemp(suffix=u'-nxdrive-file-to-upload',
dir=self.upload_tmp_dir)
with open(path, "wb") as f:
f.write(content)
os.close(fd)
return path
def test_dataError(self, *args, **kw):
import aio
# read more data than available
q = aio.Queue(1)
fd = os.open(TEST_FILENAME, os.O_DIRECT)
self.assertRaises(IOError, q.scheduleRead, fd, 0, 1, 4096 * 4096)
os.close(fd)
def getFile(self, *args, **kwds):
''' Keeping this as an alternative for the code.
We don't use it because it's not possible to know if the call was a
PK-one (and so we push the content of a temporary filename to fd or
file) or a non-PK-one (in which case nothing should be done).
filename = None
fd = None
file = None
if use_pycups:
if len(kwds) != 1:
use_pycups = True
elif kwds.has_key('filename'):
filename = kwds['filename']
elif kwds.has_key('fd'):
fd = kwds['fd']
elif kwds.has_key('file'):
file = kwds['file']
else:
use_pycups = True
if fd or file:
'''
file_object = None
fd = None
if len(args) == 2:
(use_pycups, resource, filename) = self._args_to_tuple([str, str], *args)
else:
(use_pycups, resource) = self._args_to_tuple([str], *args)
if 'filename' in kwds:
filename = kwds['filename']
elif 'fd' in kwds:
fd = kwds['fd']
elif 'file' in kwds:
file_object = kwds['file']
else:
if not use_pycups:
raise TypeError()
else:
filename = None
if (not use_pycups) and (fd is not None or file_object is not None):
# Create the temporary file in /tmp to ensure that
# cups-pk-helper-mechanism is able to write to it.
(tmpfd, tmpfname) = tempfile.mkstemp(dir="/tmp")
os.close (tmpfd)
pk_args = (resource, tmpfname)
self._call_with_pk_and_fallback(use_pycups,
'FileGet', pk_args,
self._connection.getFile,
*args, **kwds)
tmpfd = os.open (tmpfname, os.O_RDONLY)
tmpfile = os.fdopen (tmpfd, 'rt')
tmpfile.seek (0)
if fd is not None:
os.lseek (fd, 0, os.SEEK_SET)
line = tmpfile.readline()
while line != '':
os.write (fd, line.encode('UTF-8'))
line = tmpfile.readline()
else:
file_object.seek (0)
line = tmpfile.readline()
while line != '':
file_object.write (line.encode('UTF-8'))
line = tmpfile.readline()
tmpfile.close ()
os.remove (tmpfname)
else:
pk_args = (resource, filename)
self._call_with_pk_and_fallback(use_pycups,
'FileGet', pk_args,
self._connection.getFile,
*args, **kwds)
def main ():
options = docopt.docopt(usage, help=False)
options['--env'] = '' # exabgp compatibility
root = root_folder(options,['/bin/exabgpcli','/sbin/exabgpcli','/lib/exabgp/application/cli.py'])
prefix = '' if root == '/usr' else root
etc = prefix + '/etc/exabgp'
envfile = get_envfile(options,etc)
env = get_env(envfile)
pipename = env['api']['pipename']
if options['--help']:
sys.stdout.write(usage)
sys.stdout.flush()
sys.exit(0)
if not options['<command>']:
sys.stdout.write(usage)
sys.stdout.flush()
sys.exit(0)
command = ' '.join(options['<command>'])
pipes = named_pipe(root)
if len(pipes) != 1:
sys.stdout.write('could not find ExaBGP\'s named pipes (%s.in and %s.out) for the cli\n' % (pipename, pipename))
sys.stdout.write('we scanned the following folders (the number is your PID):\n - ')
sys.stdout.write('\n - '.join(pipes))
sys.stdout.flush()
sys.exit(1)
send = pipes[0] + pipename + '.in'
recv = pipes[0] + pipename + '.out'
if not check_fifo(send):
sys.stdout.write('could not find write named pipe to connect to ExaBGP')
sys.stdout.flush()
sys.exit(1)
if not check_fifo(recv):
sys.stdout.write('could not find read named pipe to connect to ExaBGP')
sys.stdout.flush()
sys.exit(1)
def write_timeout(signum, frame):
sys.stderr.write('could not send command to ExaBGP')
sys.stderr.flush()
sys.exit(1)
signal.signal(signal.SIGALRM, write_timeout)
signal.alarm(2)
try:
writer = os.open(send, os.O_WRONLY | os.O_EXCL)
os.write(writer,command.encode('utf-8') + b'\n')
os.close(writer)
except OSError as exc:
if exc.errno == errno.ENXIO:
sys.stdout.write('ExaBGP is not running / using the configured named pipe')
sys.stdout.flush()
sys.exit(1)
sys.stdout.write('could not communicate with ExaBGP')
sys.stdout.flush()
sys.exit(1)
except IOError as exc:
sys.stdout.write('could not communicate with ExaBGP')
sys.stdout.flush()
sys.exit(1)
signal.alarm(0)
if command == 'reset':
sys.exit(0)
def read_timeout(signum, frame):
sys.stderr.write('could not read answer to ExaBGP')
sys.stderr.flush()
sys.exit(1)
signal.signal(signal.SIGALRM, read_timeout)
try:
signal.alarm(5)
reader = os.open(recv, os.O_RDONLY | os.O_EXCL)
signal.alarm(0)
buf = b''
done = False
while not done:
try:
raw = os.read(reader,4096)
buf += raw
while b'\n' in buf:
line,buf = buf.split(b'\n',1)
if line == b'done':
done = True
break
if line == b'shutdown':
sys.stderr.write('ExaBGP is shutting down, command aborted\n')
sys.stderr.flush()
done = True
break
if line == b'error':
done = True
sys.stderr.write('ExaBGP returns an error\n')
sys.stderr.flush()
break
sys.stdout.write('%s\n' % line.decode())
sys.stdout.flush()
select.select([reader],[],[],0.01)
except OSError as exc:
if exc.errno in error.block:
break
except IOError as exc:
if exc.errno in error.block:
break
os.close(reader)
sys.exit(0)
except IOError:
sys.stdout.write('could not read answer from ExaBGP')
sys.stdout.flush()
except OSError:
sys.stdout.write('could not read answer from ExaBGP')
sys.stdout.flush()
pass
try:
import termios
except ImportError:
termios = None
USE_INOTIFY = False
try:
# Test whether inotify is enabled and likely to work
import pyinotify
fd = pyinotify.INotifyWrapper.create().inotify_init()
if fd >= 0:
USE_INOTIFY = True
os.close(fd)
except ImportError:
pass
RUN_RELOADER = True
FILE_MODIFIED = 1
I18N_MODIFIED = 2
_mtimes = {}
_win = (sys.platform == "win32")
_exception = None
_error_files = []
_cached_modules = set()
_cached_filenames = []
def tearDown(self):
os.close(self.db_fd)
os.remove(self.db_path)
def _daemon(cls, jupyter_notebook_filename):
from clearml import Task
# load jupyter notebook package
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from nbconvert.exporters.script import ScriptExporter
_script_exporter = ScriptExporter()
except Exception as ex:
_logger.warning('Could not read Jupyter Notebook: {}'.format(ex))
return
# load pigar
# noinspection PyBroadException
try:
from ....utilities.pigar.reqs import get_installed_pkgs_detail, file_import_modules
from ....utilities.pigar.modules import ReqsModules
from ....utilities.pigar.log import logger
logger.setLevel(logging.WARNING)
except Exception:
file_import_modules = None
# load IPython
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from IPython import get_ipython
except Exception:
# should not happen
get_ipython = None
# setup local notebook files
if jupyter_notebook_filename:
notebook = Path(jupyter_notebook_filename)
local_jupyter_filename = jupyter_notebook_filename
else:
notebook = None
fd, local_jupyter_filename = mkstemp(suffix='.ipynb')
os.close(fd)
last_update_ts = None
counter = 0
prev_script_hash = None
# noinspection PyBroadException
try:
from ....version import __version__
our_module = cls.__module__.split('.')[0], __version__
except Exception:
our_module = None
# noinspection PyBroadException
try:
import re
replace_ipython_pattern = re.compile(r'\n([ \t]*)get_ipython\(\)')
except Exception:
replace_ipython_pattern = None
# main observer loop, check if we need to exit
while not cls._exit_event.wait(timeout=0.):
# wait for timeout or sync event
cls._sync_event.wait(cls._sample_frequency if counter else cls.
_first_sample_frequency)
cls._sync_event.clear()
counter += 1
# noinspection PyBroadException
try:
# if there is no task connected, do nothing
task = Task.current_task()
if not task:
continue
script_code = None
fmodules = None
current_cell = None
# if we have a local file:
if notebook:
if not notebook.exists():
continue
# check if notebook changed
if last_update_ts is not None and notebook.stat(
).st_mtime - last_update_ts <= 0:
continue
last_update_ts = notebook.stat().st_mtime
else:
# serialize notebook to a temp file
if cls._jupyter_history_logger:
script_code, current_cell = cls._jupyter_history_logger.history_to_str(
)
else:
# noinspection PyBroadException
try:
# noinspection PyBroadException
try:
os.unlink(local_jupyter_filename)
except Exception:
pass
get_ipython().run_line_magic(
'history',
'-t -f {}'.format(local_jupyter_filename))
with open(local_jupyter_filename, 'r') as f:
script_code = f.read()
# load the modules
from ....utilities.pigar.modules import ImportedModules
fmodules = ImportedModules()
for nm in set(
[str(m).split('.')[0] for m in sys.modules]):
fmodules.add(nm, 'notebook', 0)
except Exception:
continue
# get notebook python script
if script_code is None and local_jupyter_filename:
script_code, _ = _script_exporter.from_filename(
local_jupyter_filename)
if cls._store_notebook_artifact:
# also upload the jupyter notebook as artifact
task.upload_artifact(
name='notebook',
artifact_object=Path(local_jupyter_filename),
preview='See `notebook preview` artifact',
metadata={
'UPDATE':
datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
},
wait_on_upload=True,
)
# noinspection PyBroadException
try:
from nbconvert.exporters import HTMLExporter # noqa
html, _ = HTMLExporter().from_filename(
filename=local_jupyter_filename)
local_html = Path(
gettempdir()) / 'notebook_{}.html'.format(
task.id)
with open(local_html.as_posix(), 'wt') as f:
f.write(html)
task.upload_artifact(
name='notebook preview',
artifact_object=local_html,
preview='Click `FILE PATH` link',
metadata={
'UPDATE':
datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S')
},
delete_after_upload=True,
wait_on_upload=True,
)
except Exception:
pass
current_script_hash = hash(script_code + (current_cell or ''))
if prev_script_hash and prev_script_hash == current_script_hash:
continue
# remove ipython direct access from the script code
# we will not be able to run them anyhow
if replace_ipython_pattern:
script_code = replace_ipython_pattern.sub(
r'\n# \g<1>get_ipython()', script_code)
requirements_txt = ''
conda_requirements = ''
# parse jupyter python script and prepare pip requirements (pigar)
# if backend supports requirements
if file_import_modules and Session.check_min_api_version(
'2.2'):
if fmodules is None:
fmodules, _ = file_import_modules(
notebook.parts[-1] if notebook else 'notebook',
script_code)
if current_cell:
cell_fmodules, _ = file_import_modules(
notebook.parts[-1] if notebook else 'notebook',
current_cell)
# noinspection PyBroadException
try:
fmodules |= cell_fmodules
except Exception:
pass
# add current cell to the script
if current_cell:
script_code += '\n' + current_cell
fmodules = ScriptRequirements.add_trains_used_packages(
fmodules)
# noinspection PyUnboundLocalVariable
installed_pkgs = get_installed_pkgs_detail()
# make sure we are in installed packages
if our_module and (our_module[0] not in installed_pkgs):
installed_pkgs[our_module[0]] = our_module
# noinspection PyUnboundLocalVariable
reqs = ReqsModules()
for name in fmodules:
if name in installed_pkgs:
pkg_name, version = installed_pkgs[name]
reqs.add(pkg_name, version, fmodules[name])
requirements_txt, conda_requirements = ScriptRequirements.create_requirements_txt(
reqs)
# update script
prev_script_hash = current_script_hash
data_script = task.data.script
data_script.diff = script_code
data_script.requirements = {
'pip': requirements_txt,
'conda': conda_requirements
}
# noinspection PyProtectedMember
task._update_script(script=data_script)
# update requirements
# noinspection PyProtectedMember
task._update_requirements(requirements=requirements_txt)
except Exception:
pass
def get_sequencepage(request):
if request.method == 'POST':
# getting values from post
requirement1 = request.POST.get('requirement')
requirement1=requirement1.replace("'","")
requirement = nltk.sent_tokenize(requirement1)
usecases = Sequence.ExtractMultiMessages(requirement)
print("check yyyyyyyyyyyyyyyyyyyyyyy")
print(usecases)
# UsecaseName=Usecase.Name
# adding the values in a context variable
context = {
'requirement': usecases,
}
connectionObject = sqlite3.connect(":memory:")
cursorObject = connectionObject.cursor()
createTableActors = "CREATE TABLE Sequence_Components(sender varchar(32),reciever varchar(32),Message varchar(32),loop INTEGER,MessageType varchar(32),conditions varchar(32),conditionMsg varchar(32),elsemsg varchar(32),sender_else varchar(32),reciver_else varchar(32),conditionBit INTEGER,If_loop INTEGER,else_loop INTEGER, SeqId INTEGER PRIMARY KEY)"
cursorObject.execute(createTableActors)
NoOfNulls=0
NullList={}
NullKeys=[]
NoOfNulls_else = 0
NullList_else = {}
NullKeys_else = []
for index,values in enumerate(usecases):
#for the not if conditions or if have if conditions but not else part
#for the null recivers
if(values[10]!=1 or (values[7]=='' and values[10]==1)):
if(values[1]==''):
NoOfNulls=NoOfNulls+1
NullKeys.append(values[13])
# NullList[values[13]] =values[2]
NullList[values[13]] = values[2]
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], '', values[2], values[3], values[4], values[5], values[6], values[7], values[8],
values[9], values[10], values[11], values[12], values[13]])
# print(key)
elif((values[7]!='' and values[10]==1)):
if (values[1] == ''):
NoOfNulls = NoOfNulls + 1
NullKeys.append(values[13])
# NullList[values[13]] =values[2]
NullList[values[13]] = values[2]
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], '', values[2], values[3], values[4], values[5], values[6], values[7], values[8],
values[9], values[10], values[11], values[12], values[13]])
if(values[9]==''):
NoOfNulls_else = NoOfNulls_else + 1
NullKeys_else.append(values[13])
# NullList[values[13]] =values[2]
NullList_else[values[13]] = values[2]
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], values[1], values[2], values[3], values[4], values[5], values[6], values[7], values[8],
'', values[10], values[11], values[12], values[13]])
else:
cursorObject.execute('INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)', [values[0], values[1],values[2],values[3],values[4],values[5],values[6],values[7],values[8],values[9],values[10],values[11],values[12],values[13]])
# cursorObject.executemany('INSERT INTO Relations(relation) VALUES(?)', relations[key])
# cursorObject.executemany('INSERT INTO Relations VALUES(?,?)', relations)
# print(relations)
#if has else part
elif ((values[7] != '' and values[10] == 1)):
if (values[1] == '' and values[9] != ''):
NoOfNulls = NoOfNulls + 1
NullKeys.append(values[13])
# NullList[values[13]] =values[2]
NullList[values[13]] = values[2]
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], '', values[2], values[3], values[4], values[5], values[6], values[7], values[8],
values[9], values[10], values[11], values[12], values[13]])
elif (values[9] == '' and values[1]!=''):
NoOfNulls_else = NoOfNulls_else + 1
NullKeys_else.append(values[13])
# NullList[values[13]] =values[2]
NullList_else[values[13]] = values[7]
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], values[1], values[2], values[3], values[4], values[5], values[6], values[7],
values[8],'', values[10], values[11], values[12], values[13]])
elif(values[9] == '' and values[1]==''):
NoOfNulls = NoOfNulls + 1
NullKeys.append(values[13])
# NullList[values[13]] =values[2]
NullList[values[13]] = values[2]
NoOfNulls_else = NoOfNulls_else + 1
NullKeys_else.append(values[13])
# NullList[values[13]] =values[2]
NullList_else[values[13]] = values[7]
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], '', values[2], values[3], values[4], values[5], values[6], values[7],
values[8], '', values[10], values[11], values[12], values[13]])
else:
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[
values[0], values[1], values[2], values[3], values[4], values[5], values[6], values[7],
values[8], values[9], values[10], values[11], values[12], values[13]])
else:
cursorObject.execute(
'INSERT INTO Sequence_Components(sender,reciever,Message,loop,MessageType,conditions,conditionMsg,elsemsg,sender_else,reciver_else,conditionBit,If_loop,else_loop,SeqId) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
[values[0], values[1], values[2], values[3], values[4], values[5], values[6], values[7], values[8],
values[9], values[10], values[11],values[12],values[13]])
queryTable_usecases = "SELECT * from Sequence_Components"
queryResults_Relations_usecases = cursorObject.execute(queryTable_usecases)
usecase_list = cursorObject.fetchall()
DropTable = "Drop Table Sequence_Components"
cursorObject.execute(DropTable)
print("drop the usecases table")
print(usecase_list)
# ------------------------------------------------------------------
if request.method == 'POST':
# Open a file
if os.path.exists("uml1app/static/images/draft.png"):
try:
os.remove("uml1app/static/images/draft.png")
# os.remove("draft.png")
print("yes")
except OSError:
print("no")
pass
# Open a file
if os.path.exists("draft.txt"):
try:
os.remove("draft.txt")
except OSError:
pass
# Open a file
if os.path.exists("draft.png"):
try:
os.remove("draft.png")
except OSError:
pass
fd = os.open("draft.txt", os.O_RDWR | os.O_CREAT)
# Write one string
os.write(fd, b"@startuml\n")
#os.write(fd, b"Alice -> Bob: Authentication Request\n")
for component in usecase_list:
#os.write(fd, (b"Alice -> "+component[0]+b": Authentication Request\n").encode('ascii'))
#if conditions are null
if(component[10]==0 or component[10]=='None'):
#verify this as a self message
# if(component[1]!=''):
if(component[3]==1 or component[11]==1):
os.write(fd, ("loop\n"+str(component[0]).lower() + "->" + (str(component[1]).lower()).replace(" ","") + ":" +component[2] + "\nend\n").encode('ascii'))
else:
os.write(fd, (str(component[0]).lower() +"->" + (str(component[1]).lower()).replace(" ","") + ":"+component[2] +"\n").encode('ascii'))
# else:
# if (component[3] != 1 or component[11]!=1):
# os.write(fd, (str(component[0]).lower() + "->" + (str(component[0]).lower()).replace(" ","") + ":" + component[2] + "\n").encode('ascii'))
# else:
# os.write(fd, ("loop\n"+str(component[0]).lower() + "->" + (str(component[0]).lower()).replace(" ", "") + ":" +component[2] + "\nend\n").encode('ascii'))
# #if the sentence has conditios
else:
if(component[7]=='' or component[7]=='None'):
# if (component[1] != ''):
if (component[3] == 1 or component[11]==1):
os.write(fd, ("opt "+component[5]+"\nloop\n" + str(component[0]).lower() + "->" + (
str(component[1]).lower()).replace(" ", "") + ":" + component[2] + "\nend\nend\n").encode(
'ascii'))
else:
os.write(fd, ("opt "+component[5]+"\n"+str(component[0]).lower() + "->" + (str(component[1]).lower()).replace(" ",
"") + ":" +
component[2] + "\nend\n").encode('ascii'))
# else:
# if (component[3] != 1 or component[11]!=1):
# os.write(fd, ("opt "+component[5]+"\n"+str(component[0]).lower() + "->" + (str(component[0]).lower()).replace(" ",
# "") + ":" +
# component[6] + "\nend\n").encode('ascii'))
# else:
# os.write(fd, ("alt"+component[5]+"\nloop\n" + str(component[0]).lower() + "->" + (
# str(component[0]).lower()).replace(" ", "") + ":" + component[6] + "\nend\n").encode(
# 'ascii'))
#for the else messages
else:
# if (component[1] != ''):
if (component[3] == 1 or component[11]==1):
if(component[12]==1):
os.write(fd,
("alt " + component[5] + "\nloop\n" + str(component[0]).lower() + "->" + (
str(component[1]).lower()).replace(" ", "") + ":" + component[
6] + "\nend\nelse else\n"+str(component[8].lower()).replace(" ", "") +"->"+str(component[9].lower()).replace(" ", "") +":"+component[7]+"\nend\n").encode(
'ascii'))
else:
if (component[12] == 1):
os.write(fd,
("alt " + component[5] + "\nloop\n" + str(component[0]).lower() + "->" + (
str(component[1]).lower()).replace(" ", "") + ":" + component[
6] + "\nend\nelse else\nloop\n" + str(component[8].lower()).replace(" ",
"") + "->" + str(
component[9].lower()).replace(" ", "") + ":" + component[
7] + "\nend\nend\n").encode(
'ascii'))
else:
os.write(fd,
("alt " + component[5] + "\nloop\n" + str(
component[0]).lower() + "->" + (
str(component[1]).lower()).replace(" ", "") + ":" + component[
6] + "\nend\nelse else\n" + str(component[8].lower()).replace(" ",
"") + "->" + str(
component[9].lower()).replace(" ", "") + ":" + component[
7] + "\nend\n").encode(
'ascii'))
else:
if (component[12] == 1):
os.write(fd, ("alt " + component[5] + "\n" + str(component[0]).lower() + "->" + (
str(component[1]).lower()).replace(" ",
"") + ":" +
component[6] + "\nelse else\nloop\n"+str(component[8].lower()).replace(" ", "") +"->"+str(component[9].lower()).replace(" ", "") +":"+component[7]+ "\nend\nend\n").encode('ascii'))
else:
os.write(fd, ("alt " + component[5] + "\n" + str(component[0]).lower() + "->" + (
str(component[1]).lower()).replace(" ",
"") + ":" +
component[6] + "\nelse else\n" + str(component[8].lower()).replace(
" ", "") + "->" + str(component[9].lower()).replace(" ", "") + ":" +
component[7] + "\nend\n").encode('ascii'))
# else:
# if (component[3] !=1 or component[11]!=1):
# if(component[12] == 1):
# os.write(fd, ("alt " + component[5] + "\n" + str(component[0]).lower() + "->" + (
# str(component[0]).lower()).replace(" ",
# "") + ":" +
# component[6] + "\nelse else\nloop\n"+str(component[8].lower()).replace(" ", "") +"->"+str(component[9].lower()).replace(" ", "") +":"+component[7]+ "\nend\nend\n").encode('ascii'))
# else:
# os.write(fd, ("alt " + component[5] + "\n" + str(component[0]).lower() + "->" + (
# str(component[0]).lower()).replace(" ",
# "") + ":" +
# component[6] + "\nelse else\n" + str(component[8].lower()).replace(
# " ", "") + "->" + str(component[9].lower()).replace(" ", "") + ":" +
# component[7] + "\nend\n").encode('ascii'))
#
# else:
# if (component[12] == 1):
# os.write(fd, ("alt" + component[5] + "\nloop\n" + str(component[0]).lower() + "->" + (
# str(component[0]).lower()).replace(" ", "") + ":" + component[
# 6] +"\nend\nelse else\nloop\n"+str(component[8].lower()).replace(" ", "") +"->"+str(component[9].lower()).replace(" ", "") +":"+component[7]+"\nend\nend\n").encode(
# 'ascii'))
# else:
# os.write(fd,
# ("alt" + component[5] + "\nloop\n" + str(component[0]).lower() + "->" + (
# str(component[0]).lower()).replace(" ", "") + ":" + component[
# 6] + "\nend\nelse else\n" + str(component[8].lower()).replace(" ",
# "") + "->" + str(
# component[9].lower()).replace(" ", "") + ":" + component[
# 7] + "\nend\n").encode(
# 'ascii'))
os.write(fd, b"@enduml")
# Close opened file
os.close(fd)
# time.sleep(5)
# # for ubuntu-----------------------------------------
# os.system("python -m plantuml draft.txt")
# print("file is created successfully!!")
# os.system("cp draft.png uml1app/static/images")
# # -----------------------------------------------------
# for windows-----------------------------------------
subprocess.call("python -m plantuml draft.txt")
print("file is created successfully!!")
os.system("copy draft.png uml1app\static\images")
# -----------------------------------------------------
obj = Seq_Items.objects.values()
list1 = list(obj)
loopMessages=[]
conditionMessages = []
for items in list1:
print(items['sender'])
if (items['loop'] == '1'):
loopMessages.append(items)
if (items['conditionBit'] == '1'):
conditionMessages.append(items)
print("yyy")
print("mmmmm")
print(NullList)
# for ubuntu-----------------------------------------
os.system("python -m plantuml draft.txt")
print("file is created successfully!!")
os.system("cp draft.png uml1app/static/images")
# -----------------------------------------------------
# # for windows-----------------------------------------
subprocess.call("python -m plantuml draft.txt")
print("file is created successfully!!")
os.system("copy draft.png uml1app\static\images")
# # -----------------------------------------------------
# ------------------------------------------------------------------
# ------------------------------------------------------------------
if request.method == 'POST':
# Open a file
if os.path.exists("uml1app/static/images/antsModel.docx"):
try:
os.remove("uml1app/static/images/antsModel.docx")
# os.remove("draft.png")
print("yes")
except OSError:
print("no")
pass
# Open a file
if os.path.exists("antsModel.docx"):
try:
os.remove("antsModel.docx")
except OSError:
pass
# Open a file
if os.path.exists("antsModel.docx"):
try:
os.remove("antsModel.docx")
except OSError:
pass
fd = os.open("antsModel.docx", os.O_RDWR | os.O_CREAT)
# Write one string
os.write(fd, b"@antsuml\n")
os.write(fd, b"by Ants UML Diagram designers\n")
os.write(fd, b"Actors and usecases\n")
os.write(fd, b"\n")
for values in list1:
if(values['conditionBit']!='1' and values['loop']!='1'):
os.write(fd, ("" + values['sender'] + "--->("+values['Message']+"--->:"+values['MessageType']+")---> "+values['reciever']).encode('ascii'))
elif(values['loop']=='1' and values['conditionBit']!='1' ):
os.write(fd, ("[loop] : " + values['sender'] + "--->(" + values['Message'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciever']).encode('ascii'))
elif (values['loop'] != '1' and values['conditionBit'] == '1'):
if(values['else_loop']!='1'):
os.write(fd, b"if\n")
os.write(fd, ("[opt] : " + values['sender'] + "--->(" + values['Message'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciever']).encode('ascii'))
else:
os.write(fd, b"if\n")
os.write(fd, ("[alt] : " + values['sender'] + "--->(" + values['Message'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciever']).encode('ascii'))
os.write(fd, b"\n")
os.write(fd, b"else\n")
os.write(fd, ("[alt] : " + values['sender_else'] + "--->(" + values['elsemsg'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciver_else']).encode('ascii'))
elif (values['loop'] == '1' and values['conditionBit'] == '1'):
if (values['else_loop'] != '1'):
os.write(fd, b"if\n")
os.write(fd, ("[opt] : [loop]:" + values['sender'] + "--->(" + values['Message'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciever']).encode('ascii'))
else:
os.write(fd, b"if\n")
os.write(fd, ("[alt] : [loop]:" + values['sender'] + "--->(" + values['Message'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciever']).encode('ascii'))
os.write(fd, b"\n")
os.write(fd, b"else\n")
os.write(fd, ("[alt] :[loop]: " + values['sender_else'] + "--->(" + values['elsemsg'] + "--->:" + values[
'MessageType'] + ")---> " + values['reciver_else']).encode('ascii'))
# Close opened file
os.close(fd)
# time.sleep(5)
# # # for ubuntu-----------------------------------------
# os.system("cp antsModel.docx uml1app/static/images")
# # -----------------------------------------------------
# # for windows-----------------------------------------
os.system("copy antsModel.docx uml1app\static\images")
print(list1)
print("get these data")
print(usecase_list)
print(NoOfNulls)
print(NullList)
print(NullKeys)
print(list1)
context1 = {
'participants': usecase_list,
'NoOfNulls' :NoOfNulls,
'NullList' :NullList,
'NullKeys' :NullKeys,
'seq_items' :list1,
'loops' :loopMessages,
'conditions' :conditionMessages,
'NoOfNulls_else': NoOfNulls_else,
'NullList_else': NullList_else,
'NullKeys_else': NullKeys_else,
}
print(loopMessages)
print("context 1 print")
print(context1)
template = loader.get_template("uml1app/sequence.html")
return HttpResponse(template.render(context1, request))
def test_access(self):
f = os.open(test_support.TESTFN, os.O_CREAT | os.O_RDWR)
os.close(f)
self.assertTrue(os.access(test_support.TESTFN, os.W_OK))
def __init__(self, dir=None, suffix=''):
fd, name = tempfile.mkstemp(dir=dir, suffix=suffix)
os.close(fd)
self.name = os.path.normpath(name)
def mktemp(self, prefix=''):
f, name = tempfile.mkstemp(prefix=prefix, dir=self.tempdir)
os.close(f)
return name
def test_ext_add_multi(self):
"""Test behaviour when 'add' options are used multiple times."""
fd, cfgfile = tempfile.mkstemp()
os.close(fd)
f = open(cfgfile, 'w')
f.write('[ext]\nadd-default=two')
f.close()
args = ['--configfiles=%s' % cfgfile]
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_default, 'nowtwo')
# environment overrides config file
os.environ['TEST_EXT_ADD_DEFAULT'] = 'three'
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_default, 'nowthree')
# command line overrides environment + config file, last value wins
args.extend(['--ext-add-default=four', '--ext-add-default=five'])
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_default, 'nowfive')
del os.environ['TEST_EXT_ADD_DEFAULT']
f = open(cfgfile, 'w')
f.write('[ext]\nadd-list-default=two,three')
f.close()
args = ['--configfiles=%s' % cfgfile]
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_list_default, ['now', 'two', 'three'])
os.environ['TEST_EXT_ADD_LIST_DEFAULT'] = 'four,five'
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_list_default, ['now', 'four', 'five'])
args.extend([
'--ext-add-list-default=six',
'--ext-add-list-default=seven,eight',
])
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_list_default, ['now', 'seven', 'eight'])
del os.environ['TEST_EXT_ADD_LIST_DEFAULT']
f = open(cfgfile, 'w')
f.write('[ext]\nadd-list-flex=two,,three')
f.close()
args = ['--configfiles=%s' % cfgfile]
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_list_flex, ['two', 'x', 'y', 'three'])
os.environ['TEST_EXT_ADD_LIST_FLEX'] = 'four,,five'
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_list_flex, ['four', 'x', 'y', 'five'])
args.extend([
'--ext-add-list-flex=six,',
'--ext-add-list-flex=seven,,eight',
'--ext-add-list-flex=,last',
])
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_list_flex, ['x', 'y', 'last'])
del os.environ['TEST_EXT_ADD_LIST_FLEX']
f = open(cfgfile, 'w')
f.write('[ext]\nadd-pathlist-flex=two::three')
f.close()
args = ['--configfiles=%s' % cfgfile]
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_pathlist_flex, ['two', 'p2', 'p3', 'three'])
os.environ['TEST_EXT_ADD_PATHLIST_FLEX'] = 'four::five'
args = ['--configfiles=%s' % cfgfile]
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_pathlist_flex, ['four', 'p2', 'p3', 'five'])
args.extend([
'--ext-add-pathlist-flex=six:',
'--ext-add-pathlist-flex=:last',
'--ext-add-pathlist-flex=seven::eight',
])
topt = TestOption1(go_args=args, envvar_prefix='TEST')
self.assertEqual(topt.options.ext_add_pathlist_flex, ['seven', 'p2', 'p3', 'eight'])
del os.environ['TEST_EXT_ADD_PATHLIST_FLEX']
def convert_quickbook_pages(self, refresh = False):
try:
subprocess.check_call(['quickbook', '--version'])
except:
print("Problem running quickbook, will not convert quickbook articles.")
return
bb_parser = boost_site.boostbook_parser.BoostBookParser()
for page in self.pages:
page_data = self.pages[page]
if page_data.page_state or refresh:
xml_file = tempfile.mkstemp('', '.xml')
os.close(xml_file[0])
xml_filename = xml_file[1]
try:
print("Converting " + page + ":")
subprocess.check_call(['quickbook', '--output-file', xml_filename, '-I', 'feed', page])
page_data.load(bb_parser.parse(xml_filename), refresh)
finally:
os.unlink(xml_filename)
template_vars = {
'history_style' : '',
'full_title_xml' : page_data.full_title_xml,
'title_xml' : page_data.title_xml,
'note_xml' : '',
'web_date' : page_data.web_date(),
'documentation_para' : '',
'download_table' : page_data.download_table(),
'description_xml' : page_data.description_xml
}
if page_data.type == 'release' and 'released' not in page_data.flags and 'beta' not in page_data.flags:
template_vars['note_xml'] = """
<div class="section-note"><p>Note: This release is
still under development. Please don't use this page as
a source of information, it's here for development
purposes only. Everything is subject to
change.</p></div>"""
if page_data.documentation:
template_vars['documentation_para'] = '<p><a href="' + boost_site.util.htmlencode(page_data.documentation) + '">Documentation</a>'
if(page_data.location.startswith('users/history/')):
template_vars['history_style'] = """
<style type="text/css">
/*<![CDATA[*/
#content .news-description ul {
list-style: none;
}
#content .news-description ul ul {
list-style: circle;
}
/*]]>*/
</style>
"""
boost_site.util.write_template(page_data.location,
'site-tools/templates/entry-template.html',
template_vars)
def __del__(self):
if self._fd:
os.close(self._fd)
def test_signature_verification(self):
"Test that signing and verification works"
logger.debug("test_signature_verification begins")
key = self.generate_key("Andrew", "Able", "alpha.com")
self.gpg.encoding = 'latin-1'
if gnupg._py3k:
data = 'Hello, André!'
else:
data = unicode('Hello, André', self.gpg.encoding)
data = data.encode(self.gpg.encoding)
sig = self.gpg.sign(data, keyid=key.fingerprint, passphrase='bbrown')
self.assertFalse(sig, "Bad passphrase should fail")
sig = self.gpg.sign(data, keyid=key.fingerprint, passphrase='aable')
self.assertTrue(sig, "Good passphrase should succeed")
self.assertTrue(sig.hash_algo)
verified = self.gpg.verify(sig.data)
if key.fingerprint != verified.fingerprint: # pragma: no cover
logger.debug("key: %r", key.fingerprint)
logger.debug("ver: %r", verified.fingerprint)
self.assertEqual(key.fingerprint, verified.fingerprint,
"Fingerprints must match")
self.assertEqual(verified.trust_level, verified.TRUST_ULTIMATE)
self.assertEqual(verified.trust_text, 'TRUST_ULTIMATE')
data_file = open(self.test_fn, 'rb')
sig = self.gpg.sign_file(data_file, keyid=key.fingerprint,
passphrase='aable')
data_file.close()
self.assertTrue(sig, "File signing should succeed")
self.assertTrue(sig.hash_algo)
try:
file = gnupg._make_binary_stream(sig.data, self.gpg.encoding)
verified = self.gpg.verify_file(file)
except UnicodeDecodeError: # pragma: no cover
# sometimes happens in Python 2.6
from io import BytesIO
verified = self.gpg.verify_file(BytesIO(sig.data))
if key.fingerprint != verified.fingerprint: # pragma: no cover
logger.debug("key: %r", key.fingerprint)
logger.debug("ver: %r", verified.fingerprint)
self.assertEqual(key.fingerprint, verified.fingerprint,
"Fingerprints must match")
data_file = open(self.test_fn, 'rb')
sig = self.gpg.sign_file(data_file, keyid=key.fingerprint,
passphrase='aable', detach=True)
data_file.close()
self.assertTrue(sig, "File signing should succeed")
self.assertTrue(sig.hash_algo)
try:
file = gnupg._make_binary_stream(sig.data, self.gpg.encoding)
verified = self.gpg.verify_file(file, self.test_fn)
except UnicodeDecodeError: # pragma: no cover
# sometimes happens in Python 2.6
from io import BytesIO
verified = self.gpg.verify_file(BytesIO(sig.data))
if key.fingerprint != verified.fingerprint: # pragma: no cover
logger.debug("key: %r", key.fingerprint)
logger.debug("ver: %r", verified.fingerprint)
self.assertEqual(key.fingerprint, verified.fingerprint,
"Fingerprints must match")
# Test in-memory verification
data_file = open(self.test_fn, 'rb')
data = data_file.read()
data_file.close()
fd, fn = tempfile.mkstemp()
os.write(fd, sig.data)
os.close(fd)
try:
verified = self.gpg.verify_data(fn, data)
finally:
os.unlink(fn)
if key.fingerprint != verified.fingerprint: # pragma: no cover
logger.debug("key: %r", key.fingerprint)
logger.debug("ver: %r", verified.fingerprint)
self.assertEqual(key.fingerprint, verified.fingerprint,
"Fingerprints must match")
logger.debug("test_signature_verification ends")
def close(self):
os.close(self.watchdog_pipe[0])
os.close(self.watchdog_pipe[1])
os.close(self.eintr_pipe[0])
os.close(self.eintr_pipe[1])
def resulttempfile(self):
fd, path = tempfile.mkstemp(prefix="pyfai_", suffix=".out")
os.close(fd)
os.remove(path)
yield path
os.remove(path)
def train(X, Y, val_X=None, val_Y=None, test_X=None, test_Y=None):
'''
train()
Train a Conditional Random Field for sequence tagging.
@param X. List of sparse-matrix sequences. Each sequence is one sentence.
@param Y. List of sequence tags. Each sequence is the sentence's per-token tags.
@param val_X. More X data, but a heldout dev set.
@param val_Y. More Y data, but a heldout dev set.
@return A tuple of encoded parameter weights and hyperparameters for predicting.
'''
# Sanity Check detection: features & label
#with open('a','w') as f:
# for xline,yline in zip(X,Y):
# for x,y in zip(xline,yline):
# print >>f, y, '\t', x.nonzero()[1][0]
# print >>f
# Format features fot crfsuite
feats = format_features(X, Y)
# Create a Trainer object.
trainer = pycrfsuite.Trainer(verbose=False)
for xseq, yseq in pycrf_instances(feats, labeled=True):
trainer.append(xseq, yseq)
# Train the model
os_handle, tmp_file = tempfile.mkstemp(dir=tmp_dir, suffix="crf_temp")
trainer.train(tmp_file)
# Read the trained model into a string (so it can be pickled)
model = ''
with open(tmp_file, 'rb') as f:
model = f.read()
os.close(os_handle)
# Remove the temporary file
os.remove(tmp_file)
######################################################################
# information about fitting the model
scores = {}
# how well does the model fir the training data?
train_pred = predict(model, X)
train_stats = compute_performance_stats('train', train_pred, Y)
scores['train'] = train_stats
if val_X:
val_pred = predict(model, val_X)
val_stats = compute_performance_stats('dev', val_pred, val_Y)
scores['dev'] = val_stats
if test_X:
test_pred = predict(model, test_X)
test_stats = compute_performance_stats('test', test_pred, test_Y)
scores['test'] = test_stats
# keep track of which external modules were used for building this model!
scores['hyperparams'] = {}
enabled_mods = enabled_modules()
for module, enabled in enabled_mods.items():
e = bool(enabled)
scores['hyperparams'][module] = e
return model, scores
def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
# Write the combined cert+chain to a temporary path and validate that it is OK.
# The certificate always goes above the chain.
import tempfile
fd, fn = tempfile.mkstemp('.pem')
os.write(fd, (ssl_cert + '\n' + ssl_chain).encode("ascii"))
os.close(fd)
# Do validation on the certificate before installing it.
ssl_private_key = os.path.join(
os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem'))
cert_status, cert_status_details = check_certificate(
domain, fn, ssl_private_key)
if cert_status != "OK":
if cert_status == "SELF-SIGNED":
cert_status = "This is a self-signed certificate. I can't install that."
os.unlink(fn)
if cert_status_details is not None:
cert_status += " " + cert_status_details
return cert_status
# Where to put it?
# Make a unique path for the certificate.
from cryptography.hazmat.primitives import hashes
from binascii import hexlify
cert = load_pem(load_cert_chain(fn)[0])
all_domains, cn = get_certificate_domains(cert)
path = "%s-%s-%s.pem" % (
safe_domain_name(
cn
), # common name, which should be filename safe because it is IDNA-encoded, but in case of a malformed cert make sure it's ok to use as a filename
cert.not_valid_after.date().isoformat().replace("-",
""), # expiration date
hexlify(cert.fingerprint(
hashes.SHA256())).decode("ascii")[0:8], # fingerprint prefix
)
ssl_certificate = os.path.join(
os.path.join(env["STORAGE_ROOT"], 'ssl', path))
# Install the certificate.
os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
shutil.move(fn, ssl_certificate)
ret = ["OK"]
# When updating the cert for PRIMARY_HOSTNAME, symlink it from the system
# certificate path, which is hard-coded for various purposes, and then
# restart postfix and dovecot.
if domain == env['PRIMARY_HOSTNAME']:
# Update symlink.
system_ssl_certificate = os.path.join(
os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
os.unlink(system_ssl_certificate)
os.symlink(ssl_certificate, system_ssl_certificate)
# Restart postfix and dovecot so they pick up the new file.
shell('check_call', ["/usr/sbin/service", "postfix", "restart"])
shell('check_call', ["/usr/sbin/service", "dovecot", "restart"])
ret.append("mail services restarted")
# The DANE TLSA record will remain valid so long as the private key
# hasn't changed. We don't ever change the private key automatically.
# If the user does it, they must manually update DNS.
# Update the web configuration so nginx picks up the new certificate file.
from web_update import do_web_update
ret.append(do_web_update(env))
if raw: return ret
return "\n".join(ret)
def workerScript(jobStore,
config,
jobName,
jobStoreID,
redirectOutputToLogFile=True):
"""
Worker process script, runs a job.
:param str jobName: The "job name" (a user friendly name) of the job to be run
:param str jobStoreLocator: Specifies the job store to use
:param str jobStoreID: The job store ID of the job to be run
:return int: 1 if a job failed, or 0 if all jobs succeeded
"""
configureRootLogger()
setLogLevel(config.logLevel)
##########################################
#Create the worker killer, if requested
##########################################
logFileByteReportLimit = config.maxLogFileSize
if config.badWorker > 0 and random.random() < config.badWorker:
# We need to kill the process we are currently in, to simulate worker
# failure. We don't want to just send SIGKILL, because we can't tell
# that from a legitimate OOM on our CI runner. We're going to send
# SIGUSR1 so our terminations are distinctive, and then SIGKILL if that
# didn't stick. We definitely don't want to do this from *within* the
# process we are trying to kill, so we fork off. TODO: We can still
# leave the killing code running after the main Toil flow is done, but
# since it's now in a process instead of a thread, the main Python
# process won't wait around for its timeout to expire. I think this is
# better than the old thread-based way where all of Toil would wait
# around to be killed.
killTarget = os.getpid()
sleepTime = config.badWorkerFailInterval * random.random()
if os.fork() == 0:
# We are the child
# Let the parent run some amount of time
time.sleep(sleepTime)
# Kill it gently
os.kill(killTarget, signal.SIGUSR1)
# Wait for that to stick
time.sleep(0.01)
try:
# Kill it harder. Hope the PID hasn't already been reused.
# If we succeeded the first time, this will OSError
os.kill(killTarget, signal.SIGKILL)
except OSError:
pass
# Exit without doing any of Toil's cleanup
os._exit(0)
# We don't need to reap the child. Either it kills us, or we finish
# before it does. Either way, init will have to clean it up for us.
##########################################
#Load the environment for the jobGraph
##########################################
#First load the environment for the jobGraph.
with jobStore.readSharedFileStream("environment.pickle") as fileHandle:
environment = safeUnpickleFromStream(fileHandle)
env_blacklist = {
"TMPDIR", "TMP", "HOSTNAME", "HOSTTYPE", "HOME", "LOGNAME", "USER",
"DISPLAY", "JAVA_HOME"
}
for i in environment:
if i == "PATH":
# Handle path specially. Sometimes e.g. leader may not include
# /bin, but the Toil appliance needs it.
if i in os.environ and os.environ[i] != '':
# Use the provided PATH and then the local system's PATH
os.environ[i] = environment[i] + ':' + os.environ[i]
else:
# Use the provided PATH only
os.environ[i] = environment[i]
elif i not in env_blacklist:
os.environ[i] = environment[i]
# sys.path is used by __import__ to find modules
if "PYTHONPATH" in environment:
for e in environment["PYTHONPATH"].split(':'):
if e != '':
sys.path.append(e)
toilWorkflowDir = Toil.getLocalWorkflowDir(config.workflowID,
config.workDir)
##########################################
#Setup the temporary directories.
##########################################
# Dir to put all this worker's temp files in.
localWorkerTempDir = tempfile.mkdtemp(dir=toilWorkflowDir)
os.chmod(localWorkerTempDir, 0o755)
##########################################
#Setup the logging
##########################################
#This is mildly tricky because we don't just want to
#redirect stdout and stderr for this Python process; we want to redirect it
#for this process and all children. Consequently, we can't just replace
#sys.stdout and sys.stderr; we need to mess with the underlying OS-level
#file descriptors. See <http://stackoverflow.com/a/11632982/402891>
#When we start, standard input is file descriptor 0, standard output is
#file descriptor 1, and standard error is file descriptor 2.
# Do we even want to redirect output? Let the config make us not do it.
redirectOutputToLogFile = redirectOutputToLogFile and not config.disableWorkerOutputCapture
#What file do we want to point FDs 1 and 2 to?
tempWorkerLogPath = os.path.join(localWorkerTempDir, "worker_log.txt")
if redirectOutputToLogFile:
# Announce that we are redirecting logging, and where it will now go.
# This is important if we are trying to manually trace a faulty worker invocation.
logger.info("Redirecting logging to %s", tempWorkerLogPath)
sys.stdout.flush()
sys.stderr.flush()
# Save the original stdout and stderr (by opening new file descriptors
# to the same files)
origStdOut = os.dup(1)
origStdErr = os.dup(2)
# Open the file to send stdout/stderr to.
logFh = os.open(tempWorkerLogPath,
os.O_WRONLY | os.O_CREAT | os.O_APPEND)
# Replace standard output with a descriptor for the log file
os.dup2(logFh, 1)
# Replace standard error with a descriptor for the log file
os.dup2(logFh, 2)
# Since we only opened the file once, all the descriptors duped from
# the original will share offset information, and won't clobber each
# others' writes. See <http://stackoverflow.com/a/5284108/402891>. This
# shouldn't matter, since O_APPEND seeks to the end of the file before
# every write, but maybe there's something odd going on...
# Close the descriptor we used to open the file
os.close(logFh)
debugging = logging.getLogger().isEnabledFor(logging.DEBUG)
##########################################
#Worker log file trapped from here on in
##########################################
jobAttemptFailed = False
statsDict = MagicExpando()
statsDict.jobs = []
statsDict.workers.logsToMaster = []
blockFn = lambda: True
listOfJobs = [jobName]
job = None
try:
#Put a message at the top of the log, just to make sure it's working.
logger.info("---TOIL WORKER OUTPUT LOG---")
sys.stdout.flush()
logProcessContext(config)
##########################################
#Connect to the deferred function system
##########################################
deferredFunctionManager = DeferredFunctionManager(toilWorkflowDir)
##########################################
#Load the jobGraph
##########################################
jobGraph = jobStore.load(jobStoreID)
listOfJobs[0] = str(jobGraph)
logger.debug("Parsed job wrapper")
##########################################
#Cleanup from any earlier invocation of the jobGraph
##########################################
if jobGraph.command == None:
logger.debug("Wrapper has no user job to run.")
# Cleanup jobs already finished
f = lambda jobs: [
z for z in [[y for y in x if jobStore.exists(y.jobStoreID)]
for x in jobs] if len(z) > 0
]
jobGraph.stack = f(jobGraph.stack)
jobGraph.services = f(jobGraph.services)
logger.debug(
"Cleaned up any references to completed successor jobs")
#This cleans the old log file which may
#have been left if the job is being retried after a job failure.
oldLogFile = jobGraph.logJobStoreFileID
if oldLogFile != None:
jobGraph.logJobStoreFileID = None
jobStore.update(jobGraph) #Update first, before deleting any files
jobStore.deleteFile(oldLogFile)
##########################################
# If a checkpoint exists, restart from the checkpoint
##########################################
# The job is a checkpoint, and is being restarted after previously completing
if jobGraph.checkpoint != None:
logger.debug("Job is a checkpoint")
# If the checkpoint still has extant jobs in its
# (flattened) stack and services, its subtree didn't
# complete properly. We handle the restart of the
# checkpoint here, removing its previous subtree.
if len([i for l in jobGraph.stack
for i in l]) > 0 or len(jobGraph.services) > 0:
logger.debug("Checkpoint has failed.")
# Reduce the retry count
assert jobGraph.remainingRetryCount >= 0
jobGraph.remainingRetryCount = max(
0, jobGraph.remainingRetryCount - 1)
jobGraph.restartCheckpoint(jobStore)
# Otherwise, the job and successors are done, and we can cleanup stuff we couldn't clean
# because of the job being a checkpoint
else:
logger.debug(
"The checkpoint jobs seems to have completed okay, removing any checkpoint files to delete."
)
#Delete any remnant files
list(
map(
jobStore.deleteFile,
list(
filter(jobStore.fileExists,
jobGraph.checkpointFilesToDelete))))
##########################################
#Setup the stats, if requested
##########################################
if config.stats:
startClock = getTotalCpuTime()
startTime = time.time()
while True:
##########################################
#Run the jobGraph, if there is one
##########################################
if jobGraph.command is not None:
assert jobGraph.command.startswith("_toil ")
logger.debug("Got a command to run: %s" % jobGraph.command)
#Load the job
job = Job._loadJob(jobGraph.command, jobStore)
# If it is a checkpoint job, save the command
if job.checkpoint:
jobGraph.checkpoint = jobGraph.command
# Create a fileStore object for the job
fileStore = AbstractFileStore.createFileStore(
jobStore,
jobGraph,
localWorkerTempDir,
blockFn,
caching=not config.disableCaching)
with job._executor(jobGraph=jobGraph,
stats=statsDict if config.stats else None,
fileStore=fileStore):
with deferredFunctionManager.open() as defer:
with fileStore.open(job):
# Get the next block function to wait on committing this job
blockFn = fileStore.waitForCommit
job._runner(jobGraph=jobGraph,
jobStore=jobStore,
fileStore=fileStore,
defer=defer)
# When the job succeeds, start committing files immediately.
fileStore.startCommit(jobState=False)
# Accumulate messages from this job & any subsequent chained jobs
statsDict.workers.logsToMaster += fileStore.loggingMessages
else:
#The command may be none, in which case
#the jobGraph is either a shell ready to be deleted or has
#been scheduled after a failure to cleanup
logger.debug("No user job to run, so finishing")
break
if AbstractFileStore._terminateEvent.isSet():
raise RuntimeError("The termination flag is set")
##########################################
#Establish if we can run another jobGraph within the worker
##########################################
successorJobGraph = nextChainableJobGraph(jobGraph, jobStore)
if successorJobGraph is None or config.disableChaining:
# Can't chain any more jobs.
# TODO: why don't we commit the last job's file store? Won't
# its async uploads never necessarily finish?
# If we do call startCommit here it messes with the job
# itself and Toil thinks the job needs to run again.
break
##########################################
#We have a single successor job that is not a checkpoint job.
#We transplant the successor jobGraph command and stack
#into the current jobGraph object so that it can be run
#as if it were a command that were part of the current jobGraph.
#We can then delete the successor jobGraph in the jobStore, as it is
#wholly incorporated into the current jobGraph.
##########################################
# add the successor to the list of jobs run
listOfJobs.append(str(successorJobGraph))
#Clone the jobGraph and its stack
jobGraph = copy.deepcopy(jobGraph)
#Remove the successor jobGraph
jobGraph.stack.pop()
#Transplant the command and stack to the current jobGraph
jobGraph.command = successorJobGraph.command
jobGraph.stack += successorJobGraph.stack
# include some attributes for better identification of chained jobs in
# logging output
jobGraph.unitName = successorJobGraph.unitName
jobGraph.jobName = successorJobGraph.jobName
assert jobGraph.memory >= successorJobGraph.memory
assert jobGraph.cores >= successorJobGraph.cores
#Build a fileStore to update the job
fileStore = AbstractFileStore.createFileStore(
jobStore,
jobGraph,
localWorkerTempDir,
blockFn,
caching=not config.disableCaching)
#Update blockFn
blockFn = fileStore.waitForCommit
#Add successorJobGraph to those to be deleted
fileStore.jobsToDelete.add(successorJobGraph.jobStoreID)
#This will update the job once the previous job is done
fileStore.startCommit(jobState=True)
#Clone the jobGraph and its stack again, so that updates to it do
#not interfere with this update
jobGraph = copy.deepcopy(jobGraph)
logger.debug("Starting the next job")
##########################################
#Finish up the stats
##########################################
if config.stats:
totalCPUTime, totalMemoryUsage = getTotalCpuTimeAndMemoryUsage()
statsDict.workers.time = str(time.time() - startTime)
statsDict.workers.clock = str(totalCPUTime - startClock)
statsDict.workers.memory = str(totalMemoryUsage)
# log the worker log path here so that if the file is truncated the path can still be found
if redirectOutputToLogFile:
logger.info(
"Worker log can be found at %s. Set --cleanWorkDir to retain this log",
localWorkerTempDir)
logger.info(
"Finished running the chain of jobs on this node, we ran for a total of %f seconds",
time.time() - startTime)
##########################################
#Trapping where worker goes wrong
##########################################
except: #Case that something goes wrong in worker
traceback.print_exc()
logger.error("Exiting the worker because of a failed job on host %s",
socket.gethostname())
AbstractFileStore._terminateEvent.set()
##########################################
#Wait for the asynchronous chain of writes/updates to finish
##########################################
blockFn()
##########################################
#All the asynchronous worker/update threads must be finished now,
#so safe to test if they completed okay
##########################################
if AbstractFileStore._terminateEvent.isSet():
jobGraph = jobStore.load(jobStoreID)
jobAttemptFailed = True
##########################################
#Cleanup
##########################################
# Close the worker logging
# Flush at the Python level
sys.stdout.flush()
sys.stderr.flush()
if redirectOutputToLogFile:
# Flush at the OS level
os.fsync(1)
os.fsync(2)
# Close redirected stdout and replace with the original standard output.
os.dup2(origStdOut, 1)
# Close redirected stderr and replace with the original standard error.
os.dup2(origStdErr, 2)
# sys.stdout and sys.stderr don't need to be modified at all. We don't
# need to call redirectLoggerStreamHandlers since they still log to
# sys.stderr
# Close our extra handles to the original standard output and standard
# error streams, so we don't leak file handles.
os.close(origStdOut)
os.close(origStdErr)
# Now our file handles are in exactly the state they were in before.
# Copy back the log file to the global dir, if needed.
# Note that we work with bytes instead of characters so we can seek
# relative to the end (since Python won't decode Unicode backward, or even
# interpret seek offsets in characters for us). TODO: We may get invalid or
# just different Unicode by breaking up a character at the boundary!
if jobAttemptFailed and redirectOutputToLogFile:
jobGraph.logJobStoreFileID = jobStore.getEmptyFileStoreID(
jobGraph.jobStoreID, cleanup=True)
jobGraph.chainedJobs = listOfJobs
with jobStore.updateFileStream(jobGraph.logJobStoreFileID) as w:
with open(tempWorkerLogPath, 'rb') as f:
if os.path.getsize(
tempWorkerLogPath) > logFileByteReportLimit != 0:
if logFileByteReportLimit > 0:
f.seek(-logFileByteReportLimit,
2) # seek to last tooBig bytes of file
elif logFileByteReportLimit < 0:
f.seek(logFileByteReportLimit,
0) # seek to first tooBig bytes of file
# Dump the possibly-invalid-Unicode bytes into the log file
w.write(f.read()) # TODO load file using a buffer
jobStore.update(jobGraph)
elif ((debugging or (config.writeLogsFromAllJobs
and not jobName.startswith(CWL_INTERNAL_JOBS)))
and redirectOutputToLogFile): # write log messages
with open(tempWorkerLogPath, 'rb') as logFile:
if os.path.getsize(
tempWorkerLogPath) > logFileByteReportLimit != 0:
if logFileByteReportLimit > 0:
logFile.seek(-logFileByteReportLimit,
2) # seek to last tooBig bytes of file
elif logFileByteReportLimit < 0:
logFile.seek(logFileByteReportLimit,
0) # seek to first tooBig bytes of file
# Make sure lines are Unicode so they can be JSON serialized as part of the dict.
# We may have damaged the Unicode text by cutting it at an arbitrary byte so we drop bad characters.
logMessages = [
line.decode('utf-8', 'skip')
for line in logFile.read().splitlines()
]
statsDict.logs.names = listOfJobs
statsDict.logs.messages = logMessages
if (debugging or config.stats or statsDict.workers.logsToMaster
) and not jobAttemptFailed: # We have stats/logging to report back
if USING_PYTHON2:
jobStore.writeStatsAndLogging(
json.dumps(statsDict, ensure_ascii=True))
else:
jobStore.writeStatsAndLogging(
json.dumps(statsDict, ensure_ascii=True).encode())
#Remove the temp dir
cleanUp = config.cleanWorkDir
if cleanUp == 'always' or (cleanUp == 'onSuccess' and
not jobAttemptFailed) or (cleanUp == 'onError'
and jobAttemptFailed):
shutil.rmtree(localWorkerTempDir)
#This must happen after the log file is done with, else there is no place to put the log
if (not jobAttemptFailed) and jobGraph.command == None and len(
jobGraph.stack) == 0 and len(jobGraph.services) == 0:
# We can now safely get rid of the jobGraph
jobStore.delete(jobGraph.jobStoreID)
if jobAttemptFailed:
return 1
else:
return 0
def _clone_local(self, meter, size_bytes):
if self._input_path == "/dev/null": # pragma: no cover
# Not really sure why this check is here,
# but keeping for compat
log.debug("Source dev was /dev/null. Skipping")
return
if self._input_path == self._output_path:
log.debug("Source and destination are the same. Skipping.")
return
# If a destination file exists and sparse flag is True,
# this priority takes an existing file.
if (not os.path.exists(self._output_path) and self._sparse):
clone_block_size = 4096
sparse = True
fd = None
try:
fd = os.open(self._output_path, os.O_WRONLY | os.O_CREAT,
0o640)
os.ftruncate(fd, size_bytes)
finally:
if fd:
os.close(fd)
else:
clone_block_size = 1024 * 1024 * 10
sparse = False
log.debug("Local Cloning %s to %s, sparse=%s, block_size=%s",
self._input_path, self._output_path,
sparse, clone_block_size)
zeros = b'\0' * 4096
src_fd, dst_fd = None, None
try:
try:
src_fd = os.open(self._input_path, os.O_RDONLY)
dst_fd = os.open(self._output_path,
os.O_WRONLY | os.O_CREAT, 0o640)
if (True):
#size = int(self.get_size() * 1024 * 1024 * 1024 * 10)
size = _get_size(self._path)
log.debug(("USING REF {size}", size), exc_info=True)
while size > 0:
ret = os.copy_file_range(src_fd, dst_fd, size)
size -= ret
else:
i = 0
while 1:
l = os.read(src_fd, clone_block_size)
s = len(l)
if s == 0:
meter.end(size_bytes)
break
# check sequence of zeros
if sparse and zeros == l:
os.lseek(dst_fd, s, 1)
else:
b = os.write(dst_fd, l)
if s != b: # pragma: no cover
meter.end(i)
break
i += s
if i < size_bytes:
meter.update(i)
except OSError as e: # pragma: no cover
log.debug("Error while cloning", exc_info=True)
msg = (_("Error cloning diskimage "
"%(inputpath)s to %(outputpath)s: %(error)s") %
{"inputpath": self._input_path,
"outputpath": self._output_path,
"error": str(e)})
raise RuntimeError(msg) from None
finally:
if src_fd is not None:
os.close(src_fd)
if dst_fd is not None:
os.close(dst_fd)
def _instance_cleanup(self, instance): if instance._first_instance: if self._has_pm_qos: os.close(self._cpu_latency_fd) if instance._load_monitor is not None: self._monitors_repository.delete(instance._load_monitor)
def postForkChild(self):
"post fork handling in child"
os.close(self.rfd)
self.rfd = None
def close(self):
"""
Close the write end of the pipe.
"""
os.close(self.fd_write)
def close_source_fd(self):
# FIXME: Only close source fds that come from pipes instead of this hack...
if isinstance(self.source, int) and not self.borrowed:
log.log("CLOSE SOURCE %s" % (self.source,), "fd")
os.close(self.source)
def __exit__(self, exc_type, exc_value, traceback):
self._new_stdout.flush()
os.dup2(self.oldstdout_fno, 1)
os.close(self.oldstdout_fno)
def tearDown(self):
dbo.reset()
os.close(self.version_fd)
os.remove(self.version_file)
def postForkParent(self):
"post fork handling in parent"
os.close(self.wfd)
self.wfd = None
def _communicate(ioproc_ref, proc, readPipe, writePipe):
real_ioproc = ioproc_ref()
if real_ioproc is None:
return
# Keeps the name for logging in this thread.
ioproc_name = real_ioproc.name
real_ioproc._started.set()
dataSender = None
pendingRequests = {}
responseReader = ResponseReader(readPipe)
err = proc.stderr.fileno()
poller = poll()
# When closing the ioprocess there might be race for closing this fd
# using a copy solves this
try:
try:
evtReciever = os.dup(real_ioproc._eventFdReciever)
except OSError:
evtReciever = -1
return
poller.register(err, INPUT_READY_FLAGS)
poller.register(evtReciever, INPUT_READY_FLAGS)
poller.register(readPipe, INPUT_READY_FLAGS)
poller.register(writePipe, ERROR_FLAGS)
while True:
real_ioproc = None
pollres = NoIntrPoll(poller.poll, 5)
real_ioproc = ioproc_ref()
if real_ioproc is None:
break
if not real_ioproc._isRunning:
_log.info("(%s) Shutdown requested", ioproc_name)
break
for fd, event in pollres:
if event & ERROR_FLAGS:
raise PollError(fd, event & ERROR_FLAGS)
if fd == err:
real_ioproc._processLogs(os.read(fd, 1024))
continue
if fd == readPipe:
if not responseReader.process():
continue
res = responseReader.pop()
reqId = res['id']
pendingReq = pendingRequests.pop(reqId, None)
if pendingReq is not None:
pendingReq.result = res
pendingReq.event.set()
else:
_log.warning("(%s) Unknown request id %d",
ioproc_name, reqId)
continue
if fd == evtReciever:
os.read(fd, 1)
if dataSender:
continue
try:
cmd, resObj = real_ioproc._commandQueue.get_nowait()
except Empty:
continue
reqId = real_ioproc._getRequestId()
pendingRequests[reqId] = resObj
reqString = real_ioproc._requestToBytes(cmd, reqId)
dataSender = DataSender(writePipe, reqString)
poller.modify(writePipe, OUTPUT_READY_FLAGS)
continue
if fd == writePipe:
if dataSender.process():
dataSender = None
poller.modify(writePipe, ERROR_FLAGS)
real_ioproc._pingPoller()
except PollError as e:
# Normal during shutdown - don't log an error.
_log.info("(%s) %s", ioproc_name, e)
_cleanup(pendingRequests)
except:
# Unexpected error.
_log.exception("(%s) Communication thread failed", ioproc_name)
_cleanup(pendingRequests)
finally:
os.close(readPipe)
os.close(writePipe)
if (evtReciever >= 0):
os.close(evtReciever)
rc = proc.poll()
if rc is None:
_log.info("(%s) Killing ioprocess", ioproc_name)
if IOProcess._DEBUG_VALGRIND:
os.kill(proc.pid, signal.SIGTERM)
else:
proc.kill()
rc = proc.wait()
if rc < 0:
_log.info("(%s) ioprocess was terminated by signal %s",
ioproc_name, -rc)
else:
_log.info("(%s) ioprocess terminated with code %s",
ioproc_name, rc)
real_ioproc = ioproc_ref()
if real_ioproc is not None:
with real_ioproc._lock:
if real_ioproc._isRunning:
real_ioproc._run()
def run(config=None, plugin_providers=None, product_name='ajenti', dev_mode=False,
debug_mode=False, autologin=False):
"""
A global entry point for Ajenti.
:param config: config file implementation instance to use
:type config: :class:`aj.config.BaseConfig`
:param plugin_providers: list of plugin providers to load plugins from
:type plugin_providers: list(:class:`aj.plugins.PluginProvider`)
:param str product_name: a product name to use
:param bool dev_mode: enables dev mode (automatic resource recompilation)
:param bool debug_mode: enables debug mode (verbose and extra logging)
:param bool autologin: disables authentication and logs everyone in as the user running the panel. This is EXTREMELY INSECURE.
"""
if config is None:
raise TypeError('`config` can\'t be None')
reload_module(sys)
if hasattr(sys, 'setdefaultencoding'):
sys.setdefaultencoding('utf8')
aj.product = product_name
aj.debug = debug_mode
aj.dev = dev_mode
aj.dev_autologin = autologin
aj.init()
aj.log.set_log_params(tag='master', master_pid=os.getpid())
aj.context = Context()
aj.config = config
aj.plugin_providers = plugin_providers or []
logging.info('Loading config from %s', aj.config)
aj.config.load()
aj.config.ensure_structure()
if aj.debug:
logging.warn('Debug mode')
if aj.dev:
logging.warn('Dev mode')
try:
locale.setlocale(locale.LC_ALL, '')
except locale.Error:
logging.warning('Couldn\'t set default locale')
# install a passthrough gettext replacement since all localization is handled in frontend
# and _() is here only for string extraction
__builtins__['_'] = lambda x: x
logging.info('Ajenti Core %s', aj.version)
logging.info('Detected platform: %s / %s', aj.platform, aj.platform_string)
# Load plugins
PluginManager.get(aj.context).load_all_from(aj.plugin_providers)
if len(PluginManager.get(aj.context)) == 0:
logging.warn('No plugins were loaded!')
if aj.config.data['bind']['mode'] == 'unix':
path = aj.config.data['bind']['socket']
if os.path.exists(path):
os.unlink(path)
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
listener.bind(path)
except OSError:
logging.error('Could not bind to %s', path)
sys.exit(1)
if aj.config.data['bind']['mode'] == 'tcp':
host = aj.config.data['bind']['host']
port = aj.config.data['bind']['port']
listener = socket.socket(
socket.AF_INET6 if ':' in host else socket.AF_INET, socket.SOCK_STREAM
)
if aj.platform not in ['freebsd', 'osx']:
try:
listener.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1)
except socket.error:
logging.warn('Could not set TCP_CORK')
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
logging.info('Binding to [%s]:%s', host, port)
try:
listener.bind((host, port))
except socket.error as e:
logging.error('Could not bind: %s', str(e))
sys.exit(1)
# Fix stupid socketio bug (it tries to do *args[0][0])
socket.socket.__getitem__ = lambda x, y: None
listener.listen(10)
gateway = GateMiddleware.get(aj.context)
application = HttpRoot(HttpMiddlewareAggregator([gateway])).dispatch
aj.server = SocketIOServer(
listener,
log=open(os.devnull, 'w'),
application=application,
handler_class=RequestHandler,
policy_server=False,
transports=[
str('websocket'),
str('flashsocket'),
str('xhr-polling'),
str('jsonp-polling'),
],
)
if aj.config.data['ssl']['enable'] and aj.config.data['bind']['mode'] == 'tcp':
aj.server.ssl_args = {'server_side': True}
cert_path = aj.config.data['ssl']['certificate']
if aj.config.data['ssl']['fqdn_certificate']:
fqdn_cert_path = aj.config.data['ssl']['fqdn_certificate']
else:
fqdn_cert_path = cert_path
context = gevent.ssl.SSLContext(ssl.PROTOCOL_TLS)
context.load_cert_chain(certfile=fqdn_cert_path, keyfile=fqdn_cert_path)
context.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
context.set_ciphers('ALL:!ADH:!EXP:!LOW:!RC2:!3DES:!SEED:!RC4:+HIGH:+MEDIUM')
if aj.config.data['ssl']['client_auth']['enable']:
logging.info('Enabling SSL client authentication')
context.load_verify_locations(cafile=cert_path)
if aj.config.data['ssl']['client_auth']['force']:
context.verify_mode = ssl.CERT_REQUIRED
else:
context.verify_mode = ssl.CERT_OPTIONAL
## Test callback : client_certificate_callback must return None to get forward
# context.set_servername_callback(AuthenticationService.get(aj.context).client_certificate_callback)
aj.server.wrap_socket = lambda socket, **args:context.wrap_socket(sock=socket, server_side=True)
logging.info('SSL enabled')
# auth.log
try:
syslog.openlog(
ident=str(aj.product),
facility=syslog.LOG_AUTH,
)
except:
syslog.openlog(aj.product)
def cleanup():
if hasattr(cleanup, 'started'):
return
cleanup.started = True
logging.info('Process %s exiting normally', os.getpid())
gevent.signal(signal.SIGINT, lambda: None)
gevent.signal(signal.SIGTERM, lambda: None)
if aj.master:
gateway.destroy()
p = psutil.Process(os.getpid())
for c in p.children(recursive=True):
try:
os.killpg(c.pid, signal.SIGTERM)
os.killpg(c.pid, signal.SIGKILL)
except OSError:
pass
def signal_handler():
cleanup()
sys.exit(0)
gevent.signal(signal.SIGINT, signal_handler)
gevent.signal(signal.SIGTERM, signal_handler)
aj.server.serve_forever()
if not aj.master:
# child process, server is stopped, wait until killed
gevent.wait()
if hasattr(aj.server, 'restart_marker'):
logging.warn('Restarting by request')
cleanup()
fd = 20 # Close all descriptors. Creepy thing
while fd > 2:
try:
os.close(fd)
logging.debug('Closed descriptor #%i', fd)
except OSError:
pass
fd -= 1
logging.warn('Will restart the process now')
if '-d' in sys.argv:
sys.argv.remove('-d')
os.execv(sys.argv[0], sys.argv)
else:
if aj.master:
logging.debug('Server stopped')
cleanup()
def preprocess(path):
with open(path, "rb") as fh:
lines = []
it = iter(fh)
for l in it:
# zstd.h includes <stddef.h>, which is also included by cffi's
# boilerplate. This can lead to duplicate declarations. So we strip
# this include from the preprocessor invocation.
#
# The same things happens for including zstd.h, so give it the same
# treatment.
#
# We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
# #define in zstdmt_compress.h and results in a compiler warning. So drop
# the inline #define.
if l.startswith((
b"#include <stddef.h>",
b'#include "zstd.h"',
b"#define ZSTD_STATIC_LINKING_ONLY",
)):
continue
# The preprocessor environment on Windows doesn't define include
# paths, so the #include of limits.h fails. We work around this
# by removing that import and defining INT_MAX ourselves. This is
# a bit hacky. But it gets the job done.
# TODO make limits.h work on Windows so we ensure INT_MAX is
# correct.
if l.startswith(b"#include <limits.h>"):
l = b"#define INT_MAX 2147483647\n"
# ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
# important so just filter it out.
if l.startswith(b"ZSTDLIB_API"):
l = l[len(b"ZSTDLIB_API "):]
lines.append(l)
fd, input_file = tempfile.mkstemp(suffix=".h")
os.write(fd, b"".join(lines))
os.close(fd)
try:
env = dict(os.environ)
# cffi attempts to decode source as ascii. And the preprocessor
# may insert non-ascii for some annotations. So try to force
# ascii output via LC_ALL.
env["LC_ALL"] = "C"
if getattr(compiler, "_paths", None):
env["PATH"] = compiler._paths
process = subprocess.Popen(args + [input_file],
stdout=subprocess.PIPE,
env=env)
output = process.communicate()[0]
ret = process.poll()
if ret:
raise Exception("preprocessor exited with error")
return output
finally:
os.unlink(input_file)
elif thisarg == 'write_flash':
write_addr = sys.argv.pop(0)
binary = sys.argv.pop(0)
elif thisarg:
cmdline = cmdline + [thisarg]
cmdline = cmdline + ['write_flash']
if write_option:
cmdline = cmdline + [write_option]
cmdline = cmdline + ['--flash_size', 'detect']
cmdline = cmdline + [write_addr, binary]
erase_file = ''
if erase_addr:
# Generate temporary empty (0xff) file
eraser = tempfile.mkstemp()
erase_file = eraser[1]
os.write(eraser[0], bytearray([0xff] * int(erase_len, 0)))
os.close(eraser[0])
cmdline = cmdline + [erase_addr, erase_file]
try:
esptool.main(cmdline)
except esptool.FatalError as e:
sys.stderr.write('\nA fatal esptool.py error occurred: %s' % e)
finally:
if erase_file:
os.remove(erase_file)
if any(sys.exc_info()):
sys.exit(2)
