def __init__(self, mode, *args): read_fd, write_fd = os.pipe() self.pid = os.fork() self.cmd = args[0] if self.pid: # Parent if mode == 'w': os.close(read_fd) self.file = os.fdopen(write_fd, 'w') else: os.close(write_fd) self.file = os.fdopen(read_fd, 'r') else: # Child if mode == 'w': os.close(write_fd) os.dup2(read_fd, 0) os.close(read_fd) else: os.close(read_fd) os.dup2(write_fd, 1) os.close(write_fd) try: os.execl(self.cmd, os.path.basename(args[0]), *args[1:]) except OSError, (eno, estr): print >> sys.stderr, 'exec %s: %s' % (self.cmd, estr) os._exit(1)
def __init__(self, ui, packdir, version=0): self._checkversion(version) opener = vfsmod.vfs(packdir) opener.createmode = 0o444 self.opener = opener self.entries = {} shallowutil.mkstickygroupdir(ui, packdir) self.packfp, self.packpath = opener.mkstemp( suffix=self.PACKSUFFIX + '-tmp') self.idxfp, self.idxpath = opener.mkstemp( suffix=self.INDEXSUFFIX + '-tmp') self.packfp = os.fdopen(self.packfp, 'w+') self.idxfp = os.fdopen(self.idxfp, 'w+') self.sha = hashlib.sha1() self._closed = False # The opener provides no way of doing permission fixup on files created # via mkstemp, so we must fix it ourselves. We can probably fix this # upstream in vfs.mkstemp so we don't need to use the private method. opener._fixfilemode(opener.join(self.packpath)) opener._fixfilemode(opener.join(self.idxpath)) # Write header # TODO: make it extensible (ex: allow specifying compression algorithm, # a flexible key/value header, delta algorithm, fanout size, etc) versionbuf = struct.pack('!B', self.VERSION) # unsigned 1 byte int self.writeraw(versionbuf)
def readf_win32(f, m='r', encoding='ISO8859-1'): flags = os.O_NOINHERIT | os.O_RDONLY if 'b' in m: flags |= os.O_BINARY if '+' in m: flags |= os.O_RDWR try: fd = os.open(f, flags) except OSError: raise IOError('Cannot read from %r' % f) if sys.hexversion > 0x3000000 and not 'b' in m: m += 'b' f = os.fdopen(fd, m) try: txt = f.read() finally: f.close() if encoding: txt = txt.decode(encoding) else: txt = txt.decode() else: f = os.fdopen(fd, m) try: txt = f.read() finally: f.close() return txt
def run(self): """ Execute Package building action. """ header = self._build_execution_header_output() print_info( header + "spawning package build: %s" % ( " ".join(self._packages),)) std_env = self._build_standard_environment( repository=self._params["repository"]) matter_package_names = " ".join(self._packages) std_env["MATTER_PACKAGE_NAMES"] = matter_package_names # run pkgpre, if any pkgpre = self._params["pkgpre"] if pkgpre is not None: print_info("spawning --pkgpre: %s" % (pkgpre,)) tmp_fd, tmp_path = mkstemp() with os.fdopen(tmp_fd, "wb") as tmp_f: with open(pkgpre, "rb") as pkgpre_f: tmp_f.write(pkgpre_f.read()) try: # now execute os.chmod(tmp_path, 0o700) exit_st = subprocess.call([tmp_path], env = std_env) if exit_st != 0: return exit_st finally: os.remove(tmp_path) dirs_cleanup = [] exit_st = self._run_builder(dirs_cleanup) print_info("builder terminated, exit status: %d" % (exit_st,)) # cleanup temporary directories registered on the queue for tmp_dir in dirs_cleanup: self.__cleanup_dir(tmp_dir) # run pkgpost, if any pkgpost = self._params["pkgpost"] if pkgpost is not None: print_info("spawning --pkgpost: %s" % (pkgpost,)) tmp_fd, tmp_path = mkstemp() with os.fdopen(tmp_fd, "wb") as tmp_f: with open(pkgpost, "rb") as pkgpost_f: tmp_f.write(pkgpost_f.read()) try: # now execute os.chmod(tmp_path, 0o700) post_exit_st = subprocess.call([tmp_path, str(exit_st)], env = std_env) if post_exit_st != 0: return post_exit_st finally: os.remove(tmp_path) return exit_st
def tag_raw_data(raw) : (name, temp) = tempfile.mkstemp() (rname, rtemp) = tempfile.mkstemp() f = os.fdopen(name, 'w') #weird function because we are using a tempfile f.writelines([r + '\n' for r in raw]) f.close() #call external tagger print(TAG_COMMAND.format(temp)) f = os.fdopen(rname) result = subprocess.call(TAG_COMMAND.format(temp),stdout=f, cwd=TAGGER_PATH, shell=True) #read them back in f.seek(0) result = f.readlines() f.close() result = [x.strip() for x in result] final_result = [] buf = [] for i in range(len(result)) : if (result[i] == '') : final_result.append(tuple(buf)) #you have to cut of that extra ' ' buf = [] else : pieces = result[i].split('\t') buf.append( (pieces[0],pieces[1] ) ) if (buf != []) : final_result.append(buf) return final_result
def write_env_var_to_conf_dir(var, value, conf_dir): env_jsonfile_path = os.path.join(conf_dir, 'environment.json') if var in CORE_VAR_NAMES: try: with open(env_jsonfile_path) as fd: env_vars = json.load(fd) except: env_vars = {} if value is None and var in env_vars: del env_vars[var] else: env_vars[var] = value # Make sure the file has 600 permissions try: os.remove(env_jsonfile_path) except: pass with os.fdopen(os.open(env_jsonfile_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as fd: json.dump(env_vars, fd, indent=4) fd.write("\n") else: # DX_CLI_WD, DX_USERNAME, DX_PROJECT_CONTEXT_NAME # Make sure the file has 600 permissions try: os.remove(os.path.join(conf_dir, var)) except: pass with os.fdopen(os.open(os.path.join(conf_dir, var), os.O_CREAT | os.O_WRONLY, 0o600), 'w') as fd: fd.write(value.encode(sys_encoding) if is_py2 else value) if not os.path.exists(os.path.expanduser('~/.dnanexus_config/') + 'unsetenv'): with open(os.path.expanduser('~/.dnanexus_config/') + 'unsetenv', 'w') as fd: for var in CORE_VAR_NAMES: fd.write('unset ' + var + '\n')
def ndiff(scan_a, scan_b): """Run Ndiff on two scan results, which may be filenames or NmapParserSAX objects, and return a running NdiffCommand object.""" temporary_filenames = [] if isinstance(scan_a, NmapParserSAX): fd, filename_a = tempfile.mkstemp( prefix=APP_NAME + "-diff-", suffix=".xml" ) temporary_filenames.append(filename_a) f = os.fdopen(fd, "wb") scan_a.write_xml(f) f.close() else: filename_a = scan_a if isinstance(scan_b, NmapParserSAX): fd, filename_b = tempfile.mkstemp( prefix=APP_NAME + "-diff-", suffix=".xml" ) temporary_filenames.append(filename_b) f = os.fdopen(fd, "wb") scan_b.write_xml(f) f.close() else: filename_b = scan_b return NdiffCommand(filename_a, filename_b, temporary_filenames)
def worker(sock): """ Called by a worker process after the fork(). """ signal.signal(SIGHUP, SIG_DFL) signal.signal(SIGCHLD, SIG_DFL) signal.signal(SIGTERM, SIG_DFL) # restore the handler for SIGINT, # it's useful for debugging (show the stacktrace before exit) signal.signal(SIGINT, signal.default_int_handler) # Read the socket using fdopen instead of socket.makefile() because the latter # seems to be very slow; note that we need to dup() the file descriptor because # otherwise writes also cause a seek that makes us miss data on the read side. infile = os.fdopen(os.dup(sock.fileno()), "a+", 65536) outfile = os.fdopen(os.dup(sock.fileno()), "a+", 65536) exit_code = 0 try: worker_main(infile, outfile) except SystemExit as exc: exit_code = compute_real_exit_code(exc.code) finally: try: outfile.flush() except Exception: pass return exit_code
def setUp(self): """ Create a list of files separated in time so we can do dependency checking """ import tempfile import time self.files = list() job_history = open_job_history(history_file) for i in range(6): #test_file =tempfile.NamedTemporaryFile(delete=False, prefix='testing_tmp') #self.files.append (test_file.name) # test_file.close() fh, temp_file_name = tempfile.mkstemp(suffix='.dot') self.files.append(temp_file_name) os.fdopen(fh, "w").close() # Save modify time in history file mtime = os.path.getmtime(temp_file_name) epoch_seconds = time.time() # Use epoch seconds unless there is a > 1 second discrepancy between system clock # and file system clock if epoch_seconds > mtime and epoch_seconds - mtime < 1.1: mtime = epoch_seconds else: # file system clock out of sync: # Use file modify times: slow down in case of low counter resolution # (e.g. old versions of NFS and windows) time.sleep(2) chksum = task.JobHistoryChecksum( temp_file_name, mtime, "", dummy_task()) job_history[os.path.relpath(temp_file_name)] = chksum
def __init__(self, impl=None): self._impl = impl or _poll() if hasattr(self._impl, 'fileno'): self._set_close_exec(self._impl.fileno()) self._handlers = {} self._events = {} self._callbacks = [] self._timeouts = [] self._running = False self._stopped = False self._blocking_signal_threshold = None # Create a pipe that we send bogus data to when we want to wake # the I/O loop when it is idle if os.name != 'nt': r, w = os.pipe() self._set_nonblocking(r) self._set_nonblocking(w) self._set_close_exec(r) self._set_close_exec(w) self._waker_reader = os.fdopen(r, "rb", 0) self._waker_writer = os.fdopen(w, "wb", 0) else: self._waker_reader = self._waker_writer = win32_support.Pipe() r = self._waker_writer.reader_fd self.add_handler(r, self._read_waker, self.READ)
def _write_tmpfile(self, entry): """ Write the file data to a temp file """ filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with # /tmp set nosetuid while creating files that are supposed to # be setuid try: (newfd, newfile) = \ tempfile.mkstemp(prefix=os.path.basename(entry.get("name")), dir=os.path.dirname(entry.get("name"))) except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to create temp file in %s: %s" % (os.path.dirname(entry.get('name')), err)) return False try: if isinstance(filedata, str) and str != unicode: os.fdopen(newfd, 'w').write(filedata) else: os.fdopen(newfd, 'wb').write( filedata.encode(Bcfg2.Options.setup.encoding)) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " "%s: %s" % (newfile, entry.get("name"), err)) return False return newfile
def record_site(self, job_status): """ Need a doc string here. """ job_status_name = None for name, code in JOB_RETURN_CODES._asdict().iteritems(): if code == job_status: job_status_name = name try: with os.fdopen( os.open( "task_statistics.%s.%s" % (self.site, job_status_name), os.O_APPEND | os.O_CREAT | os.O_RDWR, 0o644 ), "a", ) as fd: fd.write("%d\n" % (self.job_id)) except Exception as ex: self.logger.error(str(ex)) # Swallow the exception - record_site is advisory only try: with os.fdopen( os.open("task_statistics.%s" % (job_status_name), os.O_APPEND | os.O_CREAT | os.O_RDWR, 0o644), "a" ) as fd: fd.write("%d\n" % (self.job_id)) except Exception as ex: self.logger.error(str(ex))
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup( inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_badgridlabel.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) except: os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) testOK = self.getAssertTest() sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) # ------------------------------------------ # Check error after signal handler is back # ------------------------------------------ self.assertIn("\"gs1n\"", testOK)
def handle(self, *addrport, **options): # reopen stdout/stderr file descriptor with write mode # and 0 as the buffer size (unbuffered). # XXX: why? sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) if len(addrport) == 0 : raise CommandError('Usage is runserver %s' % self.args) if len(addrport) == 1 : self.run_one(addrport[0], **options) else : from multiprocessing import Process plist = [] for ap in addrport : p = Process(target=self.run_one, args=(ap,), kwargs=options) p.start() plist.append(p) # for p in plist : plist.terminate() while plist : if plist[0].exitcode is None : plist.pop(0) else : plist[0].join()
def __init__(self, cmd, capturestderr=False, bufsize=-1): """The parameter 'cmd' is the shell command to execute in a sub-process. On UNIX, 'cmd' may be a sequence, in which case arguments will be passed directly to the program without shell intervention (as with os.spawnv()). If 'cmd' is a string it will be passed to the shell (as with os.system()). The 'capturestderr' flag, if true, specifies that the object should capture standard error output of the child process. The default is false. If the 'bufsize' parameter is specified, it specifies the size of the I/O buffers to/from the child process.""" _cleanup() self.cmd = cmd p2cread, p2cwrite = os.pipe() c2pread, c2pwrite = os.pipe() if capturestderr: errout, errin = os.pipe() self.pid = os.fork() if self.pid == 0: # Child os.dup2(p2cread, 0) os.dup2(c2pwrite, 1) if capturestderr: os.dup2(errin, 2) self._run_child(cmd) os.close(p2cread) self.tochild = os.fdopen(p2cwrite, 'w', bufsize) os.close(c2pwrite) self.fromchild = os.fdopen(c2pread, 'r', bufsize) if capturestderr: os.close(errin) self.childerr = os.fdopen(errout, 'r', bufsize) else: self.childerr = None
def write_env_var_to_conf_dir(var, value, conf_dir): env_jsonfile_path = os.path.join(conf_dir, 'environment.json') std_vars = ['DX_APISERVER_HOST', 'DX_APISERVER_PORT', 'DX_APISERVER_PROTOCOL', 'DX_PROJECT_CONTEXT_ID', 'DX_WORKSPACE_ID', 'DX_SECURITY_CONTEXT'] if var in std_vars: try: with open(env_jsonfile_path) as fd: env_vars = json.load(fd) except: env_vars = {} if value is None and var in env_vars: del env_vars[var] else: env_vars[var] = value # Make sure the file has 600 permissions try: os.remove(env_jsonfile_path) except: pass with os.fdopen(os.open(env_jsonfile_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as fd: json.dump(env_vars, fd, indent=4) fd.write("\n") else: # DX_CLI_WD, DX_USERNAME, DX_PROJECT_CONTEXT_NAME # Make sure the file has 600 permissions try: os.remove(os.path.join(conf_dir, var)) except: pass with os.fdopen(os.open(os.path.join(conf_dir, var), os.O_CREAT | os.O_WRONLY, 0o600), 'w') as fd: fd.write(value) if not os.path.exists(os.path.expanduser('~/.dnanexus_config/') + 'unsetenv'): with open(os.path.expanduser('~/.dnanexus_config/') + 'unsetenv', 'w') as fd: for var in std_vars: fd.write('unset ' + var + '\n')
def __init__(self, cmd, capturestderr=False, bufsize=-1): """The parameter 'cmd' is the shell command to execute in a sub-process. The 'capturestderr' flag, if true, specifies that the object should capture standard error output of the child process. The default is false. If the 'bufsize' parameter is specified, it specifies the size of the I/O buffers to/from the child process.""" _cleanup() p2cread, p2cwrite = os.pipe() c2pread, c2pwrite = os.pipe() if capturestderr: errout, errin = os.pipe() self.pid = os.fork() if self.pid == 0: # Child os.dup2(p2cread, 0) os.dup2(c2pwrite, 1) if capturestderr: os.dup2(errin, 2) self._run_child(cmd) os.close(p2cread) self.tochild = os.fdopen(p2cwrite, 'w', bufsize) os.close(c2pwrite) self.fromchild = os.fdopen(c2pread, 'r', bufsize) if capturestderr: os.close(errin) self.childerr = os.fdopen(errout, 'r', bufsize) else: self.childerr = None _active.append(self)
def start(self): Ernie.log("Starting") input = os.fdopen(3) output = os.fdopen(4, "w") while(True): ipy = self.read_berp(input) if ipy == None: print 'Could not read BERP length header. Ernie server may have gone away. Exiting now.' exit() if len(ipy) == 4 and ipy[0] == bert.Atom('call'): mod, fun, args = ipy[1:4] self.log("-> " + ipy.__str__()) try: res = self.dispatch(mod, fun, args) opy = (bert.Atom('reply'), res) self.log("<- " + opy.__str__()) self.write_berp(output, opy) except ServerError, e: opy = (bert.Atom('error'), (bert.Atom('server'), 0, str(type(e)), str(e), '')) self.log("<- " + opy.__str__()) self.write_berp(output, opy) except Exception, e: opy = (bert.Atom('error'), (bert.Atom('user'), 0, str(type(e)), str(e), '')) self.log("<- " + opy.__str__()) self.write_berp(output, opy)
def heartbeat(self): # setup pipes read_transport, read_proto = yield from self.loop.connect_read_pipe( asynchttp.StreamProtocol, os.fdopen(self.up_read, 'rb')) write_transport, _ = yield from self.loop.connect_write_pipe( asynchttp.StreamProtocol, os.fdopen(self.down_write, 'wb')) reader = read_proto.set_parser(websocket.WebSocketParser) writer = websocket.WebSocketWriter(write_transport) tulip.Task(self.start_server(writer)) while True: try: msg = yield from reader.read() except asynchttp.EofStream: print('Superviser is dead, {} stopping...'.format(os.getpid())) self.loop.stop() break if msg.tp == websocket.MSG_PING: writer.pong() elif msg.tp == websocket.MSG_CLOSE: break elif msg.tp == websocket.MSG_TEXT: # broadcast message for wsc in self.clients: wsc.send(msg.data.strip().encode()) read_transport.close() write_transport.close()
def __init__(self, config_dir): self.app = Flask(__name__) self.service_map = {} self.event_map = {} signal.signal(signal.SIGTERM, self.term_handler) signal.signal(signal.SIGQUIT, self.term_handler) signal.signal(signal.SIGINT, self.term_handler) with open(config_dir) as config_fd: self.api_config = yaml.load(config_fd) self.api_fifo_name = str(uuid.uuid4()) + '.fifo' self.api_fifo_path = os.path.join(ApiConstants.API_PIPE_DIR, self.api_fifo_name) os.mkfifo(self.api_fifo_path) try: self.api_fifo_fd = os.open(self.api_fifo_path, os.O_NONBLOCK) self.api_fifo_file = os.fdopen(self.api_fifo_fd) except (IOError, OSError) as exc: print ("Unable to read the fifo file due to error {0} " .format(exc)) raise if not os.path.exists(self.api_config['moirai_input_fifo']): os.mkfifo(self.api_config['moirai_input_fifo']) try: self.moirai_fifo_fd = os.open(self.api_config['moirai_input_fifo'], os.O_WRONLY | os.O_NONBLOCK) self.moirai_fifo_file = os.fdopen(self.moirai_fifo_fd, 'w') except (IOError, OSError) as exc: print "Unable to connect to Moirai Server" self.moirai_fifo_fd = None self.setup_routes() self.command_id = 0
def verify(self, data, signature=None, keyrings=None, homedir=None): ''' `data` <string> the data to verify. `signature` <string> The signature, if detached from the data. `keyrings` <list of string> Additional keyrings to search in. `homedir` <string> Override the configured homedir. ''' tmpdir = tempfile.mkdtemp() data_file, data_path = tempfile.mkstemp(dir=tmpdir) data_file = os.fdopen(data_file, 'w') data_file.write(data) data_file.close() if signature: sig_file, sig_path = tempfile.mkstemp(dir=tmpdir) sig_file = os.fdopen(sig_file, 'w') sig_file.write(signature) sig_file.close() else: sig_path = None try: return self.verify_from_file( data_path, sig_path=sig_path, keyrings=keyrings, homedir=homedir ) finally: shutil.rmtree(tmpdir)
def GET(self, uid, gid): web.header("Content-type","text/plain") uid, gid = map(int, (uid, gid)) # Security if uid < 500 or gid < 500: yield "Invalid UID (%d) or GID (%d)\n" % (uid, gid) return try: pwd.getpwuid(uid) except KeyError: yield "UID (%d) does not exist\n" % (uid, gid) return for k in stop_program().GET(): yield k yield "Starting program with %d/%d\n" % (uid, gid) #p = subprocess.Popen(EXEC_SH, shell=True, # preexec_fn=lambda: change_user(uid,gid)) # This fails when running as daemon rpipe, wpipe = os.pipe() # Reference: http://ameblo.jp/oyasai10/entry-10615215673.html pid = os.fork() if pid == 0: # Child os.close(rpipe) wpipe = os.fdopen(wpipe, "w") change_user(uid,gid) p = subprocess.Popen(EXEC_SH, shell=True) wpipe.write("%d\n"%p.pid) sys.exit() else: # Parent os.close(wpipe) rpipe = os.fdopen(rpipe, "r") pid = int(rpipe.readline().strip()) open(PID_FILE, "w").write("%d"%pid) os.wait() # Wait child
def __init__(self, source_fd, sink_fd): """Create a new pipe object from the given file descriptors. ``source_fd`` is a file descriptor for the readable side of the pipe, ``sink_fd`` is a file descriptor for the writeable side.""" self.source = os.fdopen(source_fd, 'rb', 0) self.sink = os.fdopen(sink_fd, 'wb', 0)
def override_io(): old_out = sys.stdout old_err = sys.stderr fd_out = int(os.environ['TEST_WRITE_OUT']) fd_err = int(os.environ['TEST_WRITE_ERR']) if sys.platform == 'win32': import msvcrt fd_out = msvcrt.open_osfhandle(fd_out, 0) fd_err = msvcrt.open_osfhandle(fd_err, 0) api_out = os.fdopen(fd_out, 'w') api_err = os.fdopen(fd_err, 'w') class Intercept: def __init__(self, api, old): self.api = api self.old = old def write(self, data): import threading if threading.current_thread().name.startswith('APIThread'): self.api.write(data) else: self.old.write(data) def flush(self): self.api.flush() self.old.flush() sys.stdout = Intercept(api_out, old_out) sys.stderr = Intercept(api_err, old_err)
def __runOnce(self, CMD, use_try=None): ''' CMD: a R command string ''' use_try = use_try or self._DEBUG_MODE newline = self.newline tail_token = 'R command at time: %s' % repr(time.time()) #tail_token_r = re.sub(r'[\(\)\.]', r'\\\1', tail_token) tail_cmd = 'print("%s")%s' % (tail_token, newline) re_tail = re.compile(r'>\sprint\("%s"\)\r?\n\[1\]\s"%s"\r?\n$' % (tail_token.replace(' ', '\\s'), tail_token.replace(' ', '\\s')) ) if len(CMD) <= self.max_len or not self.localhost: fn = None else: fh, fn = tempfile.mkstemp() os.fdopen(fh, 'wb').write(_mybytes(CMD)) if sys.platform == 'cli': os.close(fh) # this is necessary on IronPython CMD = 'source("%s")' % fn.replace('\\', '/') CMD = (use_try and 'try({%s})%s%s' or '%s%s%s') % (CMD, newline, tail_cmd) sendAll(self.prog, CMD) rlt = '' while not re_tail.search(rlt): try: rltonce = readLine(self.prog) if rltonce: rlt = rlt + rltonce except: break else: rlt = re_tail.sub('', rlt) if rlt.startswith('> '): rlt = rlt[2:] if fn is not None: os.unlink(fn) return rlt
def downloadStream(self): readable_fh, writable_fh = os.pipe() with os.fdopen(readable_fh, 'r') as readable: with os.fdopen(writable_fh, 'w') as writable: def writer(): try: if self.content is not None: writable.write(self.content) elif self.version: headers = self._s3EncryptionHeaders() key = self.outer.filesBucket.get_key(self.fileID, validate=False) key.get_contents_to_file(writable, headers=headers, version_id=self.version) else: assert False finally: # This close() will send EOF to the reading end and ultimately cause # the yield to return. It also makes the implict .close() done by the # enclosing "with" context redundant but that should be ok since # .close() on file objects are idempotent. writable.close() thread = ExceptionalThread(target=writer) thread.start() yield readable thread.join()
def write_output(data, options): if options.output == "-": output = sys.stdout elif options.reparse_validate_gir: main_f, main_f_name = tempfile.mkstemp(suffix='.gir') main_f = os.fdopen(main_f, 'w') main_f.write(data) main_f.close() temp_f, temp_f_name = tempfile.mkstemp(suffix='.gir') temp_f = os.fdopen(temp_f, 'w') passthrough_gir(main_f_name, temp_f) temp_f.close() if not utils.files_are_identical(main_f_name, temp_f_name): _error("Failed to re-parse gir file; scanned=%r passthrough=%r" % ( main_f_name, temp_f_name)) os.unlink(temp_f_name) try: shutil.move(main_f_name, options.output) except OSError as e: if e.errno == errno.EPERM: os.unlink(main_f_name) return 0 raise return 0 else: try: output = open(options.output, "w") except IOError as e: _error("opening output for writing: %s" % (e.strerror, )) try: output.write(data) except IOError as e: _error("while writing output: %s" % (e.strerror, ))
def do_ipy(self, args): """Start an interactive Python interpreter""" from c1218.data import C1218Packet from c1219.access.general import C1219GeneralAccess from c1219.access.security import C1219SecurityAccess from c1219.access.log import C1219LogAccess from c1219.access.telephone import C1219TelephoneAccess vars = { '__version__': __version__, 'frmwk': self.frmwk, 'C1218Packet': C1218Packet, 'C1219GeneralAccess': C1219GeneralAccess, 'C1219SecurityAccess': C1219SecurityAccess, 'C1219LogAccess': C1219LogAccess, 'C1219TelephoneAccess': C1219TelephoneAccess } banner = 'The Framework Instance Is In The Variable \'frmwk\'' + os.linesep if self.frmwk.is_serial_connected(): vars['conn'] = self.frmwk.serial_connection banner = banner + 'The Connection Instance Is In The Variable \'conn\'' + os.linesep pyconsole = code.InteractiveConsole(vars) savestdin = os.dup(sys.stdin.fileno()) savestdout = os.dup(sys.stdout.fileno()) savestderr = os.dup(sys.stderr.fileno()) try: pyconsole.interact(banner) except SystemExit: sys.stdin = os.fdopen(savestdin, 'r', 0) sys.stdout = os.fdopen(savestdout, 'w', 0) sys.stderr = os.fdopen(savestderr, 'w', 0)
def update_file(path, updater, merger = lambda f: True): """update a file in a transaction-like manner""" fr = fw = None try: fd = os.open(path, os.O_CREAT|os.O_RDWR) try: fr = os.fdopen(fd, 'r+b') except: os.close(fd) raise fcntl.lockf(fr, fcntl.LOCK_EX) if not merger(fr): return tmpp = path + '.tmp.' + str(os.getpid()) fd = os.open(tmpp, os.O_CREAT|os.O_EXCL|os.O_WRONLY) try: fw = os.fdopen(fd, 'wb', 0) except: os.close(fd) raise updater(fw) os.fsync(fd) os.rename(tmpp, path) finally: for fx in (fr, fw): if fx: fx.close()
def __t_pty_tracker(self, trackerclass, **kwargs): def __drain(masterf): while True: termdata = masterf.read(1024) if len(termdata) == 0: break # # - Allocate a pty # - Create a thread to drain off the master side; without # this, the slave side will block when trying to write. # - Connect the prog tracker to the slave side # - Set it running # (master, slave) = pty.openpty() slavef = os.fdopen(slave, "w") masterf = os.fdopen(master, "r") t = threading.Thread(target=__drain, args=(masterf,)) t.start() p = trackerclass(output_file=slavef, **kwargs) progress.test_progress_tracker(p, gofast=True) slavef.close() t.join() masterf.close()
# # Enables flux in influxdb, for use with influxdb-client # Currently influxdb 2.x doesn't support 32-bit armhf (Pi 1/2/Zero) so not upgrading past 1.8 (for now) # import subprocess from os import fdopen, remove from shutil import move, copymode from tempfile import mkstemp influx_file = "/etc/influxdb/influxdb.conf" changed = False fd, abs_path = mkstemp() with fdopen(fd, 'w') as new_file: with open(influx_file, 'r') as old_file: for line in old_file: if 'flux-enabled' in line and line != 'flux-enabled = true\n': changed = True new_file.write('flux-enabled = true') else: new_file.write(line) copymode(influx_file, abs_path) remove(influx_file) move(abs_path, influx_file) if changed: print("Enabling Flux and restarting influxdb.") subprocess.Popen('service influxdb restart', shell=True) else: print("Flux is already enabled. No changes necessary.")
def __enter__(self): self.readable_fh, writable_fh = os.pipe() self.writable = os.fdopen(writable_fh, 'wb') self.thread = ExceptionalThread(target=self._reader) self.thread.start() return self.writable
log.debug('Changing ownership of %s with: %s' % (path, ' '.join(cmd))) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() assert p.returncode == 0, stderr except Exception, e: log.warning('Changing ownership of uploaded file %s failed: %s' % (path, str(e))) # TODO: json_file should go in the working directory json_file = tempfile.mkstemp() json_file_path = json_file[1] json_file = os.fdopen(json_file[0], 'w') for uploaded_dataset in uploaded_datasets: data = uploaded_dataset.data if uploaded_dataset.type == 'composite': # we need to init metadata before the job is dispatched data.init_meta() for meta_name, meta_value in uploaded_dataset.metadata.iteritems(): setattr(data.metadata, meta_name, meta_value) trans.sa_session.add(data) trans.sa_session.flush() json = dict(file_type=uploaded_dataset.file_type, dataset_id=data.dataset.id, dbkey=uploaded_dataset.dbkey, type=uploaded_dataset.type, metadata=uploaded_dataset.metadata, primary_file=uploaded_dataset.primary_file,
def main(args): parser = OptionParser() parser.add_option('--tmpmdpath', default=None, help="path where the outputs should be dumped for this worker") parser.add_option('--pkglist', default=None, help="file to read the pkglist from in lieu of all of them on the cli") parser.add_option("--pkgoptions", default=[], action='append', help="pkgoptions in the format of key=value") parser.add_option("--quiet", default=False, action='store_true', help="only output errors and a total") parser.add_option("--verbose", default=False, action='store_true', help="output errors and a total") parser.add_option("--globalopts", default=[], action='append', help="general options in the format of key=value") opts, pkgs = parser.parse_args(args) external_data = {'_packagenumber': 1} globalopts = {} for strs in opts.pkgoptions: k,v = strs.split('=', 1) if v in ['True', 'true', 'yes', '1', 1]: v = True elif v in ['False', 'false', 'no', '0', 0]: v = False elif v in ['None', 'none', '']: v = None external_data[k] = v for strs in opts.globalopts: k,v = strs.split('=', 1) if v in ['True', 'true', 'yes', '1', 1]: v = True elif v in ['False', 'false', 'no', '0', 0]: v = False elif v in ['None', 'none', '']: v = None globalopts[k] = v # turn off buffering on stdout sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) reldir = external_data['_reldir'] ts = rpmUtils.transaction.initReadOnlyTransaction() if opts.tmpmdpath: files = [open(opts.tmpmdpath + '/%s.xml' % i, 'w') for i in ('primary', 'filelists', 'other')] def output(*xml): for fh, buf in zip(files, xml): fh.write(buf) else: def output(*xml): buf = ' '.join(str(len(i)) for i in xml) sys.stdout.write('*** %s\n' % buf) for buf in xml: sys.stdout.write(buf) if opts.pkglist: for line in open(opts.pkglist,'r').readlines(): line = line.strip() if re.match('^\s*\#.*', line) or re.match('^\s*$', line): continue pkgs.append(line) clog_limit=globalopts.get('clog_limit', None) if clog_limit is not None: clog_limit = int(clog_limit) for pkgfile in pkgs: pkgpath = reldir + '/' + pkgfile if not os.path.exists(pkgpath): print >> sys.stderr, "File not found: %s" % pkgpath output() continue try: if not opts.quiet and opts.verbose: print "reading %s" % (pkgfile) pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath, sumtype=globalopts.get('sumtype', None), external_data=external_data) output(pkg.xml_dump_primary_metadata(), pkg.xml_dump_filelists_metadata(), pkg.xml_dump_other_metadata(clog_limit=clog_limit)) except yum.Errors.YumBaseError, e: print >> sys.stderr, "Error: %s" % e output() continue else: external_data['_packagenumber']+=1
def _cache(self): if self._access_token_cache_file is not None: with os.fdopen(os.open(self._access_token_cache_file, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as f: json.dump(self._res, f)
def get_tempfilename(suffix='.tmp'): fh, fname = tempfile.mkstemp(suffix=suffix) outsock = os.fdopen(fh,'w') outsock.close() return fname
def install(self, pkgplan, orig): """Client-side method that installs a file.""" mode = None try: mode = int(self.attrs.get("mode", None), 8) except (TypeError, ValueError): # Mode isn't valid, so let validate raise a more # informative error. self.validate(fmri=pkgplan.destination_fmri) owner, group = self.get_fsobj_uid_gid(pkgplan, pkgplan.destination_fmri) final_path = self.get_installed_path(pkgplan.image.get_root()) # Don't allow installation through symlinks. self.fsobj_checkpath(pkgplan, final_path) if not os.path.exists(os.path.dirname(final_path)): self.makedirs(os.path.dirname(final_path), mode=misc.PKG_DIR_MODE, fmri=pkgplan.destination_fmri) elif (not orig and not pkgplan.origin_fmri and "preserve" in self.attrs and self.attrs["preserve"] not in ("abandon", "install-only") and os.path.isfile(final_path)): # Unpackaged editable file is already present during # initial install; salvage it before continuing. pkgplan.salvage(final_path) # XXX If we're upgrading, do we need to preserve file perms from # existing file? # check if we have a save_file active; if so, simulate file # being already present rather than installed from scratch if "save_file" in self.attrs: orig = self.restore_file(pkgplan.image) # See if we need to preserve the file, and if so, set that up. # # XXX What happens when we transition from preserve to # non-preserve or vice versa? Do we want to treat a preserve # attribute as turning the action into a critical action? # # XXX We should save the originally installed file. It can be # used as an ancestor for a three-way merge, for example. Where # should it be stored? pres_type = self._check_preserve(orig, pkgplan) do_content = True old_path = None if pres_type == True or (pres_type and pkgplan.origin_fmri == pkgplan.destination_fmri): # File is marked to be preserved and exists so don't # reinstall content. do_content = False elif pres_type == "legacy": # Only rename old file if this is a transition to # preserve=legacy from something else. if orig.attrs.get("preserve", None) != "legacy": old_path = final_path + ".legacy" elif pres_type == "renameold.update": old_path = final_path + ".update" elif pres_type == "renameold": old_path = final_path + ".old" elif pres_type == "renamenew": final_path = final_path + ".new" elif pres_type == "abandon": return # If it is a directory (and not empty) then we should # salvage the contents. if os.path.exists(final_path) and \ not os.path.islink(final_path) and \ os.path.isdir(final_path): try: os.rmdir(final_path) except OSError as e: if e.errno == errno.ENOENT: pass elif e.errno in (errno.EEXIST, errno.ENOTEMPTY): pkgplan.salvage(final_path) elif e.errno != errno.EACCES: # this happens on Windows raise # XXX This needs to be modularized. if do_content and self.needsdata(orig, pkgplan): tfilefd, temp = tempfile.mkstemp(dir=os.path.dirname(final_path)) if not self.data: # The state of the filesystem changed after the # plan was prepared; attempt a one-off # retrieval of the data. self.data = self.__set_data(pkgplan) stream = self.data() tfile = os.fdopen(tfilefd, "wb") try: # Always verify using the most preferred hash hash_attr, hash_val, hash_func = \ digest.get_preferred_hash(self) shasum = misc.gunzip_from_stream(stream, tfile, hash_func) except zlib.error as e: raise ActionExecutionError( self, details=_("Error decompressing payload: " "{0}").format(" ".join([str(a) for a in e.args])), error=e) finally: tfile.close() stream.close() if shasum != hash_val: raise ActionExecutionError( self, details=_("Action data hash verification " "failure: expected: {expected} computed: " "{actual} action: {action}").format( expected=hash_val, actual=shasum, action=self)) else: temp = final_path try: os.chmod(temp, mode) except OSError as e: # If the file didn't exist, assume that's intentional, # and drive on. if e.errno != errno.ENOENT: raise else: return try: portable.chown(temp, owner, group) except OSError as e: if e.errno != errno.EPERM: raise # XXX There's a window where final_path doesn't exist, but we # probably don't care. if do_content and old_path: try: portable.rename(final_path, old_path) except OSError as e: if e.errno != errno.ENOENT: # Only care if file isn't gone already. raise # This is safe even if temp == final_path. try: portable.rename(temp, final_path) except OSError as e: raise api_errors.FileInUseException(final_path) # Handle timestamp if specified (and content was installed). if do_content and "timestamp" in self.attrs: t = misc.timestamp_to_time(self.attrs["timestamp"]) try: os.utime(final_path, (t, t)) except OSError as e: if e.errno != errno.EACCES: raise # On Windows, the time cannot be changed on a # read-only file os.chmod(final_path, stat.S_IRUSR | stat.S_IWUSR) os.utime(final_path, (t, t)) os.chmod(final_path, mode) # Handle system attributes. sattr = self.attrs.get("sysattr") if sattr: if isinstance(sattr, list): sattr = ",".join(sattr) sattrs = sattr.split(",") if len(sattrs) == 1 and \ sattrs[0] not in portable.get_sysattr_dict(): # not a verbose attr, try as a compact attr seq arg = sattrs[0] else: arg = sattrs try: portable.fsetattr(final_path, arg) except OSError as e: if e.errno != errno.EINVAL: raise warn = _("System attributes are not supported " "on the target image filesystem; 'sysattr'" " ignored for {0}").format(self.attrs["path"]) pkgplan.image.imageplan.pd.add_item_message( pkgplan.destination_fmri, misc.time_to_timestamp(time.time()), MSG_WARNING, warn) except ValueError as e: warn = _("Could not set system attributes for {path}" "'{attrlist}': {err}").format(attrlist=sattr, err=e, path=self.attrs["path"]) pkgplan.image.imageplan.pd.add_item_message( pkgplan.destination_fmri, misc.time_to_timestamp(time.time()), MSG_WARNING, warn)
def bitbake_main(configParams, configuration): # Python multiprocessing requires /dev/shm on Linux if sys.platform.startswith('linux') and not os.access('/dev/shm', os.W_OK | os.X_OK): raise BBMainException("FATAL: /dev/shm does not exist or is not writable") # Unbuffer stdout to avoid log truncation in the event # of an unorderly exit as well as to provide timely # updates to log files for use with tail try: if sys.stdout.name == '<stdout>': sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) except: pass configuration.setConfigParameters(configParams) ui_module = import_extension_module(bb.ui, configParams.ui) servermodule = import_extension_module(bb.server, configParams.servertype) if configParams.server_only: if configParams.servertype != "xmlrpc": raise BBMainException("FATAL: If '--server-only' is defined, we must set the " "servertype as 'xmlrpc'.\n") if not configParams.bind: raise BBMainException("FATAL: The '--server-only' option requires a name/address " "to bind to with the -B option.\n") if configParams.remote_server: raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" % ("the BBSERVER environment variable" if "BBSERVER" in os.environ \ else "the '--remote-server' option" )) if configParams.bind and configParams.servertype != "xmlrpc": raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must " "set the servertype as 'xmlrpc'.\n") if configParams.remote_server and configParams.servertype != "xmlrpc": raise BBMainException("FATAL: If '--remote-server' is defined, we must " "set the servertype as 'xmlrpc'.\n") if configParams.observe_only and (not configParams.remote_server or configParams.bind): raise BBMainException("FATAL: '--observe-only' can only be used by UI clients " "connecting to a server.\n") if configParams.kill_server and not configParams.remote_server: raise BBMainException("FATAL: '--kill-server' can only be used to terminate a remote server") if "BBDEBUG" in os.environ: level = int(os.environ["BBDEBUG"]) if level > configuration.debug: configuration.debug = level bb.msg.init_msgconfig(configParams.verbose, configuration.debug, configuration.debug_domains) # Ensure logging messages get sent to the UI as events handler = bb.event.LogHandler() if not configParams.status_only: # In status only mode there are no logs and no UI logger.addHandler(handler) # Clear away any spurious environment variables while we stoke up the cooker cleanedvars = bb.utils.clean_environment() featureset = [] if not configParams.server_only: # Collect the feature set for the UI featureset = getattr(ui_module, "featureSet", []) if not configParams.remote_server: # we start a server with a given configuration server = start_server(servermodule, configParams, configuration, featureset) bb.event.ui_queue = [] else: # we start a stub server that is actually a XMLRPClient that connects to a real server server = servermodule.BitBakeXMLRPCClient(configParams.observe_only, configParams.xmlrpctoken) server.saveConnectionDetails(configParams.remote_server) if not configParams.server_only: try: server_connection = server.establishConnection(featureset) except Exception as e: bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e))) # Restore the environment in case the UI needs it for k in cleanedvars: os.environ[k] = cleanedvars[k] logger.removeHandler(handler) if configParams.status_only: server_connection.terminate() return 0 if configParams.kill_server: server_connection.connection.terminateServer() bb.event.ui_queue = [] return 0 try: return ui_module.main(server_connection.connection, server_connection.events, configParams) finally: bb.event.ui_queue = [] server_connection.terminate() else: print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host, server.serverImpl.port)) return 0 return 1
<min x="''' + str(-(SIZE_X / 2)) + '''" y="0" z="1"/> <max x="''' + str(SIZE_X / 2) + '''" y="''' + str( SIZE_Y - 1) + '''" z="''' + str(SIZE_Z) + '''"/> </Grid> </ObservationFromGrid> <VideoProducer viewpoint="1"> <Width>860</Width> <Height>480</Height> </VideoProducer> </AgentHandlers> </AgentSection> </Mission>''', gridJson sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) # flush print output immediately agent_host = MalmoPython.AgentHost() try: agent_host.parse(sys.argv) except RuntimeError as e: print 'ERROR:', e print agent_host.getUsage() exit(1) if agent_host.receivedArgument("help"): print agent_host.getUsage() exit(0) num_iterations = 30000 if agent_host.receivedArgument("test"): num_iterations = 10 # Haven't got all day
def main(argv=sys.argv): log = logging.getLogger('heat-config') handler = logging.StreamHandler(sys.stderr) handler.setFormatter( logging.Formatter( '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s')) log.addHandler(handler) log.setLevel('DEBUG') prepare_dir(OUTPUTS_DIR) prepare_dir(WORKING_DIR) os.chdir(WORKING_DIR) c = json.load(sys.stdin) use_hiera = c['options'].get('enable_hiera', False) use_facter = c['options'].get('enable_facter', True) modulepath = c['options'].get('modulepath') tags = c['options'].get('tags') facts = {} hiera = {} fqdn = get_hostname_f(log) if fqdn: facts['FACTER_fqdn'] = fqdn for input in c['inputs']: input_name = input['name'] input_value = input.get('value', '') if use_facter: fact_name = 'FACTER_%s' % input_name facts[fact_name] = input_value if use_hiera: hiera[input_name] = input_value if use_hiera: prepare_dir(HIERA_DATADIR) hiera_data = os.path.join(HIERA_DATADIR, 'heat_config_%s.json' % c['name']) with os.fdopen(os.open(hiera_data, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600), 'w') as hiera_file: hiera_file.write(json.dumps(hiera).encode('utf8')) facts['FACTER_deploy_config_name'] = c['name'] fn = os.path.join(WORKING_DIR, '%s.pp' % c['id']) heat_outputs_path = os.path.join(OUTPUTS_DIR, c['id']) facts['FACTER_heat_outputs_path'] = heat_outputs_path env_debug = ' '.join('%s="%s" ' % (k, v) for k, v in facts.items()) env = os.environ.copy() env.update(facts) with os.fdopen(os.open(fn, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o700), 'w') as f: f.write(c.get('config', '').encode('utf-8')) cmd = [PUPPET_CMD, 'apply', '--detailed-exitcodes', fn] if modulepath: cmd.insert(-1, '--modulepath') cmd.insert(-1, modulepath) if tags: cmd.insert(-1, '--tags') cmd.insert(-1, tags) log.debug('Running %s %s' % (env_debug, ' '.join(cmd))) try: subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) except OSError: log.warn('puppet not installed yet') return stdout, stderr = subproc.communicate() log.info('Return code %s' % subproc.returncode) if stdout: log.info(stdout) if stderr: log.info(stderr) # returncode of 2 means there were successfull changes if subproc.returncode in (0, 2): returncode = 0 log.info('Completed %s' % fn) else: returncode = subproc.returncode log.error("Error running %s. [%s]\n" % (fn, subproc.returncode)) response = {} for output in c.get('outputs') or []: output_name = output['name'] try: with open('%s.%s' % (heat_outputs_path, output_name)) as out: response[output_name] = out.read() except IOError: pass response.update({ 'deploy_stdout': stdout, 'deploy_stderr': stderr, 'deploy_status_code': returncode, }) json.dump(response, sys.stdout)
def process_msg(message): if message["act"] == "get_topics": response = artm_bridge.model.topics elif message["act"] == "get_documents": topic_id = message["topic_id"] offset = message["offset"] limit = message["limit"] if type(topic_id) is not str: raise BridgeParamError("incorrect param type: `topic_id`") if type(offset) is not int or type(limit) is not int: raise BridgeParamError( "`limit` and `offset` fields must be integer") docs, weights = artm_bridge.get_documents_by_topic(topic_id, offset=offset, limit=limit) response = {"docs": docs, "weights": weights} elif message["act"] == "get_document": doc_id = message["doc_id"] if type(doc_id) is not str: raise BridgeParamError("incorrect param type: `doc_id`") docs = artm_bridge.data_source.get_documents_by_ids( [doc_id], with_modalities=True) if len(docs) == 0: raise BridgeParamError( "document with `doc_id` = '%s' is not found" % doc_id) doc = docs[0] if message["recommend_tags"]: doc["recommended_tags"] = artm_bridge.recommend_tags_by_doc(doc) response = doc elif message["act"] == "perform_search": query = message["query"] limit = message["limit"] if type(query) is not str: raise BridgeParamError("incorrect param type: `query`") if type(limit) is not int: raise BridgeParamError("incorrect param type: `limit`") response = dict( zip(["docs", "theta"], artm_bridge.search_documents(query, limit=limit))) elif message["act"] == "recommend_docs": doc_id = message["doc_id"] if type(doc_id) is not str: raise BridgeParamError("incorrect param type: `doc_id`") sim_docs_ids = artm_bridge.recommend_docs_by_doc(doc_id) response = artm_bridge.data_source.get_documents_by_ids( sim_docs_ids, with_texts=False) elif message["act"] == "transform_doc": doc_path = message["doc_path"] filename = message["filename"] try: # Initialize file resources doc_file = open(doc_path) vw_fd, vw_path = tempfile.mkstemp(prefix="upload", text=True) vw_file = os.fdopen(vw_fd, "w") batch_path = tempfile.mkdtemp(prefix="batch") # Parse uploaded file doc = pipeline.fit_transform(doc_file) # Save to Vowpal Wabbit file text_utils.VowpalWabbitSink(vw_file, lambda x: "upload") \ .fit_transform([doc]) # Transform uploaded document and return its Theta matrix response = {} response["filename"] = filename response["theta"] = artm_bridge.model.transform_one( vw_path, batch_path) except: raise finally: # Delete uploaded file doc_file.close() os.remove(doc_path) # Delete temporary files/dirs os.remove(vw_path) rm_flat_dir(batch_path) elif False and message["act"] == "get_next_assessment": ass_id = message["assessor_id"] ass_cnt = message["assessors_cnt"] col_name = message["collection_name"] if ass_id >= ass_cnt: response = "Incorrent `assessor_id`" else: docs_count = db["datasets"][col_name].count() min_id = int(ass_id * docs_count / ass_cnt) max_id = int((ass_id + 1) * docs_count / ass_cnt) # May take a long time for large datasets docs_ids = db["datasets"][col_name].find({}, {"_id": 1}) docs_ids = list( map(lambda x: x["_id"], docs_ids.sort([("_id", 1)]))) ass_docs_ids = docs_ids[min_id:max_id] # Get unused documents' ids used_docs_ids = db["assessment"][col_name].find({}, {"_id": 1}) used_docs_ids = list(map(lambda x: x["_id"], used_docs_ids)) unused_docs_ids = list(set(ass_docs_ids) - set(used_docs_ids)) # Form response random.shuffle(unused_docs_ids) # Use batches of 100 docs per request response = unused_docs_ids[:100] elif False and message["act"] == "assess_document": doc_id = message["doc_id"] is_relevant = message["is_relevant"] col_names = [ v for k, v in prefix_to_col_map.items() if doc_id.startswith(k + "_") ] if len(col_names) != 1: response = False else: col_name = col_names[0] dataset = db["assessment"][col_name] doc = {"is_relevant": is_relevant, "assess_time": datetime.now()} dataset.replace_one({"_id": doc_id}, doc, upsert=True) response = True else: raise BridgeParamError("unknown query") return response
checksum = http_download_and_hash(rustup_url(platform, version)) if validate and checksum != rustup_hash(platform): print('mismatch:\n script: %s\n server: %s' % ( RUSTUP_HASHES[platform], checksum)) else: print('OK') hashes.append((platform, checksum)) return hashes if __name__ == '__main__': '''Allow invoking the module as a utility to update checksums.''' # Unbuffer stdout so our two-part 'Checking...' messages print correctly # even if there's network delay. sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) # Hook the requests module from the greater source tree. We can't import # this at the module level since we might be imported into the bootstrap # script in standalone mode. # # This module is necessary for correct https certificate verification. mod_path = os.path.dirname(__file__) sys.path.insert(0, os.path.join(mod_path, '..', '..', 'requests')) update = False if len(sys.argv) > 1: if sys.argv[1] == '--update': update = True else: print(USAGE)
def upload_haproxy_config(self, amphora_id, listener_id): """Upload the haproxy config :param amphora_id: The id of the amphora to update :param listener_id: The id of the listener """ stream = Wrapped(flask.request.stream) # We have to hash here because HAProxy has a string length limitation # in the configuration file "peer <peername>" lines peer_name = octavia_utils.base64_sha1_string(amphora_id).rstrip('=') if not os.path.exists(util.haproxy_dir(listener_id)): os.makedirs(util.haproxy_dir(listener_id)) name = os.path.join(util.haproxy_dir(listener_id), 'haproxy.cfg.new') flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC # mode 00600 mode = stat.S_IRUSR | stat.S_IWUSR b = stream.read(BUFFER) s_io = io.StringIO() while b: # Write haproxy configuration to StringIO s_io.write(b.decode('utf8')) b = stream.read(BUFFER) # Since haproxy user_group is now auto-detected by the amphora agent, # remove it from haproxy configuration in case it was provided # by an older Octavia controller. This is needed in order to prevent # a duplicate entry for 'group' in haproxy configuration, which will # result an error when haproxy starts. new_config = re.sub(r"\s+group\s.+", "", s_io.getvalue()) # Handle any haproxy version compatibility issues new_config = haproxy_compatibility.process_cfg_for_version_compat( new_config) with os.fdopen(os.open(name, flags, mode), 'w') as file: file.write(new_config) # use haproxy to check the config cmd = "haproxy -c -L {peer} -f {config_file} -f {haproxy_ug}".format( config_file=name, peer=peer_name, haproxy_ug=consts.HAPROXY_USER_GROUP_CFG) try: subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: LOG.error("Failed to verify haproxy file: %s %s", e, e.output) # Save the last config that failed validation for debugging os.rename(name, ''.join([name, '-failed'])) return webob.Response(json=dict(message="Invalid request", details=e.output), status=400) # file ok - move it os.rename(name, util.config_path(listener_id)) try: init_system = util.get_os_init_system() LOG.debug('Found init system: %s', init_system) init_path = util.init_path(listener_id, init_system) if init_system == consts.INIT_SYSTEMD: template = SYSTEMD_TEMPLATE # Render and install the network namespace systemd service util.install_netns_systemd_service() util.run_systemctl_command( consts.ENABLE, consts.AMP_NETNS_SVC_PREFIX + '.service') elif init_system == consts.INIT_UPSTART: template = UPSTART_TEMPLATE elif init_system == consts.INIT_SYSVINIT: template = SYSVINIT_TEMPLATE init_enable_cmd = "insserv {file}".format(file=init_path) else: raise util.UnknownInitError() except util.UnknownInitError: LOG.error("Unknown init system found.") return webob.Response(json=dict( message="Unknown init system in amphora", details="The amphora image is running an unknown init " "system. We can't create the init configuration " "file for the load balancing process."), status=500) if init_system == consts.INIT_SYSTEMD: # mode 00644 mode = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) else: # mode 00755 mode = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) hap_major, hap_minor = haproxy_compatibility.get_haproxy_versions() if not os.path.exists(init_path): with os.fdopen(os.open(init_path, flags, mode), 'w') as text_file: text = template.render( peer_name=peer_name, haproxy_pid=util.pid_path(listener_id), haproxy_cmd=util.CONF.haproxy_amphora.haproxy_cmd, haproxy_cfg=util.config_path(listener_id), haproxy_user_group_cfg=consts.HAPROXY_USER_GROUP_CFG, respawn_count=util.CONF.haproxy_amphora.respawn_count, respawn_interval=( util.CONF.haproxy_amphora.respawn_interval), amphora_netns=consts.AMP_NETNS_SVC_PREFIX, amphora_nsname=consts.AMPHORA_NAMESPACE, HasIFUPAll=self._osutils.has_ifup_all(), haproxy_major_version=hap_major, haproxy_minor_version=hap_minor) text_file.write(text) # Make sure the new service is enabled on boot if init_system == consts.INIT_SYSTEMD: util.run_systemctl_command( consts.ENABLE, "haproxy-{list}".format(list=listener_id)) elif init_system == consts.INIT_SYSVINIT: try: subprocess.check_output(init_enable_cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: LOG.error( "Failed to enable haproxy-%(list)s service: " "%(err)s %(out)s", { 'list': listener_id, 'err': e, 'out': e.output }) return webob.Response(json=dict( message="Error enabling haproxy-{0} service".format( listener_id), details=e.output), status=500) res = webob.Response(json={'message': 'OK'}, status=202) res.headers['ETag'] = stream.get_md5() return res
def domain(): global statefile global state global outdir global tmpfile global debugging try: opts, args = getopt.getopt(sys.argv[1:], 'dho:s:') except getopt.GetoptError: usage() for o, a in opts: if o == '-h': usage() elif o == '-o': outdir = a elif o == '-d': debugging = True elif o == '-s': statefile = a if len(args) < 3: usage() targetfile = args[0] recip = args[1] groupname = args[2] outdir_setup() state = load_scanstate() tmpout = tempfile.mkstemp() tmpfile = os.fdopen(tmpout[0], 'w') state.register_outfile(tmpfile) hn = os.uname()[1] tmpfile.write('Subject: diffscan2 %s %s\n' % (groupname, hn)) tmpfile.write('From: diffscan2 <noreply@%s>\n' % hn) tmpfile.write('To: %s\n' % recip) tmpfile.write('\n') tmpfile.write('diffscan2 results output\n\n') run_nmap(targetfile) state.calculate() tmpfile.write('New Open Service List\n') tmpfile.write('---------------------\n') state.print_open_alerts() tmpfile.write('\n') tmpfile.write('New Closed Service List\n') tmpfile.write('---------------------\n') state.print_closed_alerts() tmpfile.write('\n') tmpfile.write('OPREV: number of times service was open in previous ' \ 'scans\n') tmpfile.write('CPREV: number of times service was closed in ' \ 'previous scans\n') tmpfile.write('maximum previous scans stored: %d\n' % state.KEEP_SCANS) tmpfile.write('current total services: %d\n' % \ state.last_scan_total_services()) tmpfile.write('previous total services: %d\n' % \ state.previous_scan_total_services()) tmpfile.write('up trend: %s\n' % state.up_trend()) tmpfile.write('down trend: %s\n' % state.down_trend()) state.clear_outfile() write_scanstate() tmpfile.close() f = open(tmpout[1], 'r') buf = f.read() f.close() if debugging: sys.stdout.write(buf) sp = subprocess.Popen(['sendmail', '-t'], stdin=subprocess.PIPE) sp.communicate(buf) os.remove(tmpout[1])
def _save(self, name, content): full_path = self.path(name) # Create any intermediate directories that do not exist. # Note that there is a race between os.path.exists and os.makedirs: # if os.makedirs fails with EEXIST, the directory was created # concurrently, and we can continue normally. Refs #16082. directory = os.path.dirname(full_path) if not os.path.exists(directory): try: if self.directory_permissions_mode is not None: # os.makedirs applies the global umask, so we reset it, # for consistency with file_permissions_mode behavior. old_umask = os.umask(0) try: os.makedirs(directory, self.directory_permissions_mode) finally: os.umask(old_umask) else: os.makedirs(directory) except OSError as e: if e.errno != errno.EEXIST: raise if not os.path.isdir(directory): raise IOError("%s exists and is not a directory." % directory) # There's a potential race condition between get_available_name and # saving the file; it's possible that two threads might return the # same name, at which point all sorts of fun happens. So we need to # try to create the file, but if it already exists we have to go back # to get_available_name() and try again. while True: try: # This file has a file path that we can move. if hasattr(content, 'temporary_file_path'): file_move_safe(content.temporary_file_path(), full_path) # This is a normal uploadedfile that we can stream. else: # This fun binary flag incantation makes os.open throw an # OSError if the file already exists before we open it. flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)) # The current umask value is masked out by os.open! fd = os.open(full_path, flags, 0o666) _file = None try: locks.lock(fd, locks.LOCK_EX) for chunk in content.chunks(): if _file is None: mode = 'wb' if isinstance(chunk, bytes) else 'wt' _file = os.fdopen(fd, mode) _file.write(chunk) finally: locks.unlock(fd) if _file is not None: _file.close() else: os.close(fd) except OSError as e: if e.errno == errno.EEXIST: # Ooops, the file exists. We need a new file name. name = self.get_available_name(name) full_path = self.path(name) else: raise else: # OK, the file save worked. Break out of the loop. break if self.file_permissions_mode is not None: os.chmod(full_path, self.file_permissions_mode) return name
def export_segments(self, segments): #check edgesmask if not self.export_folder: return cur_segments = segments for i in range(self.pool_count + 1): edges = self.edges_list[i] #indices = self.indces_list[i] filename, file_extension = os.path.splitext(self.filename) import platform if platform.system()=="Windows": file = '%s/%s_%d.vtk' % (self.export_folder, filename.split('\\')[-1], i) elif platform.system()=="Linux": file = '%s/%s_%d.vtk' % (self.export_folder, filename.split('/')[-1], i) fh, abs_path = mkstemp() edge_key = 0 faces=[] poly=False with os.fdopen(fh, 'w') as new_file: with open(file) as old_file: for line in old_file: new_file.write(line) if line.split(' ')[0] == 'SCALARS': poly = False if poly: face = np.asarray([int(j) for j in line.split(' ')[1:4]]) faces.append(face) if line.split(' ')[0] == 'POLYGONS': poly = True new_file.write("\n \nCELL_DATA %d \nSCALARS scalars double \nLOOKUP_TABLE default" % len(faces)) s_faces = np.sort(np.asarray(faces)) f01 = s_faces[:,[0,1]] f02 = s_faces[:,[0,2]] f12 = s_faces[:,[1,2]] face_labels = np.zeros(len(faces)) for edge_k,edge in enumerate(edges): idx = [] #e = np.array([indices[ed[0]], indices[ed[1]]]) idx.append( (edge==f01).all(axis=1).nonzero() ) idx.append( (edge==f02).all(axis=1).nonzero() ) idx.append( (edge==f12).all(axis=1).nonzero() ) for j in list(flatten(idx)): face_labels[j] += cur_segments[edge_k] face_labels[face_labels<1.5]=0 face_labels[face_labels>0]=1 for j,face in enumerate(faces): if j%9 == 0: new_file.write("\n") new_file.write("%d " % face_labels[j]) ''' if line[0] == 'e': new_file.write('%s %d' % (line.strip(), cur_segments[edge_key])) if edge_key < len(cur_segments): edge_key += 1 new_file.write('\n') else: new_file.write(line)''' os.remove(file) move(abs_path, file) if i < len(self.history_data['edges_mask']): cur_segments = segments[:len(self.history_data['edges_mask'][i])] cur_segments = cur_segments[self.history_data['edges_mask'][i]]
def fdopen(fd, mode="r", bufsize=-1): fp = os.fdopen(fd, mode, bufsize) return _fixseek(fp, mode)
# Get file descriptor from argument print 'spID',args.status_pipe_id pipearg = int(args.status_pipe_id) #int(sys.argv[1]) if sys.platform == "win32": pipeoutfd = msvcrt.open_osfhandle(pipearg,os.O_WRONLY) else: pipeoutfd = pipearg # Read from pipe # Note: Could be done with os.read/os.close directly, instead of os.fdopen #os.write() #os.close() if 1: pipein = os.fdopen(pipeoutfd, 'w') #for i in range(1000): #print pipein #time.sleep(5) pipein.write("{'status':%s}" % (not etl_failed)) #pprint (dir(pipein)) #pipein.flush() #time.sleep(2) pipein.close() #time.sleep(2000) except : print "Unexpected error:", sys.exc_info() #print 'error--------', e
def doCommand(self, data): """Process a single SMTP Command""" cmd = data[0:4] cmd = string.upper(cmd) keep = 1 rv = None if cmd in ["EHLO", 'HELO']: self.state = ESMTPPassthroughSession.ST_HELO elif cmd == "RSET": self.from_address = None self.to_address = None self.helo = None self.dataAccum = "" self.state = ESMTPPassthroughSession.ST_INIT elif cmd == "NOOP": pass elif cmd == "QUIT": keep = 0 return "221 bye", keep elif cmd == "MAIL": if self.state != ESMTPPassthroughSession.ST_HELO: return "503 Bad command sequence", 1 try: self.from_address = self.stripAddress(data) except: return "501 invalid address syntax", 1 self.state = ESMTPPassthroughSession.ST_MAIL elif cmd == "RCPT": if (self.state != ESMTPPassthroughSession.ST_MAIL) and ( self.state != ESMTPPassthroughSession.ST_RCPT): return "503 Bad command sequence", 1 try: rec = self.stripAddress(data) self.to_address = rec self.recipients.append(rec) except: return "501 invalid address syntax", 1 self.state = ESMTPPassthroughSession.ST_RCPT elif cmd == "DATA": if self.state != ESMTPPassthroughSession.ST_RCPT: return "503 Bad command sequence", 1 self.state = ESMTPPassthroughSession.ST_DATA self.dataAccum = "" try: (handle, tempfilename) = tempfile.mkstemp(prefix='fuglu', dir=self.config.get( 'main', 'tempdir')) self.tempfilename = tempfilename self.tempfile = os.fdopen(handle, 'w+b') except Exception as e: self.endsession(421, "could not create file: %s" % str(e)) return "354 OK, Enter data, terminated with a \\r\\n.\\r\\n", 1 if data[0:8].upper() == 'XFORWARD': self.store_xforward(data) rv = self.forwardCommand(data) return rv, keep
def disable_output_buffering(): """Disables the buffering of the stdout. Devtools command line scripts should do so, so that their stdout is consistent when not directly attached to a terminal (e.g. because another script runs devtools in a subprocess). """ sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
# print people sorted by number of posts made posts = sorted(people_list, key=lambda k: k['posts_made']) for person in posts: print(str(person)) # Print people sorted by likes sent print("SORTED BY WHO SENT THE MOST LIKES: ") sent_likes = sorted(people_list, key=lambda k: k['likes_sent']) for person in sent_likes: print(str(person)) # Print people sorted by likes received print("SORTED BY WHO GOT THE MOST LIKES: ") received_likes = sorted(people_list, key=lambda k: k['likes_received']) for person in received_likes: print(str(person)) # Print urls of pictures sent to the chat print("URLS:") for line in url_list: print(line) # Finally, write the URLS of the picuters sent to the chat to a text file curDir = os.path.dirname(os.path.realpath('__file__')) # REPLACE ****** with the filename you want to write to with os.fdopen(os.open(curDir + "/******.txt", os.O_WRONLY), "w", encoding='utf-8') as file_write: file_write.write("\n".join(url_list))
def create_tempfile(command): filed, script_file = tempfile.mkstemp(prefix='at') fileh = os.fdopen(filed, 'w') fileh.write(command) fileh.close() return script_file
def test_dialect_apply(self): class testA(csv.excel): delimiter = "\t" class testB(csv.excel): delimiter = ":" class testC(csv.excel): delimiter = "|" csv.register_dialect('testC', testC) try: fd, name = tempfile.mkstemp() fileobj = os.fdopen(fd, "w+b") try: writer = csv.writer(fileobj) writer.writerow([1, 2, 3]) fileobj.seek(0) self.assertEqual(fileobj.read(), "1,2,3\r\n") finally: fileobj.close() os.unlink(name) fd, name = tempfile.mkstemp() fileobj = os.fdopen(fd, "w+b") try: writer = csv.writer(fileobj, testA) writer.writerow([1, 2, 3]) fileobj.seek(0) self.assertEqual(fileobj.read(), "1\t2\t3\r\n") finally: fileobj.close() os.unlink(name) fd, name = tempfile.mkstemp() fileobj = os.fdopen(fd, "w+b") try: writer = csv.writer(fileobj, dialect=testB()) writer.writerow([1, 2, 3]) fileobj.seek(0) self.assertEqual(fileobj.read(), "1:2:3\r\n") finally: fileobj.close() os.unlink(name) fd, name = tempfile.mkstemp() fileobj = os.fdopen(fd, "w+b") try: writer = csv.writer(fileobj, dialect='testC') writer.writerow([1, 2, 3]) fileobj.seek(0) self.assertEqual(fileobj.read(), "1|2|3\r\n") finally: fileobj.close() os.unlink(name) fd, name = tempfile.mkstemp() fileobj = os.fdopen(fd, "w+b") try: writer = csv.writer(fileobj, dialect=testA, delimiter=';') writer.writerow([1, 2, 3]) fileobj.seek(0) self.assertEqual(fileobj.read(), "1;2;3\r\n") finally: fileobj.close() os.unlink(name) finally: csv.unregister_dialect('testC')
def _open_file(path): return os.fdopen( os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w')
def read_file(self, buffering=DEFAULT_BUFSIZE): """Get a file-like object for the read end of the pipe.""" return os.fdopen(self.read_fd, 'rb', buffering)
def createComposeFile(self, utilityobj, max_api_users=1): logging.info("Creating Compose configuration files") content = Template(''' version: "2" services: apporbit-chef: container_name: apporbit-chef image: ${APPORBIT_REGISTRY}apporbit/apporbit-chef:2.0 mem_limit: 2100000000 hostname: ${APPORBIT_HOST} dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always network_mode: "bridge" ports: - "9443:9443" environment: - UPGRADE volumes: - ${APPORBIT_LIB}/chef-server:/var/opt/chef-server:Z - ${APPORBIT_KEY}:/var/opt/chef-server/nginx/ca/:Z - chef-conf:/etc/chef-server/:Z apporbit-rmq: container_name: apporbit-rmq image: ${APPORBIT_REGISTRY}apporbit/apporbit-rmq:${APPORBIT_BUILDID} hostname: rmq dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always network_mode: "bridge" mem_limit: 2100000000 environment: - RABBITMQ_VM_MEMORY_HIGH_WATERMARK_PAGING_RATIO=0.1 volumes: - rabbitmq-data:/var/lib/rabbitmq:Z - ${APPORBIT_LOG}/rmq:/var/log/rabbitmq:Z apporbit-db: container_name: apporbit-db image: mysql:5.6.24 network_mode: "bridge" hostname: db dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always ports: - "3306" environment: - MYSQL_USER=root - MYSQL_ROOT_PASSWORD=admin - MYSQL_PASSWORD=admin - MYSQL_DATABASE=apporbit_controller volumes: - ${APPORBIT_LIB}/mysql:/var/lib/mysql:Z apporbit-consul: container_name: apporbit-consul image: ${APPORBIT_REGISTRY}apporbit/consul:${APPORBIT_BUILDID} command: -server -bootstrap --domain=${APPORBIT_DOMAIN} hostname: consul dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always network_mode: "bridge" ports: - "8400:8400" - "8500:8500" - "53:53/udp" - "53:53" - "8301:8301/udp" - "8302:8302/udp" volumes: - ${APPORBIT_LIB}/consul:/data:Z apporbit-services: container_name: apporbit-services image: ${APPORBIT_REGISTRY}apporbit/apporbit-services:${APPORBIT_BUILDID} restart: always hostname: services dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} network_mode: "bridge" environment: - TERM=xterm - GEMINI_INT_REPO=${APPORBIT_REPO} - CHEF_URL=https://${APPORBIT_CHEFHOST}:9443 - OFFLINE_MODE=${OFFLINE_MODE} - UPGRADE - MYSQL_HOST=db - MYSQL_USERNAME=root - MYSQL_PASSWORD=admin - MYSQL_DATABASE=apporbit_mist - GEMINI_STACK_IPANEMA=1 - LOG_LEVEL=${APPORBIT_LOGLEVEL} - AO_CONTROLLER_HOST=${APPORBIT_HOST} links: - apporbit-db:db - apporbit-rmq:rmq volumes_from: - apporbit-chef depends_on: - apporbit-captain volumes: - ${APPORBIT_LIB}/sshKey_root:/root:Z - ${APPORBIT_CONF}/apporbit.ini:/etc/apporbit.ini:Z - ${APPORBIT_LIB}/services:/var/lib/apporbit:Z - ${APPORBIT_LOG}/services:/var/log/apporbit:Z - ${APPORBIT_LIB}/chefconf:/opt/apporbit/chef:Z ${SERVICES_DEVMOUNT} apporbit-locator: container_name: apporbit-locator image: ${APPORBIT_REGISTRY}apporbit/locator:${APPORBIT_BUILDID} hostname: locator dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always network_mode: "bridge" ports: - "8080" environment: - CONSUL_IP_PORT=http://consul:8500 links: - apporbit-consul:consul volumes: - ${APPORBIT_LIB}/locator:/data:Z - ${APPORBIT_LOG}/locator:/var/log/apporbit:Z apporbit-svcd: container_name: apporbit-svcd image: ${APPORBIT_REGISTRY}apporbit/svcd:${APPORBIT_BUILDID} hostname: svcd dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always network_mode: "bridge" environment: - CONTROLLER_ALIAS_NAME=${APPORBIT_HOST} ports: - "8080" links: - apporbit-db:db - apporbit-locator:locator volumes: - ${APPORBIT_LOG}/svcd:/var/log/apporbit:Z apporbit-captain: container_name: apporbit-captain image: ${APPORBIT_REGISTRY}apporbit/captain:${APPORBIT_BUILDID} restart: always hostname: captain dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} network_mode: "bridge" ports: - "8080" - "8091:8091" environment: - CONTROLLER_ALIAS_NAME=${APPORBIT_HOST} - AO_REGISTRY=${DATASVC_REGISTRY} links: - apporbit-svcd:svcd volumes: - ${APPORBIT_LOG}/captain:/var/log/apporbit:Z apporbit-controller: container_name: apporbit-controller image: ${APPORBIT_REGISTRY}apporbit/apporbit-controller:${APPORBIT_BUILDID} hostname: ${APPORBIT_HOST} dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} restart: always network_mode: "bridge" ports: - "80:80" - "443:443" environment: - TERM=xterm - ON_PREM_MODE=true - OFFLINE_MODE=${OFFLINE_MODE} - THEME_NAME=apporbit-v2 - CURRENT_API_VERSION=v2 - ONPREM_EMAIL_ID=${APPORBIT_LOGINID} - LOG_LEVEL=${APPORBIT_LOGLEVEL} - MAX_POOL_SIZE=${MAX_API_USERS} - CHEF_URL=https://${APPORBIT_CHEFHOST}:9443 - AO_HOST=${APPORBIT_HOST} - CONSUL_IP=consul - CONSUL_PORT=8500 - CAPTAIN_TCP_ADDR=captain - CAPTAIN_TCP_PORT=8080 - GEMINI_INT_REPO=${APPORBIT_REPO} - MYSQL_HOST=db - MYSQL_PORT=3306 - MYSQL_USERNAME=root - MYSQL_PASSWORD=admin - MYSQL_DATABASE=apporbit_controller - AO_REGISTRY=${DATASVC_REGISTRY} links: - apporbit-db:db - apporbit-rmq:rmq - apporbit-svcd:svcd - apporbit-consul:consul - apporbit-captain:captain volumes_from: - apporbit-chef volumes: - ${APPORBIT_LOG}/controller:/var/log/apporbit:Z - ${APPORBIT_KEY}:/home/apporbit/apporbit-controller/sslkeystore:Z - ${APPORBIT_LIB}/controller/ui:/var/lib/apporbit/ui:Z ${CONTROLLER_DEVMOUNT} apporbit-docs: container_name: apporbit-docs image: ${APPORBIT_REGISTRY}apporbit/apporbit-docs:${APPORBIT_BUILDID} restart: always hostname: docs dns: - 8.8.8.8 - ${APPORBIT_DNS} dns_search: - ${APPORBIT_DNSSEARCH} network_mode: "bridge" ports: - "9080:80" depends_on: - apporbit-controller networks: default: external: name: bridge volumes: chef-conf: driver: local rabbitmq-data: driver: local ''') # Check if composeFile already exists then skip creating it. flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY try: compose_file = os.open(self.composeFile, flags) except OSError as e: if e.errno == errno.EEXIST: pass else: raise else: with os.fdopen(compose_file, 'w') as file_obj: # cat apporbit-compose-template.yml |awk '{ for(i=1; i<=NF; i++) # { if (match($i, /\$\{([a-z|A-Z|0-9|_]+)\}/)) {print substr($i,RSTART+2,RLENGTH-3)} } }' |sort |uniq if self.volume_mount: logging.warning("DEVELOPER MOUNT ENABLED") services_devmount = '- ' + self.volume_mount + '/Gemini-poc-stack:/home/apporbit/apporbit-services:z' if not os.path.isfile( self.volume_mount + "/Gemini-poc-stack/mist-cgp/run.jar"): pull_mist_binary = "wget -P " + self.volume_mount + "/Gemini-poc-stack/mist-cgp http://repos.gsintlab.com/repos/mist/integration/run.jar" utilityobj.cmdExecute(pull_mist_binary, 'pull mist binary ', True) controller_devmount = '- ' + self.volume_mount + '/Gemini-poc-mgnt:/home/apporbit/apporbit-controller:z' gemfile = self.volume_mount + "/Gemini-poc-mgnt/Gemfile" if not os.path.isfile(gemfile): rename_gemfile = "cp -f " + gemfile + "-master " + gemfile utilityobj.cmdExecute( rename_gemfile, 'copy Gemfile-master as Gemfile ', True) else: services_devmount = '' controller_devmount = '' # aoreg has / so local or full qualified images are handled in template aoreg = self.apporbit_registry if aoreg: aoreg += '/' datareg = self.apporbit_registry if self.datasvc_registry != '': datareg = self.datasvc_registry if self.apporbit_domain: domain = self.apporbit_domain else: domain = 'consul.' content = content.safe_substitute( APPORBIT_CHEFHOST=self.chef_host, APPORBIT_CONF=self.APPORBIT_CONF, APPORBIT_HOST=self.apporbit_host, APPORBIT_DOMAIN=domain, APPORBIT_DNS=self.consul_host, APPORBIT_DNSSEARCH=domain, APPORBIT_KEY=self.APPORBIT_KEY, APPORBIT_LIB=self.APPORBIT_DATA, APPORBIT_LOG=self.APPORBIT_LOG, APPORBIT_LOGINID=self.apporbit_loginid, APPORBIT_LOGLEVEL=self.log_level, APPORBIT_REGISTRY=aoreg, APPORBIT_REPO=self.apporbit_repo, APPORBIT_BUILDID=self.buildid, MAX_API_USERS=max_api_users, DATASVC_REGISTRY=datareg, SERVICES_DEVMOUNT=services_devmount, CONTROLLER_DEVMOUNT=controller_devmount, OFFLINE_MODE=self.offline_mode, ) file_obj.write(content) file_obj.close() logging.info("Create composeFile success!") return True
print(r_p_h1, w_p_h1, r_h1_p, w_h1_p, r_p_h2, w_p_h2, r_h2_p, w_h2_p) pid = os.fork() if pid > 0: # PADRE pid = os.fork() if pid > 0: # PADRE os.close(r_p_h1) os.close(w_h1_p) os.close(r_p_h2) os.close(w_h2_p) r = os.fdopen(r_h1_p) r2 = os.fdopen(r_h2_p) for i in range(10): escribir = str(random.randint(1, 20)) + "\n" os.write(w_p_h1, escribir.encode(encoding="utf-8")) linea = r.readline() print("PADRE: he recibido del hijo 1: " + linea) escribir = str(random.randint(1, 20)) + "\n" os.write(w_p_h2, escribir.encode(encoding="utf-8")) linea2 = r2.readline() print("PADRE: he recibido del hijo 2: " + linea2) os.close(w_p_h1) os.close(w_p_h2)
def write_file(self, buffering=DEFAULT_BUFSIZE): """Get a file-like object for the write end of the pipe.""" return os.fdopen(self.write_fd, 'wb', buffering)
def create_iface(self, ipv6=False, dhcpserver_opts=None): '''Create test interface with DHCP server behind it''' # run "router-side" networkd in own mount namespace to shield it from # "client-side" configuration and networkd (fd, script) = tempfile.mkstemp(prefix='networkd-router.sh') self.addCleanup(os.remove, script) with os.fdopen(fd, 'w+') as f: f.write( '''\ #!/bin/sh -eu mkdir -p /run/systemd/network mkdir -p /run/systemd/netif mount -t tmpfs none /run/systemd/network mount -t tmpfs none /run/systemd/netif [ ! -e /run/dbus ] || mount -t tmpfs none /run/dbus # create router/client veth pair cat << EOF > /run/systemd/network/test.netdev [NetDev] Name=%(ifr)s Kind=veth [Peer] Name=%(ifc)s EOF cat << EOF > /run/systemd/network/test.network [Match] Name=%(ifr)s [Network] Address=192.168.5.1/24 %(addr6)s DHCPServer=yes [DHCPServer] PoolOffset=10 PoolSize=50 DNS=192.168.5.1 %(dhopts)s EOF # run networkd as in systemd-networkd.service exec $(systemctl cat systemd-networkd.service | sed -n '/^ExecStart=/ { s/^.*=//; p}') ''' % { 'ifr': self.if_router, 'ifc': self.iface, 'addr6': ipv6 and 'Address=2600::1/64' or '', 'dhopts': dhcpserver_opts or '' }) os.fchmod(fd, 0o755) subprocess.check_call([ 'systemd-run', '--unit=networkd-test-router.service', '-p', 'InaccessibleDirectories=-/etc/systemd/network', '-p', 'InaccessibleDirectories=-/run/systemd/network', '-p', 'InaccessibleDirectories=-/run/systemd/netif', '--service-type=notify', script ]) # wait until devices got created for _ in range(50): out = subprocess.check_output( ['ip', 'a', 'show', 'dev', self.if_router]) if b'state UP' in out and b'scope global' in out: break time.sleep(0.1)
def dump_sigtask(self, fn, task, stampbase, runtime): tid = fn + ":" + task referencestamp = stampbase if isinstance(runtime, str) and runtime.startswith("customfile"): sigfile = stampbase referencestamp = runtime[11:] elif runtime and tid in self.taskhash: sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[tid] else: sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[tid] bb.utils.mkdirhier(os.path.dirname(sigfile)) data = {} data['task'] = task data['basewhitelist'] = self.basewhitelist data['taskwhitelist'] = self.taskwhitelist data['taskdeps'] = self.taskdeps[fn][task] data['basehash'] = self.basehash[tid] data['gendeps'] = {} data['varvals'] = {} data['varvals'][task] = self.lookupcache[fn][task] for dep in self.taskdeps[fn][task]: if dep in self.basewhitelist: continue data['gendeps'][dep] = self.gendeps[fn][dep] data['varvals'][dep] = self.lookupcache[fn][dep] if runtime and tid in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[tid] data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.get_unihash(dep) data['taskhash'] = self.taskhash[tid] taint = self.read_taint(fn, task, referencestamp) if taint: data['taint'] = taint if runtime and tid in self.taints: if 'nostamp:' in self.taints[tid]: data['taint'] = self.taints[tid] computed_basehash = calc_basehash(data) if computed_basehash != self.basehash[tid]: bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[tid], tid)) if runtime and tid in self.taskhash: computed_taskhash = calc_taskhash(data) if computed_taskhash != self.taskhash[tid]: bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid)) sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash) fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") try: with os.fdopen(fd, "wb") as stream: p = pickle.dump(data, stream, -1) stream.flush() os.chmod(tmpfile, 0o664) os.rename(tmpfile, sigfile) except (OSError, IOError) as err: try: os.unlink(tmpfile) except OSError: pass raise err