def _asyncsavemetadata(root, nodes): '''starts a separate process that fills metadata for the nodes This function creates a separate process and doesn't wait for it's completion. This was done to avoid slowing down pushes ''' maxnodes = 50 if len(nodes) > maxnodes: return nodesargs = [] for node in nodes: nodesargs.append('--node') nodesargs.append(node) with open(os.devnull, 'w+b') as devnull: cmdline = [ util.hgexecutable(), 'debugfillinfinitepushmetadata', '-R', root ] + nodesargs # Process will run in background. We don't care about the return code subprocess.Popen(pycompat.rapply(procutil.tonativestr, cmdline), close_fds=True, shell=False, stdin=devnull, stdout=devnull, stderr=devnull)
def callconduit(ui, name, params): """call Conduit API, params is a dict. return json.loads result, or None""" host, token = readurltoken(ui) url, authinfo = util.url(b'/'.join([host, b'api', name])).authinfo() ui.debug(b'Conduit Call: %s %s\n' % (url, pycompat.byterepr(params))) params = params.copy() params[b'api.token'] = token data = urlencodenested(params) curlcmd = ui.config(b'phabricator', b'curlcmd') if curlcmd: sin, sout = procutil.popen2(b'%s -d @- %s' % (curlcmd, procutil.shellquote(url))) sin.write(data) sin.close() body = sout.read() else: urlopener = urlmod.opener(ui, authinfo) request = util.urlreq.request(pycompat.strurl(url), data=data) with contextlib.closing(urlopener.open(request)) as rsp: body = rsp.read() ui.debug(b'Conduit Response: %s\n' % body) parsed = pycompat.rapply( lambda x: encoding.unitolocal(x) if isinstance(x, pycompat.unicode) else x, json.loads(body)) if parsed.get(b'error_code'): msg = (_(b'Conduit Error (%s): %s') % (parsed[b'error_code'], parsed[b'error_info'])) raise error.Abort(msg) return parsed[b'result']
def _call_binary(self, args): p = subprocess.Popen( pycompat.rapply(procutil.tonativestr, args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout, stderr = p.communicate() returncode = p.returncode return returncode, stdout, stderr
def _execute(self, cmd, *args, **kwargs): cmdline = [self.execmd, cmd] cmdline += args cmdline = [procutil.shellquote(arg) for arg in cmdline] cmdline += ['>', os.devnull, '2>', os.devnull] cmdline = procutil.quotecommand(' '.join(cmdline)) self.ui.debug(cmdline, '\n') return os.system(pycompat.rapply(procutil.tonativestr, cmdline))
def debugcallconduit(ui, repo, name): """call Conduit API Call parameters are read from stdin as a JSON blob. Result will be written to stdout as a JSON blob. """ # json.loads only accepts bytes from 3.6+ rawparams = encoding.unifromlocal(ui.fin.read()) # json.loads only returns unicode strings params = pycompat.rapply( lambda x: encoding.unitolocal(x) if isinstance(x, pycompat.unicode) else x, json.loads(rawparams)) # json.dumps only accepts unicode strings result = pycompat.rapply( lambda x: encoding.unifromlocal(x) if isinstance(x, bytes) else x, callconduit(ui, name, params)) s = json.dumps(result, sort_keys=True, indent=2, separators=(u',', u': ')) ui.write(b'%s\n' % encoding.unitolocal(s))
def _execute(self, cmd, *args, **kwargs): cmdline = [self.execmd, cmd] cmdline += args cmdline = [procutil.shellquote(arg) for arg in cmdline] bdevnull = pycompat.bytestr(os.devnull) cmdline += [b'>', bdevnull, b'2>', bdevnull] cmdline = b' '.join(cmdline) self.ui.debug(cmdline, b'\n') return os.system(pycompat.rapply(procutil.tonativestr, cmdline))
def _systembackground(cmd, environ=None, cwd=None): ''' like 'procutil.system', but returns the Popen object directly so we don't have to wait on it. ''' cmd = procutil.quotecommand(cmd) env = procutil.shellenviron(environ) proc = subprocess.Popen(procutil.tonativestr(cmd), shell=True, close_fds=procutil.closefds, env=procutil.tonativeenv(env), cwd=pycompat.rapply(procutil.tonativestr, cwd)) return proc
def _connect(self): if self.proc: return self.proc args = [ 'watchman', '--sockname={0}'.format(self.sockpath), '--logfile=/BOGUS', '--statefile=/BOGUS', '--no-spawn', '--no-local', '--no-pretty', '-j', ] self.proc = subprocess.Popen(pycompat.rapply(procutil.tonativestr, args), stdin=subprocess.PIPE, stdout=subprocess.PIPE) return self.proc
def _resolvesockname(self): # if invoked via a trigger, watchman will set this env var; we # should use it unless explicitly set otherwise path = os.getenv('WATCHMAN_SOCK') if path: return path cmd = ['watchman', '--output-encoding=bser', 'get-sockname'] try: args = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=os.name != 'nt') if os.name == 'nt': # if invoked via an application with graphical user interface, # this call will cause a brief command window pop-up. # Using the flag STARTF_USESHOWWINDOW to avoid this behavior. startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW args['startupinfo'] = startupinfo p = subprocess.Popen(pycompat.rapply(procutil.tonativestr, cmd), **args) except OSError as e: raise WatchmanError('"watchman" executable not in PATH (%s)' % e) stdout, stderr = p.communicate() exitcode = p.poll() if exitcode: raise WatchmanError("watchman exited with code %d" % exitcode) result = bser.loads(stdout) if b'error' in result: raise WatchmanError('get-sockname error: %s' % result['error']) return result[b'sockname']
if name is None: name = '' val = 100 + len(name) unficalllog.append(val) return val #plug them on repo localrepo.localrepository.testcachedfoobar = testcachedfoobar localrepo.localrepository.testcachedunfifoobar = testcachedunfifoobar # Create an empty repo and instantiate it. It is important to run # these tests on the real object to detect regression. repopath = pycompat.fsencode(os.path.join(os.environ['TESTTMP'], 'repo')) assert subprocess.call( pycompat.rapply(procutil.tonativestr, [b'hg', b'init', repopath])) == 0 ui = uimod.ui.load() repo = hg.repository(ui, path=repopath).unfiltered() print('') print('=== property cache ===') print('') print('calllog:', calllog) print('cached value (unfiltered):', vars(repo).get('testcachedfoobar', 'NOCACHE')) print('') print('= first access on unfiltered, should do a call') print('access:', repo.testcachedfoobar) print('calllog:', calllog)
def _batchrequest(self, pointers, action): """Get metadata about objects pointed by pointers for given action Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md """ objects = [ {r'oid': pycompat.strurl(p.oid()), r'size': p.size()} for p in pointers ] requestdata = pycompat.bytesurl( json.dumps( {r'objects': objects, r'operation': pycompat.strurl(action),} ) ) url = b'%s/objects/batch' % self.baseurl batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) batchreq.add_header(r'Accept', r'application/vnd.git-lfs+json') batchreq.add_header(r'Content-Type', r'application/vnd.git-lfs+json') try: with contextlib.closing(self.urlopener.open(batchreq)) as rsp: rawjson = rsp.read() except util.urlerr.httperror as ex: hints = { 400: _( b'check that lfs serving is enabled on %s and "%s" is ' b'supported' ) % (self.baseurl, action), 404: _(b'the "lfs.url" config may be used to override %s') % self.baseurl, } hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) raise LfsRemoteError( _(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), hint=hint, ) except util.urlerr.urlerror as ex: hint = ( _(b'the "lfs.url" config may be used to override %s') % self.baseurl ) raise LfsRemoteError( _(b'LFS error: %s') % _urlerrorreason(ex), hint=hint ) try: response = pycompat.json_loads(rawjson) except ValueError: raise LfsRemoteError( _(b'LFS server returns invalid JSON: %s') % rawjson.encode("utf-8") ) if self.ui.debugflag: self.ui.debug(b'Status: %d\n' % rsp.status) # lfs-test-server and hg serve return headers in different order headers = pycompat.bytestr(rsp.info()).strip() self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) if r'objects' in response: response[r'objects'] = sorted( response[r'objects'], key=lambda p: p[r'oid'] ) self.ui.debug( b'%s\n' % pycompat.bytesurl( json.dumps( response, indent=2, separators=(r'', r': '), sort_keys=True, ) ) ) def encodestr(x): if isinstance(x, pycompat.unicode): return x.encode('utf-8') return x return pycompat.rapply(encodestr, response)
action='store_true', help='do not start the HTTP server in the background') parser.add_option('--daemon-postexec', action='append') (options, args) = parser.parse_args() signal.signal(signal.SIGTERM, lambda x, y: sys.exit(0)) if options.foreground and options.logfile: parser.error("options --logfile and --foreground are mutually " "exclusive") if options.foreground and options.pid: parser.error("options --pid and --foreground are mutually exclusive") opts = { b'pid_file': options.pid, b'daemon': not options.foreground, b'daemon_postexec': pycompat.rapply(encoding.strtolocal, options.daemon_postexec) } service = simplehttpservice(options.host, options.port) runargs = [sys.executable, __file__] + sys.argv[1:] runargs = [pycompat.fsencode(a) for a in runargs] server.runservice(opts, initfn=service.init, runfn=service.run, logfile=options.logfile, runargs=runargs)