def download_live_logs(args): logfetch_base.log(colored('Finding current live log files', 'cyan') + '\n', args, False) tasks = tasks_to_check(args) async_requests = [] all_logs = [] callbacks.progress = 0 tasks_check_progress = 0 tasks_check_goal = len(tasks) for task in tasks: metadata = files_json(args, task) if 'slaveHostname' in metadata: uri = DOWNLOAD_FILE_FORMAT.format(metadata['slaveHostname']) for log_file in base_directory_files(args, task, metadata): logfile_name = '{0}-{1}'.format(task, log_file) if not args.logtype or (args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace('logs/', ''))): if should_download(args, logfile_name, task): async_requests.append( grequests.AsyncRequest('GET',uri , callback=callbacks.generate_callback(uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent), params={'path' : '{0}/{1}/{2}'.format(metadata['fullPathToRoot'], metadata['currentDirectory'], log_file)}, headers=args.headers ) ) all_logs.append('{0}/{1}'.format(args.dest, logfile_name)) elif args.logtype: logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True) for log_file in logs_folder_files(args, task): logfile_name = '{0}-{1}'.format(task, log_file) if not args.logtype or (args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace('logs/', ''))): if should_download(args, logfile_name, task): async_requests.append( grequests.AsyncRequest('GET',uri , callback=callbacks.generate_callback(uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent), params={'path' : '{0}/{1}/logs/{2}'.format(metadata['fullPathToRoot'], metadata['currentDirectory'], log_file)}, headers=args.headers ) ) all_logs.append('{0}/{1}'.format(args.dest, logfile_name)) elif args.logtype: logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True) tasks_check_progress += 1 logfetch_base.update_progress_bar(tasks_check_progress, tasks_check_goal, 'Log Finder', args.silent) if async_requests: logfetch_base.log(colored('\nStarting {0} live logs downloads\n'.format(len(async_requests)), 'cyan'), args, False) callbacks.goal = len(async_requests) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) return all_logs
def download_s3_logs(args): if not args.silent: sys.stderr.write(colored('Checking for S3 log files', 'cyan') + '\n') callbacks.progress = 0 logs = logs_for_all_requests(args) async_requests = [] all_logs = [] for log_file in logs: filename = log_file['key'].rsplit("/", 1)[1] if logfetch_base.is_in_date_range(args, int(str(log_file['lastModified'])[0:-3])): if not args.logtype or log_matches(args, filename): logfetch_base.log(colored('Including log {0}'.format(filename), 'blue') + '\n', args, True) if not already_downloaded(args.dest, filename): async_requests.append( grequests.AsyncRequest('GET', log_file['getUrl'], callback=callbacks.generate_callback(log_file['getUrl'], args.dest, filename, args.chunk_size, args.verbose, args.silent), headers=args.headers) ) else: logfetch_base.log(colored('Log already downloaded {0}'.format(filename), 'blue') + '\n', args, True) all_logs.append('{0}/{1}'.format(args.dest, filename)) else: logfetch_base.log(colored('Excluding {0} log does not match logtype argument {1}'.format(filename, args.logtype), 'magenta') + '\n', args, True) else: logfetch_base.log(colored('Excluding {0}, not in date range'.format(filename), 'magenta') + '\n', args, True) if async_requests: logfetch_base.log(colored('Starting {0} S3 Downloads with {1} parallel fetches\n'.format(len(async_requests), args.num_parallel_fetches), 'cyan'), args, False) callbacks.goal = len(async_requests) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) else: logfetch_base.log(colored('No S3 logs to download\n', 'cyan'), args, False) logfetch_base.log(colored('All S3 logs up to date\n', 'cyan'), args, False) all_logs = modify_download_list(all_logs) return all_logs
def download_s3_logs(args): sys.stderr.write(colored('Checking for S3 log files', 'cyan') + '\n') logs = logs_for_all_requests(args) async_requests = [] zipped_files = [] all_logs = [] for log_file in logs: filename = log_file['key'].rsplit("/", 1)[1] if logfetch_base.is_in_date_range( args, int(str(log_file['lastModified'])[0:-3])): if not args.logtype or log_matches(args, filename): if not already_downloaded(args.dest, filename): async_requests.append( grequests.AsyncRequest('GET', log_file['getUrl'], callback=generate_callback( log_file['getUrl'], args.dest, filename, args.chunk_size, args.verbose), headers=args.headers)) else: if args.verbose: sys.stderr.write( colored( 'Log already downloaded {0}'.format(filename), 'magenta') + '\n') all_logs.append('{0}/{1}'.format( args.dest, filename.replace('.gz', '.log'))) zipped_files.append('{0}/{1}'.format(args.dest, filename)) else: if args.verbose: sys.stderr.write( colored( 'Excluding {0} log does not match logtype argument {1}' .format(filename, args.logtype), 'magenta') + '\n') else: if args.verbose: sys.stderr.write( colored( 'Excluding {0}, not in date range'.format(filename), 'magenta') + '\n') if async_requests: sys.stderr.write( colored( 'Starting S3 Downloads with {0} parallel fetches'.format( args.num_parallel_fetches), 'cyan')) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) else: sys.stderr.write(colored('No S3 logs to download', 'cyan')) sys.stderr.write(colored('\nUnpacking S3 logs\n', 'cyan')) all_logs = all_logs + logfetch_base.unpack_logs(args, zipped_files) sys.stderr.write(colored('All S3 logs up to date', 'cyan') + '\n') return all_logs
def do(self, action, name, args, asynchronous=False): "Perform action" if args.pop('__meta', None): return (dict(api='/api/%s/%s' % (self.version, name), method=action.upper(), args=args)) self._ensureKey() url = '%s/api/%s/%s' % (self.host, self.version, name) self._debug('> %s %s' % (action.upper(), url)) kw = {} if action == 'get': body = kw['params'] = args else: body = kw['data'] = json.dumps(args, indent=1, cls=JSONEncoder) if self.timeout: kw['timeout'] = self.timeout headers = {'content-type': 'application/json'} headers.update(self._requestAuthHeaders) for k, v in list(self._requestAuthHeaders.items()): self._debug('>> %s: %s' % (k, v)) if body: self._debug("\n%s\n" % body) if self.xml: headers['accept'] = 'application/xml' kw['verify'] = self.verifySSL headers.update(self.headers) if asynchronous: import grequests return grequests.AsyncRequest(action, url, headers=headers, **kw) try: r = self.session.request(action, url, headers=headers, **kw) except requests.ConnectionError as e: raise BeaconAPIException( code=500, message="Could not connect to server (%s): %s" % (url, e)) self._debug('<< %s %s' % (r.status_code, r.reason)) if 'x-typehint' in r.headers: self._debug('< x-typehint: %s' % r.headers['x-typehint']) if r.status_code != 200: raise BeaconAPIException(code=r.status_code, message=r.reason, body=r.content) if r.headers['content-type'] == 'application/json': return r.json() return r.content
def main(parser, args): if args.taskId: singularity_path = "task/%s" % args.taskId elif args.deployId: if not args.requestId: exit(parser, "RequestId requires DeployId") singularity_path = "request/%s/deploy/%s" % (args.requestId, args.deployId) elif args.requestId: singularity_path = "request/%s" % args.requestId else: exit(parser, "Specify one of taskId, requestId and deployId, or requestId") if not args.singularity_uri_base: exit(parser, "Specify a base uri for Singularity") uri_prefix = "" if not args.singularity_uri_base.startswith(("http://", "https://")): uri_prefix = "http://" singularity_uri = "%s%s/logs/%s" % (uri_prefix, args.singularity_uri_base, singularity_path) print "fetching log metadata from %s" % singularity_uri singularity_response = requests.get(singularity_uri) if singularity_response.status_code < 199 or singularity_response.status_code > 299: exit( None, "Singularity responded with an invalid status code (%s)" % singularity_response.status_code) s_json = singularity_response.json() print "found %s log files" % len(s_json) async_requests = [ grequests.AsyncRequest('GET', log_file['getUrl'], callback=generate_callback( log_file['getUrl'], args.dest, log_file['key'][log_file['key'].rfind('/') + 1:], args.chunk_size)) for log_file in s_json ] grequests.map(async_requests, stream=True, size=args.num_parallel_fetches)
def download_s3_logs(args): sys.stderr.write(colored('Checking for S3 log files', 'blue') + '\n') logs = get_json_response(singularity_s3logs_uri(args)) async_requests = [] all_logs = [] for log_file in logs: filename = log_file['key'].rsplit("/", 1)[1] full_log_path = '{0}/{1}'.format(args.dest, filename.replace('.gz', '.log')) full_gz_path = '{0}/{1}'.format(args.dest, filename) if in_date_range(args, filename): if not (os.path.isfile(full_log_path) or os.path.isfile(full_gz_path)): async_requests.append( grequests.AsyncRequest('GET', log_file['getUrl'], callback=generate_callback(log_file['getUrl'], args.dest, filename, args.chunk_size) ) ) all_logs.append('{0}/{1}'.format(args.dest, filename.replace('.gz', '.log'))) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) zipped_files = ['{0}/{1}'.format(args.dest, log_file['key'].rsplit("/", 1)[1]) for log_file in logs] logfetch_base.unpack_logs(zipped_files) sys.stderr.write(colored('All S3 logs up to date', 'blue') + '\n') return all_logs
def download_live_logs(args): tasks = tasks_to_check(args) async_requests = [] zipped_files = [] all_logs = [] sys.stderr.write(colored('Finding current live log files', 'cyan') + '\n') for task in tasks: metadata = files_json(args, task) if 'slaveHostname' in metadata: uri = DOWNLOAD_FILE_FORMAT.format(metadata['slaveHostname']) if args.verbose: sys.stderr.write( colored( 'Finding logs in base directory on {0}'.format( metadata['slaveHostname']), 'magenta') + '\n') for log_file in base_directory_files(args, task, metadata): logfile_name = '{0}-{1}'.format(task, log_file) if not args.logtype or ( args.logtype and logfetch_base.log_matches( log_file, args.logtype.replace('logs/', ''))): if should_download(args, logfile_name, task): async_requests.append( grequests.AsyncRequest( 'GET', uri, callback=generate_callback( uri, args.dest, logfile_name, args.chunk_size, args.verbose), params={ 'path': '{0}/{1}/{2}'.format( metadata['fullPathToRoot'], metadata['currentDirectory'], log_file) }, headers=args.headers)) if logfile_name.endswith('.gz'): zipped_files.append('{0}/{1}'.format( args.dest, logfile_name)) else: all_logs.append('{0}/{1}'.format( args.dest, logfile_name.replace('.gz', '.log'))) elif args.logtype and args.verbose: sys.stderr.write( colored( 'Excluding log {0}, doesn\'t match {1}'.format( log_file, args.logtype), 'magenta') + '\n') if args.verbose: sys.stderr.write( colored( 'Finding logs in logs directory on {0}'.format( metadata['slaveHostname']), 'magenta') + '\n') for log_file in logs_folder_files(args, task): logfile_name = '{0}-{1}'.format(task, log_file) if not args.logtype or ( args.logtype and logfetch_base.log_matches( log_file, args.logtype.replace('logs/', ''))): if should_download(args, logfile_name, task): async_requests.append( grequests.AsyncRequest( 'GET', uri, callback=generate_callback( uri, args.dest, logfile_name, args.chunk_size, args.verbose), params={ 'path': '{0}/{1}/logs/{2}'.format( metadata['fullPathToRoot'], metadata['currentDirectory'], log_file) }, headers=args.headers)) if logfile_name.endswith('.gz'): zipped_files.append('{0}/{1}'.format( args.dest, logfile_name)) else: all_logs.append('{0}/{1}'.format( args.dest, logfile_name.replace('.gz', '.log'))) elif args.logtype and args.verbose: sys.stderr.write( colored( 'Excluding log {0}, doesn\'t match {1}'.format( log_file, args.logtype), 'magenta') + '\n') if async_requests: sys.stderr.write(colored('Starting live logs downloads\n', 'cyan')) grequests.map(async_requests, stream=True, size=args.num_parallel_fetches) if zipped_files: sys.stderr.write(colored('\nUnpacking logs\n', 'cyan')) all_logs = all_logs + logfetch_base.unpack_logs(args, zipped_files) return all_logs
# -*- coding: utf-8 -*- """ grequests 是 gevent + requests """ from urls import url_list import grequests from sfm.timer import DateTimeTimer as Timer with Timer(title="use grequests"): req_list = [ grequests.AsyncRequest(method="GET", url=url) for url in url_list ] res_list = grequests.map(req_list) results = [len(res.text) for res in res_list] print(results)