def logs_for_all_requests(args):
s3_params = {'start': int(time.mktime(args.start.timetuple()) * 1000), 'end': int(time.mktime(args.end.timetuple()) * 1000)}
if args.taskId:
return logfetch_base.get_json_response(s3_task_logs_uri(args, args.taskId), args, s3_params)
else:
tasks = logfetch_base.tasks_for_requests(args)
logs = []
tasks_progress = 0
tasks_goal = len(tasks)
for task in tasks:
s3_logs = logfetch_base.get_json_response(s3_task_logs_uri(args, task), args)
logs = logs + s3_logs if s3_logs else logs
tasks_progress += 1
logfetch_base.update_progress_bar(tasks_progress, tasks_goal, 'S3 Log Finder', args.silent or args.verbose)
found_logs = []
keys = []
for log in logs:
if not log['key'] in keys:
found_logs.append(log)
keys.append(log['key'])
logfetch_base.log(colored('\nAlso searching s3 history...\n', 'cyan'), args, False)
for request in logfetch_base.all_requests(args):
s3_logs = logfetch_base.get_json_response(s3_request_logs_uri(args, request), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
for log in logs:
if not log['key'] in keys:
found_logs.append(log)
keys.append(log['key'])
return found_logs
def logs_for_all_requests(args):
s3_params = {
'start': int(time.mktime(args.start.timetuple()) * 1000),
'end': int(time.mktime(args.end.timetuple()) * 1000)
}
if args.taskId:
return logfetch_base.get_json_response(
s3_task_logs_uri(args, args.taskId), args, s3_params)
else:
tasks = logfetch_base.tasks_for_requests(args)
logs = []
tasks_progress = 0
tasks_goal = len(tasks)
for task in tasks:
s3_logs = logfetch_base.get_json_response(
s3_task_logs_uri(args, task), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
tasks_progress += 1
logfetch_base.update_progress_bar(tasks_progress, tasks_goal,
'S3 Log Finder', args.silent)
logfetch_base.log(colored('\nAlso searching s3 history...\n', 'cyan'),
args, False)
for request in logfetch_base.all_requests(args):
s3_logs = logfetch_base.get_json_response(
s3_request_logs_uri(args, request), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
return [dict(t) for t in set(tuple(l.items())
for l in logs)] # remove any duplicates
def logs_for_all_requests(args):
s3_params = {'start': int(time.mktime(args.start.timetuple()) * 1000), 'end': int(time.mktime(args.end.timetuple()) * 1000)}
if args.taskId:
return logfetch_base.get_json_response(s3_task_logs_uri(args, args.taskId), args, s3_params)
else:
tasks = logfetch_base.tasks_for_requests(args)
logs = []
tasks_progress = 0
tasks_goal = len(tasks)
for task in tasks:
s3_logs = logfetch_base.get_json_response(s3_task_logs_uri(args, task), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
tasks_progress += 1
logfetch_base.update_progress_bar(tasks_progress, tasks_goal, 'S3 Log Finder', args.silent)
logfetch_base.log(colored('\nAlso searching s3 history...\n', 'cyan'), args, False)
for request in logfetch_base.all_requests(args):
s3_logs = logfetch_base.get_json_response(s3_request_logs_uri(args, request), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
found_logs = []
keys = []
for log in logs:
if not log['key'] in keys:
found_logs.append(log)
keys.append(log['key'])
return found_logs
def find_cached_logs(args):
matching_logs = []
log_fn_match = get_matcher(args)
for filename in os.listdir(args.dest):
if fnmatch.fnmatch(filename, log_fn_match) and in_date_range(args, filename):
log(colored('Including log {0}\n'.format(filename), 'blue'), args, True)
matching_logs.append('{0}/{1}'.format(args.dest, filename))
else:
log(colored('Excluding log {0}, not in date range\n'.format(filename), 'magenta'), args, True)
return matching_logs
def start_tail(args):
if args.requestId:
if not args.silent:
sys.stderr.write('Fetching tasks\n')
tasks = [str(t) for t in logfetch_base.tasks_for_requests(args)]
else:
tasks = [args.taskId]
log(colored('Tailing logs for tasks:\n', 'green'), args, True)
for t in tasks:
log(colored('{0}\n'.format(t), 'yellow'), args, True)
log(colored('ctrl+c to exit\n', 'cyan'), args, False)
try:
threads = []
for task in tasks:
thread = LogStreamer(args, task)
threads.append(thread)
thread.start()
for t in threads:
t.join(THREAD_TIMEOUT
) #Need a timeout otherwise can't be killed by ctrl+c
if not t.isAlive:
break
except KeyboardInterrupt:
log(colored('Stopping tail', 'magenta') + '\n', args, False)
sys.exit(0)
def should_download(args, filename, task):
if args.use_cache and already_downloaded(args, filename):
logfetch_base.log(
colored('Using cached version of file {0}\n'.format(filename),
'magenta'), args, True)
return False
if filename.endswith('.gz') and already_downloaded(args, filename):
logfetch_base.log(
colored(
'Using cached version of file {0}, zipped file has not changed\n'
.format(filename), 'magenta'), args, True)
return False
history = task_history(args, task)
if not task_still_running(args, task, history) and already_downloaded(
args, filename) and file_not_too_old(args, history, filename):
logfetch_base.log(
colored(
'Using cached version of file {0}, {1}, file has not changed\n'
.format(filename, history['taskUpdates'][-1]['taskState']),
'magenta'), args, True)
else:
logfetch_base.log(
colored(
'Will download file {0}, version on the server is newer than cached version\n'
.format(filename), 'magenta'), args, True)
return True
def grep_files(args, all_logs):
log('\n', args, False)
if args.grep:
if all_logs:
all_logs.sort()
grep_cmd = grep_command(args)
log(
colored('Running grep command ({0})\n'.format(grep_cmd),
'cyan'), args, False)
for filename in all_logs:
log('=> ' + colored(filename, 'cyan') + '\n', args, True)
content = subprocess.Popen(['cat', filename],
stdout=subprocess.PIPE)
if filename.endswith('.gz'):
zcat = subprocess.Popen('zcat',
stdin=content.stdout,
stdout=subprocess.PIPE)
grep = subprocess.Popen(grep_cmd,
stdin=zcat.stdout,
shell=True)
else:
grep = subprocess.Popen(grep_cmd,
stdin=content.stdout,
shell=True)
grep.communicate()
log(colored('Finished grep, exiting', 'green') + '\n', args, False)
else:
sys.stderr.write(colored('No logs found\n', 'magenta'))
def find_cached_logs(args):
matching_logs = []
log_fn_match = get_matcher(args)
for filename in os.listdir(args.dest):
if fnmatch.fnmatch(filename, log_fn_match) and in_date_range(
args, filename):
log(colored('Including log {0}\n'.format(filename), 'blue'), args,
True)
matching_logs.append('{0}/{1}'.format(args.dest, filename))
else:
log(
colored(
'Excluding log {0}, not in date range\n'.format(filename),
'magenta'), args, True)
return matching_logs
def logs_for_all_requests(args):
s3_params = {'start': int(time.mktime(args.start.timetuple()) * 1000), 'end': int(time.mktime(args.end.timetuple()) * 1000)}
if args.taskId:
return logfetch_base.get_json_response(s3_task_logs_uri(args, args.taskId), args, s3_params)
else:
tasks = logfetch_base.tasks_for_requests(args)
logs = []
tasks_progress = 0
tasks_goal = len(tasks)
for task in tasks:
s3_logs = logfetch_base.get_json_response(s3_task_logs_uri(args, task), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
tasks_progress += 1
logfetch_base.update_progress_bar(tasks_progress, tasks_goal, 'S3 Log Finder', args.silent)
logfetch_base.log(colored('\nAlso searching s3 history...\n', 'cyan'), args, False)
for request in logfetch_base.all_requests(args):
s3_logs = logfetch_base.get_json_response(s3_request_logs_uri(args, request), args, s3_params)
logs = logs + s3_logs if s3_logs else logs
return [dict(t) for t in set([tuple(l.items()) for l in logs])] # remove any duplicates
def should_download(args, filename, task):
if args.use_cache and already_downloaded(args, filename):
logfetch_base.log(colored('Using cached version of file {0}\n'.format(filename), 'magenta'), args, True)
return False
if filename.endswith('.gz') and already_downloaded(args, filename):
logfetch_base.log(colored('Using cached version of file {0}, zipped file has not changed\n'.format(filename), 'magenta'), args, True)
return False
history = task_history(args, task)
if not task_still_running(args, task, history) and already_downloaded(args, filename) and file_not_too_old(args, history, filename):
logfetch_base.log(colored('Using cached version of file {0}, {1}, file has not changed\n'.format(filename, history['taskUpdates'][-1]['taskState']), 'magenta'), args, True)
else:
logfetch_base.log(colored('Will download file {0}, version on the server is newer than cached version\n'.format(filename), 'magenta'), args, True)
return True
def download_live_logs(args):
logfetch_base.log(colored('Finding current live log files', 'cyan') + '\n', args, False)
tasks = tasks_to_check(args)
async_requests = []
all_logs = []
callbacks.progress = 0
tasks_check_progress = 0
tasks_check_goal = len(tasks)
for task in tasks:
metadata = files_json(args, task)
if 'slaveHostname' in metadata:
uri = DOWNLOAD_FILE_FORMAT.format(metadata['slaveHostname'])
for log_file in base_directory_files(args, task, metadata):
logfile_name = '{0}-{1}'.format(task, log_file)
if not args.logtype or (args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace('logs/', ''))):
if should_download(args, logfile_name, task):
async_requests.append(
grequests.AsyncRequest('GET',uri ,
callback=callbacks.generate_callback(uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent),
params={'path' : '{0}/{1}/{2}'.format(metadata['fullPathToRoot'], metadata['currentDirectory'], log_file)},
headers=args.headers
)
)
all_logs.append('{0}/{1}'.format(args.dest, logfile_name))
elif args.logtype:
logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True)
for log_file in logs_folder_files(args, task):
logfile_name = '{0}-{1}'.format(task, log_file)
if not args.logtype or (args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace('logs/', ''))):
if should_download(args, logfile_name, task):
async_requests.append(
grequests.AsyncRequest('GET',uri ,
callback=callbacks.generate_callback(uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent),
params={'path' : '{0}/{1}/logs/{2}'.format(metadata['fullPathToRoot'], metadata['currentDirectory'], log_file)},
headers=args.headers
)
)
all_logs.append('{0}/{1}'.format(args.dest, logfile_name))
elif args.logtype:
logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True)
tasks_check_progress += 1
logfetch_base.update_progress_bar(tasks_check_progress, tasks_check_goal, 'Log Finder', args.silent)
if async_requests:
logfetch_base.log(colored('\nStarting {0} live logs downloads\n'.format(len(async_requests)), 'cyan'), args, False)
callbacks.goal = len(async_requests)
grequests.map(async_requests, stream=True, size=args.num_parallel_fetches)
return all_logs
def download_live_logs(args):
logfetch_base.log(colored('Finding current live log files', 'cyan') + '\n', args, False)
tasks = tasks_to_check(args)
async_requests = []
all_logs = []
callbacks.progress = 0
tasks_check_progress = 0
tasks_check_goal = len(tasks)
for task in tasks:
metadata = files_json(args, task)
if 'slaveHostname' in metadata:
uri = DOWNLOAD_FILE_FORMAT.format(metadata['slaveHostname'])
for log_file in base_directory_files(args, task, metadata):
logfile_name = '{0}-{1}'.format(task, log_file)
if not args.logtype or (args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace('logs/', ''))):
if should_download(args, logfile_name, task):
async_requests.append(
grequests.AsyncRequest('GET',uri ,
callback=callbacks.generate_callback(uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent),
params={'path' : '{0}/{1}/{2}'.format(metadata['fullPathToRoot'], metadata['currentDirectory'], log_file)},
headers=args.headers
)
)
all_logs.append('{0}/{1}'.format(args.dest, logfile_name))
elif args.logtype:
logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True)
for log_file in logs_folder_files(args, task):
logfile_name = '{0}-{1}'.format(task, log_file)
if not args.logtype or (args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace('logs/', ''))):
if should_download(args, logfile_name, task):
async_requests.append(
grequests.AsyncRequest('GET',uri ,
callback=callbacks.generate_callback(uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent),
params={'path' : '{0}/{1}/logs/{2}'.format(metadata['fullPathToRoot'], metadata['currentDirectory'], log_file)},
headers=args.headers
)
)
all_logs.append('{0}/{1}'.format(args.dest, logfile_name))
elif args.logtype:
logfetch_base.log(colored('Excluding log {0}, doesn\'t match {1}'.format(log_file, args.logtype), 'magenta') + '\n', args, True)
tasks_check_progress += 1
logfetch_base.update_progress_bar(tasks_check_progress, tasks_check_goal, 'Log Finder', args.silent or args.verbose)
if async_requests:
logfetch_base.log(colored('\nStarting {0} live logs downloads\n'.format(len(async_requests)), 'cyan'), args, False)
callbacks.goal = len(async_requests)
grequests.map(async_requests, stream=True, size=args.num_parallel_fetches)
return all_logs
def grep_files(args, all_logs):
log('\n', args, False)
if args.grep:
if all_logs:
all_logs.sort()
grep_cmd = grep_command(args)
log(colored('Running grep command ({0})\n'.format(grep_cmd), 'cyan'), args, False)
for filename in all_logs:
log(colored(get_timestamp_string(filename) + ' => ' + filename, 'cyan') + '\n', args, not args.show_file_info)
content = subprocess.Popen(['cat', filename], stdout=subprocess.PIPE)
if filename.endswith('.gz'):
zcat = subprocess.Popen('zcat', stdin=content.stdout, stdout=subprocess.PIPE)
grep = subprocess.Popen(grep_cmd, stdin=zcat.stdout, shell=True)
else:
grep = subprocess.Popen(grep_cmd, stdin=content.stdout, shell=True)
grep.communicate()
log(colored('Finished grep, exiting', 'green') + '\n', args, False)
else:
sys.stderr.write(colored('No logs found\n', 'magenta'))
def should_download(args, filename, task):
if args.use_cache and already_downloaded(args, filename):
logfetch_base.log(colored("Using cached version of file {0}\n".format(filename), "magenta"), args, True)
return False
if filename.endswith(".gz") and already_downloaded(args, filename):
logfetch_base.log(
colored("Using cached version of file {0}, zipped file has not changed\n".format(filename), "magenta"),
args,
True,
)
return False
history = task_history(args, task)
if (
not task_still_running(args, task, history)
and already_downloaded(args, filename)
and file_not_too_old(args, history, filename)
):
logfetch_base.log(
colored(
"Using cached version of file {0}, {1}, file has not changed\n".format(
filename, history["taskUpdates"][-1]["taskState"]
),
"magenta",
),
args,
True,
)
else:
logfetch_base.log(
colored(
"Will download file {0}, version on the server is newer than cached version\n".format(filename),
"magenta",
),
args,
True,
)
return True
def cat_files(args, all_logs):
log('\n', args, False)
if all_logs:
all_logs.sort()
for filename in all_logs:
log(colored(get_timestamp_string(filename) + ' => ' + filename, 'cyan') + '\n', args, not args.show_file_info)
if filename.endswith('.gz'):
cat = subprocess.Popen(['cat', filename], stdout=subprocess.PIPE)
content = subprocess.Popen(['zcat'], stdin=cat.stdout)
content.communicate()
else:
cat = subprocess.Popen(['cat', filename])
cat.communicate()
sys.stdout.write('\n')
else:
log(colored('No log files found\n', 'magenta'), args, False)
def cat_files(args, all_logs):
log('\n', args, False)
if all_logs:
all_logs.sort()
for filename in all_logs:
log('=> ' + colored(filename, 'cyan') + '\n', args, False)
if filename.endswith('.gz'):
cat = subprocess.Popen(['cat', filename],
stdout=subprocess.PIPE)
content = subprocess.Popen(['zcat'], stdin=cat.stdout)
content.communicate()
else:
cat = subprocess.Popen(['cat', filename])
cat.communicate()
sys.stdout.write('\n')
else:
log(colored('No log files found\n', 'magenta'), args, False)
def start_tail(args):
if args.requestId:
if not args.silent:
sys.stderr.write('Fetching tasks\n')
tasks = [str(t) for t in logfetch_base.tasks_for_requests(args)]
else:
tasks = [args.taskId]
log(colored('Tailing logs for tasks:\n', 'green'), args, True)
for t in tasks:
log(colored('{0}\n'.format(t), 'yellow'), args, True)
log(colored('ctrl+c to exit\n', 'cyan'), args, False)
try:
threads = []
for task in tasks:
thread = LogStreamer(args, task)
threads.append(thread)
thread.start()
for t in threads:
t.join(THREAD_TIMEOUT) #Need a timeout otherwise can't be killed by ctrl+c
if not t.isAlive:
break
except KeyboardInterrupt:
log(colored('Stopping tail', 'magenta') + '\n', args, False)
sys.exit(0)
def download_s3_logs(args):
if not args.silent:
sys.stderr.write(colored('Checking for S3 log files', 'cyan') + '\n')
callbacks.progress = 0
logs = logs_for_all_requests(args)
async_requests = []
all_logs = []
for log_file in logs:
filename = log_file['key'].rsplit("/", 1)[1]
if logfetch_base.is_in_date_range(
args, int(str(log_file['lastModified'])[0:-3])):
if not args.logtype or log_matches(args, filename):
logfetch_base.log(
colored('Including log {0}'.format(filename), 'blue') +
'\n', args, True)
if not already_downloaded(args.dest, filename):
async_requests.append(
grequests.AsyncRequest(
'GET',
log_file['getUrl'],
callback=callbacks.generate_callback(
log_file['getUrl'], args.dest, filename,
args.chunk_size, args.verbose, args.silent),
headers=args.headers))
else:
logfetch_base.log(
colored('Log already downloaded {0}'.format(filename),
'blue') + '\n', args, True)
all_logs.append('{0}/{1}'.format(args.dest, filename))
else:
logfetch_base.log(
colored(
'Excluding {0} log does not match logtype argument {1}'
.format(filename, args.logtype), 'magenta') + '\n',
args, True)
else:
logfetch_base.log(
colored('Excluding {0}, not in date range'.format(filename),
'magenta') + '\n', args, True)
if async_requests:
logfetch_base.log(
colored(
'Starting {0} S3 Downloads with {1} parallel fetches\n'.format(
len(async_requests), args.num_parallel_fetches), 'cyan'),
args, False)
callbacks.goal = len(async_requests)
grequests.map(async_requests,
stream=True,
size=args.num_parallel_fetches)
else:
logfetch_base.log(colored('No S3 logs to download\n', 'cyan'), args,
False)
logfetch_base.log(colored('All S3 logs up to date\n', 'cyan'), args, False)
return all_logs
def download_s3_logs(args):
if not args.silent:
sys.stderr.write(colored("Checking for S3 log files", "cyan") + "\n")
callbacks.progress = 0
logs = logs_for_all_requests(args)
async_requests = []
all_logs = []
for log_file in logs:
filename = log_file["key"].rsplit("/", 1)[1]
if logfetch_base.is_in_date_range(args, int(str(log_file["lastModified"])[0:-3])):
if not args.logtype or log_matches(args, filename):
logfetch_base.log(colored("Including log {0}".format(filename), "blue") + "\n", args, True)
if not already_downloaded(args.dest, filename):
async_requests.append(
grequests.AsyncRequest(
"GET",
log_file["getUrl"],
callback=callbacks.generate_callback(
log_file["getUrl"], args.dest, filename, args.chunk_size, args.verbose, args.silent
),
headers=args.headers,
)
)
else:
logfetch_base.log(colored("Log already downloaded {0}".format(filename), "blue") + "\n", args, True)
all_logs.append("{0}/{1}".format(args.dest, filename))
else:
logfetch_base.log(
colored(
"Excluding {0} log does not match logtype argument {1}".format(filename, args.logtype),
"magenta",
)
+ "\n",
args,
True,
)
else:
logfetch_base.log(
colored("Excluding {0}, not in date range".format(filename), "magenta") + "\n", args, True
)
if async_requests:
logfetch_base.log(
colored(
"Starting {0} S3 Downloads with {1} parallel fetches\n".format(
len(async_requests), args.num_parallel_fetches
),
"cyan",
),
args,
False,
)
callbacks.goal = len(async_requests)
grequests.map(async_requests, stream=True, size=args.num_parallel_fetches)
else:
logfetch_base.log(colored("No S3 logs to download\n", "cyan"), args, False)
logfetch_base.log(colored("All S3 logs up to date\n", "cyan"), args, False)
return all_logs
def download_s3_logs(args):
if not args.silent:
sys.stderr.write(colored('Checking for S3 log files', 'cyan') + '\n')
callbacks.progress = 0
logs = logs_for_all_requests(args)
async_requests = []
all_logs = []
for log_file in logs:
filename = log_file['key'].rsplit("/", 1)[1]
if log_file_in_date_range(args, log_file):
if not args.logtype or log_matches(args, filename):
logfetch_base.log(colored('Including log {0}'.format(filename), 'blue') + '\n', args, True)
if not already_downloaded(args.dest, filename):
async_requests.append(
grequests.AsyncRequest('GET', log_file['getUrl'], callback=callbacks.generate_callback(log_file['getUrl'], args.dest, filename, args.chunk_size, args.verbose, args.silent), headers=args.headers)
)
else:
logfetch_base.log(colored('Log already downloaded {0}'.format(filename), 'blue') + '\n', args, True)
all_logs.append('{0}/{1}'.format(args.dest, filename))
else:
logfetch_base.log(colored('Excluding {0} log does not match logtype argument {1}'.format(filename, args.logtype), 'magenta') + '\n', args, True)
else:
logfetch_base.log(colored('Excluding {0}, not in date range'.format(filename), 'magenta') + '\n', args, True)
if async_requests:
logfetch_base.log(colored('Starting {0} S3 Downloads with {1} parallel fetches\n'.format(len(async_requests), args.num_parallel_fetches), 'cyan'), args, False)
callbacks.goal = len(async_requests)
grequests.map(async_requests, stream=True, size=args.num_parallel_fetches)
else:
logfetch_base.log(colored('No S3 logs to download\n', 'cyan'), args, False)
logfetch_base.log(colored('All S3 logs up to date\n', 'cyan'), args, False)
all_logs = modify_download_list(all_logs)
return all_logs
def download_live_logs(args):
logfetch_base.log(colored("Finding current live log files", "cyan") + "\n", args, False)
tasks = tasks_to_check(args)
async_requests = []
all_logs = []
callbacks.progress = 0
tasks_check_progress = 0
tasks_check_goal = len(tasks)
for task in tasks:
metadata = files_json(args, task)
if "slaveHostname" in metadata:
uri = DOWNLOAD_FILE_FORMAT.format(metadata["slaveHostname"])
for log_file in base_directory_files(args, task, metadata):
logfile_name = "{0}-{1}".format(task, log_file)
if not args.logtype or (
args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace("logs/", ""))
):
if should_download(args, logfile_name, task):
async_requests.append(
grequests.AsyncRequest(
"GET",
uri,
callback=callbacks.generate_callback(
uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent
),
params={
"path": "{0}/{1}/{2}".format(
metadata["fullPathToRoot"], metadata["currentDirectory"], log_file
)
},
headers=args.headers,
)
)
all_logs.append("{0}/{1}".format(args.dest, logfile_name))
elif args.logtype:
logfetch_base.log(
colored("Excluding log {0}, doesn't match {1}".format(log_file, args.logtype), "magenta")
+ "\n",
args,
True,
)
for log_file in logs_folder_files(args, task):
logfile_name = "{0}-{1}".format(task, log_file)
if not args.logtype or (
args.logtype and logfetch_base.log_matches(log_file, args.logtype.replace("logs/", ""))
):
if should_download(args, logfile_name, task):
async_requests.append(
grequests.AsyncRequest(
"GET",
uri,
callback=callbacks.generate_callback(
uri, args.dest, logfile_name, args.chunk_size, args.verbose, args.silent
),
params={
"path": "{0}/{1}/logs/{2}".format(
metadata["fullPathToRoot"], metadata["currentDirectory"], log_file
)
},
headers=args.headers,
)
)
all_logs.append("{0}/{1}".format(args.dest, logfile_name))
elif args.logtype:
logfetch_base.log(
colored("Excluding log {0}, doesn't match {1}".format(log_file, args.logtype), "magenta")
+ "\n",
args,
True,
)
tasks_check_progress += 1
logfetch_base.update_progress_bar(tasks_check_progress, tasks_check_goal, "Log Finder", args.silent)
if async_requests:
logfetch_base.log(
colored("\nStarting {0} live logs downloads\n".format(len(async_requests)), "cyan"), args, False
)
callbacks.goal = len(async_requests)
grequests.map(async_requests, stream=True, size=args.num_parallel_fetches)
return all_logs