def run(self):
tasks = []
get_url_list_task = Task(target=self.get_url_list)
tasks.append(get_url_list_task)
for i in range(0, 5):
get_html_task = Task(target=self.get_html)
tasks.append(get_html_task)
for i in range(0, 3):
get_items_task = Task(target=self.get_items)
tasks.append(get_items_task)
save_item_task = Task(target=self.save_item)
tasks.append(save_item_task)
for task in tasks:
# task.setDaemon(True)
task.daemon = True
task.start()
# 保证子线程运行优先于主线程
time.sleep(1)
self.url_queue.join()
self.html_queue.join()
self.item_queue.join()
pass
def run(self, steps, ongeneration=None):
#TODO: add a sync=True parameter to run a task in local thread or all task in paralell
# create cores
if self.engine.lower() == "process":
from multiprocessing import Process as Task
elif self.engine.lower() == "thread":
from threading import Thread as Task
else:
raise ValueError("Unknown engine {}".format(self.engine))
for i in range(self.cores):
# tasks.append(Process(target=self._loop, args=(i, steps, ongeneration, results)))
self.running_cores.append(Task(target=self._loop, args=(i, steps, ongeneration, self.result_queue)))
# launch
for task in self.running_cores:
task.start()
# create local process
# tasks.append(Process(target=self._loop, args=(0, steps, ongeneration, results)))
# self._loop(0, steps, ongeneration, results)
# join launched processes
# for task in tasks:
# task.join()
# return [results.get() for _ in range(results.qsize())]
return self
def main(totals):
"""
It prints n progress bars given a dictionary with n elements with a bar id
as a key and an integer between 1 and 100 as a value.
:param totals: dict of progress bar ids and integer values
:type totals: dict
"""
for element in totals.values():
if element > 100 or element < 1:
sys.stdout.write('One of the elements is too big or smalll.\n')
sys.exit()
status = Queue()
progress = collections.OrderedDict()
workers = []
for bar in BARS:
child = Task(target=bars, args=(status, bar))
child.start()
workers.append(child)
progress[bar] = 0.0
while any(i.is_alive() for i in workers):
time.sleep(0.1)
while not status.empty():
bar, percent = status.get()
progress[bar] = percent
print_progress(progress, totals)
def run(anjia_obj):
all_ts_url = anjia_obj.get_ts_url()
all_ts_url_group_list = split_group_by(all_ts_url, split_num=5)
tasks = []
for ts_url in all_ts_url_group_list:
task = Task(target=anjia_obj.download_all_ts,
args=(ts_url, ),
daemon=True)
tasks.append(task)
task.start()
for task in tasks:
task.join()
anjia_obj.merge_file()
def startDownLoadTask(singleDownLoader, url, destination, status, workers,
progress):
""" Start a child process to download from given url
Parameters:
singleDownLoader -- download method to download given url
url -- url string
destination -- path of the directory to save the file
status -- status Queue of workers
workers -- array of workers
progress -- progress object containing download percent of each item
"""
child = Task(target=singleDownLoader, args=(url, destination, status))
child.start()
workers.append(child)
progress[url] = 0.0
def main(argv):
global rank2domain, input_type, access_control, find_sub_pages_script, get_max_size_script, fetch_doc_log_script, fire_events_script, num_instances, parent_pid, process_list, log_f, log_dir, script_dir, fetch_asg_logs_script, chrome_newtab_url, about_blank_url, chrome_extension_prefix, chrome_extension_suffix, rank
signal.signal(signal.SIGTERM, signal_term_handler)
start_time = time.time()
try:
opts, args = getopt.getopt(argv, 'hu:d:i:n:p:t:s:e:', [
'help', 'user_dir=', 'log_dir=', 'num=', 'process=', 'type=',
'input_file=', 'start=', 'end='
])
except getopt.GetoptError:
usage()
sys.exit(2)
user_dir = None
access_control = True
input_type = 'domain'
input_file = 'top-1m.csv'
num_instances = 512
maximum_process_num = 8 # Change to 1 for debugging purposeF
start = 0
end = None
exp_dir = "exps"
log_dir = None
for opt, arg in opts:
if opt in ('-u', '--user_dir'):
user_dir = arg
elif opt in ('-d', '--dir'):
log_dir = arg
elif opt in ('-i', '--input_file'):
input_file = arg
elif opt in ('-n', '--num'):
num_instances = int(arg)
elif opt in ('-p', '--process'):
maximum_process_num = int(arg)
elif opt in ('-t', '--type'):
input_type = arg
elif opt in ('-s', '--start'):
start = int(arg)
elif opt in ('-e', '--end'):
end = int(arg)
elif opt in ('-h', '--help'):
usage()
sys.exit(0)
if user_dir is None or log_dir is None:
usage()
sys.exit(0)
try:
os.mkdir(exp_dir)
except OSError as e:
pass
try:
os.mkdir(log_dir)
except OSError as e:
pass
parent_pid = os.getpid()
script_dir = os.getcwd()
restart_parent_flag = False
log_dir = os.path.join(script_dir, log_dir)
log_file = 'exp_%s.log' % (user_dir)
log_file = os.path.join(log_dir, log_file)
log_f = codecs.open(log_file, encoding='utf-8', mode='a')
current_time = getlocaltime()
status = "PARENT SCRIPT STARTED! PARENT PID=[%d]" % parent_pid
string = '%s\t%s\n' % (current_time, status)
print(string)
log_f.write(string)
string = "%s\tProcess started, argv=%s\n" % (current_time, argv)
log_f.write(string)
completed_list = set()
completion_reg = re.compile(
'Process [0-9\s]+task ([0-9]+)/[0-9]+ PID \[\d+\] completed.')
with codecs.open(log_file, encoding='utf-8', mode='r') as input_f:
for line in input_f:
match = re.search(completion_reg, line)
if match:
task = int(match.group(1)) - 1
completed_list.add(task)
completed_list = set()
input_f = open(input_file, 'r')
lines = input_f.read().split('\n')[:-1]
input_f.close()
rank2domain = dict()
for line in lines:
data = line.split(',')
rank = data[0]
rank = int(rank)
url = ','.join(data[1:])
rank2domain[rank] = url
if end is None:
end = len(rank2domain)
try:
os.chdir(exp_dir)
except OSError as e:
pass
# Remove temp files
gc_chrome_tmp_files(force=True)
process_name = 'chrome'
kill_processes_by_name(process_name)
tasks = [i for i in range(num_instances - 1, -1, -1)]
status_queue = Queue()
try:
length = len(tasks)
head = 'Preparing [%d] task ...' % (length)
final_status_set = set()
progress = dict()
for j in range(maximum_process_num, 0, -1):
progress[j] = ''
id_pool = [j for j in range(maximum_process_num, 0, -1)]
process_num = 0
process2status = dict()
running_processes = set()
process2id = dict()
process2index = dict()
process2start_time = dict()
id2index = dict()
id2task = dict()
index2task = dict()
round_num = 0
process_list = list()
killed_process_list = list()
dead_num = 0
dead_ratio = 0
alive_check_timeout = 60
dead_ratio_list = []
alive_count = 0
while len(tasks) > 0 or len(running_processes) > 0:
current_time = getlocaltime()
num_alive_processes = sum(1 for process in process_list
if process.is_alive())
status = '[%d] processes are still alive, [%d] are running ...' % (
num_alive_processes, len(running_processes))
string = '%s\t%s\n' % (current_time, status)
print(string)
sys.stdout.flush()
while len(running_processes) < maximum_process_num and len(
tasks) > 0:
group = tasks.pop()
task = group
if task in completed_list:
continue
user_dir_group = '%s_%d' % (user_dir, group)
process_index = process_num
try:
process_list.append(
Task(target=measure,
args=(user_dir_group, task, length, start, end,
status_queue, process_index)))
process = process_list[-1]
process.start()
except OSError as e:
tasks.append(group)
time.sleep(5)
continue
process_num += 1
running_processes.add(process)
process2index[process] = process_index
process2start_time[process] = time.time()
index2task[process_index] = task
current_time = getlocaltime()
process_status = 'Process %-4d task %d/%d created. PID=%d ...' % (
process_index, task + 1, length, process.pid)
string = '%s\t%s' % (current_time, process_status)
print(string)
sys.stdout.flush()
if process_num % 20 == 0:
break
#break
#time.sleep(1)
flag = False
while any(process.is_alive() for process in process_list):
time.sleep(1)
current_time = getlocaltime()
alive_count += 1
num_alive_processes = sum(1 for process in process_list
if process.is_alive())
#flag = False
while not status_queue.empty():
process_index, process_status = status_queue.get()
string = '%s\t%s\n' % (current_time, process_status)
log_f.write(string)
if 'completed' in process_status:
flag = True
if process_status not in final_status_set:
final_status_set.add(process_status)
if alive_count % alive_check_timeout == 0:
status = '[%d] processes are still alive ...' % (
num_alive_processes)
string = '%s\t%s\n' % (current_time, status)
print(string)
sys.stdout.flush()
gc_chrome_tmp_files()
current_timestamp = time.time()
elapse = current_timestamp - start_time
dead_num = 0
# We need to get a list. Otherwise, we will receive an Exception: RuntimeError: Set changed size during iteration
for process in list(running_processes):
process_index = process2index[process]
group = index2task[process_index]
if not process.is_alive():
flag = True
process_status = 'Process %-4d task %d/%d is no longer alive...' % (
process_index, group + 1, length)
else:
process_status = 'Process %-4d task %d/%d is still alive...' % (
process_index, group + 1, length)
string = '%s\t%s\n' % (current_time, process_status)
log_f.write(string)
# Start checking log file modification time after 10 minutes
if elapse >= 60 * 10:
process_start_time = process2start_time[process]
process_elapse = current_timestamp - process_start_time
user_dir_group = '%s_%d' % (user_dir, group)
user_dir_log_file = '%s.log' % (user_dir_group)
user_dir_log_file = os.path.join(
log_dir, user_dir_log_file)
#mtime = current_timestamp
ctime = current_timestamp
try:
#mtime = os.path.getmtime(user_dir_log_file) # https://docs.python.org/2/library/os.path.html#os.path.getmtime
ctime = os.path.getctime(
user_dir_log_file
) # https://docs.python.org/2/library/os.path.html#os.path.getctime
except OSError as e:
pass
if current_timestamp - ctime >= 60 * 10 and process_elapse >= 60 * 5:
dead_num += 1
process_status = 'Process %-4d task %d/%d PID [%d] seems to be dead. Terminating and restarting process...' % (
process_index, group + 1, length,
process.pid)
string = '%s\t%s\n' % (current_time,
process_status)
log_f.write(string)
gone, alive = kill_child_processes(
parent_pid=process.pid)
process_name = 'user-data-dir=%s' % (
user_dir_group)
kill_processes_by_name(process_name)
process_name = 'log-path=webdriver_%s.log' % (
user_dir_group)
kill_processes_by_name(process_name)
running_processes.remove(process)
tasks.append(group)
flag = True
current_timestamp = time.time()
dead_ratio = 1.0 * dead_num / maximum_process_num
if len(dead_ratio_list) >= 5:
dead_ratio_list.pop(0)
dead_ratio_list.append(dead_ratio)
avg_dead_ratio = sum(dead_ratio_list) / float(
len(dead_ratio_list))
if avg_dead_ratio >= 0.1:
status = "Too many tasks are dead! Average dead ratio is %.2f!" % (
avg_dead_ratio)
string = '%s\t%s\n' % (current_time, status)
print(string)
log_f.write(string)
raise TooManyTasksDead(
"Too many tasks are dead! Average dead ratio is %.2f!"
% (avg_dead_ratio))
if flag == True or (num_alive_processes < maximum_process_num
and (len(tasks) > 0 or alive_count %
alive_check_timeout == 0)):
break
# We need to get a list. Otherwise, we will receive an Exception: RuntimeError: Set changed size during iteration
#for process in list(running_processes):
for process in process_list:
if not process.is_alive():
if process in running_processes:
running_processes.remove(process)
except (KeyboardInterrupt, Exception) as e:
current_time = getlocaltime()
status = "PARENT SCRIPT Exception %s" % type(e)
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
if not isinstance(e, KeyboardInterrupt) and not isinstance(
e, TooManyTasksDead):
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
print(type(e), "PARENT")
print(''.join('!! ' + line for line in lines))
status = ''.join('!! ' + line for line in lines)
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
restart_parent_flag = isinstance(e, TooManyTasksDead)
if restart_parent_flag:
os.chdir(script_dir)
#for process in process_list:
#try:
#process.terminate()
#except Exception:
#pass
#process_name = 'chrome'
#kill_processes_by_name(process_name)
kill_all_processes(restart_parent_flag)
while not status_queue.empty():
process_index, process_status = status_queue.get()
string = '%s\t%s\n' % (current_time, process_status)
log_f.write(string)
for process in process_list:
try:
process.join()
except Exception:
pass
gone, alive = kill_child_processes(parent_pid=parent_pid)
timeout = 10
while timeout:
time.sleep(1)
timeout -= 1
if not mp.active_children():
break
gc_chrome_tmp_files()
current_time = getlocaltime()
status = "PARENT SCRIPT COMPLETED! PARENT PID=[%d]" % parent_pid
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
log_f.close()
def main(argv):
global raw_data_dir, processed_data_dir, num_instances, parent_pid, process_list, log_f, conflicting_rank_set
signal.signal(signal.SIGTERM, signal_term_handler)
parent_pid = os.getpid()
try:
opts, args = getopt.getopt(argv, 'hu:d:i:n:p:s:e:t:', [
'help', 'user_dir=', 'exp_dir=', 'num=', 'process=', 'start=',
'end=', 'type='
])
except getopt.GetoptError:
usage()
sys.exit(2)
conflicting_rank_set = set()
user_dir = None
num_instances = 512
maximum_process_num = 8 # Change to 1 for debugging purpose
start = 0
end = None
exp_dir = "exps"
extract = False
clean = False
send = False
#input_type = 'info2index2script'
input_type = 'url2index'
for opt, arg in opts:
if opt in ('-u', '--user_dir'):
user_dir = arg
elif opt in ('-d', '--dir'):
exp_dir = arg
elif opt in ('-n', '--num'):
num_instances = int(arg)
elif opt in ('-p', '--process'):
maximum_process_num = int(arg)
elif opt in ('-s', '--start'):
start = int(arg)
elif opt in ('-e', '--end'):
end = int(arg)
elif opt in ('-t', '--type'):
input_type = arg
elif opt in ('-h', '--help'):
usage()
sys.exit(0)
if user_dir is None:
usage()
sys.exit(0)
input_file = 'top-1m.csv'
#task_queue = get_task_queue(input_file)
raw_data_dir = exp_dir
processed_data_dir = os.path.join(exp_dir, 'conflicting_data')
if not os.path.isdir(processed_data_dir):
try:
os.mkdir(processed_data_dir)
except Exception as e:
print(e)
log_file = 'convert_asg_logs.log'
log_file = os.path.join(exp_dir, log_file)
log_f = open(log_file, mode='w')
current_time = getlocaltime()
status = "PARENT SCRIPT STARTED! PARENT PID=[%d]" % parent_pid
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
string = "%s\tProcess started, argv=%s\n" % (current_time, argv)
log_f.write(string)
completed_list = set()
completion_reg = re.compile(
'Process [0-9\s]+task ([0-9]+)/[0-9]+ PID \[\d+\] completed.')
with codecs.open(log_file, encoding='utf-8', mode='r') as input_f:
for line in input_f:
m = re.search(completion_reg, line)
if m:
task = int(m.group(1)) - 1
completed_list.add(task)
completed_list = set()
try:
os.chdir(exp_dir)
except OSError as e:
print(e)
sys.exit(1)
#if end is None:
# end = max([max(q, key=lambda x:x[0]) for q in task_queue])[0]
tasks = [i for i in range(num_instances - 1, -1, -1)]
try:
length = len(tasks)
status_queue = Queue()
final_status_set = set()
process_num = 0
process2status = dict()
running_processes = set()
process2index = dict()
index2task = dict()
round_num = 0
process_list = list()
killed_process_list = list()
alive_check_timeout = 10
alive_count = 0
while len(tasks) > 0 or len(running_processes) > 0:
current_time = getlocaltime()
num_alive_processes = sum(1 for process in process_list
if process.is_alive())
status = '[%d] processes are still alive, [%d] are running ...' % (
num_alive_processes, len(running_processes))
string = '%s\t%s\n' % (current_time, status)
print(string)
sys.stdout.flush()
while len(running_processes) < maximum_process_num and len(
tasks) > 0:
group = tasks.pop()
task = group
if task in completed_list:
continue
user_dir_group = '%s_%d' % (user_dir, group)
process_index = process_num
#task_list = task_queue[task]
try:
process_list.append(
Task(target=measure,
args=(user_dir_group, task, length, start, end,
status_queue, process_index)))
process = process_list[-1]
process.start()
except OSError as e:
tasks.append(group)
time.sleep(5)
continue
process_num += 1
running_processes.add(process_list[-1])
process2index[process_list[-1]] = process_index
index2task[process_index] = task
current_time = getlocaltime()
process_status = 'Process %-4d task %d/%d created. PID=%d ...' % (
process_index, task + 1, length, process.pid)
string = '%s\t%s' % (current_time, process_status)
print(string)
sys.stdout.flush()
if process_num % 32 == 0:
break
flag = False
while any(process.is_alive() for process in process_list):
time.sleep(1)
current_time = getlocaltime()
alive_count += 1
num_alive_processes = sum(1 for process in process_list
if process.is_alive())
while not status_queue.empty():
process_index, process_status = status_queue.get()
string = '%s\t%s\n' % (current_time, process_status)
log_f.write(string)
if 'completed' in process_status:
flag = True
if process_status not in final_status_set:
final_status_set.add(process_status)
if alive_count % alive_check_timeout == 0:
status = '[%d] processes are still alive ...' % (
num_alive_processes)
string = '%s\t%s\n' % (current_time, status)
print(string)
sys.stdout.flush()
# We need to get a list. Otherwise, we will receive an exception: RuntimeError: Set changed size during iteration
for process in list(running_processes):
process_index = process2index[process]
group = index2task[process_index]
if not process.is_alive():
flag = True
process_status = 'Process %-4d task %d/%d is no longer alive...' % (
process_index, group + 1, length)
else:
process_status = 'Process %-4d task %d/%d is still alive...' % (
process_index, group + 1, length)
string = '%s\t%s\n' % (current_time, process_status)
log_f.write(string)
if flag == True or (num_alive_processes < maximum_process_num
and (len(tasks) > 0 or alive_count %
alive_check_timeout == 0)):
break
for process in process_list:
if not process.is_alive():
if process in running_processes:
running_processes.remove(process)
except (KeyboardInterrupt, Exception) as e:
current_time = getlocaltime()
status = "PARENT SCRIPT exception %s" % type(e)
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
if not isinstance(e, KeyboardInterrupt):
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
print(type(e), "PARENT")
print(''.join('!! ' + line for line in lines))
status = ''.join('!! ' + line for line in lines)
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
kill_all_processes()
while not status_queue.empty():
process_index, process_status = status_queue.get()
string = '%s\t%s\n' % (current_time, process_status)
log_f.write(string)
for process in process_list:
try:
process.join()
except Exception:
pass
gone, alive = kill_child_processes(parent_pid=parent_pid)
timeout = 10
while timeout:
time.sleep(1)
timeout -= 1
if not mp.active_children():
break
current_time = getlocaltime()
status = "PARENT SCRIPT COMPLETED! PARENT PID=[%d]" % parent_pid
string = '%s\t%s\n' % (current_time, status)
log_f.write(string)
log_f.close()
print(conflicting_rank_set)
class StorageVolumeManager(ParametricSingleton):
"""
Storage volume manager to a manage volumes on
local or remote storage pool manager
"""
# ============================
# parametric singleton stuff
# ============================
@staticmethod
def depends_on(cls, *args, **kwargs):
# singleton is based on the uri and the pool,
# extracted from the libvirt handler
(conn_handler, pool_handler) = args[0]
# get libvirt.virConnect instance from conn_handler
if isinstance(conn_handler, str):
conn_handler = ConnectionManager(conn_handler)
if isinstance(conn_handler, ConnectionManager):
conn_handler = conn_handler.connection
# TODO: in the future, also handle StoragePool objects
# get libvirt.virStoragePool instance from pool_handler
if isinstance(pool_handler, str):
manager = StoragePoolManager(conn_handler)
pool_handler = manager.lookup(pool_handler)
# get attribute and add to singleton
try:
uri = conn_handler.getURI()
pool_name = None
if pool_handler:
pool_name = pool_handler.name()
except (AttributeError, libvirt.libvirtError) as e:
log.exception(e)
raise StorageVolumeManagerError(e)
# return keys for parametric singleton
return (uri, pool_name)
# ===========
# constants
# ===========
# maximum allocation
# (determined empirically)
MAX_ALLOCATION = 197632
# create/clone flags
# libvirt.VIR_STORAGE_VOL_CREATE_PREALLOC_METADATA
# is not defined in libvirt.py
CREATE_PREALLOC_METADATA = 1
# ==============
# resize flags
# ==============
# force allocation of new size
RESIZE_ALLOCATE = libvirt.VIR_STORAGE_VOL_RESIZE_ALLOCATE
# size is relative to current
RESIZE_DELTA = libvirt.VIR_STORAGE_VOL_RESIZE_DELTA
# allow decrease in capacity
RESIZE_SHRINK = libvirt.VIR_STORAGE_VOL_RESIZE_SHRINK
# =================
# Wipe algorithms
# =================
# 1-pass, all zeroes
WIPE_ALG_ZERO = libvirt.VIR_STORAGE_VOL_WIPE_ALG_ZERO
# 4-pass NNSA Policy Letter NAP-14.1-C (XVI-8)
WIPE_ALG_NNSA = libvirt.VIR_STORAGE_VOL_WIPE_ALG_NNSA
# 4-pass DoD 5220.22-M section 8-306 procedure
WIPE_ALG_DOD = libvirt.VIR_STORAGE_VOL_WIPE_ALG_DOD
# 9-pass method recommended by the German Center of
# Security in Information Technologies
WIPE_ALG_BSI = libvirt.VIR_STORAGE_VOL_WIPE_ALG_BSI
# The canonical 35-pass sequence
WIPE_ALG_GUTMANN = libvirt.VIR_STORAGE_VOL_WIPE_ALG_GUTMANN
# 7-pass method described by Bruce Schneier "Applied Cryptography" (1996)
WIPE_ALG_SCHNEIER = libvirt.VIR_STORAGE_VOL_WIPE_ALG_SCHNEIER
# 7-pass random
WIPE_ALG_PFITZNER7 = libvirt.VIR_STORAGE_VOL_WIPE_ALG_PFITZNER7
# 33-pass random
WIPE_ALG_PFITZNER33 = libvirt.VIR_STORAGE_VOL_WIPE_ALG_PFITZNER33
# 1-pass random
WIPE_ALG_RANDOM = libvirt.VIR_STORAGE_VOL_WIPE_ALG_RANDOM
# ==================================
# Constructor and destructor stuff
# ==================================
def __init__(self, connection, pool, prefetch=False):
"""Instantiate a storage pool manager for specified connection
:param connection: either a ``basestring`` from which will be created
a connection, an instance of a ``ConnectionManager`` or directly
a libvirt connection handler
:param prefetch: ``True`` if prefetching storage pool handlers for this
connection is required. Set to ``False`` by default
:raises: StorageVolumeManagerError
"""
# create handle cache
self._cache = {'name': {}, 'key': {}, 'path': {}}
# get libvirt.virConnection from connection
self._set_drv(connection)
# get libvirt.virStoragePool from virStoragePool
# TODO: in the future, also handle StoragePool objects
self._set_pool(pool)
if isinstance(self._pool, str):
manager = StoragePoolManager(conn_handler)
self._pool = manager.lookup(self._pool)
# prefetch list of handlers
if self._pool and prefetch:
# extra flags; not used yet, so callers should always pass 0
self._pool.refresh(flags=0)
list(map(lambda name: self._lookupByName(name), self.list()))
# =======================
# context manager stuff
# =======================
def __enter__(self):
return self
# ==================
# internal helpers
# ==================
# TODO: update when new cache pattern will be implemented
def _cache_handle(self, cache, entry, where):
if where and entry:
for key, value in list(where.items()):
if key in cache:
cache[key][value] = entry
# libvirt.virConnection from connection
def _set_drv(self, connection):
self._drv = connection
if isinstance(self._drv, str):
self._drv = ConnectionManager(self._drv)
if isinstance(self._drv, ConnectionManager):
self._drv = self._drv.connection
# libvirt.virStoragePool from virStoragePool
def _set_pool(self, pool):
self._pool = pool
if isinstance(self._pool, str):
manager = StoragePoolManager(self._drv)
self._pool = manager.lookup(self._pool)
def _update_pool(self, pool):
opool = getattr(self, "_pool", None)
self._set_pool(pool)
# update parametric singleton
if opool != self._pool:
drv_uri = self._drv.getURI()
opool_name = opool
if isinstance(opool_name, libvirt.virStoragePool):
opool_name = opool_name.name()
pool_name = self._pool.name()
StorageVolumeManager.update_key((drv_uri, opool_name),
(drv_uri, pool_name))
def _lookupByName(self, name):
handle = None
# check if storage volume has already been cached
if name in self._cache['name']:
handle = self._cache['name'][name]
# storage volume not in cache, retrieve and cache it
else:
if self._pool:
try:
handle = self._pool.storageVolLookupByName(name)
key = handle.key()
path = handle.path()
where = {'name': name, 'key': key, 'path': path}
self._cache_handle(self._cache, handle, where)
except libvirt.libvirtError as e:
# do not raise an exception here, we put a simple warning
log.warn(e)
else:
log.warn("pool is not set, skipping storageVolLookupByName()")
return handle
def _lookupByKey(self, key):
handle = None
# check if storage volume has already been cached
if key in self._cache['key']:
handle = self._cache['key'][key]
# storage volume not in cache, retrieve and cache it
else:
try:
handle = self._drv.storageVolLookupByKey(key)
name = handle.name()
path = handle.path()
where = {'name': name, 'key': key, 'path': path}
self._cache_handle(self._cache, handle, where)
except libvirt.libvirtError as e:
# do not raise an exception here, we put a simple warning
log.warn(e)
return handle
def _lookupByPath(self, path):
handle = None
# check if storage volume has already been cached
if path in self._cache['path']:
handle = self._cache['path'][path]
# storage volume not in cache, retrieve and cache it
else:
try:
handle = self._drv.storageVolLookupByPath(path)
name = handle.name()
key = handle.key()
where = {'name': name, 'key': key, 'path': path}
self._cache_handle(self._cache, handle, where)
except libvirt.libvirtError as e:
# do not raise an exception here, we put a simple warning
log.warn(e)
return handle
def _lookup_volume(self, name):
volume = self.lookup(name)
if not volume:
e = "Volume '{0}' not found".format(name)
log.error(e)
raise StorageVolumeManagerError(e)
return volume
# ================
# public methods
# ================
pool = property(None, _update_pool)
def lookup(self, volume):
"""lookup a volume and return the corresponding libvirt.virStoragePool
:param connection: either a ``basestring`` with the label of
the virStorageVol to find, an instance of a ``StorageVolume``
:returns: an instance of virStorageVol
"""
handle = None
# TODO: move refresh when there is a cache miss of flush
if self._pool:
self._pool.refresh(flags=0)
# perform lookup first by name, then by key and finally by path
if isinstance(volume, str):
handle = self._lookupByName(volume)
if not handle:
handle = self._lookupByKey(volume)
if not handle:
handle = self._lookupByPath(volume)
elif isinstance(volume, StorageVolume):
volume = getattr(volume, 'name', None)
if volume:
handle = self._lookupByName(volume)
volume = getattr(volume, 'key', None)
if not handle and volume:
handle = self._lookupByKey(volume)
volume = getattr(volume, 'path', None)
if not handle and volume:
handle = self._lookupByPath(volume)
# warn if no handle has been found
if not handle:
log.warn("Unable to find volume " +
"{0} on {1}".format(volume, self._drv.getURI()))
# return handle
return handle
def list(self):
"""list storage volume for this pool
:returns: a tuple containing storage volume names
"""
labels = list()
try:
labels.extend(self._pool.listVolumes())
except libvirt.libvirtError as e:
# do not raise an exception here, we simply log the exception
log.exception(e)
return tuple(labels)
def info(self, volume):
if isinstance(volume, str):
volume = self._lookup_volume(volume)
try:
# extra flags; not used yet, so callers should always pass 0
xml = volume.XMLDesc(0)
return StorageVolume.parse(xml)
except (libvirt.libvirtError, StorageVolumeError) as e:
log.exception(e)
raise StorageVolumeManagerError(e)
def create(self, volume, flags=0):
# TODO: enable more input formats ?
# check if volume already exists
handle = self.lookup(volume.name)
if handle:
e = "'{0} already exists.".format(handle.path())
log.error(e)
raise StorageVolumeManagerError(e)
# creating new volume
try:
volume_xml = volume.unparse()
# add a warning for allocation
allocation = volume.allocation
max_alloc = StorageVolumeManager.MAX_ALLOCATION
if allocation and allocation["#text"] > max_alloc:
size = allocation["#text"]
unit = allocation["@unit"]
log.warning("allocation of {0}{1} ".format(size, unit) +
"asked for {0}, ".format(volume.name) +
"only {0} ".format(max_alloc) +
"really allocated.")
# sanitizing flags
flags = flags & StorageVolumeManager.CREATE_PREALLOC_METADATA
self._pool.createXML(volume_xml, flags=flags)
except (libvirt.libvirtError, StorageVolumeError) as e:
log.exception(e)
raise StorageVolumeManagerError(e)
# update cache
return self.lookup(volume.name)
def clone(self, origin, clone, flags=0):
# TODO: enable more formats ?
# find origin volume
if isinstance(origin, StorageVolume):
origin_volume = self._lookup_volume(origin.name)
elif isinstance(origin, libvirt.virStorageVol):
origin_volume = origin
else:
origin_volume = self._lookup_volume(origin)
# perform cloning
try:
# clone object and remove key
origin_obj = self.info(origin_volume.name())
clone_obj = origin_obj
clone_obj.key = None
clone_obj.name = clone
# rebuild xml corresponding to clone object and create
clone_xml = clone_obj.unparse()
# sanitizing flags
flags = flags & StorageVolumeManager.CREATE_PREALLOC_METADATA
clone_vol = self._pool.createXMLFrom(clone_xml,
origin_volume,
flags=flags)
except (libvirt.libvirtError, StorageVolumeError) as e:
log.exception(e)
raise StorageVolumeManagerError(e)
# update cache
return self.lookup(clone)
def delete(self, name):
# TODO: enable more formats ?
# find volume to delete
if isinstance(name, StorageVolume):
volume = self._lookup_volume(name.name)
elif isinstance(name, libvirt.virStorageVol):
volume = name
else:
volume = self._lookup_volume(name)
# delete the volume
try:
# extra flags; not used yet, so callers should always pass 0
volume.delete(flags=0)
except libvirt.libvirtError as e:
log.exception(e)
raise StorageVolumeManagerError(e)
def resize(self, name, capacity, flags=0):
# TODO: enable more formats ?
# find volume to resize
if isinstance(name, StorageVolume):
volume = self._lookup_volume(name.name)
elif isinstance(name, libvirt.virStorageVol):
volume = name
else:
volume = self._lookup_volume(name)
# resize the volume
try:
# sanitizing flags
mask = (StorageVolumeManager.RESIZE_ALLOCATE
| StorageVolumeManager.RESIZE_DELTA
| StorageVolumeManager.RESIZE_SHRINK)
flags = flags & mask
volume.resize(capacity, flags=flags)
except libvirt.libvirtError as e:
log.exception(e)
raise StorageVolumeManagerError(e)
def wipe(self, name, algorithm=None, flags=0):
# TODO: enable more formats ?
# find volume to wipe
if isinstance(name, StorageVolume):
volume = self._lookup_volume(name.name)
elif isinstance(name, libvirt.virStorageVol):
volume = name
else:
volume = self._lookup_volume(name)
# perform volume wipe
try:
# future flags, use 0 for now
flags = flags & 0
if algorithm:
# sanitize algorithm
mask = (StorageVolumeManager.WIPE_ALG_ZERO
| StorageVolumeManager.WIPE_ALG_NNSA
| StorageVolumeManager.WIPE_ALG_DOD
| StorageVolumeManager.WIPE_ALG_BSI
| StorageVolumeManager.WIPE_ALG_GUTMANN
| StorageVolumeManager.WIPE_ALG_SCHNEIER
| StorageVolumeManager.WIPE_ALG_PFITZNER7
| StorageVolumeManager.WIPE_ALG_PFITZNER33
| StorageVolumeManager.WIPE_ALG_RANDOM)
algorithm = algorithm & mask
volume.wipePattern(algorithm, flags=flags)
else:
volume.wipe(flags=flags)
except libvirt.libvirtError as e:
log.exception(e)
raise StorageVolumeManagerError(e)
def download(self,
volume,
filename,
offset=0,
length=0,
flags=0,
async=False):
def _recv_handler(stream, buffer, opaque):
filedes, status = opaque
filedes.write(buffer)
if status:
status[0] += len(buffer)
def _download_worker(volume, filename, offset, length, flags, status):
try:
if isinstance(filename, str):
if os.path.exists(filename):
e = "file {0} already exists".format(filename)
log.error(e)
raise StorageVolumeManagerError(e)
file = open(filename, 'wb')
else:
file = filename
stream = self._drv.newStream(flags=0)
# extra flags; not used yet, so callers should always pass 0
mime = volume.download(stream, offset, length, flags=0)
# perform download
stream.recvAll(_recv_handler, (file, status))
stream.finish()
file.close()
except (IOError, libvirt.libvirtError) as e:
stream.abort()
file.close()
if os.path.exists(filename):
os.unlink(filename)
log.exception(e)
raise StorageVolumeManagerError(e)
# TODO: enable more formats ?
if isinstance(volume, StorageVolume):
volume = self._lookup_volume(volume.name)
elif isinstance(volume, str):
volume = self._lookup_volume(volume)
# determining byte count to download
bytecount = length
if not length:
type, capacity, bytecount = volume.info()
# creating shared memory to get status
status = Array('L', [0, bytecount]) if async else None
task = Task(target=_download_worker,
args=(volume, filename, offset, length, flags, status))
task.start()
# if not asynchronous, wait for the task to finish
if async:
# NOTE: (status[0], status[1]) contain
# (already copied, total) bytes count
return task, status
else:
task.join()
file.close()
except (IOError, libvirt.libvirtError) as e:
stream.abort()
file.close()
log.exception(e)
raise StorageVolumeManagerError(e)
# TODO: enable more formats ?
if isinstance(name, StorageVolume):
volume = self._lookup_volume(name.name)
elif isinstance(name, libvirt.virStorageVol):
volume = name
else:
volume = self._lookup_volume(name)
# determining byte count to upload
bytecount = length
if not length:
type, capacity, bytecount = volume.info()
# creating shared memory to get status
status = Array('L', [0, bytecount]) if async else None
task = Task(target=_upload_worker,
args=(volume, filename, offset, length, flags, status))
task.start()
# if not asynchronous, wait for the task to finish
if async:
# NOTE: (status[0], status[1]) contain
# (already copied, total) bytes count
return task, status
else:
task.join()
def virement(self, money):
with self.amount.get_lock():
# get current amount
current_amount = self.amount.value
# network slow time
sleep(0.01)
# doing computation
current_amount += money
# hard drive latency
sleep(0.01)
self.amount.value = current_amount
my_account = BankAccount("Me", 1.0)
print("Start:", my_account)
tasks = []
for ind in range(0, 100):
t = Task(target=my_account.virement, args=(100, ))
t.start()
tasks.append(t)
print('Waiting end of tasks')
[t.join() for t in tasks]
print("End:", my_account)
dataFile = sys.argv[2] # Output file for data
f = open(dataFile, "w+") # Create file if it doesn't exist
f.close()
sumOfProcesses = 0 # The summed time total of all processed wavs
# For each wav in the directory specified...
for wav in wavs:
# print ("total Duration:", getTotalDuration(wav))
workers = []
# Specify and start the task
child = Task(target=callPraat, args=(wav, dataFile))
child.start()
workers.append(child)
# These few lines just give a command line counter so it seems more interactive
progress = 0
while any(i.is_alive() for i in workers):
progress += 1
print("Running time (s):", progress, wav.split("/")[-1], end="\r")
time.sleep(1)
sumOfProcesses += progress
print("\nCurrent total (s):", sumOfProcesses)
print("Total analysis time (s):", sumOfProcesses)
print("Removing undefined measurements")