def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name=name)
if num_threads is None:
num_threads = SystemOption.get(DBSession()).inventory_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name=name)
db_session = DBSession()
if num_threads is None:
num_threads = SystemOption.get(db_session).install_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name = name)
if num_threads is None:
num_threads = SystemOption.get(DBSession()).inventory_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name = name)
db_session = DBSession()
if num_threads is None:
num_threads = SystemOption.get(db_session).install_threads
# Set up the thread pool
self.pool = Pool(num_threads)
class InventoryManager(threading.Thread):
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name=name)
if num_threads is None:
num_threads = SystemOption.get(DBSession()).inventory_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def run(self):
while 1:
# This will be configurable
time.sleep(20)
self.dispatch()
def dispatch(self):
db_session = DBSession()
try:
inventory_jobs = db_session.query(InventoryJob).filter(
InventoryJob.pending_submit == True).all()
if len(inventory_jobs) > 0:
for inventory_job in inventory_jobs:
self.submit_job(inventory_job.id)
except:
logger.exception('Unable to dispatch inventory job')
finally:
db_session.close()
def submit_job(self, job_id):
with lock:
# If another inventory job for the same host is already in progress,
# the inventory job will not be queued for processing
if job_id in in_progress_jobs:
return False
in_progress_jobs[job_id] = job_id
self.pool.submit(InventoryWorkUnit(job_id))
return True
class InventoryManager(threading.Thread):
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name = name)
if num_threads is None:
num_threads = SystemOption.get(DBSession()).inventory_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def run(self):
while 1:
# This will be configurable
time.sleep(20)
self.dispatch()
def dispatch(self):
db_session = DBSession()
try:
inventory_jobs = db_session.query(InventoryJob).filter(InventoryJob.pending_submit == True).all()
if len(inventory_jobs)> 0:
for inventory_job in inventory_jobs:
self.submit_job(inventory_job.id)
except:
logger.exception('Unable to dispatch inventory job')
finally:
db_session.close()
def submit_job(self, job_id):
with lock:
# If another inventory job for the same host is already in progress,
# the inventory job will not be queued for processing
if job_id in in_progress_jobs:
return False
in_progress_jobs[job_id] = job_id
self.pool.submit(InventoryWorkUnit(job_id))
return True
class DownloadManager(threading.Thread):
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name=name)
if num_threads is None:
num_threads = SystemOption.get(DBSession()).download_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def run(self):
while 1:
# This will be configurable
time.sleep(20)
self.dispatch()
def dispatch(self):
db_session = DBSession()
try:
download_jobs = db_session.query(DownloadJob).all()
for download_job in download_jobs:
if download_job.status != JobStatus.FAILED:
with lock:
# If another download job for the same image name is already in progress,
# the download job will not be queued for processing
if download_job.cco_filename in in_progress_downloads:
continue
in_progress_downloads[
download_job.cco_filename] = download_job.cco_filename
self.pool.submit(DownloadWorkUnit(download_job.id))
except:
logger.exception('Unable to dispatch download job')
finally:
db_session.close()
class DownloadManager(threading.Thread):
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name = name)
if num_threads is None:
num_threads = SystemOption.get(DBSession()).download_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def run(self):
while 1:
# This will be configurable
time.sleep(20)
self.dispatch()
def dispatch(self):
db_session = DBSession()
try:
download_jobs = db_session.query(DownloadJob).all()
for download_job in download_jobs:
if download_job.status != JobStatus.FAILED:
with lock:
# If another download job for the same image name is already in progress,
# the download job will not be queued for processing
if download_job.cco_filename in in_progress_downloads:
continue
in_progress_downloads[download_job.cco_filename] = download_job.cco_filename
self.pool.submit(DownloadWorkUnit(download_job.id))
except:
logger.exception('Unable to dispatch download job')
finally:
db_session.close()
def fetch():
url = "http://pages.cs.wisc.edu/~remzi/OSTEP/"
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
}
try:
content = get_html_content(url, headers)
except FetchFailed:
print(f"Error: failed to get response from {url=}", file=sys.stderr)
exit(1)
except Exception as e:
print(f"Error: failed to run, caused by \"{e}\"", file=sys.stderr)
exit(1)
root_dir = ROOTDIR
if not os.path.isdir(root_dir):
os.mkdir(root_dir)
soup = BeautifulSoup(content, "lxml")
background_colors = ["yellow", "#f88017", "#00aacc", "#4CC417", "#3EA99F"]
number = 0
sub_dir = root_dir
pool = Pool(10)
pool.run()
for idx, color in enumerate(background_colors):
for td in soup.find_all("td", bgcolor=color):
if (b := td.b) and b.text:
sub_dir = f"{root_dir}/{idx}-{b.text}"
if not os.path.isdir(sub_dir):
os.mkdir(sub_dir)
if s := td.small:
number = int(s.text) + 1
if a := td.a:
name = f"{str(number).zfill(2)} {a['href']}"
fpath = f"{sub_dir}/{name}"
file_url = f"{url}{a['href']}"
pool.submit(work, url=file_url, headers=headers, fpath=fpath)
number += 1
if p.ref:
b = p.process(out[p.ref.idx], idx=j)
else:
b = p.process(buf, idx=j)
except NormalizeException, e: #mark errors and report them later. we are in threading unfortunately
self.errors.append(j + 1)
out.append(None)
continue
out.append(b)
if p.save:
print "saving file", f_out, p.save
write_file(f_out % str(p), b)
p = Pool(4)
try:
p.map(
handle,
enumerate(
self.path_iter(self.path, os.path.join(self.path, "%s"))))
except Exception, e:
p.terminate()
print self.errors
raise e
print "handling"
print self.errors
p.terminate()
for i, p in enumerate(self.processors):
p.finalize()
class SoftwareManager(threading.Thread):
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name=name)
db_session = DBSession()
if num_threads is None:
num_threads = SystemOption.get(db_session).install_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def run(self):
while 1:
time.sleep(20)
self.dispatch()
"""
In order for a scheduled install job to proceed, its dependency must be successfully completed
and is present in the InstallJobHistory table. It is possible that the dependency (install_job_id)
has multiple entries in the table. This can happen when it takes multiple tries for the dependency
to become successful (i.e. after couple failed attempts).
"""
def get_install_job_dependency_completed(self, db_session, install_job):
return db_session.query(InstallJobHistory).filter(
and_(InstallJobHistory.install_job_id == install_job.dependency,
InstallJobHistory.host_id == install_job.host_id,
InstallJobHistory.status == JobStatus.COMPLETED)).all()
def dispatch(self):
db_session = DBSession()
try:
# Check if Scheduled Installs are allowed to run.
if not can_install(db_session):
return
resultSet = db_session.query(InstallJob).filter(
InstallJob.scheduled_time <= datetime.datetime.utcnow()).all()
download_job_key_dict = get_download_job_key_dict()
if len(resultSet) > 0:
for install_job in resultSet:
if install_job.status != JobStatus.FAILED:
# If there is pending download, don't submit the install job
if is_pending_on_download(download_job_key_dict,
install_job):
continue
# This install job has a dependency, check if the expected criteria is met
if install_job.dependency is not None:
dependency_completed = self.get_install_job_dependency_completed(
db_session, install_job)
# If the dependency has not been completed, don't proceed
if len(dependency_completed) == 0:
continue
with lock:
# If another install job for the same host is already in progress,
# the install job will not be queued for processing
if install_job.host_id in in_progress_hosts:
continue
in_progress_hosts.append(install_job.host_id)
# Allow the install job to proceed
install_work_unit = InstallWorkUnit(
install_job.id, install_job.host_id)
self.pool.submit(install_work_unit)
except:
# Purpose ignore. Otherwise, it may generate continue exception
pass
finally:
db_session.close()
if p.ref:
b = p.process(out[p.ref.idx], idx=j)
else:
b = p.process(buf, idx=j)
except NormalizeException, e: #mark errors and report them later. we are in threading unfortunately
self.errors.append(j+1)
out.append(None)
continue
out.append(b)
if p.save:
print "saving file", f_out, p.save
write_file(f_out % str(p), b)
p = Pool(4)
try:
p.map(handle, enumerate(self.path_iter(self.path, os.path.join(self.path, "%s"))))
except Exception, e:
p.terminate()
print self.errors
raise e
print "handling"
print self.errors
p.terminate()
for i, p in enumerate(self.processors):
p.finalize()
if __name__ == "__main__":
# this is a sample workflow that reads some information about the traces to process
class SoftwareManager(threading.Thread):
def __init__(self, name, num_threads=None):
threading.Thread.__init__(self, name = name)
db_session = DBSession()
if num_threads is None:
num_threads = SystemOption.get(db_session).install_threads
# Set up the thread pool
self.pool = Pool(num_threads)
def run(self):
while 1:
time.sleep(20)
self.dispatch()
"""
In order for a scheduled install job to proceed, its dependency must be successfully completed
and is present in the InstallJobHistory table. It is possible that the dependency (install_job_id)
has multiple entries in the table. This can happen when it takes multiple tries for the dependency
to become successful (i.e. after couple failed attempts).
"""
def get_install_job_dependency_completed(self, db_session, install_job):
return db_session.query(InstallJobHistory).filter(and_(
InstallJobHistory.install_job_id == install_job.dependency,
InstallJobHistory.host_id == install_job.host_id,
InstallJobHistory.status == JobStatus.COMPLETED)).all()
def dispatch(self):
db_session = DBSession()
try:
# Check if Scheduled Installs are allowed to run.
if not can_install(db_session):
return
resultSet = db_session.query(InstallJob).filter(InstallJob.scheduled_time <= datetime.datetime.utcnow()).all()
download_job_key_dict = get_download_job_key_dict()
if len(resultSet)> 0:
for install_job in resultSet:
if install_job.status != JobStatus.FAILED:
# If there is pending download, don't submit the install job
if is_pending_on_download(download_job_key_dict, install_job):
continue
# This install job has a dependency, check if the expected criteria is met
if install_job.dependency is not None:
dependency_completed = self.get_install_job_dependency_completed(db_session, install_job)
# If the dependency has not been completed, don't proceed
if len(dependency_completed) == 0:
continue
with lock:
# If another install job for the same host is already in progress,
# the install job will not be queued for processing
if install_job.host_id in in_progress_hosts:
continue
in_progress_hosts.append(install_job.host_id)
# Allow the install job to proceed
install_work_unit = InstallWorkUnit(install_job.id, install_job.host_id)
self.pool.submit(install_work_unit)
except:
# Purpose ignore. Otherwise, it may generate continue exception
pass
finally:
db_session.close()