def process_execute(*args, **kwargs):
"""
Executes func from keyword arguments with values from them.
Args and kwargs are directly passed to multiprocessing.Process.
"""
from django.db import transaction
# Commit current transaction so that new process will have up-to-date DB.
if transaction.is_dirty():
transaction.commit()
proc = Process(*args, **kwargs)
proc.start()
def process_execute(*args, **kwargs):
"""
Executes func from keyword arguments with values from them.
Args and kwargs are directly passed to multiprocessing.Process.
"""
from django.db import transaction
# Commit current transaction so that new process will have up-to-date DB.
if transaction.is_dirty():
transaction.commit()
proc = Process(*args, **kwargs)
proc.start()
def generate_synthethic_users_and_traffic(number_of_users = NUMBER_OF_SYNTHETIC_USERS):
# Makes the random numbers predictable, to make the experiment reproducible.
seed(1234)
user_generator = UserDistribution(number_of_users)
# generate_and_write_synthetic_traffic is a very light method, thus it is not necessary to create a pool of processes and join them later on
for user in user_generator.users():
Process(target=user.generate_and_write_synthetic_traffic).start()
def test_highly_parallel_read_access(self):
pool = [] # Process
# pool = Pool(processes=4) # Pool
results = Queue() # Process
# results = [] # Pool
for i in range(10):
# Process() approach
pool.append(Process(target=read_mpsio, args=(self.mpsio, results)))
# Pool approach
# results.append(pool.apply_async(read_mpsio, args=(self.mpsio,)))
# Process() approach
map(lambda x: x.start(), pool)
map(lambda x: x.join(), pool)
results_hash = {}
while not results.empty():
results_hash[results.get()] = True
unique_results = results_hash.keys()
# Pool approach
# pool.close()
# pool.join()
# results = [result.get() for result in results]
# unique_results = list(set(results))
print('Unique results: %s' % unique_results)
self.assertEqual(len(unique_results), 1)
self.assertEqual(unique_results[0], self.expected_value['read'])
def submit(self, *args, **kwargs):
kwargs["queue"] = self.queue
p = Process(target=self.func, args=args, kwargs=kwargs)
self.processes.append(p)
p.start()
if self.single:
p.join()
def scrape_period_drucksachen(period):
"""Scrape the website for Drucksachen in the given period.
Arguments:
period -- A models.database.Wahlperiode object
"""
pool = ThreadPool(processes=DOWNLOAD_WORKERS)
workqueue = []
for number in range(0 + 1, 20000):
# We do not start from the highest already scraped Drucksache because
# the download code also checks if the file was successfully downloaded
# as a PDF file. Thus, if any error sneaks through on one pass, e.g.
# because the server is unavailable for a moment, it will get fixed
# automatically on the next pass.
# As the metadata remains unchanged, we don't need to reprocess these
# files in the database.
number = "%05d" % number
prefix = number[:3]
# Prepare url and path
url = BASEURL_DOC_DRUCKSACHE.format(period.period_no, prefix, number)
file = BASEPATH_FILE_DRUCKSACHE.format(period.period_no, number)
# Queue up
workqueue += [(url, file)]
# TODO Add progress bars
# Perform download
print "INFO: Downloading Drucksachen...",
stdout.flush()
results = pool.map(_download_tuple, workqueue)
print "DONE."
# Close and terminate pool
pool.close()
pool.join()
# Create background process for pdftotext operations
p = Process(target=pdf_to_text, args=(results, ), name="conv_druck")
p.start()
print "INFO: Inserting into database...",
stdout.flush()
for result in results:
process_drucksache(period, result)
print "DONE"
# Wait for text file conversion to finish
print "INFO: Waiting for text file conversion to finish...",
stdout.flush()
p.join()
print "DONE."
def run():
t1 = Process(target=dev_id1)
t2 = Process(target=dev_id2)
t3 = Process(target=dev_id3)
t4 = Process(target=dev_id4)
t5 = Process(target=dev_id5)
t6 = Process(target=dev_id6)
t1.start()
t2.start()
t3.start()
t4.start()
t5.start()
t6.start()
t1.join()
t2.join()
t3.join()
t4.join()
t5.join()
t6.join()
divev()
deviv()
print "\n\033[97;1m ==[ \033[96;1m Selesai......\033[97;1m ]== \n"
def run():
th = Process(target=dev_id1)
tk = Process(target=dev_id2)
tv = Process(target=dev_id3)
th.start()
tk.start()
tv.start()
th.join()
tk.join()
tv.join()
divev()
deviv()
print "\n\033[97;1m ==[ \033[96;1m Selesai......\033[97;1m ]== \n"
def process(target, *args, **kwargs):
return Process(target=LogExceptions(target),
name=target.__name__,
args=args,
kwargs=kwargs)