def tif_to_jp2(
_threads,
_app,
_source,
_destination,
_broken,
_options,
_verbose
):
testApp(_app)
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_source):
subpath = root.replace(_source, '').lstrip('/')
if _broken not in subpath:
jp2Path = os.path.join(_destination,subpath)
makeDir(jp2Path)
if any(".tif" in s for s in files):
print >>emaillog, 'Converting contents of ' + subpath + ' from TIF to JP2'
for file in files:
if file.endswith('.tif'):
tiff = os.path.join(root, file)
jp2 = os.path.join(_destination, subpath,
os.path.splitext(file)[0] + '.jp2')
tiffcopy = os.path.join(_destination,subpath,file)
command = _app + ' -i ' + tiff + ' -o ' + jp2 + ' ' \
+ _options
command_post = 'shutil.move(\'' + tiff + '\',\'' + tiffcopy + '\')'
if _verbose == True:
print 'Creating ' + jp2
t.add_task(executeConversion,command,command_post,tiff,_destination,_broken,file,jp2)
t.await_completion()
def jp2_to_jpeg(
_threads,
_app,
_source,
_destination,
_broken,
_jpegs,
_verbose
):
testApp(_app)
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_destination):
subpath = root.replace(_destination, '').lstrip('/')
if _broken not in subpath:
if any(".jp2" in s for s in files):
print >>emaillog, 'Converting contents of ' + subpath + ' from JP2 to JPEG'
for (output_file, size) in _jpegs:
for file in files:
if file.endswith('.jp2'):
jp2 = os.path.join(root, file)
newfile = os.path.join(root,
os.path.splitext(file)[0]) + '_' \
+ output_file
command = _app + ' -size ' + size + " " + jp2 \
+ ' -resize ' + size + ' ' + newfile
if _verbose == True:
print 'Creating ' + newfile
t.add_task(executeConversion,command,None,jp2,_source,_broken,file,newfile)
t.await_completion()
def pool_time(thread_num):
start = time.clock()
tp = ThreadPool(thread_num)
for i in range(5):
tp.add_task(time.sleep, i)
tp.wait_completion()
return time.clock() - start
def jp2_to_jpeg(_threads, _app, _source, _destination, _broken, _jpegs,
_verbose):
testApp(_app)
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_destination):
subpath = root.replace(_destination, '').lstrip('/')
if _broken not in subpath:
if any(".jp2" in s for s in files):
print >> emaillog, 'Converting contents of ' + subpath + ' from JP2 to JPEG'
for (output_file, size) in _jpegs:
for file in files:
if file.endswith('.jp2'):
jp2 = os.path.join(root, file)
newfile = os.path.join(root,
os.path.splitext(file)[0]) + '_' \
+ output_file
command = _app + ' -size ' + size + " " + jp2 \
+ ' -resize ' + size + ' ' + newfile
if _verbose == True:
print 'Creating ' + newfile
t.add_task(executeConversion, command, None, jp2,
_source, _broken, file, newfile)
t.await_completion()
def tif_to_jp2(_threads, _app, _source, _destination, _broken, _options,
_verbose):
testApp(_app)
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_source):
subpath = root.replace(_source, '').lstrip('/')
if _broken not in subpath:
jp2Path = os.path.join(_destination, subpath)
makeDir(jp2Path)
if any(".tif" in s for s in files):
print >> emaillog, 'Converting contents of ' + subpath + ' from TIF to JP2'
for file in files:
if file.endswith('.tif'):
tiff = os.path.join(root, file)
jp2 = os.path.join(_destination, subpath,
os.path.splitext(file)[0] + '.jp2')
tiffcopy = os.path.join(_destination, subpath, file)
command = _app + ' -i ' + tiff + ' -o ' + jp2 + ' ' \
+ _options
command_post = 'shutil.move(\'' + tiff + '\',\'' + tiffcopy + '\')'
if _verbose == True:
print 'Creating ' + jp2
t.add_task(executeConversion, command, command_post, tiff,
_destination, _broken, file, jp2)
t.await_completion()
def iterate(_source, _ignore, _patron, _patron_zip, _threads):
print 'Descend into ' + _source
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_source):
t.add_task(patron_bundle, _patron, _patron_zip, root)
t.await_completion()
def iterate(_source,_ignore,_patron,_patron_zip,_threads):
print 'Descend into ' + _source
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_source):
t.add_task(patron_bundle,_patron,_patron_zip,root)
t.await_completion()
def articles():
error = None
if request.method == 'POST':
query = request.form['query'].lower()
query = query.replace(" ", "%20")
if query in app.cache:
return Response(json.dumps(app.cache[query]), mimetype='application/json')
url = GOOGLE_NEWS_RSS+query
response = requests.get(url).text
# Get article links from the RSS
links = []
utils.find_links(links, response)
if len(links) > 2:
links = links[2:]
# Get article titles from the RSS
titles = []
utils.find_titles(titles, response)
if len(titles) > 2:
titles = titles[2:]
num_links = len(links)
num_titles = len(titles)
if num_titles < num_links:
links = links[:num_titles]
if num_links < num_titles:
titles = titles[:num_links]
articles = {}
pool = ThreadPool(num_links)
for i in range(num_links):
pool.add_task(
utils.get_article_sentiment, links[i], titles[i], articles)
pool.wait_completion()
sentiments = []
result = {}
result["articles"] = []
index = 0
for key in articles:
info = {}
info["id"] = index
index += 1
info["title"] = articles[key]["title"]
info["link"] = key
sentiments.append(articles[key]["sentiment"])
info["sentiment"] = articles[key]["sentiment"]
info["snippet"] = articles[key]["snippet"]
result["articles"].append(info)
average_sentiment = utils.average_sentiment(
sentiments,
len(sentiments))
result["sentiment"] = average_sentiment
app.cache[query] = deepcopy(result)
return Response(json.dumps(result), mimetype='application/json')
def sentiment():
error = None
if request.method == 'POST':
query = request.form['query']
query = query.replace(" ", "%20")
if query in app.sentiment_cache:
return Response(
json.dumps(app.sentiment_cache[query]),
mimetype='application/json')
url = GOOGLE_NEWS_RSS+query
response = requests.get(url).text
# Get article links from the RSS
links = []
utils.find_links(links, response)
if len(links) > 2:
links = links[2:]
# Get article titles from the RSS
titles = []
utils.find_titles(titles, response)
if len(titles) > 2:
titles = titles[2:]
num_links = len(links)
num_titles = len(titles)
if num_titles > 3:
num_titles = 3
if num_titles < num_links:
links = links[:num_titles]
if num_links > 3:
num_links = 3
if num_links < num_titles:
titles = titles[:num_links]
articles = {}
pool = ThreadPool(num_links)
for i in range(num_links):
pool.add_task(
utils.get_article_sentiment, links[i], titles[i], articles)
pool.wait_completion()
sentiments = []
result = {}
for key in articles:
print query, ':', articles[key]["sentiment"]
sentiments.append(articles[key]["sentiment"])
average_sentiment = utils.average_sentiment(
sentiments,
len(sentiments))
print average_sentiment
result["sentiment"] = average_sentiment
result["query"] = request.form['query']
app.sentiment_cache[query] = deepcopy(result)
return Response(json.dumps(result), mimetype='application/json')
def main1():
thread_pool = ThreadPool(20)
thread_pool.start()
session = get_session()
topic_query = \
session.query(VideoTopic).filter(VideoTopic.video_type == 1)
for topic in topic_query:
thread_pool.add_task(job, topic.henhen_id)
session.close()
thread_pool.wait_done()
def test_results(self):
def my_add(a,b):
return a+b
tp = ThreadPool(5)
for i in range(5):
tp.add_task(my_add, i, i)
d = tp.wait_completion()
vals = d.values()
vals.sort()
assert vals == [0, 2, 4, 6, 8]
def main():
pool = ThreadPool(20)
pool.start()
session = get_session()
topic_query = session.query(PicTopic).filter(PicTopic.pic_type
== 'dongmantupian').order_by(PicTopic.id.desc())
for pic_topic in topic_query:
pool.add_task(dump_job, pic_topic)
session.close()
pool.wait_done()
class BaseBot:
def __init__(self, service):
self.service = service
self.error_count = 0
self.thread_pool = ThreadPool()
# thread-safe 하다고 나와있긴 하지만 보장이 되는지 고민해 봐야함
# callback 패턴으로 바꾸는 것 고려
def handle_event(self, events):
for event in events:
if isinstance(event, ChatInitEvent):
#print(event.user_index)
self.handle_entered_room(event.team_index, event.room_index)
elif isinstance(event, ChatMessageEvent):
chat = self.service.get_chat_summary(event.room_index,
event.msg_index)
if chat:
self.handle_chat(event.team_index, event.room_index, chat)
elif isinstance(event, UserDropEvent) \
and self.service.my_index == event.user_index:
logger.error("봇 계정이 탈퇴되었습니다.")
sys.exit()
elif isinstance(event, UserPasswordChangedEvent) \
and self.service.my_index == event.user_index:
logger.error("봇 계정의 비밀번호가 바뀌었습니다.")
sys.exit()
def run(self):
while True:
try:
events = self.service.get_events()
if events:
self.thread_pool.add_task(self.handle_event, events)
time.sleep(self.service.config['lp_idle_time'])
except Exception as e:
self.error_count += 1
if self.error_count > 3:
logger.error("오류가 발생했습니다. 프로그램을 종료합니다.")
#sys.exit()
else:
logger.error("오류가 발생했습니다.:" + e)
time.sleep(5)
def handle_entered_room(self, team_index, room_index):
raise NotImplementedError()
def handle_chat(self, team_index, room_index, chat):
raise NotImplementedError()
def main():
thread_pool = ThreadPool(50)
thread_pool.start()
video_type = '7'
base_url = 'http://www.toutoulu.com/vodlist/%s_%s.html'
# init task
for page_num in range(1, page_info[video_type] + 1):
url = base_url % (video_type, page_num)
print 'add task %s' % url
thread_pool.add_task(thread_pool_job, url, video_type)
thread_pool.wait_done()
def test_thread_pool():
"""
thread pool should be able to handle task processing
"""
thread_pool = ThreadPool()
result = []
def populate_result_task():
result.extend([i for i in range(0, 10)])
return
thread_pool.add_task(populate_result_task)
thread_pool.tasks.join()
thread_pool.terminate_all_workers()
assert result == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
class DnsServer:
def __init__(self, config_name="config.json"):
self.__config = ServerConfig(config_name)
logging.basicConfig(filename=self.__config.log_file,
level=logging.DEBUG,
format='%(asctime)s %(message)s')
self.thread_pool = ThreadPool()
cache_dir = Path.cwd() / self.__config.cache_dir
self.cache = CacheStorage(cache_dir)
self.request_handler = RequestHandler(self.cache)
def run(self):
"""Binds, listens, processing DNS requests on socket"""
signal.signal(signal.SIGINT, self.__handle_exit)
s = socket(AF_INET, SOCK_DGRAM)
s.bind((self.__config.server_host, self.__config.server_port))
s.settimeout(self.__config.server_timeout)
logging.info(
f'Launched at {self.__config.server_host}:{self.__config.server_port}'
)
while True:
try:
data, addr = s.recvfrom(self.__config.recv_buff_size)
except SystemExit as e:
s.close()
break
except Exception as e:
logging.info(str(e))
s.close()
break
self.thread_pool.add_task(self.__process_request, data, addr, s)
def __process_request(self, data, addr, s_socket):
response = self.request_handler.handle_query(data)
s_socket.sendto(response, addr)
def __handle_exit(self, signal, frame):
logging.info("Received SIGINT, shutting down threads...")
print("shutting down...")
self.thread_pool.tasks.join()
self.thread_pool.terminate_all_workers()
logging.info("Threads stopped, updating cache")
self.cache.cleanup()
sys.exit(0)
def scrap_connections():
areas=set()
airports = []
with open('csv_files/top_100_airports.csv', 'rb') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
for airport in row:
airports.append(airport)
origins = dests = airports
dates = []
for i in range(100):
new_date = datetime.date(2014, 11, 02) + timedelta(days=i)
dates.append(new_date)
pool = ThreadPool(20)
pool.start()
for dest in dests:
for origin in origins:
if dest != origin:
date = dates[randint(0, len(dates)-1)]
pool.add_task(get_connections, origin, dest, date)
areas.add(get_area(origin, dest))
pool.wait_completion()
#arrange all connections in single set per area instead of list
areas_conn = dict()
for area in areas:
conn_list = flight_resp_dal.get_connections_in_area(area)
connections = set()
for conn in conn_list:
connections.update(set(conn))
areas_conn[area] = connections
flight_resp_dal.clean_areas_to_connections_table()
for area in areas:
flight_resp_dal.add_connections_to_area(area, areas_conn[area])
def test_thread_pool_with_exception():
"""
thread pool should be able to handle task processing
even if there were exceptions in some tasks
"""
thread_pool = ThreadPool()
result = []
def throw_ex_task():
raise Exception()
def populate_result_task():
result.extend([i for i in range(0, 10)])
return
thread_pool.add_task(throw_ex_task)
thread_pool.add_task(populate_result_task)
thread_pool.tasks.join()
thread_pool.terminate_all_workers()
assert result == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
def proxy_server(host, port):
addr = host, port
tp = ThreadPool(WORKERS)
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(addr)
sock.listen(WORKERS)
print('Listening on', addr)
while True:
if not select.select([sock], [], [], 1)[0]:
continue
conn, client_addr = sock.accept()
tp.add_task(proxy_connection, conn, client_addr)
except socket.error:
print('Cannot run the server')
tp.wait_completion()
class VayantConnector(object):
def __init__(self):
self.flights_resp_dal = FlightsRespDAL()
self.pool = ThreadPool(30)
self.pool.start()
def get_flight_price_async(self, trip):
response = AsyncResponse(self.do_after_done)
self.pool.add_task(self.calculate_flight_info, trip, response)
return response
def get_flight_from_cache(self, key):
cached_resp = self.flights_resp_dal.get(key)
while self.flights_resp_dal.has_key(key) and cached_resp is None:
time.sleep(5)
cached_resp = self.flights_resp_dal.get(key)
return cached_resp
def calculate_flight_info(self, trip, response):
resp = self.get_flights_info(trip)
response.set_response_value(resp)
def get_flights_info(self, trip):
resp = None
key = self._create_cache_key_from_trip(trip)
cached_resp = self.get_flight_from_cache(key)
if cached_resp:
return cached_resp
try:
self.flights_resp_dal.set(key, None)
request_json = self.build_trip_request(trip_data)
header = {"Content-Type": "application/JSON ", "Accept-encoding": "gzip"}
req = urllib2.Request("http://fs-json.demo.vayant.com:7080/", data=json.dumps(request_json), headers=header)
response = urllib2.urlopen(req)
resp = self._decompress_and_extract_json(response)
trip = self._get_flights_from_vayant_response(response)
if trip:
self.flights_resp_dal.remove(key)
return None
self.flights_resp_dal.set(key, trip)
finally:
if not resp:
self.flights_resp_dal.remove(key)
return resp
def _create_cache_key_from_trip(self, trip):
key = ""
for single_trip in trip:
key += single_trip["Origin"][0] + "-"
key += single_trip["Destination"][0] + "-"
for date in single_trip["DepartureDates"]:
key += date["Date"] + "-"
key += ":"
return key
def _decompress_and_extract_json(self, response):
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
json_resp = ""
while True:
data = response.read(8192)
if not data: break
if response.info().get('Content-Encoding') == 'gzip':
data = decompressor.decompress(data)
json_resp += data
return json.loads(json_resp)
# def print_single_flight(self, flight):
# response = "\n"
# response += flight["Fares"][0]["Origin"] + " -> " + flight["Fares"][0]["Destination"] + ":\n"
# response += "total price = {}".format(flight['Price']['Total']['Amount']) + "\n"
# response += "flights details: "+ "\n"
# for leg in flight["Flights"]:
# response += "\t" + leg["Origin"] + " -> " + leg["Destination"] + ":"+ "\n"
# response += "\t departure: " + leg["Departure"]+ "\n"
# response += "\t arrival: " + leg["Arrival"]+ "\n"
# if self.flights_resp_dal.get_airline(leg["OperatingCarrier"]) is not None:
# response += "\t carrier: " + self.flights_resp_dal.get_airline(leg["OperatingCarrier"])+ "\n"
# else:
# response += "\t carrier: " + leg["OperatingCarrier"]+ "\n"
# return response
def get_departure_flight_date(self, trip_response):
return trip_response['Flights'][0]['Departure'][0:10]
def get_return_flight_date(self, trip_response):
return trip_response['Flights'][-1]['Departure'][0:10]
def extract_cheapest_price(self, resp):
sorted_response = sorted(resp['Journeys'], key=lambda trip: trip[0]['Price']['Total']['Amount'])
try:
return sorted_response[0][0]['Price']['Total']['Amount']
except:
print "ERROR getting the price", resp, sorted_response
return 0
def get_connections_list(self, trip):
connections = set()
for single in trip['Journeys']:
if len(single[0]["Flights"]) == 2:
if single[0]["Flights"][0]["Destination"] == single[0]["Flights"][1]["Origin"]:
connections.add(single[0]["Flights"][0]["Destination"])
return connections
def get_flight(self, trip, index):
try:
x = trip['Journeys'][index][0]
return x
except:
return None
def get_price(self, flight):
if flight:
return flight['Price']['Total']['Amount']
return 99999
def get_dest_flights_in_two_way(self, trip, connection):
for i in xrange(len(trip['Flights'])-1):
if trip['Flights'][i]['Destination'] == connection:
return trip['Flights'][i], trip['Flights'][i + 1]
return None, None
def get_price_round_trip(self, origin, dest, depart_dates, arrive_dates, get_full_response = False):
first_trip = self.build_trip(origin, dest, depart_dates, 1)
second_trip = self.build_trip(dest, origin, arrive_dates, 2)
return self.get_flight_price_async([first_trip, second_trip])
def get_price_one_way(self, origin, dest, depart_dates):
first_trip = self.build_trip(origin, dest, depart_dates, 1)
return self.get_flight_price_async([first_trip])
def do_after_done(self, resp):
trip_data = resp
if trip_data and trip_data.has_key("Journeys") and trip_data['Journeys'] and len(trip_data['Journeys']) > 0:
return self.extract_cheapest_price(trip_data), trip_data
return (None, None)
def entities():
error = None
if request.method == 'POST':
query = request.form['query'].lower()
query = query.replace(" ", "%20")
if query in app.entity_cache:
return Response(json.dumps(app.entity_cache[query]), mimetype='application/json')
url = GOOGLE_NEWS_RSS+query
response = requests.get(url).text
# Get article links from the RSS
links = []
utils.find_links(links, response)
if len(links) > 1:
links = links[2:]
# Get article titles from the RSS
titles = []
utils.find_titles(titles, response)
if len(titles) > 1:
titles = titles[2:]
num_links = len(links)
entities = []
pool = ThreadPool(num_links)
for i in range(num_links):
pool.add_task(
utils.get_article_entities, links[i], entities)
pool.wait_completion()
result = {}
num_entities = len(entities)
entity_dict = defaultdict(int)
for entity_list in entities:
for entity in entity_list:
entity_dict[entity] += 1
entities_list = reversed(list(sorted(entity_dict.items(), key=lambda x: x[1])))
entities_list = [x[0] for x in entities_list]
if len(entities_list) > 5:
entities_list = entities_list[:5]
fix_entities = False
if not entities_list:
fix_entities = True
if num_entities > 0:
entities_set = entities[0]
if fix_entities:
if len(entities[0]) > 3:
entities_list = [word.title() for word in list(entities[0])[:3]]
else:
entities_list = [word.title() for word in entities[0]]
result[0] = [word.title() for word in entities[0]]
for i in range(1, num_entities):
if fix_entities and len(entities_list) < 3:
if len(entities[i]) + len(entities_list) > 3:
entities_list.extend([word.title() for word in list(entities[i])[:3-len(entities_list)]])
else:
entities_list.extend([word.title() for word in entities[i]])
result[i] = [word.title() for word in entities[i]]
result["entities"] = entities_list
app.entity_cache[query] = deepcopy(result)
return Response(json.dumps(result), mimetype='application/json')
return_dates = [depart_date + timedelta(days=21) for depart_date in depart_dates]
return_dates += [depart_date + timedelta(days=7) for depart_date in depart_dates]
return_dates += [depart_date + timedelta(days=80) for depart_date in depart_dates]
if __name__ == "__main__":
pool = ThreadPool(20, "flight_checker", FlightChecker)
for dest in dest_list:
for single_connection in connections_list:
for origin in origin_list:
for depart_date in depart_dates:
for return_date in return_dates:
if depart_date < return_date :
if origin != dest != single_connection != origin:
pool.add_task(check_flights, origin, dest, single_connection, depart_date, return_date,
final_prices)
print "Number of tasks", pool.task_number
pool.start()
print "Waiting for completion"
pool.wait_completion()
print final_prices
print len(final_prices)
for cities, price in final_prices.iteritems():
if True:#"Round Trip" not in price[0]:
print "{}, {}, price = {}, flights information is: \n".format(cities, price[0], price[1])
for flight in price[2]:
Pricer("Vayant").flights_provider.print_single_flight(flight)
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
'''
import threading
from thread_pool import ThreadPool
# Function to be executed by the thread pool
def sum_function(*args):
_sum_ = 0
thread = threading.currentThread().getName()
for arg in args:
_sum_ += arg
print(thread, "sum:", _sum_)
# Instantiate a thread pool with 50 worker threads
pool = ThreadPool(50)
# Add the functions to be executed by the threadpool
for _ in range(900):
pool.add_task(sum_function, 1, 2, 3)
# This is to prevent the threads from still running in the background
pool.kill_all_threads()
import os
hh = hackhttp.hackhttp(hackhttp.httpconpool(500))
tp = ThreadPool(500)
package = "wooyun"
if not os.path.exists(package):
os.mkdir(package)
def vlun(wid):
print "[+]%s" % wid
if os.path.isfile(wid + ".html"):
return
_, _, html, _, _ = hh.http(url="http://wooyun.org/bugs/%s" % wid,
cookcookie=False)
open(package + "/" + wid + '.html', 'wb').write(html)
def catalog(page):
_, _, html, _, _ = hh.http(
url="http://wooyun.org/bugs/new_public/page/%d" % page,
cookcookie=False)
for wid in re.findall(r'href="/bugs/(wooyun-\d+-\d+)">', html):
tp.add_task(vlun, wid)
if page > 0:
tp.add_task(catalog, page - 1)
tp.add_task(catalog, 1925)
pool_size = [x for x in range(min_thread, max_thread, thread_step)]
# Create dict with thread sizes to keep track of time
for thread_count in pool_size: times[thread_count] = []
for i in pool_size:
if ovrld.overloaded:
i = ovrld.opt_work_threads
if need_count and i == pool_size[-1] \
or need_count and ovrld.overloaded and i == ovrld.opt_work_threads:
clients = i * calculate_needed()
else:
clients= i * count
pool = ThreadPool(i)
# Clients is the final goal, it'll run the thread count for "count" iterations
# count is from config
sched_clients += clients
while clients:
# Change to your desired function...
pool.add_task(time_event, xmlrpc_call, i)
clients -= 1
total_clients += 1
if errors.error_count > errors_threshold:
quit()
pool.wait_completion()
avg_time = sum(times[i]) / len(times[i])
ovrld.calc_time(avg_time, i)
quit()
class HttpServer:
__request_parser: RequestParser = RequestParser()
__request_processor: RequestHandler = RequestHandler()
def __init__(self, config_name="config.json"):
self.__config = ServerConfig(config_name)
logging.basicConfig(filename=self.__config.log_file,
level=logging.DEBUG,
format='%(asctime)s %(message)s')
self.thread_pool = ThreadPool()
def run(self):
"""Binds, listens, processing HTTP requests on socket"""
s = socket(AF_INET, SOCK_STREAM)
s.bind((self.__config.host, self.__config.port))
s.listen(self.__config.queue_size)
logging.info(f'Launched at {self.__config.port}')
while True:
try:
client_connection, _ = s.accept()
except Exception as e:
logging.info(e)
s.close()
break
client_connection.settimeout(self.__config.max_req_time)
self.thread_pool.add_task(self.__route_request, client_connection)
def __route_request(self, client):
"""Routes request to handler if exists, then closes the connection"""
if self.__config.proxy_pass_mode:
__proxy_passer = ProxyPasser(client)
__proxy_passer.run()
return
while True:
try:
raw_request = self.__read_from_socket(client)
except timeout:
logging.info("Caught timeout waiting for socket connection")
break
except ReadSocketError:
bad_response = Response(code=400)
client.sendall(bad_response.get_bytes())
client.close()
logging.info(f'Failed to read request. Returned response'
f' {bad_response.code}')
return
req = self.__request_parser.parse(raw_request)
if req.method == "GET":
response_func, response = self.__request_processor.handle_get(
req)
logging.info(f'Received GET {req.path}, '
f'returned response {response.code}')
response_func(client=client)
if "Connection" not in req.headers \
or req.headers["Connection"].lower() != "keep-alive":
break
client.close()
def __read_from_socket(self, client):
"""Reads request data from socket. If request method or protocol
are not supported, rejects it"""
result = bytearray()
req = None
head_len = 0
total_len = None
while not total_len or head_len < total_len:
chunk = client.recv(8192)
if not chunk:
break
result += chunk
head_len += len(chunk)
if not req:
req = self.__request_parser.try_get_headers(result)
if not req:
continue
if req.method not in self.__config.supported_methods or \
req.proto not in self.__config.supported_protos:
logging.info(f'Received unsupported request {req}')
raise ReadSocketError("Request of this type not supported")
total_len = req.headers.get("Content-Length")
if not total_len:
break
return result
# Create dict with thread sizes to keep track of time
for thread_count in pool_size:
times[thread_count] = []
for i in pool_size:
if ovrld.overloaded:
i = ovrld.opt_work_threads
if need_count and i == pool_size[-1] \
or need_count and ovrld.overloaded and i == ovrld.opt_work_threads:
clients = i * calculate_needed()
else:
clients = i * count
pool = ThreadPool(i)
# Clients is the final goal, it'll run the thread count for "count" iterations
# count is from config
sched_clients += clients
while clients:
# Change to your desired function...
pool.add_task(time_event, xmlrpc_call, i)
clients -= 1
total_clients += 1
if errors.error_count > errors_threshold:
quit()
pool.wait_completion()
avg_time = sum(times[i]) / len(times[i])
ovrld.calc_time(avg_time, i)
quit()
class ProxyServer:
__request_parser: RequestParser = RequestParser()
__response_parser: ResponseParser = ResponseParser()
def __init__(self, config_name="config.json"):
self.__config = ServerConfig(config_name)
logging.basicConfig(filename=self.__config.log_file,
level=logging.DEBUG,
format='%(asctime)s %(message)s')
self.thread_pool = ThreadPool()
def run(self):
"""Binds, listens, processing HTTP requests on socket"""
signal.signal(signal.SIGINT, self.__handle_exit)
s = socket(AF_INET, SOCK_STREAM)
s.bind((self.__config.host, self.__config.port))
s.listen(self.__config.queue_size)
logging.info(f'Launched at {self.__config.port}')
while True:
try:
client_connection, _ = s.accept()
except SystemExit as e:
s.close()
break
except Exception as e:
logging.info(e)
s.close()
break
# client_connection.settimeout(self.__config.max_req_time)
self.thread_pool.add_task(self.__route_request, client_connection)
def __route_request(self, client):
"""Routes request to handler if exists, then closes the connection"""
proxy_passer = ProxyPasser(client)
proxy_passer.run()
def __read_from_socket(self, sock, is_response):
"""Reads request data from socket. If request method or protocol
are not supported, rejects it"""
result = bytearray()
headers = None
head_len = 0
total_len = None
while not total_len or head_len < total_len:
chunk = sock.recv(8192)
if not chunk:
break
result += chunk
head_len += len(chunk)
if not headers:
headers = self.try_get_headers(result, is_response)
if not headers:
continue
total_len_raw = headers.get("Content-Length")
if total_len_raw:
total_len = int(total_len_raw)
else:
continue
return result
def try_get_headers(self, data, is_response):
try:
if is_response:
headers, parsed_msg = self.__response_parser.parse(data)
else:
headers, parsed_msg = self.__request_parser.parse(data)
if parsed_msg:
return headers
except ValueError:
return None
def __handle_exit(self, signal, frame):
logging.info("Received SIGINT, shutting down threads...")
print("shutting down...")
self.thread_pool.tasks.join()
self.thread_pool.terminate_all_workers()
logging.info("Threads stopped")
sys.exit(0)
topo.del_switchs()
resault = topo.buildNet_Topo(interval)
if resault:
docker = docker_operate()
docker.resolve(interval)
else:
docker = docker_operate()
docker.resolve(interval)
if re == 1:
topo.del_switchs()
print interval
if __name__ == '__main__':
client = zmq.Context()
print 'Connect to server......'
socket = client.socket(zmq.REQ)
socket.connect("tcp://127.0.0.1:6662")
dealThread = ThreadPool(3)
while True:
# get message
socket.send("word")
print 'listen......'
message = socket.recv(102400)
data = json.loads(message)
# deal(data)
dealThread.add_task(deal, data)
# socket.send("word")
# thread.start_new_thread(deal,(data,))
from thread_pool import ThreadPool
from db.flights_resp import FlightsRespDAL
from flights_data.flight_checks import FlightChecker
from datetime import date
airports_error = []
def check_flights(origin, dest, depart_date, flight_checker):
if not flight_checker.pricer.get_price_one_way(origin, dest, str(depart_date))[0]:
airports_error.append(dest)
pool = ThreadPool(20, "flight_checker", FlightChecker)
flight_resp_dal = FlightsRespDAL()
depart = date(2014, 8, 02)
dests = flight_resp_dal.get_all_airports()
print len(dests)
for dest in dests:
pool.add_task(check_flights, "JFK", dest, depart)
pool.start()
pool.wait_completion()
print airports_error
for y in airports_error:
flight_resp_dal.airport_collection.remove({"airport_code": y})
titles = []
utils.find_titles(titles, response)
if len(titles) > 2:
titles = titles[2:]
num_links = len(links)
num_titles = len(titles)
if num_titles < num_links:
links = links[:num_titles]
if num_links < num_titles:
titles = titles[:num_links]
articles = {}
pool = ThreadPool(num_links)
for i in range(num_links):
pool.add_task(
utils.get_article_sentiment, links[i], titles[i], articles)
pool.wait_completion()
sentiments = []
result = {}
result["articles"] = []
for key in articles:
info = {}
info["title"] = articles[key]["title"]
info["link"] = key
sentiments.append(articles[key]["sentiment"])
info["sentiment"] = articles[key]["sentiment"]
info["snippet"] = articles[key]["snippet"]
result["articles"].append(info)
import re
import os
hh = hackhttp.hackhttp(hackhttp.httpconpool(500))
tp = ThreadPool(500)
package = "wooyun"
if not os.path.exists(package):
os.mkdir(package)
def vlun(wid):
print "[+]%s" % wid
if os.path.isfile(wid + ".html"):
return
_, _, html, _, _ = hh.http(
url="http://wooyun.org/bugs/%s" % wid, cookcookie=False)
open(package + "/" + wid + '.html', 'wb').write(html)
def catalog(page):
_, _, html, _, _ = hh.http(
url="http://wooyun.org/bugs/new_public/page/%d" % page,
cookcookie=False)
for wid in re.findall(r'href="/bugs/(wooyun-\d+-\d+)">', html):
tp.add_task(vlun, wid)
if page > 0:
tp.add_task(catalog, page - 1)
tp.add_task(catalog, 1925)
import logging
from threading import Thread
import asynctask
from thread_pool import ThreadPool
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s|%(levelname)s|%(threadName)s|%(message)s',
)
def foo(x):
if x == 3:
raise Exception('More than 2 but less than 4?')
logging.info(x)
if __name__ == '__main__':
thread_pool = ThreadPool(4)
for i in xrange(100):
thread_pool.add_task(func=foo, args=(i, ))
thread_pool.join_and_stop()
