def __init__(self,
file_list,
mode,
database_path,
settings,
temp_dir=None):
# Have the GUI pass a list of files straight to here
# Then, on the basis of what is needed, pass the
# filenames to the requisite functions
# This includes getting file info for the database
# Parsing for the reader proper
# Caching upon closing
self.file_list = [i for i in file_list if os.path.exists(i)]
self.statistics = [0, (len(file_list))]
self.hashes_and_paths = {}
self.work_mode = mode[0]
self.addition_mode = mode[1]
self.database_path = database_path
self.auto_tags = settings['auto_tags']
self.auto_cover = settings['auto_cover']
self.temp_dir = temp_dir
if database_path:
self.database_hashes()
self.threading_completed = []
self.queue = Manager().Queue()
self.errors = Manager().list()
self.processed_books = []
if self.work_mode == 'addition':
progress_object_generator()
def check_open_ports(url: str,
ip: str,
path: Optional[str] = None) -> List[Result]:
results = []
# create processing pool
pool = Pool(os.cpu_count() * 2)
mgr = Manager()
queue = mgr.Queue()
# read the data in from the data directory
if path is None:
file_path = pkg_resources.resource_filename(
"yawast", "resources/common_ports.json")
else:
file_path = path
with open(file_path) as json_file:
data = json.load(json_file)
for rec in data:
pool.apply_async(_is_port_open, (url, ip, rec, queue))
pool.close()
pool.join()
while not queue.empty():
val = queue.get()
if val is not None:
results.append(val)
return results
def new_send_email(email):
if not IS_ACTIVE:
new_send_email.queue = Manager().Queue()
process = Process(target=process_sent_queue,
args=(new_send_email.queue, ))
process.daemon = True
process.start()
models.IS_ACTIVE = True
new_send_email.queue.put(email)
def main():
manager = Manager()
lock = manager.Lock()
partial_get_index = functools.partial(get_index, lock)
pool = Pool()
pool.map(partial_get_index, [i for i in range(START_PAGE, END_PAGE + 1)])
pool.close()
pool.join()
print('over')
def __init__(self, file_list, n_treads, q):
assert isinstance(file_list, (list, tuple))
assert isinstance(n_treads, int)
assert n_treads >= 0
assert type(q) is Manager().Queue
self.is_active = True
self.file_list = file_list
self.n_treads = n_treads
self.q = q
def calc(self, computeOptions, **args):
tiles = self._get_tile_service().get_tiles_bounded_by_box(computeOptions.get_min_lat(), computeOptions.get_max_lat(),
computeOptions.get_min_lon(), computeOptions.get_max_lon(),
computeOptions.get_dataset()[0],
computeOptions.get_start_time(),
computeOptions.get_end_time())
if len(tiles) == 0:
raise NexusProcessingException.NoDataException(reason="No data found for selected timeframe")
maxprocesses = int(self.algorithm_config.get("multiprocessing", "maxprocesses"))
results = []
if maxprocesses == 1:
calculator = LongitudeHofMoellerCalculator()
for x, tile in enumerate(tiles):
result = calculator.longitude_time_hofmoeller_stats(tile, x)
results.append(result)
else:
manager = Manager()
work_queue = manager.Queue()
done_queue = manager.Queue()
for x, tile in enumerate(tiles):
work_queue.put(
('longitude_time_hofmoeller_stats', tile, x))
[work_queue.put(SENTINEL) for _ in range(0, maxprocesses)]
# Start new processes to handle the work
pool = Pool(maxprocesses)
[pool.apply_async(pool_worker, (LONGITUDE, work_queue, done_queue)) for _ in range(0, maxprocesses)]
pool.close()
# Collect the results
for x, tile in enumerate(tiles):
result = done_queue.get()
try:
error_str = result['error']
logger.error(error_str)
raise NexusProcessingException(reason="Error calculating longitude_time_hofmoeller_stats.")
except KeyError:
pass
results.append(result)
pool.terminate()
manager.shutdown()
results = sorted(results, key=lambda entry: entry["time"])
results = self.applyDeseasonToHofMoeller(results, pivot="lons")
result = HoffMoellerResults(results=results, computeOptions=computeOptions, type=HoffMoellerResults.LONGITUDE)
return result
def get_daily_difference_average_for_box(self, min_lat, max_lat, min_lon,
max_lon, dataset1, dataset2,
start_time, end_time):
daysinrange = self._tile_service.find_days_in_range_asc(
min_lat, max_lat, min_lon, max_lon, dataset1, start_time, end_time)
maxprocesses = int(
self.algorithm_config.get("multiprocessing", "maxprocesses"))
if maxprocesses == 1:
calculator = DailyDifferenceAverageCalculator()
averagebyday = []
for dayinseconds in daysinrange:
result = calculator.calc_average_diff_on_day(
min_lat, max_lat, min_lon, max_lon, dataset1, dataset2,
dayinseconds)
averagebyday.append((result[0], result[1]))
else:
# Create a task to calc average difference for each day
manager = Manager()
work_queue = manager.Queue()
done_queue = manager.Queue()
for dayinseconds in daysinrange:
work_queue.put(
('calc_average_diff_on_day', min_lat, max_lat, min_lon,
max_lon, dataset1, dataset2, dayinseconds))
[work_queue.put(SENTINEL) for _ in xrange(0, maxprocesses)]
# Start new processes to handle the work
pool = Pool(maxprocesses)
[
pool.apply_async(pool_worker, (work_queue, done_queue))
for _ in xrange(0, maxprocesses)
]
pool.close()
# Collect the results as [(day (in ms), average difference for that day)]
averagebyday = []
for i in xrange(0, len(daysinrange)):
result = done_queue.get()
if result[0] == 'error':
print >> sys.stderr, result[1]
raise NexusProcessingException(
reason="Error calculating average by day.")
rdata = result
averagebyday.append((rdata[0], rdata[1]))
pool.terminate()
manager.shutdown()
return averagebyday
def main():
old_folder_name = "Test"
new_folder_name = old_folder_name + "_copy"
os.mkdir(new_folder_name)
pool = Pool(5)
queue = Manager().Queue()
file_list = os.listdir(old_folder_name)
file_size = len(file_list)
for file in file_list:
pool.apply_async(copy_file_worker, (file, old_folder_name, new_folder_name, queue))
num = 0
while num < file_size:
queue.get()
num += 1
print("Copy Rate: %.2f%%" % (num / file_size * 100))
print("Done!")
def parallel_parsing(directory):
trees = list()
input_files = [java_file for java_file in get_java_files(directory)]
# pool = Pool(processes=8)
# x = pool.apply_async(create_project_parse_tree, args=(input_files[0], trees))
# for java_file in get_java_files(directory):
# res = pool.apply_async(create_project_parse_tree, (java_file,)) # runs in *only* one process
# print(type(res.get(timeout=1))) #
# num = Value('d', 0.0)
# arr = Array('i', range(500))
with Manager() as manager:
d = manager.dict()
q = manager.Queue(10)
p = Process(target=create_project_parse_tree, args=(input_files[0], q))
p.start()
p.join()
print(q)
return True
# return "done"
def _upload_pool(self):
pool = ThreadPool(processes=self.n_treads)
res = pool.map(self._upload_file, self.file_list)
n_res = len(res)
res_dict = Counter(res)
succes = res_dict[True]
self.q.put("Uploaded {}/{}".format(succes, n_res))
self.is_active = False
def start(self):
proc = Process(target=self._upload_pool)
proc.start()
if __name__ == '__main__':
q = Manager().Queue()
files_list = [
os.path.join(os.getcwd(), "test_data", i)
for i in os.listdir("./test_data")
]
uploader = Uploader(files_list, 2, q)
uploader.start()
while uploader.is_active:
progress = q.get()
print(progress)
def getTimeSeriesStatsForBoxSingleDataSet(self,
min_lat,
max_lat,
min_lon,
max_lon,
ds,
start_time=0,
end_time=-1,
applySeasonalFilter=True,
applyLowPass=True):
daysinrange = self._get_tile_service().find_days_in_range_asc(
min_lat, max_lat, min_lon, max_lon, ds, start_time, end_time)
if len(daysinrange) == 0:
raise NoDataException(
reason="No data found for selected timeframe")
maxprocesses = int(
self.algorithm_config.get("multiprocessing", "maxprocesses"))
results = []
if maxprocesses == 1:
calculator = TimeSeriesCalculator()
for dayinseconds in daysinrange:
result = calculator.calc_average_on_day(
min_lat, max_lat, min_lon, max_lon, ds, dayinseconds)
results.append(result)
else:
# Create a task to calc average difference for each day
manager = Manager()
work_queue = manager.Queue()
done_queue = manager.Queue()
for dayinseconds in daysinrange:
work_queue.put(('calc_average_on_day', min_lat, max_lat,
min_lon, max_lon, ds, dayinseconds))
[work_queue.put(SENTINEL) for _ in range(0, maxprocesses)]
# Start new processes to handle the work
pool = Pool(maxprocesses)
[
pool.apply_async(pool_worker, (work_queue, done_queue))
for _ in range(0, maxprocesses)
]
pool.close()
# Collect the results as [(day (in ms), average difference for that day)]
for i in range(0, len(daysinrange)):
result = done_queue.get()
try:
error_str = result['error']
logger.error(error_str)
raise NexusProcessingException(
reason="Error calculating average by day.")
except KeyError:
pass
results.append(result)
pool.terminate()
manager.shutdown()
results = sorted(results, key=lambda entry: entry["time"])
filt.applyAllFiltersOnField(results,
'mean',
applySeasonal=applySeasonalFilter,
applyLowPass=applyLowPass)
filt.applyAllFiltersOnField(results,
'max',
applySeasonal=applySeasonalFilter,
applyLowPass=applyLowPass)
filt.applyAllFiltersOnField(results,
'min',
applySeasonal=applySeasonalFilter,
applyLowPass=applyLowPass)
return results, {}
import json
import logging
from multiprocessing.dummy import Manager
import random
from urllib.parse import urlparse, parse_qs
from config.config import BASEPYCURLCONFIG
from core import MySpider
from core.Spider import CrawlJob
__author__ = 'Florian'
m = Manager()
crawled = m.dict()
logger = logging.getLogger("logger")
fh = logging.FileHandler("clubs.jsonl", 'a+')
simpleFormat = logging.Formatter("%(message)s")
fh.setFormatter(simpleFormat)
fh.setLevel(logging.WARNING)
ch = logging.StreamHandler()
ch.setFormatter(simpleFormat)
ch.setLevel(logging.DEBUG)
logger.addHandler(fh)
logger.addHandler(ch)
logger.setLevel(logging.DEBUG)
@MySpider.QueueInitializer.register()
def seeds():
jobs = []
logger = logging.getLogger("logger")
for federation in [
import datetime
from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing.dummy import Manager
import openalpr_api
from openalpr_api.rest import ApiException
from sinesp_client import SinespClient
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import requests
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# initiating globals
now = datetime.datetime.now()
sc = SinespClient()
pool = ThreadPool(4) # number of threads for parallelism
lock = Manager().Lock()
def process_cars(car_images):
return pool.map(process_car_image, car_images)
def process_car_image(image_path):
# create an instance of the API class
api_instance = openalpr_api.DefaultApi()
secret_key = 'sk_4fb6feea66fb6b5524b2d984' # str | The secret key used to authenticate your account. You can view your secret key by visiting https://cloud.openalpr.com/
country = 'br' # str | Defines the training data used by OpenALPR. \"us\" analyzes North-American style plates. \"eu\" analyzes European-style plates. This field is required if using the \"plate\" task You may use multiple datasets by using commas between the country codes. For example, 'au,auwide' would analyze using both the Australian plate styles. A full list of supported country codes can be found here https://github.com/openalpr/openalpr/tree/master/runtime_data/config
recognize_vehicle = 0 # int | If set to 1, the vehicle will also be recognized in the image This requires an additional credit per request (optional) (default to 0)
state = '' # str | Corresponds to a US state or EU country code used by OpenALPR pattern recognition. For example, using \"md\" matches US plates against the Maryland plate patterns. Using \"fr\" matches European plates against the French plate patterns. (optional) (default to )
return_image = 0 # int | If set to 1, the image you uploaded will be encoded in base64 and sent back along with the response (optional) (default to 0)
topn = 10 # int | The number of results you would like to be returned for plate candidates and vehicle classifications (optional) (default to 10)
prewarp = '' # str | Prewarp configuration is used to calibrate the analyses for the angle of a particular camera. More information is available here http://doc.openalpr.com/accuracy_improvements.html#calibration (optional) (default to )
dealdata = tuple(tuple([y for y in x]) for x in data)
try:
# 执行sql语句
cursor.executemany(sql, dealdata)
# 提交到数据库执行
db.commit()
except pm.Warning as w:
print(repr(w))
# 如果发生错误则回滚
db.rollback()
if __name__ == '__main__':
# pool = ThreadPool(4)
# page = [x + 1 for x in range(18)]
manager = Manager()
lock = manager.Lock()
# pool = ThreadPool(processes=8)
for i in range(1, 20):
# pool.apply_async(spider, (i, lock))
spider(i)
print(list)
for i in list:
if i == 21:
continue
else:
dealdata(i)
# results = pool.map(spider ,page)
# pool.close()
# pool.join()
def __init__(self, cpu_count=cpu_count(), **kwargs):
self.m = Manager()
self.pool = self.m.Pool(cpu_count)
self.kwargs = kwargs
self.queue = self.m.Queue()
self.log_queue = self.m.Queue()
def getTimeSeriesStatsForBoxSingleDataSet(self, bounding_polygon, ds, start_seconds_from_epoch,
end_seconds_from_epoch,
apply_seasonal_cycle_filter=True, apply_low_pass_filter=True):
the_time = datetime.now()
daysinrange = self._get_tile_service().find_days_in_range_asc(bounding_polygon.bounds[1],
bounding_polygon.bounds[3],
bounding_polygon.bounds[0],
bounding_polygon.bounds[2],
ds,
start_seconds_from_epoch,
end_seconds_from_epoch)
logger.info("Finding days in range took %s for dataset %s" % (str(datetime.now() - the_time), ds))
if len(daysinrange) == 0:
raise NoDataException(reason="No data found for selected timeframe")
the_time = datetime.now()
maxprocesses = int(self.algorithm_config.get("multiprocessing", "maxprocesses"))
results = []
if maxprocesses == 1:
calculator = TimeSeriesCalculator()
for dayinseconds in daysinrange:
result = calculator.calc_average_on_day(bounding_polygon.wkt, ds, dayinseconds)
results += [result] if result else []
else:
# Create a task to calc average difference for each day
manager = Manager()
work_queue = manager.Queue()
done_queue = manager.Queue()
for dayinseconds in daysinrange:
work_queue.put(
('calc_average_on_day', bounding_polygon.wkt, ds, dayinseconds))
[work_queue.put(SENTINEL) for _ in range(0, maxprocesses)]
# Start new processes to handle the work
pool = Pool(maxprocesses)
[pool.apply_async(pool_worker, (work_queue, done_queue)) for _ in range(0, maxprocesses)]
pool.close()
# Collect the results as [(day (in ms), average difference for that day)]
for i in range(0, len(daysinrange)):
result = done_queue.get()
try:
error_str = result['error']
logger.error(error_str)
raise NexusProcessingException(reason="Error calculating average by day.")
except KeyError:
pass
results += [result] if result else []
pool.terminate()
manager.shutdown()
results = sorted(results, key=lambda entry: entry["time"])
logger.info("Time series calculation took %s for dataset %s" % (str(datetime.now() - the_time), ds))
if apply_seasonal_cycle_filter:
the_time = datetime.now()
for result in results:
month = datetime.utcfromtimestamp(result['time']).month
month_mean, month_max, month_min = self.calculate_monthly_average(month, bounding_polygon.wkt, ds)
seasonal_mean = result['mean'] - month_mean
seasonal_min = result['min'] - month_min
seasonal_max = result['max'] - month_max
result['meanSeasonal'] = seasonal_mean
result['minSeasonal'] = seasonal_min
result['maxSeasonal'] = seasonal_max
logger.info(
"Seasonal calculation took %s for dataset %s" % (str(datetime.now() - the_time), ds))
the_time = datetime.now()
filtering.applyAllFiltersOnField(results, 'mean', applySeasonal=False, applyLowPass=apply_low_pass_filter)
filtering.applyAllFiltersOnField(results, 'max', applySeasonal=False, applyLowPass=apply_low_pass_filter)
filtering.applyAllFiltersOnField(results, 'min', applySeasonal=False, applyLowPass=apply_low_pass_filter)
if apply_seasonal_cycle_filter and apply_low_pass_filter:
try:
filtering.applyFiltersOnField(results, 'meanSeasonal', applySeasonal=False, applyLowPass=True,
append="LowPass")
filtering.applyFiltersOnField(results, 'minSeasonal', applySeasonal=False, applyLowPass=True,
append="LowPass")
filtering.applyFiltersOnField(results, 'maxSeasonal', applySeasonal=False, applyLowPass=True,
append="LowPass")
except Exception as e:
# If it doesn't work log the error but ignore it
tb = traceback.format_exc()
logger.warn("Error calculating SeasonalLowPass filter:\n%s" % tb)
logger.info(
"LowPass filter calculation took %s for dataset %s" % (str(datetime.now() - the_time), ds))
return results, {}
import boto3
import botocore
import random
from botocore.client import Config
from botocore.endpoint import MAX_POOL_CONNECTIONS
from multiprocessing import TimeoutError
from multiprocessing.dummy import Pool as ThreadPool, Manager, Value
from enumerate_iam.utils.remove_metadata import remove_metadata
from enumerate_iam.utils.json_utils import json_encoder
from enumerate_iam.bruteforce_tests import BRUTEFORCE_TESTS
MAX_THREADS = 12
CLIENT_POOL = {}
MANAGER = Manager()
STOP_SIGNAL = MANAGER.Value('i', 0)
def report_arn(candidate):
"""
Attempt to extract and slice up an ARN from the input string
"""
logger = logging.getLogger()
arn_search = re.search(r'.*(arn:aws:.*?) .*', candidate)
if arn_search:
arn = arn_search.group(1)
arn_id = arn.split(':')[4]