def deploy_cloud_roles(self, roles): pool = ThreadPool(4) results = pool.map(self.deploy_role, roles) pool.close() pool.join() return results
import skimage.transform import numpy as np import time import os import pdb import cPickle as pickle import random from scipy import ndimage from utils import * from bleu import evaluate, evaluate_captions_ciderD, evaluate_for_particular_captions from multiprocessing.dummy import Pool as ThreadPool import sys reload(sys) sys.setdefaultencoding('utf-8') pool = ThreadPool(10) eps = 1e-10 def names2data(features, names): data = np.asarray(pool.map(lambda x: features[x][:], names)) return data class CaptioningSolver(object): def __init__(self, model, data, val_data, **kwargs): """ Required Arguments: - model: Show Attend and Tell caption generating model - data: Training data; dictionary with the following keys: - features: Feature vectors of shape (82783, 196, 512)
print "初始化解析,用时:%.1f秒" % (end - begin) def countFinish(): while True: print len(finish) time.sleep(5) def download(item): time.sleep(random.randint(1,3)) link, package_path = item try: r = requests.get(link, timeout=(20,120)) if r.status_code == 200: with open(os.path.join(package_path, link.split("/")[-1]), "wb") as f: f.write(r.content) finish.append(1) else: print "下载失败:", link, r.status_code, r.reason except Exception as e: print "下载失败:", link, str(e) if __name__ == "__main__": t = Thread(target=countFinish) t.setDaemon(True) t.start() pool = ThreadPool(3) pool.map(download, source_list[2429:])
action='store_true', help='pass LM hash as password') parser.add_argument('-d', '--domain', default='.', help='domain. default is local') parser.add_argument('-w', '--threads', type=int, default=1, help='default 1') parser.add_argument('-t', '--timeout', type=int, default=3, help='socket timeout. default 3s') parser.add_argument('-f', '--file', help='hosts file, 1 per line') parser.add_argument('hosts', nargs='*', help='hostnames or addresses') args = parser.parse_args() if args.file: for addr in open(args.file): args.hosts.append(addr.strip()) if args.prompt: args.password = getpass.getpass() socket.setdefaulttimeout(args.timeout) pool = ThreadPool(args.threads) pool.map(auth_thread, [(h, args) for h in set(args.hosts)])
def _parallel_augments(cls, action_fx, data): pool = ThreadPool(len(data)) results = pool.map(action_fx, data) pool.close() pool.join() return results
def multiprocessing(): if max_threads < 1: pool = ThreadPool() else: pool = ThreadPool(max_threads) return pool
return Request def writeRespone(request, fileposition): Response = urllib2.urlopen(request, timeout=10) fd2 = os.dup(fd) os.lseek(fd2, fileposition, os.SEEK_SET) print os.write(fd2, Response.read()) os.close(fd2) def openURL(start): Request = returnRequest(start) try: writeRespone(Request, start) except socket.error, e: print e, print("%d have error, download later " % start) LaterDownList.append(start) except urllib2.HTTPError, e: print e, print("resource unavailable") os.remove(File) sys.exit(127) pool = ThreadPool(processes=1) fd = os.open(File, os.O_WRONLY|os.O_CREAT) print "go" pool.map(openURL, LenList) pool.close() pool.join() os.close(fd)
def parse(self, pages=None): self.bar = ProgressBar(total=len(pages)) pool = ThreadPool(64) pool.map(self.parse_page, pages) pool.close() pool.join()
def simple_bridge_loops(graph, start_overlap_reads, end_overlap_reads, minimap_alignments, read_dict, scoring_scheme, threads, segments_to_bridge): bridges = [] ra = get_right_arrow() zero_loops = 'A' + ra + 'C' + ra + 'B' one_loop = 'A' + ra + 'C' + ra + 'D' + ra + 'C' + ra + 'B' two_loops = 'A' + ra + 'C' + ra + 'D' + ra + 'C' + ra + 'D' + ra + 'C' + ra + 'B' log.log_explanation('Simple loops are parts of the graph where two contigs (A and B) ' 'are connected via a repeat (C) which loops back to itself (via D). It ' 'is possible to traverse the loop zero times (' + zero_loops + '), one ' 'time (' + one_loop + '), two times (' + two_loops + '), etc. ' 'Long reads which span the loop inform which is the correct number of ' 'times through. In this step, such reads are found and each is aligned ' 'against alternative loop counts. A reads casts its "vote" for the loop ' 'count it agrees best with, and Unicycler creates a bridge using the ' 'most voted for count.') loops = sorted(graph.find_all_simple_loops()) seg_nums_to_bridge = set(x.number for x in segments_to_bridge) loops = [x for x in loops if abs(x[0]) in seg_nums_to_bridge and abs(x[1]) in seg_nums_to_bridge and abs(x[3]) not in seg_nums_to_bridge] loops = [x for x in loops if abs(x[0]) != abs(x[1])] if not loops: log.log('No suitable simple loops present') return [] col_widths = [5, 6, 6, 5, 5, 18, 5, 7] loop_table_header = ['Start', 'Repeat', 'Middle', 'End', 'Read count', 'Read votes', 'Loop count', 'Bridge quality'] print_table([loop_table_header], fixed_col_widths=col_widths, left_align_header=False, alignments='RRRRRLRR', indent=0) for start, end, middle, repeat in loops: if middle is None: loop_table_row = [start, repeat, '', end] else: loop_table_row = [start, repeat, middle, end] forward_strand_reads = end_overlap_reads[start] & start_overlap_reads[end] reverse_strand_reads = end_overlap_reads[-end] & start_overlap_reads[-start] all_reads = list(forward_strand_reads) + list(reverse_strand_reads) strands = ['F'] * len(forward_strand_reads) + ['R'] * len(reverse_strand_reads) loop_table_row.append(len(all_reads)) # This dictionary will collect the votes. The key is the number of times through the loop # and the value is the vote count. Votes for -1 times through the loop occur for reads # which don't conform to the loop assumption. votes = defaultdict(int) # We'll try a range of repeat counts. The segment depth gives us a first guess as to the # repeat count, which guides how high we should test. mean_start_end_depth = (graph.segments[abs(start)].depth + graph.segments[abs(end)].depth) / 2 if middle is None: repeat_depth = graph.segments[abs(repeat)].depth best_repeat_guess = int(round(repeat_depth / mean_start_end_depth)) - 1 else: middle_depth = graph.segments[abs(middle)].depth best_repeat_guess = int(round(middle_depth / mean_start_end_depth)) best_repeat_guess = max(1, best_repeat_guess) max_tested_loop_count = (best_repeat_guess + 1) * 2 # Use a simple loop if we only have one thread. if threads == 1: for read, strand in zip(all_reads, strands): vote = get_read_loop_vote(start, end, middle, repeat, strand, minimap_alignments, read, read_dict, graph, max_tested_loop_count, scoring_scheme) votes[vote] += 1 # Use a thread pool if we have more than one thread. else: pool = ThreadPool(threads) arg_list = [] for read, strand in zip(all_reads, strands): arg_list.append((start, end, middle, repeat, strand, minimap_alignments, read, read_dict, graph, max_tested_loop_count, scoring_scheme)) for vote in pool.imap_unordered(get_read_loop_vote_one_arg, arg_list): votes[vote] += 1 # Format the vote totals nicely for the table. vote_str = '' for loop_count in sorted(votes.keys()): if loop_count == -1: vote_str += 'bad: ' elif loop_count == 1: vote_str += '1 loop: ' else: vote_str += str(loop_count) + ' loops: ' vote_count = votes[loop_count] vote_str += str(vote_count) + ' vote' + ('s' if vote_count != 1 else '') + ' ' loop_table_row.append(vote_str.strip()) # Determine the repeat count which wins! results = sorted(list(votes.items()), key=lambda x: x[1], reverse=True) if not results: loop_table_row += ['no reads', ''] else: winning_loop_count = results[0][0] winning_votes = results[0][1] if len(results) == 1: second_best_votes = 0 votes_against = 0 else: second_best_votes = results[1][1] votes_against = sum(r[1] for r in results) - winning_votes if winning_loop_count == -1: loop_table_row += ['bad reads', ''] elif winning_votes == second_best_votes: loop_table_row += ['tie vote', ''] else: # If we got here, then we're good to bridge! loop_table_row.append(str(winning_loop_count)) bridge_path = [repeat] for _ in range(winning_loop_count): if middle is not None: bridge_path.append(middle) bridge_path.append(repeat) bridges.append(SimpleLongReadBridge(graph, start, end, bridge_path, winning_votes, votes_against)) loop_table_row.append(float_to_str(bridges[-1].quality, 1)) print_table([loop_table_row], fixed_col_widths=col_widths, header_format='normal', alignments='RRRRRLRR', left_align_header=False, bottom_align_header=False, sub_colour={'bad reads': 'red', 'no reads': 'red', 'tie vote': 'red'}, indent=0) return bridges
def _send_bulk(smss, uses_multiprocessing=True, log_level=None, threads=4): # Multiprocessing does not play well with database connection # Fix: Close connections on forking process # https://groups.google.com/forum/#!topic/django-users/eCAIY9DAfG0 if uses_multiprocessing: db_connection.close() if log_level is None: log_level = get_log_level() sent_smses = [] failed_smses = [] sms_count = len(smss) logger.info('Process started, sending %s sms' % sms_count) def send(sms): try: sms.dispatch(log_level=log_level, commit=False) sent_smses.append(sms) logger.debug('Successfully sent sms #%d' % sms.id) except Exception as e: logger.debug('Failed to send sms #%d' % sms.id) failed_smses.append((sms, e)) number_of_threads = min(threads, sms_count) pool = ThreadPool(number_of_threads) pool.map(send, smss) pool.close() pool.join() pool.terminate() # update statuses of sent and failed_smses for sms in sent_smses: sms.save() for (sms, _) in failed_smses: sms.save() if log_level >= 1: logs = [] for (sms, exception) in failed_smses: logs.append( Log(sms=sms, status=STATUS.failed, message=str(exception), exception_type=type(exception).__name__)) if logs: Log.objects.bulk_create(logs) if log_level == 2: logs = [] for sms in sent_smses: logs.append(Log(sms=sms, status=STATUS.sent)) if logs: Log.objects.bulk_create(logs) sent_count = len(sent_smses) failed_count = len(failed_smses) logger.info('Process finished, %s attempted, %s sent, %s failed' % (sms_count, sent_count, failed_count)) return (sent_count, failed_count)
pre_embed_dim = data.type_num * info_section output_dim = max(data.train_list[:, 1].tolist()) + 1 # 隐藏单元节点数 两层 DEVICE = "cuda" if torch.cuda.is_available() else "cpu" # 获取预处理数据 assert select_method in ["end_node", "all_node"] x = data.x train_list = data.train_list # 训练节点/数据对应的标签 test_list = data.test_list # 测试节点/数据对应的索引 val_list = data.val_list # 验证节点/数据对应的索引 criterion = nn.CrossEntropyLoss().to(DEVICE) num_thread = 12 pool = ThreadPool(num_thread) # 此处枚举metapath并初始化选择模型 metapath_name = mp.enum_metapath_name(data.get_metapath_name(), data.get_metapath_dict(), metapath_length) metapath_list = mp.enum_longest_metapath_index(data.get_metapath_name(), data.get_metapath_dict(), metapath_length) select_model = GraphMSE(metapath_list=metapath_name, input_dim=input_dim, pre_embed_dim=pre_embed_dim, select_dim=output_dim, mlp_settings=mlp_settings).to(DEVICE) train(model=select_model, epochs=100, method="all_node", ablation=ablation)
def main(): p = ThreadPool() p.map_async(goto_wc, range(50)) p.close() p.join()
removal_list=Removal_list, isotypes=Isotypes) B = SpadeGating(filename=B_Filename, gates=Gates, parameters=Parameters, source=Source, target=Target, removal_list=Removal_list, isotypes=Isotypes) plate_plate = pd.Series() antigens_to_test = [ i for i in A.data.columns if i not in backbone_antigens ] pool = ThreadPool(num_workers) costs = pool.map(worker, antigens_to_test) pool.close() pool.join() output["{}vs{}".format(Normal, Base)] = dict(zip(antigens_to_test, costs)) writing_out = pd.DataFrame(output) writing_out.to_csv(output_file) c = [i for i in writing_out.columns if i not in ['Untitled: 0']] #writing_out['mean'] = writing_out.mean(axis=1) #writing_out['rms'] = np.sqrt(sum(output[c]*output[c],axis=1)) #writing_out.to_csv(output_file)
def calculateParallel(numbers, threads=10): pool = ThreadPool(threads) results = pool.map(squareNumber, numbers) pool.close() #pool.join() return results
# For trajectory storage import h5py import uuid # OpenBLAS(used by OpenCV) changes CPU affinity os.sched_setaffinity(0, range(os.cpu_count())) def setaff(): os.sched_setaffinity(0, range(os.cpu_count())) # for Multi-threading from multiprocessing.dummy import Pool as ThreadPool pool = ThreadPool(5, setaff) # ======================================================================= def filter_trajs_displacement(trajs): num_trajs = len(trajs) disp_stor = np.empty((num_trajs, ), np.float32) for ii in range(num_trajs): disp_stor[ii] = np.sum( np.sqrt(np.sum((trajs[1:, :] - trajs[0:-1, :])**2, 1))) # Remove trajectories that have very low displacement good_trajs = np.flatnonzero(disp_stor > -1) return good_trajs
from PIL import Image, ImageFilter from multiprocessing.dummy import Pool as ThreadPool from skimage.filters import gaussian import numpy as np import scipy NUM_OF_THREADS = 1 SIGMA = 12 FINAL_SLICE_SIZE = 512 filename = 'k' dirname = './'+filename BLUR_DIRNAME = dirname+'_blurred' CROP_DIRNAME = dirname+'_cropped' pool = ThreadPool(NUM_OF_THREADS) # Load every image file in the provided directory filenames = [os.path.join(dirname, fname) for fname in os.listdir(dirname) if fname.endswith('.jpg')] filenames = np.asarray(filenames) filenames_split = np.array_split(filenames, NUM_OF_THREADS) print(filenames) if not os.path.exists(BLUR_DIRNAME): os.mkdir(BLUR_DIRNAME) if not os.path.exists(CROP_DIRNAME): os.mkdir(CROP_DIRNAME)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--api', help='Which API to use. Choose from (gc/azure/aws)', type=str) parser.add_argument('--api-key', help='API_KEY or the path to the Key fiile', type=str) parser.add_argument('--output-dir', help='Where to store the generated audio samples', type=str) parser.add_argument('--length', help='Length of captchas in characters', type=int) parser.add_argument('--count', help='How many captchas to generate', type=int) parser.add_argument('--scramble', help='Whether to scramble image names', default=False, action='store_true') parser.add_argument('--symbols', help='File with the symbols to use in captchas', type=str) args = parser.parse_args() if args.api is None: print("Please specify the cloud platform to use") exit(1) if args.api_key is None: print("Please specify the API_KEY or the path to the Key fiile") exit(1) if args.output_dir is None: print("Please specify the samples output directory") exit(1) if args.length is None: print("Please specify the captcha length") exit(1) if args.count is None: print("Please specify the captcha count to generate") exit(1) if args.symbols is None: print("Please specify the captcha symbols file") exit(1) symbols_file = open(args.symbols, 'r') symbols = symbols_file.readline().strip() if not os.path.exists(args.output_dir): print("Creating output directory " + args.output_dir) os.makedirs(args.output_dir) for i in range(args.count): captcha_text = ''.join( [random.choice(symbols) for j in range(args.length)]) # GOOGLE Cloud TTS API if args.api.lower() == "gc": os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = args.api_key client = texttospeech.TextToSpeechClient() voice_list = [] voices = client.list_voices() for voice in voices.voices: gender = texttospeech.enums.SsmlVoiceGender(voice.ssml_gender) if format(gender.name) == 'FEMALE': for language_code in voice.language_codes: if re.match('^en-.*', language_code) is not None: voice_list.append(format(voice.name)) for voice_name in voice_list: print('Google TextToSpeech API voice ', voice_name) response = gc_synthesize_text(client, captcha_text, voice_name).audio_content file_name = create_filename(args.scramble, args.output_dir, captcha_text) write_audio(response, file_name) sleep(0.2) # MICROSOFT Azure Cloud Cognitive Speeh API elif args.api.lower() == "azure": voice_list = [ "Microsoft Server Speech Text to Speech Voice (en-AU, Catherine)", "Microsoft Server Speech Text to Speech Voice (en-AU, HayleyRUS)", "Microsoft Server Speech Text to Speech Voice (en-CA, Linda)", "Microsoft Server Speech Text to Speech Voice (en-CA, HeatherRUS)", "Microsoft Server Speech Text to Speech Voice (en-GB, Susan, Apollo)", "Microsoft Server Speech Text to Speech Voice (en-GB, HazelRUS)", "Microsoft Server Speech Text to Speech Voice (en-US, ZiraRUS)", "Microsoft Server Speech Text to Speech Voice (en-US, JessaRUS)", "Microsoft Server Speech Text to Speech Voice (en-US, Jessa24kRUS)" ] speech_key, service_region = args.api_key, "westeurope" speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region) for voice in voice_list: file_name = create_filename(args.scramble, args.output_dir, captcha_text) azure_speech_synthesis_to_mp3_file(speech_config, " ".join(captcha_text), voice, file_name) sleep(0.5) # AWS Polly Speeh API elif args.api.lower() == 'aws': voice_list = [ 'Salli', 'Joanna', 'Kendra', 'Kimberly', 'Amy', 'Emma', 'Nicole', 'Raveena', 'Aditi' ] polly_client = boto3.Session( aws_access_key_id="AKIA3TQLBVKRT5JS7Y4O", aws_secret_access_key=args.api_key, region_name='eu-west-1').client('polly') for voice in voice_list: response = polly_client.synthesize_speech( VoiceId=voice, Engine="standard", OutputFormat='mp3', Text=" ".join(captcha_text)) response = response['AudioStream'].read() file_name = create_filename(args.scramble, args.output_dir, captcha_text) write_audio(response, file_name) # GTTS Library elif args.api.lower() == 'gtts': pool = ThreadPool(4) pool.map(my_function, my_array) gtts_instance = gTTS(text=captcha_text, lang='en', slow=False) file_name = create_filename(args.scramble, args.output_dir, captcha_text) gtts_instance.save(file_name)
def calculateParallel(flist, threads=2): pool = ThreadPool(threads) results = pool.map(do_prune, flist) pool.close() pool.join() return results
nlp = spacy.load('de') from lib import AgdistisEntityLinker linker = AgdistisEntityLinker() nlp.add_pipe(linker) import time start_time = time.time() from multiprocessing.dummy import Pool as ThreadPool #print(nlp.pipe_names) # Default processing components for en model pool = ThreadPool(8) file = sys.argv[1] file2 = sys.argv[2] line2 = open(file2,"w"); def annotate(text): #for text in open("testfile.txt", 'r+'): try: newtext = text; doc = nlp(newtext) position = 1; for ent in doc.ents:
def forward_request(input_json, master_name, pretty, debug): """ Forwards a request to the node who has all available information to answer it. This function is called when a distributed_master function is used. Only the master node calls this function. An API request will only be forwarded to worker nodes. :param input_json: API request: Example: {"function": "/agents", "arguments":{"limit":5}, "ossec_path": "/var/ossec", "from_cluster":false} :param master_name: Name of the master node. Necessary to check whether to forward it to a worker node or not. :param pretty: JSON pretty print :param debug: Debug :return: a JSON response. """ def forward(node_name, return_none=False): """ Forwards a request to a node. :param node_name: Node to forward a request to. :param return_none: Whether to return an error message or nothing (if there's an error forwarding the request). :return: a JSON response """ if node_name == 'unknown' or node_name == '': # if the agent is never connected or pending (i.e. its node name is unknown or empty), do the request locally response = json.loads( distribute_function(copy.deepcopy(input_json))) else: # if not, check if the node the request is being forwarded to is the master or a worker. command = 'dapi_forward {}'.format( node_name) if node_name != master_name else 'dapi' if command == 'dapi': # if it's the master, execute the request directly response = json.loads( distribute_function(copy.deepcopy(input_json), debug=debug)) else: # if it's a worker, forward it response = i_s.execute( '{} {}'.format(command, json.dumps(input_json)), input_json['arguments']['wait_for_complete']) if not isinstance(response, dict): # If there's an error and the flag return_none is not set, return a dictionary with the response. response = { 'error': 3016, 'message': str(WazuhException(3016, response)) } if not return_none else None return response def forward_list(item): """ Function called when there are multiple nodes to forward a request to. :param item: A dictionary with {node_name: [list of agents ids]} :return: JSON response of a single node """ name, agent_ids = item if agent_ids: input_json['arguments']['agent_id'] = agent_ids return forward(name, agent_ids == []) # get the node(s) who has all available information to answer the request. node_name, is_list = get_solver_node(input_json, master_name) input_json['from_cluster'] = True if is_list: # if there are multiple nodes to forward the request, create a ThreadPool and forward it in parallel. pool = ThreadPool(len(node_name)) responses = list( filter(lambda x: x is not None, pool.map(forward_list, node_name.items()))) pool.close() pool.join() final_json = {} response = merge_results(responses, final_json, input_json) else: response = forward(node_name) data, error = __split_response_data(response) return print_json(data=data, pretty=pretty, error=error)
incremented = 0 parameter = 0 q = Queue.Queue() threadQ = Queue.Queue() def putToQ(tupleValQ): time.sleep(0.1) global threadQ tupleValQ[1].put(tupleValQ[0]) threadQ.put(threading.current_thread()) #queue.put(val) val, queue return tupleValQ[0] pool = ThreadPool(7) results = pool.map(putToQ, zip([i for i in range(20)], [q] * 20)) #threads start not in array order print "queue: ", while not q.empty(): print q.get(), print " " print "threadQueue: ", while not threadQ.empty(): print threadQ.get() print "results: ", results
import socket from multiprocessing.dummy import Pool as ThreadPool import time #server_IP = '127.0.0.1' server_IP = '106.14.161.204' server_Port = 3389 def tryOnceConnect(args): print("thread args = %s"%args) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((server_IP, server_Port)) print(s.recv(1024).decode('utf-8')) for data in [b'Michael', b'Tracy', b'Sarah']: s.send(data) print(s.recv(1024).decode('utf-8')) s.send((b'exit')) s.close() #main args = [] for i in range(0,1024): args.append(i) pool = ThreadPool(1024) time_start = time.time() result = pool.map(tryOnceConnect, args) pool.close() pool.join() time_end = time.time() print "%s 100线程 共消耗:"%str(time_end-time_start)
def _parallel_augment(cls, action_fx, data, n, num_thread=2): pool = ThreadPool(num_thread) results = pool.map(action_fx, [data] * n) pool.close() pool.join() return results
with open('summary.txt', "w+") as f: pass ds = [a for a in AnmalZoo] ds.remove(AnmalZoo.Hamming) ds.remove(AnmalZoo.Levenshtein) ds.remove(AnmalZoo.EntityResolution) ds.remove(AnmalZoo.Dotstar) ds.remove(AnmalZoo.PowerEN) ds.remove(AnmalZoo.Brill) ds.remove(AnmalZoo.RandomForest) ds.remove(AnmalZoo.Dotstar03) ds.remove(AnmalZoo.Dotstar06) ds.remove(AnmalZoo.Dotstar09) ds.remove(AnmalZoo.Protomata) ds.remove(AnmalZoo.Ranges05) ds.remove(AnmalZoo.Bro217) ds.remove(AnmalZoo.Ranges1) ds.remove(AnmalZoo.ExactMath) ds.remove(AnmalZoo.Custom) ds.remove(AnmalZoo.TCP) ds.remove(AnmalZoo.Synthetic_BlockRings) thread_count = 8 t_pool = ThreadPool(thread_count) results = t_pool.map(process_single_ds, ds) t_pool.close() t_pool.join()
def main(): import sys import getopt from glob import glob args, img_mask = getopt.getopt(sys.argv[1:], '', ['debug=', 'square_size=', 'threads=']) args = dict(args) args.setdefault('--debug', './output/') args.setdefault('--square_size', 1.0) args.setdefault('--threads', 4) if not img_mask: img_mask = 'C:/Users/wangl/GRIP/images/*.jpeg' # default # img_mask = '../data/left??.jpg' # default else: img_mask = img_mask[0] img_names = glob(img_mask) debug_dir = args.get('--debug') if debug_dir and not os.path.isdir(debug_dir): os.mkdir(debug_dir) square_size = float(args.get('--square_size')) pattern_size = (9, 6) pattern_points = np.zeros((np.prod(pattern_size), 3), np.float32) pattern_points[:, :2] = np.indices(pattern_size).T.reshape(-1, 2) pattern_points *= square_size obj_points = [] img_points = [] h, w = cv.imread(img_names[0], cv.IMREAD_GRAYSCALE ).shape[:2] # TODO: use imquery call to retrieve results def processImage(fn): print('processing %s... ' % fn) img = cv.imread(fn, 0) if img is None: print("Failed to load", fn) return None assert w == img.shape[1] and h == img.shape[0], ( "size: %d x %d ... " % (img.shape[1], img.shape[0])) found, corners = cv.findChessboardCorners(img, pattern_size) if found: term = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_COUNT, 30, 0.1) cv.cornerSubPix(img, corners, (5, 5), (-1, -1), term) if debug_dir: vis = cv.cvtColor(img, cv.COLOR_GRAY2BGR) cv.drawChessboardCorners(vis, pattern_size, corners, found) _path, name, _ext = splitfn(fn) outfile = os.path.join(debug_dir, name + '_chess.png') cv.imwrite(outfile, vis) if not found: print('chessboard not found') return None print(' %s... OK' % fn) return (corners.reshape(-1, 2), pattern_points) threads_num = int(args.get('--threads')) if threads_num <= 1: chessboards = [processImage(fn) for fn in img_names] else: print("Run with %d threads..." % threads_num) from multiprocessing.dummy import Pool as ThreadPool pool = ThreadPool(threads_num) chessboards = pool.map(processImage, img_names) chessboards = [x for x in chessboards if x is not None] for (corners, pattern_points) in chessboards: img_points.append(corners) obj_points.append(pattern_points) # calculate camera distortion rms, camera_matrix, dist_coefs, _rvecs, _tvecs = cv.calibrateCamera( obj_points, img_points, (w, h), None, None) print("\nRMS:", rms) print("camera matrix:\n", camera_matrix) print("distortion coefficients: ", dist_coefs.ravel()) # undistort the image with the calibration print('') for fn in img_names if debug_dir else []: _path, name, _ext = splitfn(fn) img_found = os.path.join(debug_dir, name + '_chess.png') outfile = os.path.join(debug_dir, name + '_undistorted.png') img = cv.imread(img_found) if img is None: continue h, w = img.shape[:2] newcameramtx, roi = cv.getOptimalNewCameraMatrix( camera_matrix, dist_coefs, (w, h), 1, (w, h)) dst = cv.undistort(img, camera_matrix, dist_coefs, None, newcameramtx) # crop and save the image x, y, w, h = roi dst = dst[y:y + h, x:x + w] print('Undistorted image written to: %s' % outfile) cv.imwrite(outfile, dst) print('Done')
import logging import itertools SIO_configuration = configuration.SIOconfiguration() FIO_instance = FIO('10.139.218.26') SCLI = scli.SCLI(sio_config=SIO_configuration) SIOInfraHandler = scli.SIOInfraHandler() SIOInfraGather = scli.SIOInfraGather(SCLI, SIOInfraHandler) IntegrationConfigInstance = configuration.Integration() MainLogger = logger_init.logging_config( integration_config=IntegrationConfigInstance, logging_mode='DEBUG', log_to_file=False, executable_path=__file__) pool = ThreadPool(5) base_volume = 'vol1' vol_size_in_tb = 32 sdc_ip_A = '192.168.247.16' sdc_ip_M = '10.139.218.26' SCLI.login() def make_mh_full_snap(base_volume_func: str, snapshot_name_func: str, sdc_ip_a_func: str, sdc_ip_m_func: str, vol_size_in_tb_func: int): scini_guid = False write_offset = 8796093022208 logger = logging.getLogger()
batch_start = i batch_end = i + seg * batch_size with open(vid_data_file, data_mode) as f: #Open data file # for vids_in_run in VidBatcherSingle( batch_start, batch_end, seg ):#loop through this segment of ids # vids_in_run = [ str(i) for i in vids_in_run ] # records = crawler(vids_in_run) # # with ThreadPool(2) as p: # records = p.map(crawler, [vids_in_run]) # # file_cache = file_cache + records[0] # print(file_cache) with ThreadPool(prs) as p: records = p.map(crawler, [[ str(i) for i in vids_in_run ] for vids_in_run in VidBatcherSingle(batch_start, batch_end, seg) ]) # print('records length: ' + str( len(records) ) + '\n' ) # print('each record length: ' + str( len(records[0]) ) + '\n' ) # print('0 the element of the first record length: ' + str( len(records[0][0]) ) + '\n' ) # print(records) records_flatten = [item for sublist in records for item in sublist] VidDumpBatch(records_flatten, f) print('job finished at: ' + str(datetime.datetime.now()) + '\n')
def probMask(A, P, X, ii, jj): def cross_product(a, b): return a[0] * b[1] - a[1] * b[0] def iscross(pairData): A, B = pairData[0] C, D = pairData[1] AC = C - A AD = D - A BC = C - B BD = D - B CA = -AC CB = -BC DA = -AD DB = -BD return 1 if cross_product(AC, AD) * cross_product( BC, BD) < 0 and cross_product(CA, CB) * cross_product( DA, DB) < 0 else 0 def calprobMask(pairData): Adj, Pro, Xfe, ii, jj = pairData Adj = Adj[:ii, :ii] Pro = Pro[:jj - ii, :] Xfe = Xfe[:jj] # existed ridges edgesList = [] edgeSet = set() edgesIdxList = [] # G = nx.Graph() for i in range(ii): for j in range(i): if Adj[i][j] == 1 and (i, j) not in edgeSet: edgesList.append([Xfe[i][:2], Xfe[j][:2]]) edgesIdxList.append((i, j)) # G.add_edge(i, j) edgeSet.add((i, j)) newNodes = Xfe[ii:jj] newp = Pro for idx in range(len(newNodes)): for pidx in range(len(Pro[idx])): newedge = [Xfe[pidx][:2], Xfe[ii + idx][:2]] # edgesIdxList.append((pidx,ii+idx)) for i, edge in enumerate(edgesList): if pidx in edgesIdxList[i]: continue if iscross([edge, newedge]): newp[idx, pidx] = 0 break return newp newP = [] pool = ThreadPool(4) tasks = [[A[i], P[i], X[i], ii, jj] for i in range(len(A))] newP = pool.map(calprobMask, tasks) pool.close() pool.join() return newP
print(driver.title) driver.quit() def gethtml1(url): prefs = {"profile.managed_default_content_settings.images": 2} chrome_options = Options() chrome_options.add_argument('--headless') chrome_options.add_experimental_option("prefs", prefs) driver = webdriver.Chrome(chrome_options=chrome_options) driver.get(url) print(driver.title) driver.quit() #driver.close() myurls = [ 'http://www.gzzbw.cn/trade/?category=affiche', ] #gethtml("http://www.gzzbw.cn/trade/?category=affiche") pool = ThreadPool(4) #results = pool.map(urllib.request.urlopen, urls) #results = pool.map(gethtml, myurls) #gethtml1('https://www.python.org/community/awards/') results = pool.map(gethtml1, urls) results = pool.map(gethtml, myurls) pool.close() pool.join()
line = fp.readline() if line: libs.append(line.strip()) else: break fp.close() return libs def redisCheck(ip): try: r = redis.Redis(host=ip,port=6379,db=0,socket_timeout=120) return {ip:r.ping()} except: return {ip:False} if __name__ == '__main__': socket.setdefaulttimeout(10) filepath,threed_num = argv[1:] ips = readText(filepath) pool = ThreadPool(int(threed_num)) results = pool.map(redisCheck,ips) pool.close() pool.join() result = {} for x in results: result = dict(result,**x) fp = open('redisOut.txt','w') for i in result: fp.write(i + ' ' + str(result[i]) + '\n') fp.close