def login(self, server, username=None, password=None, server_data=None): if username is None: username = getpass.getuser() try: print('Logging in to {}@{}'.format(username, server)) self._ssh = SSHClient() self._ssh.connect(server, username, password) self._ssh.get_transport().set_keepalive(30) except: # noqa: E722 self._ssh = None raise if server_data is not None: try: sys.stdout.flush() print( 'Please wait for a new verification code before logging in to {}' .format(server_data), file=sys.stderr, flush=True) print('Logging in to {}@{}'.format(username, server_data)) self._ssh_data = SSHClient() self._ssh_data.connect(server_data, username, password) self._ssh_data.get_transport().set_keepalive(30) except: # noqa: E722 self._ssh_data = None raise return self
def handle_selectchallenge(self, sess): response_page = BeautifulSoup(sess.text, 'html.parser') challenges = [] for i in response_page.select('form[data-challengeentry]'): action = i.attrs.get("action") if "challenge/totp/" in action: challenges.append([ 'TOTP (Google Authenticator)', i.attrs.get("data-challengeentry") ]) elif "challenge/ipp/" in action: challenges.append(['SMS', i.attrs.get("data-challengeentry")]) elif "challenge/iap/" in action: challenges.append( ['SMS other phone', i.attrs.get("data-challengeentry")]) elif "challenge/sk/" in action: challenges.append( ['YubiKey', i.attrs.get("data-challengeentry")]) elif "challenge/az/" in action: challenges.append( ['Google Prompt', i.attrs.get("data-challengeentry")]) print('Choose MFA method from available:') for i, mfa in enumerate(challenges, start=1): print("{}: {}".format(i, mfa[0])) selected_challenge = input("Enter MFA choice number (1): ") or None if selected_challenge is not None and int(selected_challenge) <= len( challenges): selected_challenge = int(selected_challenge) - 1 else: selected_challenge = 0 challenge_id = challenges[selected_challenge][1] print("MFA Type Chosen: {}".format(challenges[selected_challenge][0])) # We need the specific form of the challenge chosen challenge_form = response_page.find( 'form', {'data-challengeentry': challenge_id}) payload = {} for tag in challenge_form.find_all('input'): if tag.get('name') is None: continue payload[tag.get('name')] = tag.get('value') if response_page.find('input', {'name': 'TrustDevice'}) is not None: payload['TrustDevice'] = 'on' # POST to google with the chosen challenge return self.post(self.base_url + challenge_form.get('action'), data=payload)
def parse_saml(self): if self.session_state is None: raise RuntimeError( 'You must use do_login() before calling parse_saml()') parsed = BeautifulSoup(self.session_state.text, 'html.parser') try: saml_element = parsed.find('input', { 'name': 'SAMLResponse' }).get('value') except: if self.save_failure: print( "SAML lookup failed, storing failure page to 'saml.html' to assist with debugging." ) with open("saml.html", 'w') as out: out.write(self.session_state.text.encode('utf-8')) raise ExpectedGoogleException( 'Something went wrong - Could not find SAML response, check your credentials or use --save-failure-html to debug.' ) return base64.b64decode(saml_element)
def get(self, url): try: response = self.check_for_failure(self.session.get(url)) except requests.exceptions.ConnectionError as e: print( 'There was a connection error, check your network settings: {}'. format(e)) sys.exit(1) except requests.exceptions.Timeout as e: print('The connection timed out, please try again: {}'.format(e)) sys.exit(1) except requests.exceptions.TooManyRedirects as e: print('The number of redirects exceeded the maximum allowed: {}'. format(e)) sys.exit(1) return response
def interactive_mode(args): """Interactive mode""" print('Start interactive mode', file=sys.stderr) parser = StandaloneParser(args.parser, args.load_model, args.example_preprocessor, beam_size=args.beam_size, cuda=args.cuda) while True: utterance = input('Query:').strip() hypotheses = parser.parse(utterance, debug=True) for hyp_id, hyp in enumerate(hypotheses): print('------------------ Hypothesis %d ------------------' % hyp_id) print(hyp.code)
from __future__ import print_function from . import _version import sys import requests import json import base64 from bs4 import BeautifulSoup from six.moves import urllib_parse, input from six import print_ as print # The U2F USB Library is optional, if it's there, include it. try: from . import u2f except ImportError: print("Failed to import U2F libraries, U2F login unavailable. Other " "methods can still continue.") class ExpectedGoogleException(Exception): def __init__(self, *args): super(ExpectedGoogleException, self).__init__(*args) class Google: def __init__(self, config): """The Google object holds authentication state for a given session. You need to supply: username: FQDN Google username, eg [email protected] password: obvious idp_id: Google's assigned IdP identifier for your G-suite account
def do_login(self): self.session = requests.Session() self.session.headers[ 'User-Agent'] = "AWS Sign-in/{} (Cevo aws-google-auth)".format( self.version) sess = self.get(self.login_url) # Collect information from the page source first_page = BeautifulSoup(sess.text, 'html.parser') gxf = first_page.find('input', {'name': 'gxf'}).get('value') self.cont = first_page.find('input', {'name': 'continue'}).get('value') page = first_page.find('input', {'name': 'Page'}).get('value') sign_in = first_page.find('input', {'name': 'signIn'}).get('value') account_login_url = first_page.find('form', { 'id': 'gaia_loginform' }).get('action') payload = { 'bgresponse': 'js_disabled', 'checkConnection': '', 'checkedDomains': 'youtube', 'continue': self.cont, 'Email': self.config.username, 'gxf': gxf, 'identifier-captcha-input': '', 'identifiertoken': '', 'identifiertoken_audio': '', 'ltmpl': 'popup', 'oauth': 1, 'Page': page, 'Passwd': '', 'PersistentCookie': 'yes', 'ProfileInformation': '', 'pstMsg': 0, 'sarp': 1, 'scc': 1, 'SessionState': '', 'signIn': sign_in, '_utf8': '?', } # GALX is sometimes not there try: galx = first_page.find('input', {'name': 'GALX'}).get('value') payload['GALX'] = galx except: pass # POST to account login info page, to collect profile and session info sess = self.post(account_login_url, data=payload) self.session.headers['Referer'] = sess.url # Collect ProfileInformation, SessionState, signIn, and Password Challenge URL challenge_page = BeautifulSoup(sess.text, 'html.parser') profile_information = challenge_page.find('input', { 'name': 'ProfileInformation' }).get('value') session_state = challenge_page.find('input', { 'name': 'SessionState' }).get('value') sign_in = challenge_page.find('input', {'name': 'signIn'}).get('value') passwd_challenge_url = challenge_page.find('form', { 'id': 'gaia_loginform' }).get('action') # Update the payload payload['SessionState'] = session_state payload['ProfileInformation'] = profile_information payload['signIn'] = sign_in payload['Passwd'] = self.config.password # POST to Authenticate Password sess = self.post(passwd_challenge_url, data=payload) response_page = BeautifulSoup(sess.text, 'html.parser') error = response_page.find(class_='error-msg') cap = response_page.find('input', {'name': 'logincaptcha'}) # Were there any errors logging in? Could be invalid username or password # There could also sometimes be a Captcha, which means Google thinks you, # or someone using the same outbound IP address as you, is a bot. if error is not None: raise ExpectedGoogleException('Invalid username or password') self.check_extra_step(response_page) if cap is not None: raise ExpectedGoogleException( 'Captcha Required. Manually Login to remove this.') self.session.headers['Referer'] = sess.url if "selectchallenge/" in sess.url: sess = self.handle_selectchallenge(sess) # Was there an MFA challenge? if "challenge/totp/" in sess.url: error_msg = "" while error_msg is not None: sess = self.handle_totp(sess) error_msg = self.parse_error_message(sess) if error_msg is not None: print(error_msg) elif "challenge/ipp/" in sess.url: sess = self.handle_sms(sess) elif "challenge/az/" in sess.url: sess = self.handle_prompt(sess) elif "challenge/sk/" in sess.url: sess = self.handle_sk(sess) elif "challenge/iap/" in sess.url: sess = self.handle_iap(sess) elif "challenge/ootp/5" in sess.url: raise NotImplementedError( 'Offline Google App OOTP not implemented') # ... there are different URLs for backup codes (printed) # and security keys (eg yubikey) as well # save for later self.session_state = sess
def handle_prompt(self, sess): response_page = BeautifulSoup(sess.text, 'html.parser') challenge_url = sess.url.split("?")[0] data_key = response_page.find('div', { 'data-api-key': True }).get('data-api-key') data_tx_id = response_page.find('div', { 'data-tx-id': True }).get('data-tx-id') # Need to post this to the verification/pause endpoint await_url = "https://content.googleapis.com/cryptauth/v1/authzen/awaittx?alt=json&key={}".format( data_key) await_body = {'txId': data_tx_id} self.check_prompt_code(response_page) print( "Open the Google App, and tap 'Yes' on the prompt to sign in ...") self.session.headers['Referer'] = sess.url retry = True response = None while retry: try: response = self.post(await_url, json_data=await_body) retry = False except requests.exceptions.HTTPError as ex: if not ex.response.status_code == 500: raise ex parsed_response = json.loads(response.text) payload = { 'challengeId': response_page.find('input', { 'name': 'challengeId' }).get('value'), 'challengeType': response_page.find('input', { 'name': 'challengeType' }).get('value'), 'continue': response_page.find('input', { 'name': 'continue' }).get('value'), 'scc': response_page.find('input', { 'name': 'scc' }).get('value'), 'sarp': response_page.find('input', { 'name': 'sarp' }).get('value'), 'checkedDomains': 'youtube', 'checkConnection': 'youtube:1295:1', 'pstMsg': 0, 'TL': response_page.find('input', { 'name': 'TL' }).get('value'), 'gxf': response_page.find('input', { 'name': 'gxf' }).get('value'), 'token': parsed_response['txToken'], 'action': response_page.find('input', { 'name': 'action' }).get('value'), 'TrustDevice': 'on', } return self.post(challenge_url, data=payload)
nb_top=nb_top, multi_index=multi_index, nb_try=nb_try, verbose=verbose) job[i]['solutions'] = solutions comm.send(job, dest=0) stop = comm.recv(source=0) print('worker %d is exiting' % rank) if __name__ == '__main__': # save command command_list = sys.argv command = ' '.join(command_list) with open('spind.com', 'w') as f: f.write('%s\n' % command) comm = MPI.COMM_WORLD size = comm.Get_size() if size == 1: print('Run SPIND with at least 2 processes!') sys.exit() rank = comm.Get_rank() args = docopt(__doc__) if rank == 0: print(args, flush=True) master_run(args) else: worker_run(args)
# This script shows how to use the client in anonymous mode # against jira.atlassian.com. from six import print_ as print from jira.client import GreenHopper # By default, the client will connect to a JIRA instance started from the Atlassian Plugin SDK # (see https://developer.atlassian.com/display/DOCS/Installing+the+Atlassian+Plugin+SDK for details). # Override this with the options parameter. # GreenHopper is a plugin in a JIRA instance options = { 'server': 'https://jira.atlassian.com' } gh = GreenHopper(options) # Get all boards viewable by anonymous users. boards = gh.boards() # Get the sprints in a specific board board_id = 441 print("GreenHopper board: %s (%s)" % (boards[0].name, board_id)) sprints = gh.sprints(board_id) # List the incomplete issues in each sprint for sprint in sprints: sprint_id = sprint.id print("Sprint: %s" % sprint.name) incompleted_issues = gh.incompleted_issues(board_id, sprint_id) print("Incomplete issues: %s" % ', '.join(issue.key for issue in incompleted_issues))
import time import six from chainer import serializers from chainer import cuda import chainer.optimizers as O import numpy as np from lda2vec import prepare_topics, print_top_words_per_topic from lda2vec import utils from lda import LDA gpu_id = int(os.getenv('CUDA_GPU', 0)) cuda.get_device(gpu_id).use() six.print("Using GPU " + str(gpu_id)) vocab = pickle.load(open('vocab.pkl', 'rb')) corpus = pickle.load(open('corpus.pkl', 'rb')) bow = np.load("bow.npy").astype('float32') # Remove bow counts on the first two tokens, which <SKIP> and <EOS> bow[:, :2] = 0 # Normalize bag of words to be a probability # bow = bow / bow.sum(axis=1)[:, None] # Number of docs n_docs = bow.shape[0] # Number of unique words in the vocabulary n_vocab = bow.shape[1] # Number of dimensions in a single word vector n_units = 256
def handle_captcha(self, sess, payload): response_page = BeautifulSoup(sess.text, 'html.parser') # Collect ProfileInformation, SessionState, signIn, and Password Challenge URL profile_information = response_page.find('input', { 'name': 'ProfileInformation' }).get('value') session_state = response_page.find('input', { 'name': 'SessionState' }).get('value') sign_in = response_page.find('input', {'name': 'signIn'}).get('value') passwd_challenge_url = response_page.find('form', { 'id': 'gaia_loginform' }).get('action') # Update the payload payload['SessionState'] = session_state payload['ProfileInformation'] = profile_information payload['signIn'] = sign_in payload['Passwd'] = self.config.password # Get all captcha challenge tokens and urls captcha_container = response_page.find('div', {'class': 'captcha-container'}) captcha_logintoken = captcha_container.find('input', { 'name': 'logintoken' }).get('value') captcha_url = captcha_container.find('input', { 'name': 'url' }).get('value') captcha_logintoken_audio = captcha_container.find( 'input', { 'name': 'logintoken_audio' }).get('value') captcha_url_audio = captcha_container.find('input', { 'name': 'url_audio' }).get('value') open_image = True # Check if there is a display utility installed as Image.open(f).show() do not raise any exception if not # if neither xv or display are available just display the URL for the user to visit. if os.name == 'posix' and sys.platform != 'darwin': if find_executable('xv') is None and find_executable( 'display') is None: open_image = False print("Please visit the following URL to view your CAPTCHA: {}".format( captcha_url)) if open_image: try: with requests.get(captcha_url) as url: with io.BytesIO(url.content) as f: Image.open(f).show() except Exception: pass try: captcha_input = raw_input("Captcha (case insensitive): ") or None except NameError: captcha_input = input("Captcha (case insensitive): ") or None # Update the payload payload['logincaptcha'] = captcha_input payload['logintoken'] = captcha_logintoken payload['url'] = captcha_url payload['logintoken_audio'] = captcha_logintoken_audio payload['url_audio'] = captcha_url_audio return self.post(passwd_challenge_url, data=payload)
def handle_selectchallenge(self, sess): response_page = BeautifulSoup(sess.text, 'html.parser') # Known mfa methods, 5 is disabled till its implemented auth_methods = { 2: 'TOTP (Google Authenticator)', 3: 'SMS', 4: 'OOTP (Google Prompt)' # 5: 'OOTP (Google App Offline Security Code)' } unavailable_challenge_ids = [ int(i.attrs.get('data-unavailable')) for i in response_page.find_all( lambda tag: tag.name == 'form' and 'data-unavailable' in tag.attrs ) ] # ootp via google app offline code isn't implemented. make sure its not valid. unavailable_challenge_ids.append(5) challenge_ids = [ int(i.get('value')) for i in response_page.find_all('input', {'name': 'challengeId'}) if int(i.get('value')) not in unavailable_challenge_ids ] challenge_ids.sort() auth_methods = { k: auth_methods[k] for k in challenge_ids if k in auth_methods and k not in unavailable_challenge_ids } print('Choose MFA method from available:') print('\n'.join( '{}: {}'.format(*i) for i in list(auth_methods.items()))) selected_challenge = input("Enter MFA choice number ({}): ".format( challenge_ids[-1:][0])) or None if selected_challenge is not None and int( selected_challenge) in challenge_ids: challenge_id = int(selected_challenge) else: # use the highest index as that will default to prompt, then sms, then totp, etc. challenge_id = challenge_ids[-1:][0] print("MFA Type Chosen: {}".format(auth_methods[challenge_id])) # We need the specific form of the challenge chosen challenge_form = response_page.find( 'form', {'data-challengeentry': challenge_id}) payload = { 'challengeId': challenge_id, 'challengeType': challenge_form.find('input', { 'name': 'challengeType' }).get('value'), 'continue': challenge_form.find('input', { 'name': 'continue' }).get('value'), 'scc': challenge_form.find('input', { 'name': 'scc' }).get('value'), 'sarp': challenge_form.find('input', { 'name': 'sarp' }).get('value'), 'checkedDomains': challenge_form.find('input', { 'name': 'checkedDomains' }).get('value'), 'pstMsg': challenge_form.find('input', { 'name': 'pstMsg' }).get('value'), 'TL': challenge_form.find('input', { 'name': 'TL' }).get('value'), 'gxf': challenge_form.find('input', { 'name': 'gxf' }).get('value'), 'subAction': challenge_form.find('input', { 'name': 'subAction' }).get('value'), } if challenge_form.find('input', {'name': 'SendMethod'}) is not None: payload['SendMethod'] = challenge_form.find( 'input', { 'name': 'SendMethod' }).get('value') # POST to google with the chosen challenge return self.post( self.base_url + challenge_form.get('action'), data=payload)
# save last cxi batch if not empty if len(batch) > 0: # save full cxi cxi_file = os.path.join(hit_dir, '%s-rank%d-job%d.cxi' % (prefix, rank, count)) util.save_full_cxi(batch, cxi_file, cxi_entry_id=cxi_entry_id, mask_file=mask_file, extra_datasets=extra_datasets, cxi_dtype=cxi_dtype, shuffle=shuffle) sys.stdout.flush() done = True comm.send(done, dest=0) if __name__ == '__main__': comm = MPI.COMM_WORLD size = comm.Get_size() if size == 1: print('Run batch peak2cxi with at least 2 processes!') sys.exit() rank = comm.Get_rank() args = docopt(__doc__) if rank == 0: master_run(args) else: worker_run(args)
def master_run(args): flush = args['--flush'] # mkdir if not exist hit_dir = args['<hit-dir>'] if not os.path.isdir(hit_dir): os.mkdir(hit_dir) peak_file = args['<peak-file>'] peak_info = np.load(peak_file) batch_size = int(args['--batch-size']) nb_jobs = int(np.ceil(len(peak_info) / batch_size)) # collect jobs ids = np.array_split(np.arange(len(peak_info)), nb_jobs) jobs = [] for i in range(len(ids)): jobs.append(peak_info[ids[i]]) print('%d jobs, %d frames to be processed' % (nb_jobs, len(peak_info)), flush=flush) # other parameters buffer_size = int(args['--buffer-size']) update_freq = int(args['--update-freq']) # dispatch jobs job_id = 0 reqs = {} workers = set(range(1, size)) finished_workers = set() time_start = time.time() for worker in workers: if job_id < nb_jobs: job = jobs[job_id] else: job = [] # dummy job comm.isend(job, dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) print('job %d/%d --> slave %d' % (job_id, nb_jobs, worker), flush=flush) job_id += 1 while job_id < nb_jobs: stop = False workers -= finished_workers time.sleep(0.1) # take a break for worker in workers: finished, result = reqs[worker].test() if finished: if job_id < nb_jobs: print('job %d/%d --> slave %d' % (job_id, nb_jobs, worker), flush=flush) comm.isend(stop, dest=worker) comm.isend(jobs[job_id], dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) job_id += 1 else: stop = True comm.isend(stop, dest=worker) print('stop signal --> %d' % worker, flush=flush) finished_workers.add(worker) if job_id % update_freq == 0: # update stat progress = float(job_id) / nb_jobs * 100 stat_dict = { 'progress': '%.2f%%' % progress, 'duration/sec': 'not finished', 'total jobs': nb_jobs, } stat_file = os.path.join(hit_dir, 'peak2cxi.yml') with open(stat_file, 'w') as f: yaml.dump(stat_dict, f, default_flow_style=False) all_done = False while not all_done: time.sleep(0.1) workers -= finished_workers all_done = True for worker in workers: finished, result = reqs[worker].test() if finished: stop = True comm.isend(stop, dest=worker) finished_workers.add(worker) else: all_done = False time_end = time.time() duration = time_end - time_start stat_dict = { 'progress': 'done', 'duration/sec': duration, 'total jobs': nb_jobs, } stat_file = os.path.join(hit_dir, 'peak2cxi.yml') with open(stat_file, 'w') as f: yaml.dump(stat_dict, f, default_flow_style=False) print('All Done!', flush=flush) MPI.Finalize()
def master_run(args): flush = args['--flush'] # mkdir if not exist hit_dir = args['<hit-dir>'] if not os.path.isdir(hit_dir): os.makedirs(hit_dir) input_lst = args['<input-lst>'] min_peaks = int(args['--min-peaks']) with open(input_lst) as f: _files = f.readlines() files = [f.strip() for f in _files] # load hit finding configuration file conf_file = args['<conf-file>'] with open(conf_file) as f: conf = yaml.load(f) dataset = conf['dataset'] # save hit conf and mask in hit dir mask_file = conf['mask file'] if mask_file is not None: copyfile(mask_file, '%s/mask.npy' % hit_dir) copyfile(conf_file, '%s/hit_conf.yml' % hit_dir) batch_size = int(args['--batch-size']) buffer_size = int(args['--buffer-size']) jobs, nb_frames = util.collect_jobs(files, dataset, batch_size) nb_jobs = len(jobs) print('%d frames, %d jobs to be processed' % (nb_frames, nb_jobs), flush=flush) update_freq = int(args['--update-freq']) prefix = os.path.basename(input_lst).split('.')[0] # dispatch jobs job_id = 0 reqs = {} results = [] workers = set(range(1, size)) finished_workers = set() time_start = time.time() for worker in workers: if job_id < nb_jobs: job = jobs[job_id] else: job = [] # dummy job comm.isend(job, dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) print('job %d/%d --> worker %d' % (job_id, nb_jobs, worker), flush=flush) job_id += 1 while job_id < nb_jobs: stop = False time.sleep(0.1) workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: results += result if job_id < nb_jobs: print('job %d/%d --> worker %d' % (job_id, nb_jobs, worker), flush=flush) comm.isend(stop, dest=worker) comm.isend(jobs[job_id], dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) job_id += 1 else: stop = True comm.isend(stop, dest=worker) print('stop signal --> worker %d' % worker, flush=flush) finished_workers.add(worker) if job_id % update_freq == 0: # update stat progress = float(job_id) / nb_jobs * 100 df = pd.DataFrame(results) if len(df) == 0: continue processed_hits = len(df[df['nb_peak'] >= min_peaks]) processed_frames = len(df) hit_rate = float(processed_hits) / processed_frames * 100. stat_dict = { 'progress': '%.2f%%' % progress, 'processed hits': processed_hits, 'hit rate': '%.2f%%' % hit_rate, 'duration/sec': 'not finished', 'processed frames': processed_frames, 'total jobs': nb_jobs, 'total frames': nb_frames, 'time start': time_start, } stat_file = os.path.join(hit_dir, 'stat.yml') with open(stat_file, 'w') as f: yaml.dump(stat_dict, f, default_flow_style=False) all_done = False while not all_done: time.sleep(0.1) all_done = True workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: results += result stop = True print('stop signal --> worker %d' % worker, flush=flush) comm.isend(stop, dest=worker) finished_workers.add(worker) else: all_done = False time_end = time.time() duration = time_end - time_start # save stat file df = pd.DataFrame(results) if len(df) == 0: print('WARNING! Empty hit finding results!') sys.exit() processed_hits = len(df[df['nb_peak'] >= min_peaks]) processed_frames = len(df) hit_rate = float(processed_hits) / processed_frames * 100. stat_dict = { 'progress': 'done', 'processed hits': processed_hits, 'hit rate': '%.2f%%' % hit_rate, 'duration/sec': duration, 'processed frames': processed_frames, 'total jobs': nb_jobs, 'total frames': nb_frames, 'time start': time_start, } stat_file = os.path.join(hit_dir, 'stat.yml') with open(stat_file, 'w') as f: yaml.dump(stat_dict, f, default_flow_style=False) # save simple results to csv csv_file = os.path.join(hit_dir, '%s.csv' % prefix) simple_results = [] for i in range(len(results)): simple_results.append( { 'filepath': results[i]['filepath'], 'dataset': results[i]['dataset'], 'frame': results[i]['frame'], 'nb_peak': results[i]['nb_peak'] } ) df = pd.DataFrame(simple_results) df.to_csv(csv_file) # save detailed peak info to npz peak_file = os.path.join(hit_dir, '%s.npy' % prefix) np.save(peak_file, results) print('All Done!', flush=flush)
def worker_run(args): stop = False filepath = None h5_obj = None buffer_size = int(args['--buffer-size']) flush = args['--flush'] min_max_intensity = float(args['--min-max-intensity']) # hit finding parameters with open(args['<conf-file>']) as f: conf = yaml.load(f) center = conf['center'] adu_per_photon = conf['adu per photon'] epsilon = conf['epsilon'] bin_size = conf['bin size'] if conf['mask on']: mask = util.read_image(conf['mask file'])['image'] else: mask = None hit_finder = conf['hit finder'] gaussian_sigma = conf['gaussian filter sigma'] min_distance = conf['min distance'] merge_flat_peaks = conf['merge flat peaks'] min_gradient = conf['min gradient'] max_peaks = conf['max peaks'] min_snr = conf['min snr'] min_pixels = conf['min pixels'] max_pixels = conf['max pixels'] peak_refine_mode = conf['peak refine mode'] snr_mode = conf['snr mode'] sig_radius = conf['signal radius'] bg_inner_radius = conf['background inner radius'] bg_outer_radius = conf['background outer radius'] crop_size = conf['crop size'] bg_ratio = conf['background ratio'] sig_ratio = conf['signal ratio'] sig_thres = conf['signal threshold'] dataset = conf['dataset'] # perform hit finding while not stop: job = comm.recv(buf=buffer_size, source=0) for i in range(len(job)): _filepath = job[i]['filepath'] frame = job[i]['frame'] if _filepath != filepath: filepath = _filepath ext = filepath.split('.')[-1] h5_obj = h5py.File(filepath, 'r') if ext in ('cxi', 'h5') \ else None lcls_data = util.get_lcls_data(filepath) if ext == 'lcls' \ else None image_data = util.read_image( filepath, frame=frame, h5_obj=h5_obj, lcls_data=lcls_data, dataset=dataset ) data_dict = {} if ext == 'lcls': if 'event_codes' in image_data: data_dict['event_codes'] = image_data['event_codes'] if 'flow_rate' in image_data: data_dict['flow_rate'] = image_data['flow_rate'] if 'pressure' in image_data: data_dict['pressure'] = image_data['pressure'] if 'photon_energy' in image_data: data_dict['photon_energy'] = image_data['photon_energy'] if 'clen' in image_data: data_dict['clen'] = image_data['clen'] if 'fiducial' in image_data: data_dict['fiducial'] = image_data['fiducial'] if 'epics-PV' in image_data: data_dict['epics-PV'] = image_data['epics-PV'] elif ext == 'h5': if 'pump_on' in image_data: data_dict['pump_on'] = image_data['pump_on'] total_intensity, max_intensity = 0., 0. if image_data['image'] is not None: image = image_data['image'] * mask if mask is not None else image_data['image'] total_intensity = np.sum(image) max_intensity = np.max(image) if max_intensity > min_max_intensity: peaks_dict = util.find_peaks( image_data['image'], center, adu_per_photon=adu_per_photon, epsilon=epsilon, bin_size=bin_size, mask=mask, hit_finder=hit_finder, gaussian_sigma=gaussian_sigma, min_gradient=min_gradient, min_distance=min_distance, merge_flat_peaks=merge_flat_peaks, max_peaks=max_peaks, min_snr=min_snr, min_pixels=min_pixels, max_pixels=max_pixels, refine_mode=peak_refine_mode, snr_mode=snr_mode, signal_radius=sig_radius, bg_inner_radius=bg_inner_radius, bg_outer_radius=bg_outer_radius, crop_size=crop_size, bg_ratio=bg_ratio, signal_ratio=sig_ratio, signal_thres=sig_thres, ) else: peaks_dict = {} if peaks_dict.get('strong', None) is not None: job[i]['nb_peak'] = len(peaks_dict['strong']) job[i]['peak_info'] = peaks_dict['info'] else: job[i]['nb_peak'] = 0 data_dict['total_intensity'] = total_intensity data_dict['max_intensity'] = max_intensity job[i]['data_dict'] = data_dict comm.send(job, dest=0) stop = comm.recv(source=0) print('slave %d is exiting' % rank, flush=flush)
def master_run(args): # parse parameters peak_dir = args['<peak-dir>'] batch_size = int(args['--batch-size']) output_dir = args['--output'] update_freq = int(args['--update-freq']) # collect and sort jobs peak_files = glob('%s/*.txt' % peak_dir) peak_files.sort(key=lambda x: int(x.split('-')[-1].split('.')[0])) jobs = [] job = [] for i in range(len(peak_files)): job.append({'peak path': peak_files[i]}) if len(job) == batch_size: jobs.append(job) job = [] if len(job) > 0: jobs.append(job) nb_jobs = len(jobs) # dispatch jobs job_id = 0 reqs = {} results = [] workers = set(range(1, size)) finished_workers = set() for worker in workers: if job_id < nb_jobs: job = jobs[job_id] else: job = [] # dummy job comm.isend(job, dest=worker) print('%d/%d --> %d' % (job_id, nb_jobs, worker), flush=True) reqs[worker] = comm.irecv(source=worker) job_id += 1 while job_id < nb_jobs: time.sleep(0.001) workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: results += result if job_id < nb_jobs: stop = False comm.isend(stop, dest=worker) comm.isend(jobs[job_id], dest=worker) print('%d/%d --> %d' % (job_id, nb_jobs, worker), flush=True) reqs[worker] = comm.irecv(source=worker) job_id += 1 else: stop = True comm.isend(stop, dest=worker) finished_workers.add(worker) if job_id % update_freq == 0: write_summary(results, output_dir) print('indexing summary updated!', flush=True) all_done = False while not all_done: time.sleep(0.001) all_done = True workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: results += result stop = True comm.isend(stop, dest=worker) finished_workers.add(worker) else: all_done = False write_summary(results, output_dir) print('all done')
def handle_sk(self, sess): response_page = BeautifulSoup(sess.text, 'html.parser') challenge_url = sess.url.split("?")[0] challenges_txt = response_page.find('input', { 'name': "id-challenge" }).get('value') challenges = json.loads(challenges_txt) facet_url = urllib_parse.urlparse(challenge_url) facet = facet_url.scheme + "://" + facet_url.netloc app_id = challenges["appId"] u2f_challenges = [] for c in challenges["challenges"]: c["appId"] = app_id u2f_challenges.append(c) # Prompt the user up to attempts_remaining times to insert their U2F device. attempts_remaining = 5 auth_response = None while True: try: auth_response = json.dumps(u2f.u2f_auth(u2f_challenges, facet)) break except RuntimeWarning: print("No U2F device found. {} attempts remaining.".format( attempts_remaining)) if attempts_remaining <= 0: break else: input( "Insert your U2F device and press enter to try again..." ) attempts_remaining -= 1 # If we exceed the number of attempts, raise an error and let the program exit. if auth_response is None: raise ExpectedGoogleException( "No U2F device found. Please check your setup.") payload = { 'challengeId': response_page.find('input', { 'name': 'challengeId' }).get('value'), 'challengeType': response_page.find('input', { 'name': 'challengeType' }).get('value'), 'continue': response_page.find('input', { 'name': 'continue' }).get('value'), 'scc': response_page.find('input', { 'name': 'scc' }).get('value'), 'sarp': response_page.find('input', { 'name': 'sarp' }).get('value'), 'checkedDomains': response_page.find('input', { 'name': 'checkedDomains' }).get('value'), 'pstMsg': response_page.find('input', { 'name': 'pstMsg' }).get('value'), 'TL': response_page.find('input', { 'name': 'TL' }).get('value'), 'gxf': response_page.find('input', { 'name': 'gxf' }).get('value'), 'id-challenge': challenges_txt, 'id-assertion': auth_response, 'TrustDevice': 'on', } return self.post(challenge_url, data=payload)
snr_mode=snr_mode, signal_radius=sig_radius, bg_inner_radius=bg_inner_radius, bg_outer_radius=bg_outer_radius, crop_size=crop_size, bg_ratio=bg_ratio, signal_ratio=sig_ratio, signal_thres=sig_thres, ) if peaks_dict.get('strong', None) is not None: peaks += peaks_dict['strong'].tolist() comm.send(peaks, dest=0) stop = comm.recv(source=0) if stop: print('slave %d is exiting' % rank, flush=flush) if __name__ == '__main__': comm = MPI.COMM_WORLD size = comm.Get_size() if size == 1: print('Run batch hit finder with at least 2 processes!') sys.exit() rank = comm.Get_rank() argv = docopt(__doc__) if rank == 0: master_run(argv) else: worker_run(argv)
def handle_iap(self, sess): response_page = BeautifulSoup(sess.text, 'html.parser') challenge_url = sess.url.split("?")[0] phone_number = input('Enter your phone number:') or None while True: try: choice = int( input( 'Type 1 to receive a code by SMS or 2 for a voice call:' )) if choice not in [1, 2]: raise ValueError except ValueError: print("Not a valid (integer) option, try again") continue else: if choice == 1: send_method = 'SMS' elif choice == 2: send_method = 'VOICE' else: continue break payload = { 'challengeId': response_page.find('input', { 'name': 'challengeId' }).get('value'), 'challengeType': response_page.find('input', { 'name': 'challengeType' }).get('value'), 'continue': self.cont, 'scc': response_page.find('input', { 'name': 'scc' }).get('value'), 'sarp': response_page.find('input', { 'name': 'sarp' }).get('value'), 'checkedDomains': response_page.find('input', { 'name': 'checkedDomains' }).get('value'), 'pstMsg': response_page.find('input', { 'name': 'pstMsg' }).get('value'), 'TL': response_page.find('input', { 'name': 'TL' }).get('value'), 'gxf': response_page.find('input', { 'name': 'gxf' }).get('value'), 'phoneNumber': phone_number, 'sendMethod': send_method, } # Submit phone number and desired method (SMS or voice call) sess = self.post(challenge_url, data=payload) response_page = BeautifulSoup(sess.text, 'html.parser') challenge_url = sess.url.split("?")[0] token = input("Enter " + send_method + " token: G-") or None payload = { 'challengeId': response_page.find('input', { 'name': 'challengeId' }).get('value'), 'challengeType': response_page.find('input', { 'name': 'challengeType' }).get('value'), 'continue': response_page.find('input', { 'name': 'continue' }).get('value'), 'scc': response_page.find('input', { 'name': 'scc' }).get('value'), 'sarp': response_page.find('input', { 'name': 'sarp' }).get('value'), 'checkedDomains': response_page.find('input', { 'name': 'checkedDomains' }).get('value'), 'pstMsg': response_page.find('input', { 'name': 'pstMsg' }).get('value'), 'TL': response_page.find('input', { 'name': 'TL' }).get('value'), 'gxf': response_page.find('input', { 'name': 'gxf' }).get('value'), 'pin': token, } # Submit SMS/VOICE token return self.post(challenge_url, data=payload)
def worker_run(args): stop = False filepath = None h5_obj = None buffer_size = int(args['--buffer-size']) flush = args['--flush'] # hit finding parameters with open(args['<conf-file>']) as f: conf = yaml.load(f) center = conf['center'] adu_per_photon = conf['adu per photon'] epsilon = conf['epsilon'] bin_size = conf['bin size'] if conf['mask on']: mask = util.read_image(conf['mask file'])['image'] else: mask = None hit_finder = conf['hit finder'] gaussian_sigma = conf['gaussian filter sigma'] min_distance = conf['min distance'] min_gradient = conf['min gradient'] max_peaks = conf['max peaks'] min_snr = conf['min snr'] min_pixels = conf['min pixels'] peak_refine_mode = conf['peak refine mode'] snr_mode = conf['snr mode'] sig_radius = conf['signal radius'] bg_inner_radius = conf['background inner radius'] bg_outer_radius = conf['background outer radius'] crop_size = conf['crop size'] bg_ratio = conf['background ratio'] sig_ratio = conf['signal ratio'] sig_thres = conf['signal threshold'] dataset = conf['dataset'] # perform hit finding while not stop: job = comm.recv(buf=buffer_size, source=0) peaks = [] for i in range(len(job)): _filepath = job[i]['filepath'] frame = job[i]['frame'] if _filepath != filepath: filepath = _filepath ext = filepath.split('.')[-1] h5_obj = h5py.File(filepath, 'r') if ext in ('cxi', 'h5') \ else None lcls_data = util.get_lcls_data(filepath) if ext == 'lcls' \ else None image = util.read_image(filepath, frame=frame, h5_obj=h5_obj, lcls_data=lcls_data, dataset=dataset)['image'] if image is None: continue peaks_dict = util.find_peaks( image, center, adu_per_photon=adu_per_photon, epsilon=epsilon, bin_size=bin_size, mask=mask, hit_finder=hit_finder, gaussian_sigma=gaussian_sigma, min_gradient=min_gradient, min_distance=min_distance, max_peaks=max_peaks, min_snr=min_snr, min_pixels=min_pixels, refine_mode=peak_refine_mode, snr_mode=snr_mode, signal_radius=sig_radius, bg_inner_radius=bg_inner_radius, bg_outer_radius=bg_outer_radius, crop_size=crop_size, bg_ratio=bg_ratio, signal_ratio=sig_ratio, signal_thres=sig_thres, ) if peaks_dict.get('strong', None) is not None: peaks += peaks_dict['strong'].tolist() comm.send(peaks, dest=0) stop = comm.recv(source=0) if stop: print('slave %d is exiting' % rank, flush=flush)
def print_part(*args): print(*args, end="", flush=True)
def download(key, secret, version, replays_dir, download_dir, extract=False, remove=False, filter_version='keep'): # Get OAuth token from us region api = BnetAPI(key, secret) # Get meta file infos for the give client version print('Searching replay packs with client version:', version) meta_file_urls = api.search_by_client_version(version) if len(meta_file_urls) == 0: sys.exit('No matching replay packs found for the client version!') # Download replay packs. download_base_url = api.get_base_url() print('Found {} replay packs'.format(len(meta_file_urls))) print('Downloading to:', download_dir) print('Extracting to:', replays_dir) mkdirs(download_dir) for i, meta_file_url in enumerate(sorted(meta_file_urls), 1): # Construct full url to download replay packs meta_file_info = api.get(meta_file_url) archive_url = requests.compat.urljoin(download_base_url, meta_file_info['path']) print_part('{}/{}: {} ... '.format(i, len(meta_file_urls), archive_url)) file_name = archive_url.split('/')[-1] file_path = os.path.join(download_dir, file_name) with requests.get(archive_url, stream=True) as response: print_part( int(response.headers['Content-Length']) // 1024**2, 'Mb ... ') if (not os.path.exists(file_path) or os.path.getsize(file_path) != int(response.headers['Content-Length'])): with open(file_path, 'wb') as f: shutil.copyfileobj(response.raw, f) print_part('downloaded') else: print_part('found') if extract: print_part(' ... extracting') if os.path.getsize(file_path) <= 22: # Size of an empty zip file. print_part(' ... zip file is empty') else: subprocess.call([ 'unzip', '-P', 'iagreetotheeula', '-u', '-o', '-q', '-d', replays_dir, file_path ]) if remove: os.remove(file_path) print() if mpyq is not None and filter_version != 'keep': print('Filtering replays.') found_versions = collections.defaultdict(int) found_str = lambda: ', '.join( '%s: %s' % (v, c) for v, c in sorted(found_versions.items())) all_replays = [ f for f in os.listdir(replays_dir) if f.endswith('.SC2Replay') ] for i, file_name in enumerate(all_replays): if i % 100 == 0: print_part('\r%s/%s: %d%%, found: %s' % (i, len(all_replays), 100 * i / len(all_replays), found_str())) file_path = os.path.join(replays_dir, file_name) with open(file_path, "rb") as fd: try: archive = mpyq.MPQArchive(fd).extract() except KeyboardInterrupt: return except: found_versions['corrupt'] += 1 os.remove(file_path) continue metadata = json.loads( archive[b'replay.gamemetadata.json'].decode('utf-8')) game_version = '.'.join(metadata['GameVersion'].split('.')[:-1]) found_versions[game_version] += 1 if filter_version == 'sort': version_dir = os.path.join(replays_dir, game_version) if found_versions[ game_version] == 1: # First one of this version. mkdirs(version_dir) os.rename(file_path, os.path.join(version_dir, file_name)) elif filter_version == 'delete': if game_version != version: os.remove(file_path) print('\nFound replays:', found_str())
# cut data into XOR ytrue = ((Xs[:, 0] < np.mean(Xs[:, 0])) * (Xs[:, 1] < np.mean(Xs[:, 1])) + (Xs[:, 0] > np.mean(Xs[:, 0])) * ( Xs[:, 1] > np.mean(Xs[:, 1]))) * 1 ys = np.array([-1] * N) sidx = np.append(np.random.choice(np.where(ytrue == 0)[0], supevised_data_points // 2), np.random.choice(np.where(ytrue == 1)[0], supevised_data_points // 2)) ys[sidx] = ytrue[sidx] Xsupervised = Xs[ys != -1, :] ysupervised = ys[ys != -1] # compare models lbl = "Purely supervised SVM:" print(lbl) model = sklearn.svm.SVC(kernel=kernel, probability=True) model.fit(Xsupervised, ysupervised) evaluate_and_plot(model, Xs, ys, ytrue, lbl, 1) lbl = "S3VM (Gieseke et al. 2012):" print(lbl) model = scikitTSVM.SKTSVM(kernel=kernel) model.fit(Xs, ys) evaluate_and_plot(model, Xs, ys, ytrue, lbl, 2) lbl = "CPLE(pessimistic) SVM:" print(lbl) model = CPLELearningModel(sklearn.svm.SVC(kernel=kernel, probability=True), predict_from_probabilities=True) model.fit(Xs, ys) evaluate_and_plot(model, Xs, ys, ytrue, lbl, 3)
batch = [] count += 1 comm.send(job, dest=0) stop = comm.recv(source=0) # save last cxi batch if not empty if len(batch) > 0: comp_file = os.path.join(comp_dir, '%s-rank%d-job%d.cxi' % (prefix, rank, count)) util.save_cxi(batch, comp_file, comp_dataset, out_dtype=comp_dtype, shuffle=shuffle) done = True comm.send(done, dest=0) if __name__ == '__main__': comm = MPI.COMM_WORLD size = comm.Get_size() if size == 1: print('Run batch compressor with at least 2 processes!') sys.exit() rank = comm.Get_rank() args = docopt(__doc__) if rank == 0: master_run(args) else: worker_run(args)
def handle_captcha(self, sess, payload): response_page = BeautifulSoup(sess.text, 'html.parser') # Collect ProfileInformation, SessionState, signIn, and Password Challenge URL profile_information = response_page.find('input', { 'name': 'ProfileInformation' }).get('value') session_state = response_page.find('input', { 'name': 'SessionState' }).get('value') sign_in = response_page.find('input', {'name': 'signIn'}).get('value') passwd_challenge_url = response_page.find('form', { 'id': 'gaia_loginform' }).get('action') # Update the payload payload['SessionState'] = session_state payload['ProfileInformation'] = profile_information payload['signIn'] = sign_in payload['Passwd'] = self.config.password # Get all captcha challenge tokens and urls captcha_container = response_page.find('div', {'id': 'identifier-captcha'}) captcha_logintoken = captcha_container.find('input', { 'id': 'identifier-token' }).get('value') captcha_img = captcha_container.find('div', {'class': 'captcha-img'}) captcha_url = "https://accounts.google.com" + captcha_img.find( 'img').get('src') captcha_logintoken_audio = '' open_image = True # Check if there is a display utility installed as Image.open(f).show() do not raise any exception if not # if neither xv or display are available just display the URL for the user to visit. if os.name == 'posix' and sys.platform != 'darwin': if find_executable('xv') is None and find_executable( 'display') is None: open_image = False print("Please visit the following URL to view your CAPTCHA: {}".format( captcha_url)) if open_image: try: with requests.get(captcha_url) as url: with io.BytesIO(url.content) as f: Image.open(f).show() except Exception: pass try: captcha_input = raw_input("Captcha (case insensitive): ") or None except NameError: captcha_input = input("Captcha (case insensitive): ") or None # Update the payload payload['identifier-captcha-input'] = captcha_input payload['identifiertoken'] = captcha_logintoken payload['identifiertoken_audio'] = captcha_logintoken_audio payload['checkedDomains'] = 'youtube' payload['checkConnection'] = 'youtube:574:1' payload['Email'] = self.config.username response = self.post(passwd_challenge_url, data=payload) newPayload = {} auth_response_page = BeautifulSoup(response.text, 'html.parser') form = auth_response_page.find('form') for tag in form.find_all('input'): if tag.get('name') is None: continue newPayload[tag.get('name')] = tag.get('value') newPayload['Email'] = self.config.username newPayload['Passwd'] = self.config.password if newPayload.get('TrustDevice', None) is not None: newPayload['TrustDevice'] = 'on' return self.post(response.url, data=newPayload)
# and the second - how many threads the computational core # should spawn hyphyInstance = HyPhy._THyPhy (os.getcwd(),2) # the basic interface command is 'ExecuteBF' which # executes HyPhy batch language commands in HyPhy # and returns a string representation of the return value # (if any) from HYPHY # The returned object is of type _THyPhyString with # sData and sLength fields # HyPhy will take care of disposing of the memory needed # to store the result hyphyResult = hyphyInstance.ExecuteBF ("return 2+2;"); print("Testing a trivial HyPhy command. 2+2 = ", hyphyResult.sData) # an optional second argument to ExecuteBF # can be used to "flush" the current state of the system # this is the default option for the call of ExecuteBF # passing the second argument of False or 0 will preserve # the execution state print("Consecutive command exection") hyphyInstance.ExecuteBF ("z:=x+y;",False); hyphyInstance.ExecuteBF ("x=3;",False); hyphyInstance.ExecuteBF ("y=5;",False); hyphyResult = hyphyInstance.ExecuteBF ("return z;",False); print("The value of z is ", hyphyResult.sData)
def master_run(args): flush = args['--flush'] file_lst = args['<file-lst>'] with open(file_lst) as f: _files = f.readlines() # remove trailing '/n' files = [] for f in _files: if '\n' == f[-1]: files.append(f[:-1]) else: files.append(f) # load hit finding configuration file with open(args['<conf-file>']) as f: conf = yaml.load(f) # collect jobs dataset = conf['dataset'] batch_size = int(args['--batch-size']) max_frames = int(args['--max-frames']) buffer_size = int(args['--buffer-size']) jobs, nb_frames = util.collect_jobs(files, dataset, batch_size, max_frames=max_frames) nb_jobs = len(jobs) print('%d frames, %d jobs to be processed' % (nb_frames, nb_jobs), flush=flush) # dispatch jobs job_id = 0 reqs = {} peaks = [] workers = set(range(1, size)) finished_workers = set() for worker in workers: if job_id < nb_jobs: job = jobs[job_id] else: job = [] # dummy job comm.isend(job, dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) print('job %d/%d --> %d' % (job_id, nb_jobs, worker), flush=flush) job_id += 1 while job_id < nb_jobs: stop = False time.sleep(0.1) # take a break workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: peaks += result if job_id < nb_jobs: print('job %d/%d --> %d' % (job_id, nb_jobs, worker), flush=flush) comm.isend(stop, dest=worker) comm.isend(jobs[job_id], dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) job_id += 1 else: stop = True comm.isend(stop, dest=worker) print('stop signal --> %d' % worker, flush=flush) finished_workers.add(worker) all_done = False while not all_done: all_done = True workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: peaks += result stop = True print('stop signal --> %d' % worker, flush=flush) comm.isend(stop, dest=worker) finished_workers.add(worker) else: all_done = False # build and save peak powder filepath = jobs[0][0]['filepath'] frame = jobs[0][0]['frame'] h5_obj = h5py.File(filepath, 'r') image = util.read_image(filepath, frame=frame, h5_obj=h5_obj, dataset=dataset)['image'] powder = np.zeros(image.shape) peaks = np.round(np.array(peaks)).astype(np.int) powder[peaks[:, 0], peaks[:, 1]] = 1 powder_file = args['-o'] dir_ = os.path.dirname(powder_file) if not os.path.isdir(dir_): os.mkdir(dir_) np.savez(powder_file, powder_pattern=powder, powder_peaks=peaks) print('All Done!', flush=flush) MPI.Finalize()
def main(): args = parse_args() # Get OAuth token from us region api = BnetAPI(args.key, args.secret) # Get meta file infos for the give client version print('Searching replay packs with client version:', args.version) meta_file_urls = api.search_by_client_version(args.version) if len(meta_file_urls) == 0: sys.exit('No matching replay packs found for the client version!') # Download replay packs. download_base_url = api.get_base_url() print('Found {} replay packs'.format(len(meta_file_urls))) print('Downloading to:', args.download_dir) print('Extracting to:', args.replays_dir) mkdirs(args.download_dir) for i, meta_file_url in enumerate(sorted(meta_file_urls), 1): # Construct full url to download replay packs meta_file_info = api.get(meta_file_url) archive_url = requests.compat.urljoin(download_base_url, meta_file_info['path']) print_part('{}/{}: {} ... '.format(i, len(meta_file_urls), archive_url)) file_name = archive_url.split('/')[-1] file_path = os.path.join(args.download_dir, file_name) with requests.get(archive_url, stream=True) as response: print_part(int(response.headers['Content-Length']) // 1024**2, 'Mb ... ') if (not os.path.exists(file_path) or os.path.getsize(file_path) != int(response.headers['Content-Length'])): with open(file_path, 'wb') as f: shutil.copyfileobj(response.raw, f) print_part('downloaded') else: print_part('found') if args.extract: print_part(' ... extracting') if os.path.getsize(file_path) <= 22: print_part(' ... zip file is empty') else: subprocess.call(['unzip', '-P', 'iagreetotheeula', '-u', '-o', '-q', '-d', args.replays_dir, file_path]) if args.remove: os.remove(file_path) print() if args.filter_version != 'keep': print('Filtering replays.') found_versions = collections.defaultdict(int) found_str = lambda: ', '.join('%s: %s' % (v, c) for v, c in sorted(found_versions.items())) all_replays = [f for f in os.listdir(args.replays_dir) if f.endswith('.SC2Replay')] for i, file_name in enumerate(all_replays): if i % 100 == 0: print_part('\r%s/%s: %d%%, found: %s' % (i, len(all_replays), 100 * i / len(all_replays), found_str())) file_path = os.path.join(args.replays_dir, file_name) with open(file_path) as fd: try: archive = mpyq.MPQArchive(fd).extract() except ValueError: found_versions['corrupt'] += 1 os.remove(file_path) continue metadata = json.loads(archive[b'replay.gamemetadata.json'].decode('utf-8')) game_version = '.'.join(metadata['GameVersion'].split('.')[:-1]) found_versions[game_version] += 1 if args.filter_version == 'sort': version_dir = os.path.join(args.replays_dir, game_version) if found_versions[game_version] == 1: # First one of this version. mkdirs(version_dir) os.rename(file_path, os.path.join(version_dir, file_name)) elif args.filter_version == 'delete': if game_version != args.version: os.remove(file_path) print('\nFound replays:', found_str())
# This script shows how to use the client in anonymous mode # against jira.atlassian.com. from six import print_ as print from jira.client import GreenHopper # By default, the client will connect to a JIRA instance started from the Atlassian Plugin SDK # (see https://developer.atlassian.com/display/DOCS/Installing+the+Atlassian+Plugin+SDK for details). # Override this with the options parameter. # GreenHopper is a plugin in a JIRA instance options = {'server': 'https://jira.atlassian.com'} gh = GreenHopper(options) # Get all boards viewable by anonymous users. boards = gh.boards() # Get the sprints in a specific board board_id = 441 print("GreenHopper board: %s (%s)" % (boards[0].name, board_id)) sprints = gh.sprints(board_id) # List the incomplete issues in each sprint for sprint in sprints: sprint_id = sprint.id print("Sprint: %s" % sprint.name) incompleted_issues = gh.incompleted_issues(board_id, sprint_id) print("Incomplete issues: %s" % ', '.join(issue.key for issue in incompleted_issues))
def master_run(args): flush = args['--flush'] # mkdir for output if not exist comp_dir = os.path.abspath(args['<comp-dir>']) comp_lst_dir = os.path.abspath(args['<comp-lst-dir>']) if not os.path.isdir(comp_dir): os.makedirs(comp_dir) if not os.path.isdir(comp_lst_dir): os.makedirs(comp_lst_dir) # get all raw data files raw_lst = args['<raw-lst>'] with open(raw_lst) as f: _files = f.readlines() # remove trailing '/n' files = [] raw_size = 0 # total size fo input files(raw data) for f in _files: files.append(f[:-1]) raw_size += os.path.getsize(f[:-1]) # collect jobs batch_size = int(args['--batch-size']) buffer_size = int(args['--buffer-size']) raw_dataset = args['<raw-dataset>'] jobs, nb_frames = util.collect_jobs(files, raw_dataset, batch_size) nb_jobs = len(jobs) time_start = time.time() print('%d frames, %d jobs to be processed' % (nb_frames, nb_jobs), flush=flush) update_freq = int(args['--update-freq']) # distribute jobs job_id = 0 reqs = {} results = [] workers = set(range(1, size)) finished_workers = set() for worker in workers: if job_id < nb_jobs: job = jobs[job_id] else: job = [] # dummy job comm.isend(job, dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) print('job %d/%d --> %d' % (job_id, nb_jobs, worker), flush=flush) job_id += 1 while job_id < nb_jobs: stop = False time.sleep(0.1) # take a break workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: results += result if job_id < nb_jobs: print('job %d/%d --> %d' % (job_id, nb_jobs, worker), flush=flush) comm.isend(stop, dest=worker) comm.isend(jobs[job_id], dest=worker) reqs[worker] = comm.irecv(buf=buffer_size, source=worker) job_id += 1 else: stop = True comm.isend(stop, dest=worker) finished_workers.add(worker) if job_id % update_freq == 0: progress = float(job_id) / nb_jobs * 100 stat_dict = { 'time start': time_start, 'progress': '%.2f%%' % progress, 'duration/sec': 'not finished', 'total frames': nb_frames, 'total jobs': nb_jobs, 'raw size': 'not finished', 'compressed size': 'not finished', 'compression ratio': 'not finished', } stat_file = os.path.join(comp_dir, 'stat.yml') with open(stat_file, 'w') as f: yaml.dump(stat_dict, f, default_flow_style=False) all_accepted = False while not all_accepted: all_accepted = True time.sleep(0.1) # take a break workers -= finished_workers for worker in workers: finished, result = reqs[worker].test() if finished: stop = True comm.isend(stop, dest=worker) finished_workers.add(worker) else: all_accepted = False print('all jobs accepted', flush=flush) # check status of last job batch all_done = False workers = set(range(1, size)) finished_workers = set() for worker in workers: reqs[worker] = comm.irecv(source=worker) while not all_done: all_done = True time.sleep(0.1) workers -= finished_workers for worker in workers: done, _ = reqs[worker].test() if done: print('worker %d done' % worker, flush=flush) finished_workers.add(worker) else: all_done = False # save results time_end = time.time() duration = time_end - time_start # write compressed cxi file lst compressed_size = 0 prefix = os.path.basename(raw_lst).split('.')[0] comp_lst_file = open(os.path.join(comp_lst_dir, '%s.lst' % prefix), 'w') comp_files = glob('%s/*.cxi' % comp_dir) for comp_file in comp_files: compressed_size += os.path.getsize(os.path.abspath(comp_file)) comp_lst_file.write('%s\n' % comp_file) comp_lst_file.close() compression_ratio = float(raw_size) / float(compressed_size) stat_dict = { 'time start': time_start, 'progress': 'done', 'duration/sec': duration, 'total frames': nb_frames, 'total jobs': nb_jobs, 'raw size': raw_size, 'compressed size': compressed_size, 'compression ratio': '%.2f' % compression_ratio, } stat_file = os.path.join(comp_dir, 'stat.yml') with open(stat_file, 'w') as f: yaml.dump(stat_dict, f, default_flow_style=False) print('All Done!', flush=flush)
def process_auth(args, config): # Set up logging logging.getLogger().setLevel(getattr(logging, args.log_level.upper(), None)) if config.region is None: config.region = util.Util.get_input("AWS Region: ") logging.debug('%s: region is: %s', __name__, config.region) # If there is a valid cache and the user opted to use it, use that instead # of prompting the user for input (it will also ignroe any set variables # such as username or sp_id and idp_id, as those are built into the SAML # response). The user does not need to be prompted for a password if the # SAML cache is used. if args.saml_assertion: saml_xml = base64.b64decode(args.saml_assertion) elif args.saml_cache and config.saml_cache: saml_xml = config.saml_cache logging.info('%s: SAML cache found', __name__) else: # No cache, continue without. logging.info('%s: SAML cache not found', __name__) if config.username is None: config.username = util.Util.get_input("Google username: "******"Google IDP ID: ") logging.debug('%s: idp is: %s', __name__, config.idp_id) if config.sp_id is None: config.sp_id = util.Util.get_input("Google SP ID: ") logging.debug('%s: sp is: %s', __name__, config.sp_id) # There is no way (intentional) to pass in the password via the command # line nor environment variables. This prevents password leakage. keyring_password = None if config.keyring: keyring_password = keyring.get_password("aws-google-auth", config.username) if keyring_password: config.password = keyring_password else: config.password = util.Util.get_password("Google Password: "******"Google Password: "******"aws-google-auth", config.username, config.password) # We now have a new SAML value that can get cached (If the user asked # for it to be) if args.saml_cache: config.saml_cache = saml_xml # The amazon_client now has the SAML assertion it needed (Either via the # cache or freshly generated). From here, we can get the roles and continue # the rest of the workflow regardless of cache. amazon_client = amazon.Amazon(config, saml_xml) roles = amazon_client.roles # Determine the provider and the role arn (if the the user provided isn't an option) if config.role_arn in roles and not config.ask_role: config.provider = roles[config.role_arn] else: if config.account and config.resolve_aliases: aliases = amazon_client.resolve_aws_aliases(roles) config.role_arn, config.provider = util.Util.pick_a_role(roles, aliases, config.account) elif config.account: config.role_arn, config.provider = util.Util.pick_a_role(roles, account=config.account) elif config.resolve_aliases: aliases = amazon_client.resolve_aws_aliases(roles) config.role_arn, config.provider = util.Util.pick_a_role(roles, aliases) else: config.role_arn, config.provider = util.Util.pick_a_role(roles) if not config.quiet: print("Assuming " + config.role_arn) print("Credentials Expiration: " + format(amazon_client.expiration.astimezone(get_localzone()))) if config.print_creds: amazon_client.print_export_line() if config.profile: config.write(amazon_client)