def login(self, server, username=None, password=None, server_data=None):
if username is None:
username = getpass.getuser()
try:
print('Logging in to {}@{}'.format(username, server))
self._ssh = SSHClient()
self._ssh.connect(server, username, password)
self._ssh.get_transport().set_keepalive(30)
except: # noqa: E722
self._ssh = None
raise
if server_data is not None:
try:
sys.stdout.flush()
print(
'Please wait for a new verification code before logging in to {}'
.format(server_data),
file=sys.stderr,
flush=True)
print('Logging in to {}@{}'.format(username, server_data))
self._ssh_data = SSHClient()
self._ssh_data.connect(server_data, username, password)
self._ssh_data.get_transport().set_keepalive(30)
except: # noqa: E722
self._ssh_data = None
raise
return self
def handle_selectchallenge(self, sess):
response_page = BeautifulSoup(sess.text, 'html.parser')
challenges = []
for i in response_page.select('form[data-challengeentry]'):
action = i.attrs.get("action")
if "challenge/totp/" in action:
challenges.append([
'TOTP (Google Authenticator)',
i.attrs.get("data-challengeentry")
])
elif "challenge/ipp/" in action:
challenges.append(['SMS', i.attrs.get("data-challengeentry")])
elif "challenge/iap/" in action:
challenges.append(
['SMS other phone',
i.attrs.get("data-challengeentry")])
elif "challenge/sk/" in action:
challenges.append(
['YubiKey', i.attrs.get("data-challengeentry")])
elif "challenge/az/" in action:
challenges.append(
['Google Prompt',
i.attrs.get("data-challengeentry")])
print('Choose MFA method from available:')
for i, mfa in enumerate(challenges, start=1):
print("{}: {}".format(i, mfa[0]))
selected_challenge = input("Enter MFA choice number (1): ") or None
if selected_challenge is not None and int(selected_challenge) <= len(
challenges):
selected_challenge = int(selected_challenge) - 1
else:
selected_challenge = 0
challenge_id = challenges[selected_challenge][1]
print("MFA Type Chosen: {}".format(challenges[selected_challenge][0]))
# We need the specific form of the challenge chosen
challenge_form = response_page.find(
'form', {'data-challengeentry': challenge_id})
payload = {}
for tag in challenge_form.find_all('input'):
if tag.get('name') is None:
continue
payload[tag.get('name')] = tag.get('value')
if response_page.find('input', {'name': 'TrustDevice'}) is not None:
payload['TrustDevice'] = 'on'
# POST to google with the chosen challenge
return self.post(self.base_url + challenge_form.get('action'),
data=payload)
def parse_saml(self):
if self.session_state is None:
raise RuntimeError(
'You must use do_login() before calling parse_saml()')
parsed = BeautifulSoup(self.session_state.text, 'html.parser')
try:
saml_element = parsed.find('input', {
'name': 'SAMLResponse'
}).get('value')
except:
if self.save_failure:
print(
"SAML lookup failed, storing failure page to 'saml.html' to assist with debugging."
)
with open("saml.html", 'w') as out:
out.write(self.session_state.text.encode('utf-8'))
raise ExpectedGoogleException(
'Something went wrong - Could not find SAML response, check your credentials or use --save-failure-html to debug.'
)
return base64.b64decode(saml_element)
def get(self, url):
try:
response = self.check_for_failure(self.session.get(url))
except requests.exceptions.ConnectionError as e:
print(
'There was a connection error, check your network settings: {}'.
format(e))
sys.exit(1)
except requests.exceptions.Timeout as e:
print('The connection timed out, please try again: {}'.format(e))
sys.exit(1)
except requests.exceptions.TooManyRedirects as e:
print('The number of redirects exceeded the maximum allowed: {}'.
format(e))
sys.exit(1)
return response
def interactive_mode(args):
"""Interactive mode"""
print('Start interactive mode', file=sys.stderr)
parser = StandaloneParser(args.parser,
args.load_model,
args.example_preprocessor,
beam_size=args.beam_size,
cuda=args.cuda)
while True:
utterance = input('Query:').strip()
hypotheses = parser.parse(utterance, debug=True)
for hyp_id, hyp in enumerate(hypotheses):
print('------------------ Hypothesis %d ------------------' %
hyp_id)
print(hyp.code)
from __future__ import print_function
from . import _version
import sys
import requests
import json
import base64
from bs4 import BeautifulSoup
from six.moves import urllib_parse, input
from six import print_ as print
# The U2F USB Library is optional, if it's there, include it.
try:
from . import u2f
except ImportError:
print("Failed to import U2F libraries, U2F login unavailable. Other "
"methods can still continue.")
class ExpectedGoogleException(Exception):
def __init__(self, *args):
super(ExpectedGoogleException, self).__init__(*args)
class Google:
def __init__(self, config):
"""The Google object holds authentication state
for a given session. You need to supply:
username: FQDN Google username, eg [email protected]
password: obvious
idp_id: Google's assigned IdP identifier for your G-suite account
def do_login(self):
self.session = requests.Session()
self.session.headers[
'User-Agent'] = "AWS Sign-in/{} (Cevo aws-google-auth)".format(
self.version)
sess = self.get(self.login_url)
# Collect information from the page source
first_page = BeautifulSoup(sess.text, 'html.parser')
gxf = first_page.find('input', {'name': 'gxf'}).get('value')
self.cont = first_page.find('input', {'name': 'continue'}).get('value')
page = first_page.find('input', {'name': 'Page'}).get('value')
sign_in = first_page.find('input', {'name': 'signIn'}).get('value')
account_login_url = first_page.find('form', {
'id': 'gaia_loginform'
}).get('action')
payload = {
'bgresponse': 'js_disabled',
'checkConnection': '',
'checkedDomains': 'youtube',
'continue': self.cont,
'Email': self.config.username,
'gxf': gxf,
'identifier-captcha-input': '',
'identifiertoken': '',
'identifiertoken_audio': '',
'ltmpl': 'popup',
'oauth': 1,
'Page': page,
'Passwd': '',
'PersistentCookie': 'yes',
'ProfileInformation': '',
'pstMsg': 0,
'sarp': 1,
'scc': 1,
'SessionState': '',
'signIn': sign_in,
'_utf8': '?',
}
# GALX is sometimes not there
try:
galx = first_page.find('input', {'name': 'GALX'}).get('value')
payload['GALX'] = galx
except:
pass
# POST to account login info page, to collect profile and session info
sess = self.post(account_login_url, data=payload)
self.session.headers['Referer'] = sess.url
# Collect ProfileInformation, SessionState, signIn, and Password Challenge URL
challenge_page = BeautifulSoup(sess.text, 'html.parser')
profile_information = challenge_page.find('input', {
'name': 'ProfileInformation'
}).get('value')
session_state = challenge_page.find('input', {
'name': 'SessionState'
}).get('value')
sign_in = challenge_page.find('input', {'name': 'signIn'}).get('value')
passwd_challenge_url = challenge_page.find('form', {
'id': 'gaia_loginform'
}).get('action')
# Update the payload
payload['SessionState'] = session_state
payload['ProfileInformation'] = profile_information
payload['signIn'] = sign_in
payload['Passwd'] = self.config.password
# POST to Authenticate Password
sess = self.post(passwd_challenge_url, data=payload)
response_page = BeautifulSoup(sess.text, 'html.parser')
error = response_page.find(class_='error-msg')
cap = response_page.find('input', {'name': 'logincaptcha'})
# Were there any errors logging in? Could be invalid username or password
# There could also sometimes be a Captcha, which means Google thinks you,
# or someone using the same outbound IP address as you, is a bot.
if error is not None:
raise ExpectedGoogleException('Invalid username or password')
self.check_extra_step(response_page)
if cap is not None:
raise ExpectedGoogleException(
'Captcha Required. Manually Login to remove this.')
self.session.headers['Referer'] = sess.url
if "selectchallenge/" in sess.url:
sess = self.handle_selectchallenge(sess)
# Was there an MFA challenge?
if "challenge/totp/" in sess.url:
error_msg = ""
while error_msg is not None:
sess = self.handle_totp(sess)
error_msg = self.parse_error_message(sess)
if error_msg is not None:
print(error_msg)
elif "challenge/ipp/" in sess.url:
sess = self.handle_sms(sess)
elif "challenge/az/" in sess.url:
sess = self.handle_prompt(sess)
elif "challenge/sk/" in sess.url:
sess = self.handle_sk(sess)
elif "challenge/iap/" in sess.url:
sess = self.handle_iap(sess)
elif "challenge/ootp/5" in sess.url:
raise NotImplementedError(
'Offline Google App OOTP not implemented')
# ... there are different URLs for backup codes (printed)
# and security keys (eg yubikey) as well
# save for later
self.session_state = sess
def handle_prompt(self, sess):
response_page = BeautifulSoup(sess.text, 'html.parser')
challenge_url = sess.url.split("?")[0]
data_key = response_page.find('div', {
'data-api-key': True
}).get('data-api-key')
data_tx_id = response_page.find('div', {
'data-tx-id': True
}).get('data-tx-id')
# Need to post this to the verification/pause endpoint
await_url = "https://content.googleapis.com/cryptauth/v1/authzen/awaittx?alt=json&key={}".format(
data_key)
await_body = {'txId': data_tx_id}
self.check_prompt_code(response_page)
print(
"Open the Google App, and tap 'Yes' on the prompt to sign in ...")
self.session.headers['Referer'] = sess.url
retry = True
response = None
while retry:
try:
response = self.post(await_url, json_data=await_body)
retry = False
except requests.exceptions.HTTPError as ex:
if not ex.response.status_code == 500:
raise ex
parsed_response = json.loads(response.text)
payload = {
'challengeId':
response_page.find('input', {
'name': 'challengeId'
}).get('value'),
'challengeType':
response_page.find('input', {
'name': 'challengeType'
}).get('value'),
'continue':
response_page.find('input', {
'name': 'continue'
}).get('value'),
'scc':
response_page.find('input', {
'name': 'scc'
}).get('value'),
'sarp':
response_page.find('input', {
'name': 'sarp'
}).get('value'),
'checkedDomains':
'youtube',
'checkConnection':
'youtube:1295:1',
'pstMsg':
0,
'TL':
response_page.find('input', {
'name': 'TL'
}).get('value'),
'gxf':
response_page.find('input', {
'name': 'gxf'
}).get('value'),
'token':
parsed_response['txToken'],
'action':
response_page.find('input', {
'name': 'action'
}).get('value'),
'TrustDevice':
'on',
}
return self.post(challenge_url, data=payload)
nb_top=nb_top,
multi_index=multi_index,
nb_try=nb_try,
verbose=verbose)
job[i]['solutions'] = solutions
comm.send(job, dest=0)
stop = comm.recv(source=0)
print('worker %d is exiting' % rank)
if __name__ == '__main__':
# save command
command_list = sys.argv
command = ' '.join(command_list)
with open('spind.com', 'w') as f:
f.write('%s\n' % command)
comm = MPI.COMM_WORLD
size = comm.Get_size()
if size == 1:
print('Run SPIND with at least 2 processes!')
sys.exit()
rank = comm.Get_rank()
args = docopt(__doc__)
if rank == 0:
print(args, flush=True)
master_run(args)
else:
worker_run(args)
# This script shows how to use the client in anonymous mode
# against jira.atlassian.com.
from six import print_ as print
from jira.client import GreenHopper
# By default, the client will connect to a JIRA instance started from the Atlassian Plugin SDK
# (see https://developer.atlassian.com/display/DOCS/Installing+the+Atlassian+Plugin+SDK for details).
# Override this with the options parameter.
# GreenHopper is a plugin in a JIRA instance
options = {
'server': 'https://jira.atlassian.com'
}
gh = GreenHopper(options)
# Get all boards viewable by anonymous users.
boards = gh.boards()
# Get the sprints in a specific board
board_id = 441
print("GreenHopper board: %s (%s)" % (boards[0].name, board_id))
sprints = gh.sprints(board_id)
# List the incomplete issues in each sprint
for sprint in sprints:
sprint_id = sprint.id
print("Sprint: %s" % sprint.name)
incompleted_issues = gh.incompleted_issues(board_id, sprint_id)
print("Incomplete issues: %s" %
', '.join(issue.key for issue in incompleted_issues))
import time
import six
from chainer import serializers
from chainer import cuda
import chainer.optimizers as O
import numpy as np
from lda2vec import prepare_topics, print_top_words_per_topic
from lda2vec import utils
from lda import LDA
gpu_id = int(os.getenv('CUDA_GPU', 0))
cuda.get_device(gpu_id).use()
six.print("Using GPU " + str(gpu_id))
vocab = pickle.load(open('vocab.pkl', 'rb'))
corpus = pickle.load(open('corpus.pkl', 'rb'))
bow = np.load("bow.npy").astype('float32')
# Remove bow counts on the first two tokens, which <SKIP> and <EOS>
bow[:, :2] = 0
# Normalize bag of words to be a probability
# bow = bow / bow.sum(axis=1)[:, None]
# Number of docs
n_docs = bow.shape[0]
# Number of unique words in the vocabulary
n_vocab = bow.shape[1]
# Number of dimensions in a single word vector
n_units = 256
def handle_captcha(self, sess, payload):
response_page = BeautifulSoup(sess.text, 'html.parser')
# Collect ProfileInformation, SessionState, signIn, and Password Challenge URL
profile_information = response_page.find('input', {
'name': 'ProfileInformation'
}).get('value')
session_state = response_page.find('input', {
'name': 'SessionState'
}).get('value')
sign_in = response_page.find('input', {'name': 'signIn'}).get('value')
passwd_challenge_url = response_page.find('form', {
'id': 'gaia_loginform'
}).get('action')
# Update the payload
payload['SessionState'] = session_state
payload['ProfileInformation'] = profile_information
payload['signIn'] = sign_in
payload['Passwd'] = self.config.password
# Get all captcha challenge tokens and urls
captcha_container = response_page.find('div',
{'class': 'captcha-container'})
captcha_logintoken = captcha_container.find('input', {
'name': 'logintoken'
}).get('value')
captcha_url = captcha_container.find('input', {
'name': 'url'
}).get('value')
captcha_logintoken_audio = captcha_container.find(
'input', {
'name': 'logintoken_audio'
}).get('value')
captcha_url_audio = captcha_container.find('input', {
'name': 'url_audio'
}).get('value')
open_image = True
# Check if there is a display utility installed as Image.open(f).show() do not raise any exception if not
# if neither xv or display are available just display the URL for the user to visit.
if os.name == 'posix' and sys.platform != 'darwin':
if find_executable('xv') is None and find_executable(
'display') is None:
open_image = False
print("Please visit the following URL to view your CAPTCHA: {}".format(
captcha_url))
if open_image:
try:
with requests.get(captcha_url) as url:
with io.BytesIO(url.content) as f:
Image.open(f).show()
except Exception:
pass
try:
captcha_input = raw_input("Captcha (case insensitive): ") or None
except NameError:
captcha_input = input("Captcha (case insensitive): ") or None
# Update the payload
payload['logincaptcha'] = captcha_input
payload['logintoken'] = captcha_logintoken
payload['url'] = captcha_url
payload['logintoken_audio'] = captcha_logintoken_audio
payload['url_audio'] = captcha_url_audio
return self.post(passwd_challenge_url, data=payload)
def handle_selectchallenge(self, sess):
response_page = BeautifulSoup(sess.text, 'html.parser')
# Known mfa methods, 5 is disabled till its implemented
auth_methods = {
2: 'TOTP (Google Authenticator)',
3: 'SMS',
4: 'OOTP (Google Prompt)'
# 5: 'OOTP (Google App Offline Security Code)'
}
unavailable_challenge_ids = [
int(i.attrs.get('data-unavailable'))
for i in response_page.find_all(
lambda tag: tag.name == 'form' and 'data-unavailable' in tag.attrs
)
]
# ootp via google app offline code isn't implemented. make sure its not valid.
unavailable_challenge_ids.append(5)
challenge_ids = [
int(i.get('value'))
for i in response_page.find_all('input', {'name': 'challengeId'})
if int(i.get('value')) not in unavailable_challenge_ids
]
challenge_ids.sort()
auth_methods = {
k: auth_methods[k]
for k in challenge_ids
if k in auth_methods and k not in unavailable_challenge_ids
}
print('Choose MFA method from available:')
print('\n'.join(
'{}: {}'.format(*i) for i in list(auth_methods.items())))
selected_challenge = input("Enter MFA choice number ({}): ".format(
challenge_ids[-1:][0])) or None
if selected_challenge is not None and int(
selected_challenge) in challenge_ids:
challenge_id = int(selected_challenge)
else:
# use the highest index as that will default to prompt, then sms, then totp, etc.
challenge_id = challenge_ids[-1:][0]
print("MFA Type Chosen: {}".format(auth_methods[challenge_id]))
# We need the specific form of the challenge chosen
challenge_form = response_page.find(
'form', {'data-challengeentry': challenge_id})
payload = {
'challengeId':
challenge_id,
'challengeType':
challenge_form.find('input', {
'name': 'challengeType'
}).get('value'),
'continue':
challenge_form.find('input', {
'name': 'continue'
}).get('value'),
'scc':
challenge_form.find('input', {
'name': 'scc'
}).get('value'),
'sarp':
challenge_form.find('input', {
'name': 'sarp'
}).get('value'),
'checkedDomains':
challenge_form.find('input', {
'name': 'checkedDomains'
}).get('value'),
'pstMsg':
challenge_form.find('input', {
'name': 'pstMsg'
}).get('value'),
'TL':
challenge_form.find('input', {
'name': 'TL'
}).get('value'),
'gxf':
challenge_form.find('input', {
'name': 'gxf'
}).get('value'),
'subAction':
challenge_form.find('input', {
'name': 'subAction'
}).get('value'),
}
if challenge_form.find('input', {'name': 'SendMethod'}) is not None:
payload['SendMethod'] = challenge_form.find(
'input', {
'name': 'SendMethod'
}).get('value')
# POST to google with the chosen challenge
return self.post(
self.base_url + challenge_form.get('action'), data=payload)
# save last cxi batch if not empty
if len(batch) > 0:
# save full cxi
cxi_file = os.path.join(hit_dir,
'%s-rank%d-job%d.cxi' % (prefix, rank, count))
util.save_full_cxi(batch,
cxi_file,
cxi_entry_id=cxi_entry_id,
mask_file=mask_file,
extra_datasets=extra_datasets,
cxi_dtype=cxi_dtype,
shuffle=shuffle)
sys.stdout.flush()
done = True
comm.send(done, dest=0)
if __name__ == '__main__':
comm = MPI.COMM_WORLD
size = comm.Get_size()
if size == 1:
print('Run batch peak2cxi with at least 2 processes!')
sys.exit()
rank = comm.Get_rank()
args = docopt(__doc__)
if rank == 0:
master_run(args)
else:
worker_run(args)
def master_run(args):
flush = args['--flush']
# mkdir if not exist
hit_dir = args['<hit-dir>']
if not os.path.isdir(hit_dir):
os.mkdir(hit_dir)
peak_file = args['<peak-file>']
peak_info = np.load(peak_file)
batch_size = int(args['--batch-size'])
nb_jobs = int(np.ceil(len(peak_info) / batch_size))
# collect jobs
ids = np.array_split(np.arange(len(peak_info)), nb_jobs)
jobs = []
for i in range(len(ids)):
jobs.append(peak_info[ids[i]])
print('%d jobs, %d frames to be processed' % (nb_jobs, len(peak_info)),
flush=flush)
# other parameters
buffer_size = int(args['--buffer-size'])
update_freq = int(args['--update-freq'])
# dispatch jobs
job_id = 0
reqs = {}
workers = set(range(1, size))
finished_workers = set()
time_start = time.time()
for worker in workers:
if job_id < nb_jobs:
job = jobs[job_id]
else:
job = [] # dummy job
comm.isend(job, dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
print('job %d/%d --> slave %d' % (job_id, nb_jobs, worker),
flush=flush)
job_id += 1
while job_id < nb_jobs:
stop = False
workers -= finished_workers
time.sleep(0.1) # take a break
for worker in workers:
finished, result = reqs[worker].test()
if finished:
if job_id < nb_jobs:
print('job %d/%d --> slave %d' % (job_id, nb_jobs, worker),
flush=flush)
comm.isend(stop, dest=worker)
comm.isend(jobs[job_id], dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
job_id += 1
else:
stop = True
comm.isend(stop, dest=worker)
print('stop signal --> %d' % worker, flush=flush)
finished_workers.add(worker)
if job_id % update_freq == 0:
# update stat
progress = float(job_id) / nb_jobs * 100
stat_dict = {
'progress': '%.2f%%' % progress,
'duration/sec': 'not finished',
'total jobs': nb_jobs,
}
stat_file = os.path.join(hit_dir, 'peak2cxi.yml')
with open(stat_file, 'w') as f:
yaml.dump(stat_dict, f, default_flow_style=False)
all_done = False
while not all_done:
time.sleep(0.1)
workers -= finished_workers
all_done = True
for worker in workers:
finished, result = reqs[worker].test()
if finished:
stop = True
comm.isend(stop, dest=worker)
finished_workers.add(worker)
else:
all_done = False
time_end = time.time()
duration = time_end - time_start
stat_dict = {
'progress': 'done',
'duration/sec': duration,
'total jobs': nb_jobs,
}
stat_file = os.path.join(hit_dir, 'peak2cxi.yml')
with open(stat_file, 'w') as f:
yaml.dump(stat_dict, f, default_flow_style=False)
print('All Done!', flush=flush)
MPI.Finalize()
def master_run(args):
flush = args['--flush']
# mkdir if not exist
hit_dir = args['<hit-dir>']
if not os.path.isdir(hit_dir):
os.makedirs(hit_dir)
input_lst = args['<input-lst>']
min_peaks = int(args['--min-peaks'])
with open(input_lst) as f:
_files = f.readlines()
files = [f.strip() for f in _files]
# load hit finding configuration file
conf_file = args['<conf-file>']
with open(conf_file) as f:
conf = yaml.load(f)
dataset = conf['dataset']
# save hit conf and mask in hit dir
mask_file = conf['mask file']
if mask_file is not None:
copyfile(mask_file, '%s/mask.npy' % hit_dir)
copyfile(conf_file, '%s/hit_conf.yml' % hit_dir)
batch_size = int(args['--batch-size'])
buffer_size = int(args['--buffer-size'])
jobs, nb_frames = util.collect_jobs(files, dataset, batch_size)
nb_jobs = len(jobs)
print('%d frames, %d jobs to be processed' %
(nb_frames, nb_jobs), flush=flush)
update_freq = int(args['--update-freq'])
prefix = os.path.basename(input_lst).split('.')[0]
# dispatch jobs
job_id = 0
reqs = {}
results = []
workers = set(range(1, size))
finished_workers = set()
time_start = time.time()
for worker in workers:
if job_id < nb_jobs:
job = jobs[job_id]
else:
job = [] # dummy job
comm.isend(job, dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
print('job %d/%d --> worker %d' % (job_id, nb_jobs, worker), flush=flush)
job_id += 1
while job_id < nb_jobs:
stop = False
time.sleep(0.1)
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
results += result
if job_id < nb_jobs:
print('job %d/%d --> worker %d' %
(job_id, nb_jobs, worker), flush=flush)
comm.isend(stop, dest=worker)
comm.isend(jobs[job_id], dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
job_id += 1
else:
stop = True
comm.isend(stop, dest=worker)
print('stop signal --> worker %d' % worker, flush=flush)
finished_workers.add(worker)
if job_id % update_freq == 0:
# update stat
progress = float(job_id) / nb_jobs * 100
df = pd.DataFrame(results)
if len(df) == 0:
continue
processed_hits = len(df[df['nb_peak'] >= min_peaks])
processed_frames = len(df)
hit_rate = float(processed_hits) / processed_frames * 100.
stat_dict = {
'progress': '%.2f%%' % progress,
'processed hits': processed_hits,
'hit rate': '%.2f%%' % hit_rate,
'duration/sec': 'not finished',
'processed frames': processed_frames,
'total jobs': nb_jobs,
'total frames': nb_frames,
'time start': time_start,
}
stat_file = os.path.join(hit_dir, 'stat.yml')
with open(stat_file, 'w') as f:
yaml.dump(stat_dict, f, default_flow_style=False)
all_done = False
while not all_done:
time.sleep(0.1)
all_done = True
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
results += result
stop = True
print('stop signal --> worker %d' % worker, flush=flush)
comm.isend(stop, dest=worker)
finished_workers.add(worker)
else:
all_done = False
time_end = time.time()
duration = time_end - time_start
# save stat file
df = pd.DataFrame(results)
if len(df) == 0:
print('WARNING! Empty hit finding results!')
sys.exit()
processed_hits = len(df[df['nb_peak'] >= min_peaks])
processed_frames = len(df)
hit_rate = float(processed_hits) / processed_frames * 100.
stat_dict = {
'progress': 'done',
'processed hits': processed_hits,
'hit rate': '%.2f%%' % hit_rate,
'duration/sec': duration,
'processed frames': processed_frames,
'total jobs': nb_jobs,
'total frames': nb_frames,
'time start': time_start,
}
stat_file = os.path.join(hit_dir, 'stat.yml')
with open(stat_file, 'w') as f:
yaml.dump(stat_dict, f, default_flow_style=False)
# save simple results to csv
csv_file = os.path.join(hit_dir, '%s.csv' % prefix)
simple_results = []
for i in range(len(results)):
simple_results.append(
{
'filepath': results[i]['filepath'],
'dataset': results[i]['dataset'],
'frame': results[i]['frame'],
'nb_peak': results[i]['nb_peak']
}
)
df = pd.DataFrame(simple_results)
df.to_csv(csv_file)
# save detailed peak info to npz
peak_file = os.path.join(hit_dir, '%s.npy' % prefix)
np.save(peak_file, results)
print('All Done!', flush=flush)
def worker_run(args):
stop = False
filepath = None
h5_obj = None
buffer_size = int(args['--buffer-size'])
flush = args['--flush']
min_max_intensity = float(args['--min-max-intensity'])
# hit finding parameters
with open(args['<conf-file>']) as f:
conf = yaml.load(f)
center = conf['center']
adu_per_photon = conf['adu per photon']
epsilon = conf['epsilon']
bin_size = conf['bin size']
if conf['mask on']:
mask = util.read_image(conf['mask file'])['image']
else:
mask = None
hit_finder = conf['hit finder']
gaussian_sigma = conf['gaussian filter sigma']
min_distance = conf['min distance']
merge_flat_peaks = conf['merge flat peaks']
min_gradient = conf['min gradient']
max_peaks = conf['max peaks']
min_snr = conf['min snr']
min_pixels = conf['min pixels']
max_pixels = conf['max pixels']
peak_refine_mode = conf['peak refine mode']
snr_mode = conf['snr mode']
sig_radius = conf['signal radius']
bg_inner_radius = conf['background inner radius']
bg_outer_radius = conf['background outer radius']
crop_size = conf['crop size']
bg_ratio = conf['background ratio']
sig_ratio = conf['signal ratio']
sig_thres = conf['signal threshold']
dataset = conf['dataset']
# perform hit finding
while not stop:
job = comm.recv(buf=buffer_size, source=0)
for i in range(len(job)):
_filepath = job[i]['filepath']
frame = job[i]['frame']
if _filepath != filepath:
filepath = _filepath
ext = filepath.split('.')[-1]
h5_obj = h5py.File(filepath, 'r') if ext in ('cxi', 'h5') \
else None
lcls_data = util.get_lcls_data(filepath) if ext == 'lcls' \
else None
image_data = util.read_image(
filepath, frame=frame,
h5_obj=h5_obj,
lcls_data=lcls_data,
dataset=dataset
)
data_dict = {}
if ext == 'lcls':
if 'event_codes' in image_data:
data_dict['event_codes'] = image_data['event_codes']
if 'flow_rate' in image_data:
data_dict['flow_rate'] = image_data['flow_rate']
if 'pressure' in image_data:
data_dict['pressure'] = image_data['pressure']
if 'photon_energy' in image_data:
data_dict['photon_energy'] = image_data['photon_energy']
if 'clen' in image_data:
data_dict['clen'] = image_data['clen']
if 'fiducial' in image_data:
data_dict['fiducial'] = image_data['fiducial']
if 'epics-PV' in image_data:
data_dict['epics-PV'] = image_data['epics-PV']
elif ext == 'h5':
if 'pump_on' in image_data:
data_dict['pump_on'] = image_data['pump_on']
total_intensity, max_intensity = 0., 0.
if image_data['image'] is not None:
image = image_data['image'] * mask if mask is not None else image_data['image']
total_intensity = np.sum(image)
max_intensity = np.max(image)
if max_intensity > min_max_intensity:
peaks_dict = util.find_peaks(
image_data['image'], center,
adu_per_photon=adu_per_photon,
epsilon=epsilon,
bin_size=bin_size,
mask=mask,
hit_finder=hit_finder,
gaussian_sigma=gaussian_sigma,
min_gradient=min_gradient,
min_distance=min_distance,
merge_flat_peaks=merge_flat_peaks,
max_peaks=max_peaks,
min_snr=min_snr,
min_pixels=min_pixels,
max_pixels=max_pixels,
refine_mode=peak_refine_mode,
snr_mode=snr_mode,
signal_radius=sig_radius,
bg_inner_radius=bg_inner_radius,
bg_outer_radius=bg_outer_radius,
crop_size=crop_size,
bg_ratio=bg_ratio,
signal_ratio=sig_ratio,
signal_thres=sig_thres,
)
else:
peaks_dict = {}
if peaks_dict.get('strong', None) is not None:
job[i]['nb_peak'] = len(peaks_dict['strong'])
job[i]['peak_info'] = peaks_dict['info']
else:
job[i]['nb_peak'] = 0
data_dict['total_intensity'] = total_intensity
data_dict['max_intensity'] = max_intensity
job[i]['data_dict'] = data_dict
comm.send(job, dest=0)
stop = comm.recv(source=0)
print('slave %d is exiting' % rank, flush=flush)
def master_run(args):
# parse parameters
peak_dir = args['<peak-dir>']
batch_size = int(args['--batch-size'])
output_dir = args['--output']
update_freq = int(args['--update-freq'])
# collect and sort jobs
peak_files = glob('%s/*.txt' % peak_dir)
peak_files.sort(key=lambda x: int(x.split('-')[-1].split('.')[0]))
jobs = []
job = []
for i in range(len(peak_files)):
job.append({'peak path': peak_files[i]})
if len(job) == batch_size:
jobs.append(job)
job = []
if len(job) > 0:
jobs.append(job)
nb_jobs = len(jobs)
# dispatch jobs
job_id = 0
reqs = {}
results = []
workers = set(range(1, size))
finished_workers = set()
for worker in workers:
if job_id < nb_jobs:
job = jobs[job_id]
else:
job = [] # dummy job
comm.isend(job, dest=worker)
print('%d/%d --> %d' % (job_id, nb_jobs, worker), flush=True)
reqs[worker] = comm.irecv(source=worker)
job_id += 1
while job_id < nb_jobs:
time.sleep(0.001)
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
results += result
if job_id < nb_jobs:
stop = False
comm.isend(stop, dest=worker)
comm.isend(jobs[job_id], dest=worker)
print('%d/%d --> %d' % (job_id, nb_jobs, worker),
flush=True)
reqs[worker] = comm.irecv(source=worker)
job_id += 1
else:
stop = True
comm.isend(stop, dest=worker)
finished_workers.add(worker)
if job_id % update_freq == 0:
write_summary(results, output_dir)
print('indexing summary updated!', flush=True)
all_done = False
while not all_done:
time.sleep(0.001)
all_done = True
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
results += result
stop = True
comm.isend(stop, dest=worker)
finished_workers.add(worker)
else:
all_done = False
write_summary(results, output_dir)
print('all done')
def handle_sk(self, sess):
response_page = BeautifulSoup(sess.text, 'html.parser')
challenge_url = sess.url.split("?")[0]
challenges_txt = response_page.find('input', {
'name': "id-challenge"
}).get('value')
challenges = json.loads(challenges_txt)
facet_url = urllib_parse.urlparse(challenge_url)
facet = facet_url.scheme + "://" + facet_url.netloc
app_id = challenges["appId"]
u2f_challenges = []
for c in challenges["challenges"]:
c["appId"] = app_id
u2f_challenges.append(c)
# Prompt the user up to attempts_remaining times to insert their U2F device.
attempts_remaining = 5
auth_response = None
while True:
try:
auth_response = json.dumps(u2f.u2f_auth(u2f_challenges, facet))
break
except RuntimeWarning:
print("No U2F device found. {} attempts remaining.".format(
attempts_remaining))
if attempts_remaining <= 0:
break
else:
input(
"Insert your U2F device and press enter to try again..."
)
attempts_remaining -= 1
# If we exceed the number of attempts, raise an error and let the program exit.
if auth_response is None:
raise ExpectedGoogleException(
"No U2F device found. Please check your setup.")
payload = {
'challengeId':
response_page.find('input', {
'name': 'challengeId'
}).get('value'),
'challengeType':
response_page.find('input', {
'name': 'challengeType'
}).get('value'),
'continue':
response_page.find('input', {
'name': 'continue'
}).get('value'),
'scc':
response_page.find('input', {
'name': 'scc'
}).get('value'),
'sarp':
response_page.find('input', {
'name': 'sarp'
}).get('value'),
'checkedDomains':
response_page.find('input', {
'name': 'checkedDomains'
}).get('value'),
'pstMsg':
response_page.find('input', {
'name': 'pstMsg'
}).get('value'),
'TL':
response_page.find('input', {
'name': 'TL'
}).get('value'),
'gxf':
response_page.find('input', {
'name': 'gxf'
}).get('value'),
'id-challenge':
challenges_txt,
'id-assertion':
auth_response,
'TrustDevice':
'on',
}
return self.post(challenge_url, data=payload)
snr_mode=snr_mode,
signal_radius=sig_radius,
bg_inner_radius=bg_inner_radius,
bg_outer_radius=bg_outer_radius,
crop_size=crop_size,
bg_ratio=bg_ratio,
signal_ratio=sig_ratio,
signal_thres=sig_thres,
)
if peaks_dict.get('strong', None) is not None:
peaks += peaks_dict['strong'].tolist()
comm.send(peaks, dest=0)
stop = comm.recv(source=0)
if stop:
print('slave %d is exiting' % rank, flush=flush)
if __name__ == '__main__':
comm = MPI.COMM_WORLD
size = comm.Get_size()
if size == 1:
print('Run batch hit finder with at least 2 processes!')
sys.exit()
rank = comm.Get_rank()
argv = docopt(__doc__)
if rank == 0:
master_run(argv)
else:
worker_run(argv)
def handle_iap(self, sess):
response_page = BeautifulSoup(sess.text, 'html.parser')
challenge_url = sess.url.split("?")[0]
phone_number = input('Enter your phone number:') or None
while True:
try:
choice = int(
input(
'Type 1 to receive a code by SMS or 2 for a voice call:'
))
if choice not in [1, 2]:
raise ValueError
except ValueError:
print("Not a valid (integer) option, try again")
continue
else:
if choice == 1:
send_method = 'SMS'
elif choice == 2:
send_method = 'VOICE'
else:
continue
break
payload = {
'challengeId':
response_page.find('input', {
'name': 'challengeId'
}).get('value'),
'challengeType':
response_page.find('input', {
'name': 'challengeType'
}).get('value'),
'continue':
self.cont,
'scc':
response_page.find('input', {
'name': 'scc'
}).get('value'),
'sarp':
response_page.find('input', {
'name': 'sarp'
}).get('value'),
'checkedDomains':
response_page.find('input', {
'name': 'checkedDomains'
}).get('value'),
'pstMsg':
response_page.find('input', {
'name': 'pstMsg'
}).get('value'),
'TL':
response_page.find('input', {
'name': 'TL'
}).get('value'),
'gxf':
response_page.find('input', {
'name': 'gxf'
}).get('value'),
'phoneNumber':
phone_number,
'sendMethod':
send_method,
}
# Submit phone number and desired method (SMS or voice call)
sess = self.post(challenge_url, data=payload)
response_page = BeautifulSoup(sess.text, 'html.parser')
challenge_url = sess.url.split("?")[0]
token = input("Enter " + send_method + " token: G-") or None
payload = {
'challengeId':
response_page.find('input', {
'name': 'challengeId'
}).get('value'),
'challengeType':
response_page.find('input', {
'name': 'challengeType'
}).get('value'),
'continue':
response_page.find('input', {
'name': 'continue'
}).get('value'),
'scc':
response_page.find('input', {
'name': 'scc'
}).get('value'),
'sarp':
response_page.find('input', {
'name': 'sarp'
}).get('value'),
'checkedDomains':
response_page.find('input', {
'name': 'checkedDomains'
}).get('value'),
'pstMsg':
response_page.find('input', {
'name': 'pstMsg'
}).get('value'),
'TL':
response_page.find('input', {
'name': 'TL'
}).get('value'),
'gxf':
response_page.find('input', {
'name': 'gxf'
}).get('value'),
'pin':
token,
}
# Submit SMS/VOICE token
return self.post(challenge_url, data=payload)
def worker_run(args):
stop = False
filepath = None
h5_obj = None
buffer_size = int(args['--buffer-size'])
flush = args['--flush']
# hit finding parameters
with open(args['<conf-file>']) as f:
conf = yaml.load(f)
center = conf['center']
adu_per_photon = conf['adu per photon']
epsilon = conf['epsilon']
bin_size = conf['bin size']
if conf['mask on']:
mask = util.read_image(conf['mask file'])['image']
else:
mask = None
hit_finder = conf['hit finder']
gaussian_sigma = conf['gaussian filter sigma']
min_distance = conf['min distance']
min_gradient = conf['min gradient']
max_peaks = conf['max peaks']
min_snr = conf['min snr']
min_pixels = conf['min pixels']
peak_refine_mode = conf['peak refine mode']
snr_mode = conf['snr mode']
sig_radius = conf['signal radius']
bg_inner_radius = conf['background inner radius']
bg_outer_radius = conf['background outer radius']
crop_size = conf['crop size']
bg_ratio = conf['background ratio']
sig_ratio = conf['signal ratio']
sig_thres = conf['signal threshold']
dataset = conf['dataset']
# perform hit finding
while not stop:
job = comm.recv(buf=buffer_size, source=0)
peaks = []
for i in range(len(job)):
_filepath = job[i]['filepath']
frame = job[i]['frame']
if _filepath != filepath:
filepath = _filepath
ext = filepath.split('.')[-1]
h5_obj = h5py.File(filepath, 'r') if ext in ('cxi', 'h5') \
else None
lcls_data = util.get_lcls_data(filepath) if ext == 'lcls' \
else None
image = util.read_image(filepath,
frame=frame,
h5_obj=h5_obj,
lcls_data=lcls_data,
dataset=dataset)['image']
if image is None:
continue
peaks_dict = util.find_peaks(
image,
center,
adu_per_photon=adu_per_photon,
epsilon=epsilon,
bin_size=bin_size,
mask=mask,
hit_finder=hit_finder,
gaussian_sigma=gaussian_sigma,
min_gradient=min_gradient,
min_distance=min_distance,
max_peaks=max_peaks,
min_snr=min_snr,
min_pixels=min_pixels,
refine_mode=peak_refine_mode,
snr_mode=snr_mode,
signal_radius=sig_radius,
bg_inner_radius=bg_inner_radius,
bg_outer_radius=bg_outer_radius,
crop_size=crop_size,
bg_ratio=bg_ratio,
signal_ratio=sig_ratio,
signal_thres=sig_thres,
)
if peaks_dict.get('strong', None) is not None:
peaks += peaks_dict['strong'].tolist()
comm.send(peaks, dest=0)
stop = comm.recv(source=0)
if stop:
print('slave %d is exiting' % rank, flush=flush)
def print_part(*args):
print(*args, end="", flush=True)
def download(key,
secret,
version,
replays_dir,
download_dir,
extract=False,
remove=False,
filter_version='keep'):
# Get OAuth token from us region
api = BnetAPI(key, secret)
# Get meta file infos for the give client version
print('Searching replay packs with client version:', version)
meta_file_urls = api.search_by_client_version(version)
if len(meta_file_urls) == 0:
sys.exit('No matching replay packs found for the client version!')
# Download replay packs.
download_base_url = api.get_base_url()
print('Found {} replay packs'.format(len(meta_file_urls)))
print('Downloading to:', download_dir)
print('Extracting to:', replays_dir)
mkdirs(download_dir)
for i, meta_file_url in enumerate(sorted(meta_file_urls), 1):
# Construct full url to download replay packs
meta_file_info = api.get(meta_file_url)
archive_url = requests.compat.urljoin(download_base_url,
meta_file_info['path'])
print_part('{}/{}: {} ... '.format(i, len(meta_file_urls),
archive_url))
file_name = archive_url.split('/')[-1]
file_path = os.path.join(download_dir, file_name)
with requests.get(archive_url, stream=True) as response:
print_part(
int(response.headers['Content-Length']) // 1024**2, 'Mb ... ')
if (not os.path.exists(file_path) or os.path.getsize(file_path) !=
int(response.headers['Content-Length'])):
with open(file_path, 'wb') as f:
shutil.copyfileobj(response.raw, f)
print_part('downloaded')
else:
print_part('found')
if extract:
print_part(' ... extracting')
if os.path.getsize(file_path) <= 22: # Size of an empty zip file.
print_part(' ... zip file is empty')
else:
subprocess.call([
'unzip', '-P', 'iagreetotheeula', '-u', '-o', '-q', '-d',
replays_dir, file_path
])
if remove:
os.remove(file_path)
print()
if mpyq is not None and filter_version != 'keep':
print('Filtering replays.')
found_versions = collections.defaultdict(int)
found_str = lambda: ', '.join(
'%s: %s' % (v, c) for v, c in sorted(found_versions.items()))
all_replays = [
f for f in os.listdir(replays_dir) if f.endswith('.SC2Replay')
]
for i, file_name in enumerate(all_replays):
if i % 100 == 0:
print_part('\r%s/%s: %d%%, found: %s' %
(i, len(all_replays), 100 * i / len(all_replays),
found_str()))
file_path = os.path.join(replays_dir, file_name)
with open(file_path, "rb") as fd:
try:
archive = mpyq.MPQArchive(fd).extract()
except KeyboardInterrupt:
return
except:
found_versions['corrupt'] += 1
os.remove(file_path)
continue
metadata = json.loads(
archive[b'replay.gamemetadata.json'].decode('utf-8'))
game_version = '.'.join(metadata['GameVersion'].split('.')[:-1])
found_versions[game_version] += 1
if filter_version == 'sort':
version_dir = os.path.join(replays_dir, game_version)
if found_versions[
game_version] == 1: # First one of this version.
mkdirs(version_dir)
os.rename(file_path, os.path.join(version_dir, file_name))
elif filter_version == 'delete':
if game_version != version:
os.remove(file_path)
print('\nFound replays:', found_str())
# cut data into XOR
ytrue = ((Xs[:, 0] < np.mean(Xs[:, 0])) * (Xs[:, 1] < np.mean(Xs[:, 1])) + (Xs[:, 0] > np.mean(Xs[:, 0])) * (
Xs[:, 1] > np.mean(Xs[:, 1]))) * 1
ys = np.array([-1] * N)
sidx = np.append(np.random.choice(np.where(ytrue == 0)[0], supevised_data_points // 2),
np.random.choice(np.where(ytrue == 1)[0], supevised_data_points // 2))
ys[sidx] = ytrue[sidx]
Xsupervised = Xs[ys != -1, :]
ysupervised = ys[ys != -1]
# compare models
lbl = "Purely supervised SVM:"
print(lbl)
model = sklearn.svm.SVC(kernel=kernel, probability=True)
model.fit(Xsupervised, ysupervised)
evaluate_and_plot(model, Xs, ys, ytrue, lbl, 1)
lbl = "S3VM (Gieseke et al. 2012):"
print(lbl)
model = scikitTSVM.SKTSVM(kernel=kernel)
model.fit(Xs, ys)
evaluate_and_plot(model, Xs, ys, ytrue, lbl, 2)
lbl = "CPLE(pessimistic) SVM:"
print(lbl)
model = CPLELearningModel(sklearn.svm.SVC(kernel=kernel, probability=True), predict_from_probabilities=True)
model.fit(Xs, ys)
evaluate_and_plot(model, Xs, ys, ytrue, lbl, 3)
batch = []
count += 1
comm.send(job, dest=0)
stop = comm.recv(source=0)
# save last cxi batch if not empty
if len(batch) > 0:
comp_file = os.path.join(comp_dir,
'%s-rank%d-job%d.cxi' % (prefix, rank, count))
util.save_cxi(batch,
comp_file,
comp_dataset,
out_dtype=comp_dtype,
shuffle=shuffle)
done = True
comm.send(done, dest=0)
if __name__ == '__main__':
comm = MPI.COMM_WORLD
size = comm.Get_size()
if size == 1:
print('Run batch compressor with at least 2 processes!')
sys.exit()
rank = comm.Get_rank()
args = docopt(__doc__)
if rank == 0:
master_run(args)
else:
worker_run(args)
def handle_captcha(self, sess, payload):
response_page = BeautifulSoup(sess.text, 'html.parser')
# Collect ProfileInformation, SessionState, signIn, and Password Challenge URL
profile_information = response_page.find('input', {
'name': 'ProfileInformation'
}).get('value')
session_state = response_page.find('input', {
'name': 'SessionState'
}).get('value')
sign_in = response_page.find('input', {'name': 'signIn'}).get('value')
passwd_challenge_url = response_page.find('form', {
'id': 'gaia_loginform'
}).get('action')
# Update the payload
payload['SessionState'] = session_state
payload['ProfileInformation'] = profile_information
payload['signIn'] = sign_in
payload['Passwd'] = self.config.password
# Get all captcha challenge tokens and urls
captcha_container = response_page.find('div',
{'id': 'identifier-captcha'})
captcha_logintoken = captcha_container.find('input', {
'id': 'identifier-token'
}).get('value')
captcha_img = captcha_container.find('div', {'class': 'captcha-img'})
captcha_url = "https://accounts.google.com" + captcha_img.find(
'img').get('src')
captcha_logintoken_audio = ''
open_image = True
# Check if there is a display utility installed as Image.open(f).show() do not raise any exception if not
# if neither xv or display are available just display the URL for the user to visit.
if os.name == 'posix' and sys.platform != 'darwin':
if find_executable('xv') is None and find_executable(
'display') is None:
open_image = False
print("Please visit the following URL to view your CAPTCHA: {}".format(
captcha_url))
if open_image:
try:
with requests.get(captcha_url) as url:
with io.BytesIO(url.content) as f:
Image.open(f).show()
except Exception:
pass
try:
captcha_input = raw_input("Captcha (case insensitive): ") or None
except NameError:
captcha_input = input("Captcha (case insensitive): ") or None
# Update the payload
payload['identifier-captcha-input'] = captcha_input
payload['identifiertoken'] = captcha_logintoken
payload['identifiertoken_audio'] = captcha_logintoken_audio
payload['checkedDomains'] = 'youtube'
payload['checkConnection'] = 'youtube:574:1'
payload['Email'] = self.config.username
response = self.post(passwd_challenge_url, data=payload)
newPayload = {}
auth_response_page = BeautifulSoup(response.text, 'html.parser')
form = auth_response_page.find('form')
for tag in form.find_all('input'):
if tag.get('name') is None:
continue
newPayload[tag.get('name')] = tag.get('value')
newPayload['Email'] = self.config.username
newPayload['Passwd'] = self.config.password
if newPayload.get('TrustDevice', None) is not None:
newPayload['TrustDevice'] = 'on'
return self.post(response.url, data=newPayload)
# and the second - how many threads the computational core
# should spawn
hyphyInstance = HyPhy._THyPhy (os.getcwd(),2)
# the basic interface command is 'ExecuteBF' which
# executes HyPhy batch language commands in HyPhy
# and returns a string representation of the return value
# (if any) from HYPHY
# The returned object is of type _THyPhyString with
# sData and sLength fields
# HyPhy will take care of disposing of the memory needed
# to store the result
hyphyResult = hyphyInstance.ExecuteBF ("return 2+2;");
print("Testing a trivial HyPhy command. 2+2 = ", hyphyResult.sData)
# an optional second argument to ExecuteBF
# can be used to "flush" the current state of the system
# this is the default option for the call of ExecuteBF
# passing the second argument of False or 0 will preserve
# the execution state
print("Consecutive command exection")
hyphyInstance.ExecuteBF ("z:=x+y;",False);
hyphyInstance.ExecuteBF ("x=3;",False);
hyphyInstance.ExecuteBF ("y=5;",False);
hyphyResult = hyphyInstance.ExecuteBF ("return z;",False);
print("The value of z is ", hyphyResult.sData)
def master_run(args):
flush = args['--flush']
file_lst = args['<file-lst>']
with open(file_lst) as f:
_files = f.readlines()
# remove trailing '/n'
files = []
for f in _files:
if '\n' == f[-1]:
files.append(f[:-1])
else:
files.append(f)
# load hit finding configuration file
with open(args['<conf-file>']) as f:
conf = yaml.load(f)
# collect jobs
dataset = conf['dataset']
batch_size = int(args['--batch-size'])
max_frames = int(args['--max-frames'])
buffer_size = int(args['--buffer-size'])
jobs, nb_frames = util.collect_jobs(files,
dataset,
batch_size,
max_frames=max_frames)
nb_jobs = len(jobs)
print('%d frames, %d jobs to be processed' % (nb_frames, nb_jobs),
flush=flush)
# dispatch jobs
job_id = 0
reqs = {}
peaks = []
workers = set(range(1, size))
finished_workers = set()
for worker in workers:
if job_id < nb_jobs:
job = jobs[job_id]
else:
job = [] # dummy job
comm.isend(job, dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
print('job %d/%d --> %d' % (job_id, nb_jobs, worker), flush=flush)
job_id += 1
while job_id < nb_jobs:
stop = False
time.sleep(0.1) # take a break
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
peaks += result
if job_id < nb_jobs:
print('job %d/%d --> %d' % (job_id, nb_jobs, worker),
flush=flush)
comm.isend(stop, dest=worker)
comm.isend(jobs[job_id], dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
job_id += 1
else:
stop = True
comm.isend(stop, dest=worker)
print('stop signal --> %d' % worker, flush=flush)
finished_workers.add(worker)
all_done = False
while not all_done:
all_done = True
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
peaks += result
stop = True
print('stop signal --> %d' % worker, flush=flush)
comm.isend(stop, dest=worker)
finished_workers.add(worker)
else:
all_done = False
# build and save peak powder
filepath = jobs[0][0]['filepath']
frame = jobs[0][0]['frame']
h5_obj = h5py.File(filepath, 'r')
image = util.read_image(filepath,
frame=frame,
h5_obj=h5_obj,
dataset=dataset)['image']
powder = np.zeros(image.shape)
peaks = np.round(np.array(peaks)).astype(np.int)
powder[peaks[:, 0], peaks[:, 1]] = 1
powder_file = args['-o']
dir_ = os.path.dirname(powder_file)
if not os.path.isdir(dir_):
os.mkdir(dir_)
np.savez(powder_file, powder_pattern=powder, powder_peaks=peaks)
print('All Done!', flush=flush)
MPI.Finalize()
def main():
args = parse_args()
# Get OAuth token from us region
api = BnetAPI(args.key, args.secret)
# Get meta file infos for the give client version
print('Searching replay packs with client version:', args.version)
meta_file_urls = api.search_by_client_version(args.version)
if len(meta_file_urls) == 0:
sys.exit('No matching replay packs found for the client version!')
# Download replay packs.
download_base_url = api.get_base_url()
print('Found {} replay packs'.format(len(meta_file_urls)))
print('Downloading to:', args.download_dir)
print('Extracting to:', args.replays_dir)
mkdirs(args.download_dir)
for i, meta_file_url in enumerate(sorted(meta_file_urls), 1):
# Construct full url to download replay packs
meta_file_info = api.get(meta_file_url)
archive_url = requests.compat.urljoin(download_base_url, meta_file_info['path'])
print_part('{}/{}: {} ... '.format(i, len(meta_file_urls), archive_url))
file_name = archive_url.split('/')[-1]
file_path = os.path.join(args.download_dir, file_name)
with requests.get(archive_url, stream=True) as response:
print_part(int(response.headers['Content-Length']) // 1024**2, 'Mb ... ')
if (not os.path.exists(file_path) or
os.path.getsize(file_path) != int(response.headers['Content-Length'])):
with open(file_path, 'wb') as f:
shutil.copyfileobj(response.raw, f)
print_part('downloaded')
else:
print_part('found')
if args.extract:
print_part(' ... extracting')
if os.path.getsize(file_path) <= 22:
print_part(' ... zip file is empty')
else:
subprocess.call(['unzip', '-P', 'iagreetotheeula', '-u', '-o', '-q', '-d', args.replays_dir, file_path])
if args.remove:
os.remove(file_path)
print()
if args.filter_version != 'keep':
print('Filtering replays.')
found_versions = collections.defaultdict(int)
found_str = lambda: ', '.join('%s: %s' % (v, c) for v, c in sorted(found_versions.items()))
all_replays = [f for f in os.listdir(args.replays_dir) if f.endswith('.SC2Replay')]
for i, file_name in enumerate(all_replays):
if i % 100 == 0:
print_part('\r%s/%s: %d%%, found: %s' % (i, len(all_replays), 100 * i / len(all_replays), found_str()))
file_path = os.path.join(args.replays_dir, file_name)
with open(file_path) as fd:
try:
archive = mpyq.MPQArchive(fd).extract()
except ValueError:
found_versions['corrupt'] += 1
os.remove(file_path)
continue
metadata = json.loads(archive[b'replay.gamemetadata.json'].decode('utf-8'))
game_version = '.'.join(metadata['GameVersion'].split('.')[:-1])
found_versions[game_version] += 1
if args.filter_version == 'sort':
version_dir = os.path.join(args.replays_dir, game_version)
if found_versions[game_version] == 1: # First one of this version.
mkdirs(version_dir)
os.rename(file_path, os.path.join(version_dir, file_name))
elif args.filter_version == 'delete':
if game_version != args.version:
os.remove(file_path)
print('\nFound replays:', found_str())
def print_part(*args):
print(*args, end="", flush=True)
# This script shows how to use the client in anonymous mode
# against jira.atlassian.com.
from six import print_ as print
from jira.client import GreenHopper
# By default, the client will connect to a JIRA instance started from the Atlassian Plugin SDK
# (see https://developer.atlassian.com/display/DOCS/Installing+the+Atlassian+Plugin+SDK for details).
# Override this with the options parameter.
# GreenHopper is a plugin in a JIRA instance
options = {'server': 'https://jira.atlassian.com'}
gh = GreenHopper(options)
# Get all boards viewable by anonymous users.
boards = gh.boards()
# Get the sprints in a specific board
board_id = 441
print("GreenHopper board: %s (%s)" % (boards[0].name, board_id))
sprints = gh.sprints(board_id)
# List the incomplete issues in each sprint
for sprint in sprints:
sprint_id = sprint.id
print("Sprint: %s" % sprint.name)
incompleted_issues = gh.incompleted_issues(board_id, sprint_id)
print("Incomplete issues: %s" % ', '.join(issue.key
for issue in incompleted_issues))
def master_run(args):
flush = args['--flush']
# mkdir for output if not exist
comp_dir = os.path.abspath(args['<comp-dir>'])
comp_lst_dir = os.path.abspath(args['<comp-lst-dir>'])
if not os.path.isdir(comp_dir):
os.makedirs(comp_dir)
if not os.path.isdir(comp_lst_dir):
os.makedirs(comp_lst_dir)
# get all raw data files
raw_lst = args['<raw-lst>']
with open(raw_lst) as f:
_files = f.readlines()
# remove trailing '/n'
files = []
raw_size = 0 # total size fo input files(raw data)
for f in _files:
files.append(f[:-1])
raw_size += os.path.getsize(f[:-1])
# collect jobs
batch_size = int(args['--batch-size'])
buffer_size = int(args['--buffer-size'])
raw_dataset = args['<raw-dataset>']
jobs, nb_frames = util.collect_jobs(files, raw_dataset, batch_size)
nb_jobs = len(jobs)
time_start = time.time()
print('%d frames, %d jobs to be processed' % (nb_frames, nb_jobs),
flush=flush)
update_freq = int(args['--update-freq'])
# distribute jobs
job_id = 0
reqs = {}
results = []
workers = set(range(1, size))
finished_workers = set()
for worker in workers:
if job_id < nb_jobs:
job = jobs[job_id]
else:
job = [] # dummy job
comm.isend(job, dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
print('job %d/%d --> %d' % (job_id, nb_jobs, worker), flush=flush)
job_id += 1
while job_id < nb_jobs:
stop = False
time.sleep(0.1) # take a break
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
results += result
if job_id < nb_jobs:
print('job %d/%d --> %d' % (job_id, nb_jobs, worker),
flush=flush)
comm.isend(stop, dest=worker)
comm.isend(jobs[job_id], dest=worker)
reqs[worker] = comm.irecv(buf=buffer_size, source=worker)
job_id += 1
else:
stop = True
comm.isend(stop, dest=worker)
finished_workers.add(worker)
if job_id % update_freq == 0:
progress = float(job_id) / nb_jobs * 100
stat_dict = {
'time start': time_start,
'progress': '%.2f%%' % progress,
'duration/sec': 'not finished',
'total frames': nb_frames,
'total jobs': nb_jobs,
'raw size': 'not finished',
'compressed size': 'not finished',
'compression ratio': 'not finished',
}
stat_file = os.path.join(comp_dir, 'stat.yml')
with open(stat_file, 'w') as f:
yaml.dump(stat_dict, f, default_flow_style=False)
all_accepted = False
while not all_accepted:
all_accepted = True
time.sleep(0.1) # take a break
workers -= finished_workers
for worker in workers:
finished, result = reqs[worker].test()
if finished:
stop = True
comm.isend(stop, dest=worker)
finished_workers.add(worker)
else:
all_accepted = False
print('all jobs accepted', flush=flush)
# check status of last job batch
all_done = False
workers = set(range(1, size))
finished_workers = set()
for worker in workers:
reqs[worker] = comm.irecv(source=worker)
while not all_done:
all_done = True
time.sleep(0.1)
workers -= finished_workers
for worker in workers:
done, _ = reqs[worker].test()
if done:
print('worker %d done' % worker, flush=flush)
finished_workers.add(worker)
else:
all_done = False
# save results
time_end = time.time()
duration = time_end - time_start
# write compressed cxi file lst
compressed_size = 0
prefix = os.path.basename(raw_lst).split('.')[0]
comp_lst_file = open(os.path.join(comp_lst_dir, '%s.lst' % prefix), 'w')
comp_files = glob('%s/*.cxi' % comp_dir)
for comp_file in comp_files:
compressed_size += os.path.getsize(os.path.abspath(comp_file))
comp_lst_file.write('%s\n' % comp_file)
comp_lst_file.close()
compression_ratio = float(raw_size) / float(compressed_size)
stat_dict = {
'time start': time_start,
'progress': 'done',
'duration/sec': duration,
'total frames': nb_frames,
'total jobs': nb_jobs,
'raw size': raw_size,
'compressed size': compressed_size,
'compression ratio': '%.2f' % compression_ratio,
}
stat_file = os.path.join(comp_dir, 'stat.yml')
with open(stat_file, 'w') as f:
yaml.dump(stat_dict, f, default_flow_style=False)
print('All Done!', flush=flush)
def process_auth(args, config):
# Set up logging
logging.getLogger().setLevel(getattr(logging, args.log_level.upper(), None))
if config.region is None:
config.region = util.Util.get_input("AWS Region: ")
logging.debug('%s: region is: %s', __name__, config.region)
# If there is a valid cache and the user opted to use it, use that instead
# of prompting the user for input (it will also ignroe any set variables
# such as username or sp_id and idp_id, as those are built into the SAML
# response). The user does not need to be prompted for a password if the
# SAML cache is used.
if args.saml_assertion:
saml_xml = base64.b64decode(args.saml_assertion)
elif args.saml_cache and config.saml_cache:
saml_xml = config.saml_cache
logging.info('%s: SAML cache found', __name__)
else:
# No cache, continue without.
logging.info('%s: SAML cache not found', __name__)
if config.username is None:
config.username = util.Util.get_input("Google username: "******"Google IDP ID: ")
logging.debug('%s: idp is: %s', __name__, config.idp_id)
if config.sp_id is None:
config.sp_id = util.Util.get_input("Google SP ID: ")
logging.debug('%s: sp is: %s', __name__, config.sp_id)
# There is no way (intentional) to pass in the password via the command
# line nor environment variables. This prevents password leakage.
keyring_password = None
if config.keyring:
keyring_password = keyring.get_password("aws-google-auth", config.username)
if keyring_password:
config.password = keyring_password
else:
config.password = util.Util.get_password("Google Password: "******"Google Password: "******"aws-google-auth", config.username, config.password)
# We now have a new SAML value that can get cached (If the user asked
# for it to be)
if args.saml_cache:
config.saml_cache = saml_xml
# The amazon_client now has the SAML assertion it needed (Either via the
# cache or freshly generated). From here, we can get the roles and continue
# the rest of the workflow regardless of cache.
amazon_client = amazon.Amazon(config, saml_xml)
roles = amazon_client.roles
# Determine the provider and the role arn (if the the user provided isn't an option)
if config.role_arn in roles and not config.ask_role:
config.provider = roles[config.role_arn]
else:
if config.account and config.resolve_aliases:
aliases = amazon_client.resolve_aws_aliases(roles)
config.role_arn, config.provider = util.Util.pick_a_role(roles, aliases, config.account)
elif config.account:
config.role_arn, config.provider = util.Util.pick_a_role(roles, account=config.account)
elif config.resolve_aliases:
aliases = amazon_client.resolve_aws_aliases(roles)
config.role_arn, config.provider = util.Util.pick_a_role(roles, aliases)
else:
config.role_arn, config.provider = util.Util.pick_a_role(roles)
if not config.quiet:
print("Assuming " + config.role_arn)
print("Credentials Expiration: " + format(amazon_client.expiration.astimezone(get_localzone())))
if config.print_creds:
amazon_client.print_export_line()
if config.profile:
config.write(amazon_client)
