def download(self, filename, token=None):
# remove leading backlash
path = self.path
url = self.get_url(token)
self.path = None
if path.startswith('/'):
path = path[1:]
if self.credentials:
blob = self.bucket.get_blob(path)
blob.download_to_filename(filename)
else:
def _download_internal():
r = requests.get(url, stream=True)
raise_detailed_error(r)
if r.status_code == 200:
with open(filename, 'wb') as f:
for chunk in r:
f.write(chunk)
elif r.status_code >= 500:
raise HTTPServerError(r.status_code, r.text)
retry(_download_internal,
no_retries=10,
sleep_time=5,
exception_class=HTTPServerError)
def _resetDMP(sid,explicit_unmap=False,delete_nodes=False):
# If mpath has been turned on since the sr/vdi was attached, we
# might be trying to unmap it before the daemon has been started
# This is unnecessary (and will fail) so just return.
deactivate_MPdev(sid)
if not _is_mpath_daemon_running():
util.SMlog("Warning: Trying to unmap mpath device when multipathd not running")
return
# If the multipath daemon is running, but we were initially plugged
# with multipathing set to no, there may be no map for us in the multipath
# tables. In that case, list_paths will return [], but remove_map might
# throw an exception. Catch it and ignore it.
if explicit_unmap:
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-f', sid]),
maxretry = 3, period = 4)
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-W']), maxretry = 3,
period = 4)
else:
mpath_cli.ensure_map_gone(sid)
path = "/dev/mapper/%s" % sid
if not util.wait_for_nopath(path, 10):
util.SMlog("MPATH: WARNING - path did not disappear [%s]" % path)
else:
util.SMlog("MPATH: path disappeared [%s]" % path)
def _resetDMP(sid, explicit_unmap=False):
# If mpath has been turned on since the sr/vdi was attached, we
# might be trying to unmap it before the daemon has been started
# This is unnecessary (and will fail) so just return.
deactivate_MPdev(sid)
if not _is_mpath_daemon_running():
util.SMlog("Warning: Trying to unmap mpath device when multipathd not running")
return
# If the multipath daemon is running, but we were initially plugged
# with multipathing set to no, there may be no map for us in the multipath
# tables. In that case, list_paths will return [], but remove_map might
# throw an exception. Catch it and ignore it.
if explicit_unmap:
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-f', sid]),
maxretry=3, period=4)
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-W']), maxretry=3,
period=4)
else:
mpath_cli.ensure_map_gone(sid)
path = "/dev/mapper/%s" % sid
if not util.wait_for_nopath(path, 10):
util.SMlog("MPATH: WARNING - path did not disappear [%s]" % path)
else:
util.SMlog("MPATH: path disappeared [%s]" % path)
def reload(self):
if not util.confirm(
"Are you sure you want to tear down the {} ASgroup and recreate it?"
.format(self.name())):
return
util.message_integrations("Reloading ASgroup {}".format(self.name()))
self.deactivate()
util.retry(lambda: self.activate(), 60)
def proxyVotes(b, e):
vote(firstProducer, firstProducer + 1)
proxy = accounts[firstProducer]['name']
retry(args.cleos + 'system regproxy ' + proxy)
sleep(1.0)
for i in range(b, e):
voter = accounts[i]['name']
retry(args.cleos + 'system voteproducer proxy ' + voter + ' ' + proxy)
def top_tracks(self, artist_uid, count=10):
uid = artist_uid.split(':')[-1].split('/')[-1]
uri = 'https://api.spotify.com/v1/artists/%s' % uid
log.debug(uri)
resp = util.retry(uri)
doc = resp.json()
artist_name = doc['name']
limit = min(count, 50)
misses = 0
offset = 0
tracks = []
seen = set()
log.debug(artist_name.encode('utf-8'))
while len(tracks) < count and misses < 5:
prev_len = len(tracks)
uri = 'https://api.spotify.com/v1/search?%s' % urllib.urlencode(
{
'type': 'track',
'limit': limit,
'offset': offset,
'q': 'artist:%s' % artist_name.encode('utf-8')
}
)
log.debug(uri)
resp = util.retry(uri)
try:
doc = resp.json()
except:
doc = {'tracks': {'items': []}}
for track in doc.get('tracks', {}).get('items',[]):
title = kt.normalize(track['name'])
if track['artists'][0]['uri'].split(':')[-1] == uid and title not in seen:
uniq = True
for seen_title in seen:
if kt.matches(title, seen_title):
uniq = False
break
if uniq:
tracks.append(track_from_json(track))
seen.add(title)
if len(tracks) >= count:
break
offset += limit
if len(tracks) == prev_len:
misses += 1
return tracks
def finish_download():
try:
self._download_file(key, tar_filename)
except BaseException as e:
self.logger.debug(e)
if os.path.exists(tar_filename):
# first, figure out if the tar file has a base path of .
# or not
self.logger.info("Untarring {}".format(tar_filename))
listtar, _ = subprocess.Popen(['tar', '-tf', tar_filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True).communicate()
listtar = listtar.strip().split(b'\n')
listtar = [s.decode('utf-8') for s in listtar]
self.logger.info('List of files in the tar: ' + str(listtar))
if listtar[0].startswith('./'):
# Files are archived into tar from .; adjust path
# accordingly
basepath = local_path
else:
basepath = local_basepath
tarcmd = ('mkdir -p {} && ' +
'tar -xf {} -C {} --keep-newer-files') \
.format(basepath, tar_filename, basepath)
self.logger.debug('Tar cmd = {}'.format(tarcmd))
tarp = subprocess.Popen(['/bin/bash', '-c', tarcmd],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
tarout, tarerr = tarp.communicate()
if tarp.returncode != 0:
self.logger.info('tar had a non-zero return code!')
self.logger.info('tar cmd = ' + tarcmd)
self.logger.info('tar output: \n ' + str(tarout))
if len(listtar) == 1:
actual_path = os.path.join(basepath, listtar[0])
self.logger.info('Renaming {} into {}'.format(
actual_path, local_path))
retry(lambda: os.rename(actual_path, local_path),
no_retries=5,
sleep_time=1,
exception_class=OSError,
logger=self.logger)
os.remove(tar_filename)
else:
self.logger.warning(
'file {} download failed'.format(tar_filename))
def _refresh_DMP(sid, npaths):
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-r', sid]), maxretry = 3,
period = 4)
path = os.path.join(DEVMAPPERPATH, sid)
util.wait_for_path(path, 10)
if not os.path.exists(path):
raise xs_errors.XenError('DMP failed to activate mapper path')
lvm_path = "/dev/disk/by-scsid/"+sid+"/mapper"
util.wait_for_path(lvm_path, 10)
activate_MPdev(sid, path)
def activate_lc(self):
# make sure we have a launchconfig activated
lc = self.lc()
if not lc:
# might need to wait a min
util.retry(lambda: self.lc(), 60)
if lc.exists():
print "Using LaunchConfig {}".format(lc.name())
return True
print "Creating LaunchConfig {}".format(lc.name())
return lc.activate()
def update_lc(self):
oldlc = self.lc()
# get new version
lc = oldlc.update()
# set lc
asgroup = self.get_asgroup()
lcname = lc.name()
setattr(asgroup, 'launch_config_name', lcname)
asgroup.update()
# delete old
conn = util.as_conn()
if oldlc.name() is not lcname:
util.retry(lambda: conn.delete_launch_configuration(oldlc.name()), 60)
def reload(self):
# skip deactivation if it doesn't exist
asgroup = self.get_asgroup()
if not asgroup or not self.exists():
self.activate()
return
debug("In asgroup.py reload")
if not util.confirm("Are you sure you want to tear down the {} ASgroup and recreate it?".format(self.name())):
return
util.message_integrations("Reloading ASgroup {}".format(self.name()))
self.deactivate()
util.retry(lambda: self.activate(), 60)
def _refresh_DMP(sid, npaths):
if not _is_valid_multipath_device(sid):
return
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-r', sid]),
maxretry=3,
period=4)
path = os.path.join(DEVMAPPERPATH, sid)
util.wait_for_path(path, 10)
if not os.path.exists(path):
raise xs_errors.XenError('DMP failed to activate mapper path')
lvm_path = "/dev/disk/by-scsid/" + sid + "/mapper"
util.wait_for_path(lvm_path, 10)
activate_MPdev(sid, path)
def update_lc(self):
oldlc = self.lc()
# get new version
lc = oldlc.update()
# set lc
asgroup = self.get_asgroup()
lcname = lc.name()
setattr(asgroup, 'launch_config_name', lcname)
asgroup.update()
# delete old
conn = util.as_conn()
if oldlc.name() is not lcname:
util.retry(lambda: conn.delete_launch_configuration(oldlc.name()),
60)
def stop_experiment(self, experiment):
if isinstance(experiment, six.string_types):
key = experiment
else:
key = experiment.key
headers = self._get_headers()
def post_request():
request = requests.post(self.url + '/api/stop_experiment',
headers=headers,
data=json.dumps({"key": key}))
self._raise_detailed_error(request)
retry(post_request, sleep_time=10, logger=self.logger)
def activate_lc(self):
# make sure we have a launchconfig activated
lc = self.lc()
if not lc:
# might need to wait a min
util.retry(lambda: self.lc(), 60)
lc = self.lc()
if not lc:
print "Timed out waiting to create LaunchConfiguration"
return false
if lc.exists():
print "Using LaunchConfig {}".format(lc.name())
return True
print "Creating LaunchConfig {}".format(lc.name())
return lc.activate()
def reload(self):
# skip deactivation if it doesn't exist
asgroup = self.get_asgroup()
if not asgroup or not self.exists():
self.activate()
return
debug("In asgroup.py reload")
if not util.confirm(
"Are you sure you want to tear down the {} ASgroup and recreate it?"
.format(self.name())):
return
util.message_integrations("Reloading ASgroup {}".format(self.name()))
self.deactivate()
util.retry(lambda: self.activate(), 60)
def _put_file(self, path, file_object, token, userid):
request_ref = self.storage_bucket + "/o?name={0}".format(path)
def post_file(**kwargs):
def _post_file():
request_object = self.requests.post(request_ref,
data=file_object,
**kwargs)
raise_detailed_error(request_object)
return request_object
return retry(_post_file,
no_retries=10,
sleep_time=5,
exception_class=HTTPServerError)
if token:
headers = {"Authorization": "Firebase " + token}
request_object = post_file(headers=headers)
if userid:
headers['Content-Type'] = 'application/json'
def patch_owner():
patch_request = self.requests.patch(
request_ref,
headers=headers,
data=json.dumps({'metadata': {
'owner': userid
}}))
raise_detailed_error(patch_request)
return patch_request
retry(patch_owner,
no_retries=10,
sleep_time=5,
exception_class=HTTPServerError)
return request_object.json()
elif self.credentials:
blob = self.bucket.blob(path)
if isinstance(file_object, str):
return blob.upload_from_filename(filename=file_object)
else:
return blob.upload_from_file(file_obj=file_object)
else:
return post_file().json()
def deactivate(self): # a.k.a asg destroy
# NOTE
# deleting asg logic should be in its own function
# * delete ASG by reducing capacities of asg to 0
# * delete launchconfig
#
# reducing ASG capacities to 0 triggers eventual instance
# termination
debug("In asgroup.py deactivate")
asg_name = self.name()
ag = util.as_conn()
ec2 = util.ec2_conn()
asg_info = ag.describe_auto_scaling_groups( AutoScalingGroupNames = [ asg_name ] )
if not asg_info['AutoScalingGroups']:
print("ASG does not exist. Maybe it was already deleted?")
else:
# delete the ASG
num_instances = len(asg_info['AutoScalingGroups'][0]['Instances'])
if self.get_num_instances() == 0:
pprint("There are no instances in asg: " + asg_name)
print("Deleting asg: " + asg_name)
response = ag.delete_auto_scaling_group( AutoScalingGroupName=asg_name )
util.message_integrations("Deleted ASgroup {}".format(asg_name))
else:
debug("There are " + str(num_instances) + " instances that need to be removed from asg: " + asg_name)
debug("terminating instances in asg: " + asg_name)
debug("by setting to 0 MinSize, MaxSize, DesiredCapacity")
response = ag.update_auto_scaling_group(AutoScalingGroupName = asg_name, MinSize=0, MaxSize=0, DesiredCapacity=0)
debug("Waiting 30 seconds to give AWS time to terminate the instances")
if self.get_num_instances() != 0:
util.retry(lambda: ag.delete_auto_scaling_group(AutoScalingGroupName=asg_name), 300)
if self.get_num_instances() != 0 or self.get_num_instances():
print("unable to delete instances in asg.")
return False
util.message_integrations("Deleted ASgroup {}".format(asg_name))
# if launch config exists, delete it
lc = self.lc()
if not lc.exists():
print("launchconfig does not exist. Maybe you deleted it already?")
else:
lc.deactivate()
return True
def add_experiment(self, experiment, userid=None, compression=None):
headers = self._get_headers()
compression = compression if compression else self.compression
for tag, art in six.iteritems(experiment.artifacts):
if not art['mutable'] and art.get('local') is not None:
art['hash'] = HTTPArtifactStore(None, None,
compression,
self.verbose) \
.get_artifact_hash(art)
data = {}
data['experiment'] = experiment.__dict__
data['compression'] = compression
def post_request():
request = requests.post(self.url + '/api/add_experiment',
headers=headers,
data=json.dumps(data))
self._raise_detailed_error(request)
return request
request = retry(post_request,
no_retries=10,
sleep_time=10,
logger=self.logger)
artifacts = request.json()['artifacts']
self._update_artifacts(experiment, artifacts)
def get_next_prime():
log.debug("get_next_prime()")
@transactional
def get_ownership(prime):
log.debug("get_ownereship()")
key = prime.key()
last_assigned_at = prime.last_assigned_at
# get the current entity in this transaction
prime = model.Prime.get(key)
if prime.last_assigned_at != last_assigned_at:
return None
prime.last_assigned_at = datetime.now()
prime.put()
return prime
while retry(10, raise_if_failed=True):
all_primes = model.Prime.all()
primes = all_primes.order("last_assigned_at").order("number").fetch(1)
log.debug("primes retrieved")
if primes:
prime = get_ownership(primes[0])
if prime:
return prime
elif not all_primes.fetch(1):
log.debug("call Prime.ensure_number()")
sentinel = model.Prime.ensure_number(0, parent=prime_root)
else:
pass
def get_max_sieve():
log.debug("get_max_sieve()")
while retry(10, raise_if_failed=True):
all_sieves = model.Sieve.all()
sieves = all_sieves.order("-number").fetch(1)
if sieves:
return sieves[0].number
def get_issues(page):
url = ("https://api.github.com/repos/Khan/khan-exercises/issues"
"?page=%d&per_page=100" % page)
issue_data = util.retry(lambda: urllib2.urlopen(url, timeout=60),
'fetching khan-exercises issues',
lambda exc: isinstance(exc, urlfetch_errors))
# This flag is False if we should continue to the next page of
# issues and True if we should stop looking at more pages.
done = False
for issue in json.loads(issue_data.read()):
if issue["user"]["login"] == "KhanBugz":
if last_issue == -1:
# If we have no data so far, only go one page.
done = True
if issue["number"] > last_issue:
first_issue[0] = max(first_issue[0], issue["number"])
issues.append(issue)
else:
# If we've come to an issue we already saw,
# don't continue to further pages or issues
done = True
break
if ((re.findall(r'<(.*?)>; rel="(.*?)"',
issue_data.info().getheader("Link"))[0][1] == "next")
and not done):
get_issues(page + 1)
def get_issues(page):
url = ("https://api.github.com/repos/Khan/khan-exercises/issues"
"?page=%d&per_page=100" % page)
issue_data = util.retry(lambda: urllib2.urlopen(url, timeout=60),
'fetching khan-exercises issues',
lambda exc: isinstance(exc, urlfetch_errors))
# This flag is False if we should continue to the next page of
# issues and True if we should stop looking at more pages.
done = False
for issue in json.loads(issue_data.read()):
if issue["user"]["login"] == "KhanBugz":
if last_issue == -1:
# If we have no data so far, only go one page.
done = True
if issue["number"] > last_issue:
first_issue[0] = max(first_issue[0], issue["number"])
issues.append(issue)
else:
# If we've come to an issue we already saw,
# don't continue to further pages or issues
done = True
break
if ((re.findall(
r'<(.*?)>; rel="(.*?)"',
issue_data.info().getheader("Link"))[0][1] == "next") and
not done):
get_issues(page + 1)
def activate_lc(self):
debug("In asgroup.py activate_lc")
# make sure we have a launchconfig activated
lc = self.lc()
if not lc:
# might need to wait a min
util.retry(lambda: self.lc(), 60)
lc = self.lc()
if not lc:
print "Timed out waiting to create LaunchConfiguration"
return false
if lc.exists():
print "Using LaunchConfig {}".format(lc.get_lc_server_name())
return True
print "Creating LaunchConfig {}".format(lc.name())
return lc.activate()
def _refresh_DMP(sid, npaths):
if not _is_valid_multipath_device(sid):
return
path = os.path.join(DEVMAPPERPATH, sid)
# If the mapper path doesn't exist force a reload in multipath
if not os.path.exists(path):
util.retry(lambda: util.pread2(['/usr/sbin/multipath', '-r', sid]),
maxretry=3,
period=4)
util.wait_for_path(path, 10)
if not os.path.exists(path):
raise xs_errors.XenError(
'MultipathMapperPathMissing',
'Device mapper path {} not found'.format(path))
lvm_path = "/dev/disk/by-scsid/" + sid + "/mapper"
util.wait_for_path(lvm_path, 10)
activate_MPdev(sid, path)
def deactivate(self):
if not self.exists():
return
print "Deleting launchconfig..."
lc = self.get_lc()
if util.retry(lambda: lc.delete(), 500):
util.message_integrations("Deleted LaunchConfig {}".format(self.name()))
else:
util.message_integrations("Failed to delete LaunchConfig {}".format(self.name()))
def deactivate(self):
if not self.exists():
return
print "Deleting launchconfig..."
lc = self.get_lc()
if util.retry(lambda: lc.delete(), 500):
util.message_integrations("Deleted LaunchConfig {}".format(self.name()))
else:
util.message_integrations("Failed to delete LaunchConfig {}".format(self.name()))
def post_file(**kwargs):
def _post_file():
request_object = self.requests.post(request_ref,
data=file_object,
**kwargs)
raise_detailed_error(request_object)
return request_object
return retry(_post_file,
no_retries=10,
sleep_time=5,
exception_class=HTTPServerError)
def myretry(*args, **kvargs):
""" Log error and how many retries is remaining. """
def log_retries(tries_remaining, exception, delay):
global logger
logger.warn('{}; retry {} more times, next in {} sec.'\
.format(
str(exception),
tries_remaining,
delay
)
)
kvargs['hook']=log_retries
return retry(*args, **kvargs)
def myretry(*args, **kvargs):
""" Log error and how many retries is remaining. """
def log_retries(tries_remaining, exception, delay):
global logger
logger.warn('{}; retry {} more times, next in {} sec.'\
.format(
str(exception),
tries_remaining,
delay
)
)
kvargs['hook'] = log_retries
return retry(*args, **kvargs)
def createStakedAccounts(b, e):
ramFunds = round(args.ram_funds * 10000)
configuredMinStake = round(args.min_stake * 10000)
maxUnstaked = round(args.max_unstaked * 10000)
for i in range(b, e):
a = accounts[i]
funds = a['funds']
print('#' * 80)
print('# %d/%d %s %s' % (i, e, a['name'], intToCurrency(funds)))
print('#' * 80)
if funds < ramFunds:
print('skipping %s: not enough funds to cover ram' % a['name'])
continue
minStake = min(funds - ramFunds, configuredMinStake)
unstaked = min(funds - ramFunds - minStake, maxUnstaked)
stake = funds - ramFunds - unstaked
stakeNet = round(stake / 2)
stakeCpu = stake - stakeNet
print('%s: total funds=%s, ram=%s, net=%s, cpu=%s, unstaked=%s' % (a['name'], intToCurrency(a['funds']), intToCurrency(ramFunds), intToCurrency(stakeNet), intToCurrency(stakeCpu), intToCurrency(unstaked)))
assert(funds == ramFunds + stakeNet + stakeCpu + unstaked)
retry(args.cleos + 'system newaccount --transfer eosio %s %s --stake-net "%s" --stake-cpu "%s" --buy-ram "%s" ' %
(a['name'], a['pub'], intToCurrency(stakeNet), intToCurrency(stakeCpu), intToCurrency(ramFunds)))
if unstaked:
retry(args.cleos + 'transfer eosio %s "%s"' % (a['name'], intToCurrency(unstaked)))
def pop_sieves():
log.debug("pop_sieves()")
while retry(10, raise_if_failed=True):
all_sieves = model.Sieve.all()
sieves = all_sieves.order("number").fetch(1)
if sieves:
try:
return delete_sieve(sieves[0])
except (KeyError, db.TransactionFailedError):
pass
else:
@transactional
def init_sieves():
for number in xrange(2, chunk_size):
model.Sieve(number=number, parent=sieve_root).put()
init_sieves()
def deactivate(self):
if not self.exists():
return
ag = self.get_asgroup()
ag.min_size = 0
ag.max_size = 0
ag.desired_capacity = 0
ag.update()
ag.shutdown_instances()
print "Deleting... this may take a few minutes..."
if util.retry(lambda: ag.delete(), 500):
util.message_integrations("Deleted ASgroup {}".format(self.name()))
# delete launchconfig too
lc = self.lc()
lc.deactivate()
else:
util.message_integrations("Failed to delete ASgroup {}".format(self.name()))
def check(self):
import hashlib
import datetime
now = datetime.datetime.utcnow()
logging.info(now)
uid = self.system_id
s = "{uid}{date:%Y%m%d}{hour}{cl_cons}".format(date=now,
hour=now.hour % 7,
cl_cons="3456789012",
uid=self.system_id)
pokey = hashlib.md5(s.encode()).hexdigest()
url = 'https://kiosk.multidat.ru/api.php?action=CheckLichense&PoKey={key}&uid={uid}'.format(
key=pokey, uid=uid)
resp = {}
try:
resp = retry(urllib.error.URLError)(urllib.request.urlopen)(
url, timeout=5).read().decode()
resp = json.loads(resp)
except urllib.error.URLError:
exit(403)
except Exception:
exit(100)
if resp.get('err_txt'):
logging.critical(resp.get('err_txt'))
exit(int(resp.get('err_no', '403')))
else:
#assert self.system_id == resp['uid']
s = "2109876543{date:%d%m%Y}{hour}{key}".format(date=now,
hour=now.hour % 7,
key=pokey)
#print(s)
srvkey = hashlib.md5(s.encode()).hexdigest()
print(srvkey)
print(resp)
if resp['SrvKey'][:-3] != srvkey:
exit(int(resp.get('err_no', '403')))
print(resp['SrvKey'][-3:], end="")
print('ok')
t = self.checker
self.checker = Checker(int(resp['SrvKey'][-3:]), self.check)
self.checker.start()
return
def deactivate(self):
if not self.exists():
return
ag = self.get_asgroup()
ag.min_size = 0
ag.max_size = 0
ag.desired_capacity = 0
ag.update()
ag.shutdown_instances()
print "Deleting... this may take a few minutes..."
if util.retry(lambda: ag.delete(), 500):
util.message_integrations("Deleted ASgroup {}".format(self.name()))
# delete launchconfig too
lc = self.lc()
lc.deactivate()
else:
util.message_integrations("Failed to delete ASgroup {}".format(
self.name()))
def checkpoint_experiment(self, experiment):
if isinstance(experiment, six.string_types):
key = experiment
experiment = self.get_experiment(key)
else:
key = experiment.key
headers = self._get_headers()
def post_request():
request = requests.post(self.url + '/api/checkpoint_experiment',
headers=headers,
data=json.dumps({"key": key}))
self._raise_detailed_error(request)
artifacts = request.json()['artifacts']
return artifacts
artifacts = retry(post_request, sleep_time=10, logger=self.logger)
self._update_artifacts(experiment, artifacts)
experiment.time_last_checkpoint = time.time()
def get_ticket_data(start_time_t):
global JIRA_PASSWORD
if JIRA_PASSWORD is None:
with open(JIRA_PASSWORD_FILE) as f:
JIRA_PASSWORD = f.read().strip()
# Compose API call: get as many issues as possible from > start_time_t
# We can only get 1000 at a time, according to
# https://confluence.atlassian.com/display/CLOUDKB/Changing+maxResults+Parameter+for+JIRA+REST+API
# Also note that we must pass in an integer representing milliseconds
# since the epoch, according to:
# https://confluence.atlassian.com/display/JIRA/Advanced+Searching#AdvancedSearching-Created
fields = ','.join([CREATED_FIELD, EXERCISE_FIELD])
values = {'fields': fields,
'maxResults': 1000,
'project': '"Assessment items"'
' and "Issue type" != "Not translated"'
' and created > %s'
' order by created asc'
% int(1000 * start_time_t),
}
url = ('https://khanacademy.atlassian.net/rest/api/latest/search'
'?jql=%s' % urllib.urlencode(values))
request = urllib2.Request(url)
# Send base64-encoded 'user:password', according to
# https://developer.atlassian.com/display/JIRADEV/JIRA+REST+API+Example+-+Basic+Authentication
encoded_password = base64.standard_b64encode('%s:%s' % (JIRA_USER,
JIRA_PASSWORD))
request.add_unredirected_header('Authorization',
'Basic %s' % encoded_password)
request.add_header('Content-Type', 'application/json')
urlfetch_errors = (socket.error, urllib2.HTTPError, httplib.HTTPException)
data = util.retry(lambda: urllib2.urlopen(request, timeout=60),
'loading jira ticket data',
lambda exc: isinstance(exc, urlfetch_errors))
return json.load(data)
def get_ticket_data(start_time_t):
"""Given start_time to export from, call Zendesk API for ticket data."""
global ZENDESK_PASSWORD
if ZENDESK_PASSWORD is None:
with open(ZENDESK_PASSWORD_FILE) as f:
ZENDESK_PASSWORD = f.read().strip()
# According to
# http://developer.zendesk.com/documentation/rest_api/ticket_export.html
# "Requests with a start_time less than 5 minutes old will also
# be rejected."
if int(time.time()) - start_time_t <= 300:
return None
url = ('https://khanacademy.zendesk.com/api/v2/exports/tickets.json'
'?start_time=%s' % start_time_t)
request = urllib2.Request(url)
# This is the best way to set the user, according to
# http://stackoverflow.com/questions/2407126/python-urllib2-basic-auth-problem
encoded_password = base64.standard_b64encode('%s:%s' % (ZENDESK_USER,
ZENDESK_PASSWORD))
request.add_unredirected_header('Authorization',
'Basic %s' % encoded_password)
def _should_retry(exc):
if isinstance(exc, urllib2.HTTPError) and exc.code == 429:
# quota limits: try again, but wait first.
time.sleep(int(exc.headers['Retry-After']))
return isinstance(exc, (socket.error, urllib2.HTTPError,
httplib.HTTPException))
data = util.retry(lambda: urllib2.urlopen(request, timeout=60),
'loading zendesk ticket data',
_should_retry)
return json.load(data)
def get_ticket_data(start_time_t):
"""Given start_time to export from, call Zendesk API for ticket data."""
global ZENDESK_PASSWORD
if ZENDESK_PASSWORD is None:
with open(ZENDESK_PASSWORD_FILE) as f:
ZENDESK_PASSWORD = f.read().strip()
# According to
# http://developer.zendesk.com/documentation/rest_api/ticket_export.html
# "Requests with a start_time less than 5 minutes old will also
# be rejected."
if int(time.time()) - start_time_t <= 300:
return None
url = ('https://khanacademy.zendesk.com/api/v2/exports/tickets.json'
'?start_time=%s' % start_time_t)
request = urllib2.Request(url)
# This is the best way to set the user, according to
# http://stackoverflow.com/questions/2407126/python-urllib2-basic-auth-problem
encoded_password = base64.standard_b64encode('%s:%s' % (ZENDESK_USER,
ZENDESK_PASSWORD))
request.add_unredirected_header('Authorization',
'Basic %s' % encoded_password)
def _should_retry(exc):
if isinstance(exc, urllib2.HTTPError) and exc.code == 429:
# quota limits: try again, but wait first.
time.sleep(int(exc.headers['Retry-After']))
return isinstance(exc, (socket.error, urllib2.HTTPError,
httplib.HTTPException))
data = util.retry(lambda: urllib2.urlopen(request, timeout=60),
'loading zendesk ticket data',
_should_retry)
return json.load(data)
def createStakedAccounts(begin, end):
#(b,e) =(0,len(accounts))=(0,7),(0~3)users,(4~7)producers
for i in range(begin, end):
a = accounts[i]
stakeNet = 500000000000 #50000000DATX
stakeCpu = 500000000000 #50000000DATX
stakeRam = 500000000000 #50000000DATX
small_stake = 5000000 #500DATX
util.retry(
'cldatx ' +
'system newaccount --transfer datxos %s %s --stake-net "%s" --stake-cpu "%s" --buy-ram "%s" '
% (a['name'], a['pub'], intToCurrency(stakeNet),
intToCurrency(stakeCpu), intToCurrency(stakeRam)))
util.sleep(1)
for i in range(user_limit):
a = accounts[i]
util.retry(
'cldatx ' +
'push action datxos.token transfer \'["datxos",%s,"%s","vote"]\' -p datxos'
% (a['name'], intToCurrency(5000000000000)))
util.sleep(1)
util.retry('cldatx ' +
'system delegatebw datxos %s "%s" "%s" --transfer' %
(a['name'], intToCurrency(300000000000),
intToCurrency(350000000000)))
util.sleep(1)
util.run(
'cldatx ' +
'push action datxos.dtoke transfer \'{"from":"datxos.dbtc","to":"alice","quantity":"300.0000 DBTC","memo":"test"}\' -p datxos.dbtc'
)
util.run(
'cldatx ' +
'push action datxos.dtoke transfer \'{"from":"datxos.deth","to":"alice","quantity":"300.0000 DETH","memo":"test"}\' -p datxos.deth'
)
util.run(
'cldatx ' +
'push action datxos.dtoke transfer \'{"from":"datxos.deos","to":"alice","quantity":"300.0000 DEOS","memo":"test"}\' -p datxos.deos'
)
def SetSystemContract():
util.retry('cldatx ' + 'set contract datxos ' + contracts_dir +
'DatxSystem/ -x 3500')
util.sleep(1)
util.run('cldatx ' + 'push action datxos setpriv' +
util.jsonArg(['datxos.msig', 1]) + '-p datxos@active')
def reload(self):
if not util.confirm("Are you sure you want to tear down the {} ASgroup and recreate it?".format(self.name())):
return
util.message_integrations("Reloading ASgroup {}".format(self.name()))
self.deactivate()
util.retry(lambda: self.activate(), 60)
def vote(b, e):
for i in range(b, e):
voter = accounts[i]['name']
prods = random.sample(range(firstProducer, firstProducer + numProducers), num_producers_vote-1)
prods = ' '.join(map(lambda x: accounts[x]['name'], prods))
util.retry('cldatx ' + 'system voteproducer prods ' + voter + ' ' + prods)
def deactivate(self): # a.k.a asg destroy
# NOTE
# deleting asg logic should be in its own function
# * delete ASG by reducing capacities of asg to 0
# * delete launchconfig
#
# reducing ASG capacities to 0 triggers eventual instance
# termination
debug("In asgroup.py deactivate")
asg_name = self.name()
ag = util.as_conn()
ec2 = util.ec2_conn()
asg_info = ag.describe_auto_scaling_groups(
AutoScalingGroupNames=[asg_name])
if not asg_info['AutoScalingGroups']:
print("ASG does not exist. Maybe it was already deleted?")
else:
# delete the ASG
num_instances = len(asg_info['AutoScalingGroups'][0]['Instances'])
if self.get_num_instances() == 0:
pprint("There are no instances in asg: " + asg_name)
print("Deleting asg: " + asg_name)
response = ag.delete_auto_scaling_group(
AutoScalingGroupName=asg_name)
util.message_integrations(
"Deleted ASgroup {}".format(asg_name))
else:
debug("There are " + str(num_instances) +
" instances that need to be removed from asg: " +
asg_name)
debug("terminating instances in asg: " + asg_name)
debug("by setting to 0 MinSize, MaxSize, DesiredCapacity")
response = ag.update_auto_scaling_group(
AutoScalingGroupName=asg_name,
MinSize=0,
MaxSize=0,
DesiredCapacity=0)
debug(
"Waiting 30 seconds to give AWS time to terminate the instances"
)
if self.get_num_instances() != 0:
util.retry(
lambda: ag.delete_auto_scaling_group(
AutoScalingGroupName=asg_name), 300)
if self.get_num_instances() != 0 or self.get_num_instances():
print("unable to delete instances in asg.")
return False
util.message_integrations(
"Deleted ASgroup {}".format(asg_name))
# if launch config exists, delete it
lc = self.lc()
if not lc.exists():
print(
"launchconfig does not exist. Maybe you deleted it already?")
else:
lc.deactivate()
return True
def acknowledge(self, ack_id):
retry(lambda: self._client.delete_message(QueueUrl=self._queue_url,
ReceiptHandle=ack_id),
sleep_time=10,
logger=self.logger)
def stepSetSystemContract():
retry(args.cleos + 'set contract eosio ' + args.contracts_dir + 'eosio.system/')
sleep(1)
run(args.cleos + 'push action eosio setpriv' + jsonArg(['eosio.msig', 1]) + '-p eosio@active')
def RegProducers():
util.retry('cldatx system regproducer ' + producer_name + ' ' +
public_key + ' ' + producer_url + ' ' + verifier_url)
