def check_keys(mc):
log(">> Checking if current keys are in LRU")
for key in KEYLIST:
if not key_in_lru(mc, key):
raise exceptions.Exception("Expecting " + key +
" in LRU but not found")
TCHECK_LRUDELETE = len(KEYLIST) / 2
log(">> Deleting", str(TCHECK_LRUDELETE), "keys from LRU")
DELETED = []
for x in range(TCHECK_LRUDELETE):
key = pick_random_key_from_list()
log(">> DELETE", key)
KEYLIST.remove(key)
DELETED.append(key)
mc.delete(key)
wait_for_lru(mc)
log(">> Checking if deletes are reflected in LRU")
for key in KEYLIST:
if not key_in_lru(mc, key):
raise exceptions.Exception("Expecting " + key +
" in LRU but not found")
# The following needs to change (we need to check if LRU didn't pick it up)
for key in DELETED:
try:
if key_in_lru(mc, key):
raise exceptions.Exception("Not expecting " + key +
" in LRU but found")
except mc_bin_client.MemcachedError:
pass
def long_to_path(id, root=""):
"""
Converts a long to a path such that for all directiories only
a 1000 files and a 1000 subdirectories will be returned.
This method duplicates the logic in
ome.io.nio.AbstractFileSystemService.java:getPath()
"""
suffix = ""
remaining = id
dirno = 0
if id is None or id == "":
raise exceptions.Exception("Expecting a not-null id.")
id = long(id)
if id < 0:
raise exceptions.Exception("Expecting a non-negative id.")
while (remaining > 999):
remaining /= 1000
if remaining > 0:
dirno = remaining % 1000
suffix = os.path.join("Dir-%03d" % dirno, suffix)
return os.path.join(root, "%s%s" %(suffix,id))
def setUp(self):
c = omero.client(
pmap=['--Ice.Config=' + (os.environ.get("ICE_CONFIG"))])
try:
self.root_password = c.ic.getProperties().getProperty(
'omero.rootpass')
omero_host = c.ic.getProperties().getProperty('omero.host')
finally:
c.__del__()
blitz = Server.find(host=omero_host)
if blitz is None:
Server.reset()
for s in settings.SERVER_LIST:
server = (len(s) > 2) and unicode(s[2]) or None
Server(host=unicode(s[0]), port=int(s[1]), server=server)
Server.freeze()
blitz = Server.find(server=omero_host)
if len(blitz):
self.server_id = blitz[0].id
connector = Connector(self.server_id, True)
self.rootconn = connector.create_connection(
'TEST.webadmin', 'root', self.root_password)
if self.rootconn is None or not self.rootconn.isConnected(
) or not self.rootconn.keepAlive():
raise exceptions.Exception("Cannot connect")
else:
raise exceptions.Exception("'%s' is not on omero.web.server_list" %
omero_host)
def GetRepoBase():
"""Returns the repository base of the root local checkout."""
info = gclient_scm.CaptureSVNInfo('.')
root = info['Repository Root']
url = info['URL']
if not root or not url:
raise exceptions.Exception("I'm confused by your checkout")
if not url.startswith(root):
raise exceptions.Exception("I'm confused by your checkout", url, root)
return url[len(root):] + '/'
def loginAsUser(self, username, password):
blitz = Server.get(pk=self.server_id)
if blitz is not None:
connector = Connector(self.server_id, True)
conn = connector.create_connection('TEST.webadmin', username,
password)
if conn is None or not conn.isConnected() or not conn.keepAlive():
raise exceptions.Exception("Cannot connect")
return conn
else:
raise exceptions.Exception("'%s' is not on omero.web.server_list" %
omero_host)
def sendPacket(timeout):
error_code = PyTTXTalk.send(timeout)
if (error_code != 0):
raise exceptions.Exception('%d: %s' %
(error_code, ERROR_CODES[error_code]))
if (PyTTXTalk.getError() != 0):
raise exceptions.Exception((PyTTXTalk.getError(), PyTTXTalk.popU8()))
def setThalam(self, thalam):
#
# When we set the thalam, the samam value changes.
#
if (type(thalam) != type(5)):
raise exceptions.Exception("Thalam should be an integer")
if (thalam % 2 != 0):
raise exceptions.Exception(
"Thalam (in mathrai) should be a multiple of 2")
self.thalam = thalam
if (self.debug):
sys.stderr.write('Thalam: ' + str(thalam) + '\n')
self.findSamam()
self.findAnswer()
def write_key_to_compute_node(keypair, local_path, remote_path, host,
private_key):
"""Write the private key of the nova instance to the compute node
First fetches the private key from the keypair and writes it to a
temporary file in the local machine. It then sftp's the file
to the compute host.
:param keypair: nova keypair
:param local_path: path to private key file of the nova instance in the
local machine
:param remote_path: path where the private key file has to be placed
in the remote machine
:param host: compute host credentials
:param private_key: path to your private key file
:return:
"""
LOG.debug("WRITING PRIVATE KEY TO COMPUTE NODE")
k = paramiko.RSAKey.from_private_key_file(private_key)
write_key_to_local_path(keypair, local_path)
try:
transport = paramiko.Transport(host['ip'], host['port'])
except paramiko.SSHException as e:
raise exceptions.Exception(
"PARAMIKO TRANSPORT FAILED. CHECK IF THE HOST IP %s AND PORT %s "
"ARE CORRECT %s", host['ip'], host['port'], e)
try:
transport.connect(username=host['username'], pkey=k)
except paramiko.BadHostKeyException as e:
transport.close()
raise exceptions.Exception(
"BADHOSTKEY EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
except paramiko.AuthenticationException as e:
transport.close()
raise exceptions.Exception(
"AUTHENTICATION EXCEPTION WHEN CONNECTING TO %s", host["ip"], e)
except paramiko.SSHException as e:
transport.close()
raise exceptions.Exception("SSH EXCEPTION WHEN CONNECTING TO %s",
host["ip"], e)
LOG.debug("CONNECTED TO HOST <%s>", host["ip"])
try:
sftp_client = paramiko.SFTPClient.from_transport(transport)
sftp_client.put(local_path, remote_path)
except IOError as e:
raise exceptions.Exception("FILE PATH DOESN'T EXIST", e)
finally:
transport.close()
def pop(self):
# 判断是否空栈
if self.isempty():
raise exceptions.Exception('stack is empty')
else:
self.top = self.top - 1
self.stack.pop()
def import_image(self, filename = None, client = None, extra_args=None):
if filename is None:
filename = self.OmeroPy / ".." / ".." / ".." / "components" / "common" / "test" / "tinyTest.d3d.dv"
if client is None:
client = self.client
server = client.getProperty("omero.host")
port = client.getProperty("omero.port")
key = client.getSessionId()
# Search up until we find "OmeroPy"
dist_dir = self.OmeroPy / ".." / ".." / ".." / "dist"
args = [sys.executable]
args.append(str(path(".") / "bin" / "omero"))
args.extend(["-s", server, "-k", key, "-p", port, "import", "--"])
if extra_args:
args.extend(extra_args)
args.append(filename)
popen = subprocess.Popen(args, cwd=str(dist_dir), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = popen.communicate()
rc = popen.wait()
if rc != 0:
raise exceptions.Exception("import failed: [%r] %s\n%s" % (args, rc, err))
pix_ids = []
for x in out.split("\n"):
if x and x.find("Created") < 0 and x.find("#") < 0:
try: # if the line has an image ID...
imageId = str(long(x.strip()))
pix_ids.append(imageId)
except: pass
return pix_ids
def init_cached_bases():
print "Reading cached map key (SPEEDY-ISH - takes a few seconds) ..."
cached_base_colors = []
f = open('base_colors.cache', 'r')
for line in f.xreadlines():
r, g, b, n = line.split('\t')
cached_base_colors.append((col(int(r), int(g), int(b)), n[:-1]))
f.close()
cached_base_colors = tuple(cached_base_colors)
cached_base_mods = [[] for x in range(9)]
f = open('base_mods.cache', 'r')
old_level = ""
for line in f.xreadlines():
i, r, g, b, n = line.split('\t')
if old_level != i:
print "... Level %s of 8 ..." % i
old_level = i
cached_base_mods[int(i)].append((col(int(r), int(g), int(b)), n[:-1]))
f.close()
for i in range(9):
cached_base_mods[i] = tuple(cached_base_mods[i])
print "... Validating ..."
if (len(cached_base_colors) == 327842) and (len(cached_base_mods[8])
== 414081):
print "... Done!"
return [cached_base_colors, cached_base_mods]
else:
print "... ERROR in cache! ... rebuilding ..."
raise exceptions.Exception('Cache Mismatch')
def improveMap(m, goog, transl):
for g, t in zip(goog, transl):
if g == ' ':
if t != ' ':
raise ex.Exception('blank to non blank')
if t == ' ':
if g != ' ':
raise ex.Exception('non blank to blank')
if g in m:
if m[g] != t:
raise ex.Exception('inconsistence: ' +
'"' + g + '" was to "' + m[g] + '"' +
' now "' + t + '"')
else:
m[g] = t
return m
def __init__(self, prefix = "omero"):
"""
Initializes a TempFileManager instance with a userDir containing
the given prefix value, or "omero" by default. Also registers
an atexit callback to call self.cleanup() on exit.
"""
self.logger = logging.getLogger("omero.util.TempFileManager")
self.is_win32 = ( sys.platform == "win32" )
self.prefix = prefix
self.userdir = self.tmpdir() / ("%s_%s" % (self.prefix, self.username()))
"""
User-accessible directory of the form $TMPDIR/omero_$USERNAME.
If the given directory is not writable, an attempt is made
to use an alternative
"""
if not self.create(self.userdir) and not self.access(self.userdir):
i = 0
while i < 10:
t = path("%s_%s" % (self.userdir, i))
if self.create(t) or self.access(t):
self.userdir = t
break
raise exceptions.Exception("Failed to create temporary directory: %s" % self.userdir)
self.dir = self.userdir / self.pid()
"""
Directory under which all temporary files and folders will be created.
An attempt to remove a path not in this directory will lead to an
exception.
"""
# Now create the directory. If a later step throws an
# exception, we should try to rollback this change.
if not self.dir.exists():
self.dir.makedirs()
self.logger.debug("Using temp dir: %s" % self.dir)
self.lock = None
try:
self.lock = open(str(self.dir / ".lock"), "a+")
"""
.lock file under self.dir which is used to prevent other
TempFileManager instances (also in other languages) from
cleaning up this directory.
"""
try:
portalocker.lock(self.lock, portalocker.LOCK_EX|portalocker.LOCK_NB)
atexit.register(self.cleanup)
except:
lock = self.lock
self.lock = None
if lock:
self.lock.close()
raise
finally:
try:
if not self.lock:
self.cleanup()
except:
self.logger.warn("Error on cleanup after error", exc_info = True)
def readMatrix(fn):
msg = None
matrix = labels = data = None
if os.path.splitext(fn)[1] == '.pkl' or os.path.splitext(fn)[1] == '.sym':
pkl_file = open(fn, 'rb')
matrix = pickle.load(pkl_file)
data = None
#print self.matrix
if hasattr(matrix, 'items'):
data = matrix.items
pkl_file.close()
else:
#print fn
fle = open(fn)
while 1:
lne = fle.readline().strip()
if lne:
break
spl = lne.split()
try:
dim = int(spl[0])
except:
msg = "Matrix dimension expected in the first line"
raise exceptions.Exception
labeled = len(spl) > 1 and spl[1] in ["labelled", "labeled"]
matrix = orange.SymMatrix(dim)
data = None
if labeled:
labels = []
else:
labels = [""] * dim
for li, lne in enumerate(fle):
if li > dim:
if not li.strip():
continue
msg = "File too long"
raise exceptions.IndexError
spl = lne.split("\t")
if labeled:
labels.append(spl[0].strip())
spl = spl[1:]
if len(spl) > dim:
msg = "Line %i too long" % li + 2
raise exceptions.IndexError
for lj, s in enumerate(spl):
if s:
try:
matrix[li, lj] = float(s)
except:
msg = "Invalid number in line %i, column %i" % (li + 2,
lj)
if msg:
raise exceptions.Exception(msg)
return matrix, labels, data
def convert_ical2md(ical_path, export_path, from_date, to_date):
# summarize in each date
daily_md = {}
with open(ical_path) as ic:
content = ic.read()
cal = Calendar.from_ical(content)
event_cnt = 1
for event in cal.walk('vevent'):
# check event date
start_dt = event['dtstart'].dt
if from_date <= start_dt and start_dt <= to_date:
md = event2md(event)
if md != None and len(md) > 0:
start_dt_str = start_dt.strftime('%Y-%m-%d')
if start_dt_str in daily_md:
daily_md[start_dt_str].append(md)
else:
daily_md[start_dt_str] = [md]
event_cnt += 1
# iteritems is only vailable in 2.x
for md_date_str, md_list in daily_md.iteritems():
err = export_md(export_path, md_date_str, md_list)
if err != None:
return exceptions.Exception("Error while exporting md : {}".format(
str(err)))
return None
def new_user(self, group=None, perms=None, admin=False, system=False):
"""
admin: If user is to be an admin of the created group
system: If user is to be a system admin
"""
if not self.root:
raise exceptions.Exception("No root client. Cannot create user")
adminService = self.root.getSession().getAdminService()
name = self.uuid()
# Create group if necessary
if not group:
g = self.new_group(perms=perms)
group = g.name.val
else:
g, group = self.group_and_name(group)
# Create user
e = omero.model.ExperimenterI()
e.omeName = rstring(name)
e.firstName = rstring(name)
e.lastName = rstring(name)
uid = adminService.createUser(e, group)
e = adminService.lookupExperimenter(name)
if admin:
adminService.setGroupOwner(g, e)
if system:
adminService.addGroups(e, \
[omero.model.ExperimenterGroupI(0, False)])
return adminService.getExperimenter(uid)
def wait_for_namespace_creation(namespace_tag,
router_id,
hosts,
private_key,
timeout=60):
"""Wait for the namespace creation
Get into each of the controllers/compute nodes and check which one contains
the snat/qrouter namespace corresponding to rally_router. Sleep for a sec
and repeat until either the namespace is found or the namespace_creation_
time exceeded.
:param namespace_tag: which namespace ("snat_" or "qrouter_")
:param router_id: uuid of the rally_router
:param hosts: controllers or compute hosts
:param private_key: path to private key file
:param timeout: namespace creation time
:return:
"""
start_time = time.time()
while True:
for host in hosts:
namespaces = get_namespace(host, private_key)
for line in namespaces:
if line == (namespace_tag + router_id):
namespace_tag = line
return namespace_tag, host
time.sleep(1)
if time.time() - start_time > timeout:
raise exceptions.Exception("TIMEOUT WHILE WAITING FOR"
" NAMESPACES TO BE CREATED")
def _wait_for_status_change(self,
resource,
final_status,
resource_tag,
wait_timeout=60,
check_interval=1):
"""Wait for resource's status change
Wait till the status of the resource changes to final state or till
the time exceeds the wait_timeout value.
:param resource: resource whose status has to be checked
:param final_status: desired final status of the resource
:param resource_tag: to identify the resource as vpnservice or
ipser_site_connection
:param wait_timeout: timeout value in seconds
:param check_interval: time to sleep before each check for the status
change
:return: resource
"""
start_time = time.time()
while True:
resource = self._get_resource(resource_tag,
resource[resource_tag]['id'])
current_status = resource[resource_tag]['status']
if current_status == final_status:
return resource
time.sleep(check_interval)
if time.time() - start_time > wait_timeout:
self._cleanup(called_from="VpnUtils._wait_for_status_change")
raise exceptions.Exception(
"Timeout while waiting for status "
"change to %s.", final_status)
def __init__(self,pid,out_file):
'''
Constructor
'''
#self._pid = pid
#self._dir = os.path.join('/proc',str(self._pid),'task')
self._out_file = out_file
self._stats = {}
self._task_list = []
self._task_stats = {}
self._task_names = {}
self._g = None
#if not os.path.isdir(self._dir):
self._pid = self.find_pid(pid)
self._dir = None
if self._pid:
self._dir = os.path.join('/proc',str(self._pid),'task')
if not self._dir or not os.path.isdir(self._dir):
raise exceptions.Exception('No such a process %s' % str(pid))
#try:
# self._out = open(self._out_file,'wb')
#except IOError as e:
# raise exceptions.Exception('Failed to open output file %s (%s)' % (self._out_file, str(e)))
self._name = open(os.path.join('/proc',str(self._pid),'comm')).read()
print ('Processing %s' % self._name)
def _wait_for_status_change(self, resource, resource_tag, final_status,
wait_timeout=60, check_interval=1):
"""Wait for resource's status change
Wait till the status of the resource changes to final state or till
the time exceeds the wait_timeout value.
:param resource: resource whose status has to be checked
:param final_status: desired final status of the resource
:param resource_tag: to identify the resource as vpnservice or
ipsec_site_connection
:param wait_timeout: timeout value in seconds
:param check_interval: time to sleep before each check for the status
change
:return: resource
"""
LOG.debug('WAIT_FOR_%s_STATUS_CHANGE ', resource[resource_tag]['id'])
start_time = time.time()
while True:
resource = self._get_resource(
resource_tag, resource[resource_tag]['id'])
current_status = resource[resource_tag]['status']
if current_status == final_status:
return resource
time.sleep(check_interval)
if time.time() - start_time > wait_timeout:
raise exceptions.Exception(
"Timeout waiting for resource {} to change to {} status".
format(resource[resource_tag]['name'], final_status))
def __init__(self, lab_id, topology_name, is_override, is_only_xml=False):
topo_path = os.path.join(lab.TOPOLOGIES_DIR, topology_name + '.yaml')
lab.make_tmp_dir(lab.DISKS_DIR)
try:
with open(topo_path) as f:
self.topology = yaml.load(f)
self.is_only_xml = is_only_xml
self.lab_id = int(lab_id)
self.image_url_re = re.compile(r'source +file.+(http.+) *-->')
self.name_re = re.compile(r'<name>(.+)</name>')
self.is_override = is_override
except exceptions.ValueError:
raise exceptions.Exception('lab_id is supposed to be integer')
except:
raise exceptions.Exception(
'wrong topology descriptor provided: {0}'.format(topo_path))
def wait_load(self, target, timeout):
"""
wait till the specified target is uploaded
:param target: the target to wait for
:param timeout: maximum time to wait
:return: None
"""
start_time = time.time()
if target in self._uploaded:
return
wait = timeout - (time.time() - start_time)
while wait > 0:
try:
mac = self._uploaded_queue.get(timeout=wait)
logging.info("%s uploaded" % str(mac))
logging.debug("+ %s, %s" % (str(target.bssid), str(mac)))
self._uploaded.add(mac)
if target.bssid.lower() == mac.lower():
return
except multiprocessing.queues.Empty:
break
wait = timeout - (time.time() - start_time)
raise exceptions.Exception("timeout")
def _wait_for_status_change(self,
cluster_id,
final_status,
timeout=300,
check_interval=1):
"""Waits for specified change in cluster status.
Will wait until cluster status changes to a specified status within
timeout period.
:param: cluster_id: uuid, cluster id
:param final_status: str, final cluster status
:param timeout: int, max time to check for status change
:param check_interval: int, interval to check status changes in
"""
start_time = time.time()
while True:
cluster = self._get_cluster(cluster_id)
current_status = cluster['status']
if current_status == final_status:
return cluster
time.sleep(check_interval)
if time.time() - start_time > timeout:
self._delete_cluster(cluster_id)
raise exceptions.Exception(
"Timeout while waiting for status "
"change to %s.", final_status)
def resolve(symbol_1):
global NIL
if symbol_1 == NIL:
return []
elif symbol_1 == ENVELOPE:
return "ENVELOPE"
else:
raise exceptions.Exception("unknown symbol \"%s\"" % symbol_1)
def DownloadPackage(filename, md5):
localname = '/tmp/' + filename
ret = DownloadFileTo(GetUrl() + ':5009/' + filename, localname)
if ret != md5:
raise exceptions.Exception('MD5 not match!')
return localname
def getPlane(self, *args):
if self.good_calls == 0:
e = exceptions.Exception("MOCK EXCEPTION")
e.close = False
raise e
else:
self.good_calls -= 1
return "0" * (2 * self.pixels.getSizeX() * self.pixels.getSizeY())
def push(self, x):
# 判断是否满栈
if self.isfull():
raise exceptions.Exception('stack is full')
else:
self.stack.append(x)
self.top = self.top + 1
def setParameter(self, paramName, value):
"""Set parameter value"""
(setter, getter) = self._getParameterMethods(paramName)
if setter is None:
import exceptions
raise exceptions.Exception("setParameter -- parameter name '%s' does not exist in region %s of type %s" %
(paramName, self.name, self.type))
setter(paramName, value)
def __init__(self, sleeptime=60, stop_event=None):
"""
Add resources via add(object). They should have a no-arg cleanup()
and a check() method.
The check method will be called periodically (default: 60 seconds)
on each resource. The cleanup method will be called on
Resources.cleanup()
"""
self.stuff = []
self._lock = threading.RLock()
self.logger = logging.getLogger("omero.util.Resources")
self.stop_event = stop_event
if not self.stop_event:
self.stop_event = omero.util.concurrency.get_event(
name="Resources")
if sleeptime < 5:
raise exceptions.Exception(
"Sleep time should be greater than 5: %s" % sleeptime)
self.sleeptime = sleeptime
class Task(threading.Thread):
"""
Internal thread used for checking "stuff"
"""
def run(self):
ctx = self.ctx # Outer class
ctx.logger.info("Starting")
while not ctx.stop_event.isSet():
try:
ctx.logger.debug("Executing")
copy = ctx.copyStuff()
remove = ctx.checkAll(copy)
ctx.removeAll(remove)
except:
ctx.logger.error("Exception during execution",
exc_info=True)
ctx.logger.debug("Sleeping %s" % ctx.sleeptime)
# ticket:1531 - Attempting to catch threading issues
try:
ctx.stop_event.wait(ctx.sleeptime)
except ValueError:
pass
if isinstance(ctx.stop_event,
omero.util.concurrency.AtExitEvent):
if ctx.stop_event.atexit:
return # Skipping log. See #3260
ctx.logger.info("Halted")
self.thread = Task()
self.thread.ctx = self
self.thread.start()
def doStuff(bucket, md5bucketname, args):
if len(args) == 0:
showIndex(bucket)
elif len(args) == 1:
showDay(bucket, args[0])
elif len(args) == 2:
restoreFile(bucket, md5bucketname, args[0], args[1])
else:
raise exceptions.Exception("Don' know what to do with your args")