def mount(self, mountpoint=None):
mountpoint = self.makeMountPoint(mountpoint)
new_env, domain = cifutils.getCIFCredentials(self.dconf, self.session)
options = self.getMountOptions(domain)
if options:
options = ",".join(str(x) for x in options if x)
try:
util.ioretry(lambda:
util.pread(["mount.cifs", self.remoteserver,
mountpoint, "-o", options], new_env=new_env),
errlist=[errno.EPIPE, errno.EIO],
maxretry=2, nofail=True)
except util.CommandException as inst:
raise SMBException("mount failed with return code %d" % inst.code)
# Sanity check to ensure that the user has at least RO access to the
# mounted share. Windows sharing and security settings can be tricky.
try:
util.listdir(mountpoint)
except util.CommandException:
try:
self.unmount(mountpoint, True)
except SMBException:
util.logException('SMBSR.unmount()')
raise SMBException("Permission denied. "
"Please check user privileges.")
def delete(self, sr_uuid):
self.attach(sr_uuid)
cleanup.gc_force(self.session, self.uuid)
# check to make sure no VDIs are present; then remove old
# files that are non VDI's
try:
if util.ioretry(lambda: util.pathexists(self.path)):
#Load the VDI list
self._loadvdis()
for uuid in self.vdis:
if not self.vdis[uuid].deleted:
raise xs_errors.XenError('SRNotEmpty', \
opterr='VDIs still exist in SR')
# remove everything else, there are no vdi's
for name in util.ioretry(lambda: util.listdir(self.path)):
fullpath = os.path.join(self.path,name)
try:
util.ioretry(lambda: os.unlink(fullpath))
except util.CommandException, inst:
if inst.code != errno.ENOENT and \
inst.code != errno.EISDIR:
raise xs_errors.XenError('FileSRDelete', \
opterr='failed to remove %s error %d' \
% (fullpath, inst.code))
self.detach(sr_uuid)
def print_LUNs(self):
self.LUNs = {}
if os.path.exists(self.path):
for file in util.listdir(self.path):
if file.find("LUN") != -1 and file.find("_") == -1:
vdi_path = os.path.join(self.path, file)
LUNid = file.replace("LUN", "")
obj = self.vdi(self.uuid)
obj._query(vdi_path, LUNid)
self.LUNs[obj.uuid] = obj
dom = xml.dom.minidom.Document()
element = dom.createElement("iscsi-target")
dom.appendChild(element)
for uuid in self.LUNs:
val = self.LUNs[uuid]
entry = dom.createElement('LUN')
element.appendChild(entry)
for attr in ('vendor', 'serial', 'LUNid', \
'size', 'SCSIid'):
try:
aval = getattr(val, attr)
except AttributeError:
continue
if aval:
subentry = dom.createElement(attr)
entry.appendChild(subentry)
textnode = dom.createTextNode(str(aval))
subentry.appendChild(textnode)
print >> sys.stderr, dom.toprettyxml()
def get_conferences():
files = util.listdir(CONFERENCE_FOLDER)
util.mkdir(CONFERENCE_CRALWED_FOLDER)
cnt = 0
conf = util.load_json('conf_name.json')
for file_name in files:
save_path = os.path.join(CONFERENCE_CRALWED_FOLDER, file_name)
if util.exists(save_path):
continue
data = util.load_json(os.path.join(CONFERENCE_FOLDER, file_name))
if data['short'] not in conf.keys():
continue
html = util.get_page(data['url'])
subs = get_subs(data['short'], html)
data['name'] = conf[data['short']]
data['sub'] = {}
for sub in subs:
if sub not in conf.keys():
continue
html = util.get_page('http://dblp.uni-trier.de/db/conf/' + sub)
data['sub'][sub] = {}
data['sub'][sub]['pub'] = get_publications(html)
data['sub'][sub]['name'] = conf[sub]
cnt += 1
print cnt, len(files), data['short']
util.save_json(save_path, data)
def disk_info():
logdir = dsz.lp.GetLogsDirectory()
projectdir = os.path.split(logdir)[0]
infofile = os.path.join(projectdir, 'disk-version.txt')
if os.path.exists(infofile):
dsz.ui.Echo((
'Disk version already logged; if you switched disks for some reason, rename %s and restart the LP please.'
% infofile), dsz.GOOD)
return True
opsdisk_root = os.path.normpath((dsz.lp.GetResourcesDirectory() + '/..'))
dszfiles = util.listdir(opsdisk_root, '^DSZOpsDisk-.+\\.zip$')
disk = None
if (len(dszfiles) == 1):
disk = dszfiles[0]
elif (len(dszfiles) > 1):
menu = util.menu.Menu('Found mulitple DSZOpsDisk zips:', dszfiles,
None, 'Which one are you executing? ')
index = (menu.show()[0] - 1)
if ((index > 0) and (index < len(dszfiles))):
disk = dszfiles[index]
else:
dsz.ui.Echo(
'Could not determine which opsdisk is running. Version NOT recorded.',
dsz.ERROR)
return False
else:
dsz.ui.Echo(
'Could not find DSZOpsDisk zip. Disk version NOT recorded.',
dsz.ERROR)
return False
with open(infofile, 'w') as output:
output.write(('%s\n' % disk))
dsz.ui.Echo(('Disk version %s recorded to %s.' % (disk, infofile)),
dsz.GOOD)
return True
def get_conferences():
files = util.listdir(CONFERENCE_FOLDER)
util.mkdir(CONFERENCE_CRALWED_FOLDER)
cnt = 0
conf = util.load_json('conf_name.json')
for file_name in files:
save_path = os.path.join(CONFERENCE_CRALWED_FOLDER, file_name)
if util.exists(save_path):
continue
data = util.load_json(os.path.join(CONFERENCE_FOLDER, file_name))
if data['short'] not in conf.keys():
continue
html = util.get_page(data['url'])
subs = get_subs(data['short'], html)
data['name'] = conf[data['short']]
data['sub'] = {}
for sub in subs:
if sub not in conf.keys():
continue
html = util.get_page('http://dblp.uni-trier.de/db/conf/' + sub)
data['sub'][sub] = {}
data['sub'][sub]['pub'] = get_publications(html)
data['sub'][sub]['name'] = conf[sub]
cnt += 1
print cnt, len(files), data['short']
util.save_json(save_path, data)
def get_lun_scsiid_devicename_mapping(targetIQN, portal):
iscsilib.refresh_luns(targetIQN, portal)
lunToScsiId={}
path = os.path.join("/dev/iscsi",targetIQN,portal)
try:
deviceToLun = {}
deviceToScsiId = {}
for file in util.listdir(path):
realPath = os.path.realpath(os.path.join(path, file))
if file.find("LUN") == 0 and file.find("_") == -1:
lun=file.replace("LUN","")
if deviceToScsiId.has_key(realPath):
lunToScsiId[lun] = (deviceToScsiId[realPath], realPath)
else:
deviceToLun[realPath] = lun
continue
if file.find("SERIAL-") == 0:
scsiid = file.replace("SERIAL-", "")
# found the SCSI ID, check if LUN has already been seen for this SCSI ID.
if deviceToLun.has_key(realPath):
# add this to the map
lunId = deviceToLun[realPath]
lunToScsiId[lunId] = (scsiid, realPath)
else:
# The SCSI ID was seen first so add this to the map
deviceToScsiId[realPath] = scsiid
return lunToScsiId
except util.CommandException, inst:
XenCertPrint("Failed to find any LUNs for IQN: %s and portal: %s" % targetIQN, portal)
return {}
def create(self, sr_uuid, size):
# Check SCSIid not already in use by other PBDs
if util.test_SCSIid(self.session, sr_uuid, self.SCSIid):
raise xs_errors.XenError('SRInUse')
self.iscsi.attach(sr_uuid)
try:
if not self.iscsi._attach_LUN_bySCSIid(self.SCSIid):
# UPGRADE FROM GEORGE: take care of ill-formed SCSIid
upgraded = False
matchSCSIid = False
for file in filter(self.iscsi.match_lun,
util.listdir(self.iscsi.path)):
path = os.path.join(self.iscsi.path, file)
if not util.wait_for_path(path, ISCSISR.MAX_TIMEOUT):
util.SMlog(
"Unable to detect LUN attached to host [%s]" %
path)
continue
try:
SCSIid = scsiutil.getSCSIid(path)
except:
continue
try:
matchSCSIid = scsiutil.compareSCSIid_2_6_18(
self.SCSIid, path)
except:
continue
if (matchSCSIid):
util.SMlog("Performing upgrade from George")
try:
pbd = util.find_my_pbd(self.session, self.host_ref,
self.sr_ref)
device_config = self.session.xenapi.PBD.get_device_config(
pbd)
device_config['SCSIid'] = SCSIid
self.session.xenapi.PBD.set_device_config(
pbd, device_config)
self.dconf['SCSIid'] = SCSIid
self.SCSIid = self.dconf['SCSIid']
except:
continue
if not self.iscsi._attach_LUN_bySCSIid(self.SCSIid):
raise xs_errors.XenError('InvalidDev')
else:
upgraded = True
break
else:
util.SMlog(
"Not a matching LUN, skip ... scsi_id is: %s" %
SCSIid)
continue
if not upgraded:
raise xs_errors.XenError('InvalidDev')
self._pathrefresh(OCFSoISCSISR)
OCFSSR.OCFSSR.create(self, sr_uuid, size)
except Exception, inst:
self.iscsi.detach(sr_uuid)
raise xs_errors.XenError("SRUnavailable", opterr=inst)
def create(self, sr_uuid, size):
if util.ioretry(lambda: self._checkmount()):
raise xs_errors.XenError('NFSAttached')
# Set the target path temporarily to the base dir
# so that we can create the target SR directory
self.remotepath = self.dconf['serverpath']
try:
self.attach(sr_uuid)
except:
try:
os.rmdir(self.path)
except:
pass
raise xs_errors.XenError('NFSMount')
newpath = os.path.join(self.path, sr_uuid)
if util.ioretry(lambda: util.pathexists(newpath)):
if len(util.ioretry(lambda: util.listdir(newpath))) != 0:
self.detach(sr_uuid)
raise xs_errors.XenError('SRExists')
else:
try:
util.ioretry(lambda: util.makedirs(newpath))
except util.CommandException, inst:
if inst.code != errno.EEXIST:
self.detach(sr_uuid)
raise xs_errors.XenError('NFSCreate', \
opterr='remote directory creation error is %d' \
% inst.code)
def listDir(self, filesyspath, urlpath, outputStream):
f = outputStream
filelist, dirlist = util.listdir(filesyspath)
filelist.sort(lambda a, b: cmp(a.lower(), b.lower()))
dirlist.sort(lambda a, b: cmp(a.lower(), b.lower()))
f.write('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">\n')
f.write("<html>\n")
f.write(
("<head><title>Directory listing for %s</title></head>\n" % urlpath)
+ ("<body>\n<h2>Directory listing for %s</h2>\n<hr>\n" % urlpath)
)
if urlpath:
f.write('<p><a href="..">↑ parent directory</a>')
if dirlist:
f.write("<h4>Directories</h4>\n<ul>\n")
f.write("\n".join(['<li><a href="%s/">%s</a>' % (urllib.quote(name), name) for name in dirlist]))
f.write("</ul>\n")
if filelist:
f.write("<h4>Files</h4>\n<ul>\n")
f.write("\n".join(['<li><a href="%s">%s</a>' % (urllib.quote(name), name) for name in filelist]))
f.write("</ul>\n")
else:
f.write("<p>There are no files in this location.\n")
f.write("<hr>\n\n<address>Served by Snakelets</address>")
f.write("\n</body></html>")
def delete(self, sr_uuid):
if not self._checkpath(self.path):
raise xs_errors.XenError("SRUnavailable", opterr="no such directory %s" % self.path)
cleanup.gc_force(self.session, self.uuid)
# check to make sure no VDIs are present; then remove old
# files that are non VDI's
try:
if util.ioretry(lambda: util.pathexists(self.path)):
# Load the VDI list
self._loadvdis()
for uuid in self.vdis:
if not self.vdis[uuid].deleted:
raise xs_errors.XenError("SRNotEmpty", opterr="VDIs still exist in SR")
# remove everything else, there are no vdi's
for name in util.ioretry(lambda: util.listdir(self.path)):
fullpath = os.path.join(self.path, name)
try:
util.ioretry(lambda: os.unlink(fullpath))
except util.CommandException, inst:
if inst.code != errno.ENOENT and inst.code != errno.EISDIR:
raise xs_errors.XenError(
"FileSRDelete", opterr="failed to remove %s error %d" % (fullpath, inst.code)
)
except util.CommandException, inst:
raise xs_errors.XenError("FileSRDelete", opterr="error %d" % inst.code)
def _loadvdis(self):
if self.vdis:
return
for name in filter(util.match_uuid, util.ioretry( \
lambda: util.listdir(self.path))):
fields = name.split('.')
if len(fields) == 2 and fields[1] == self.sr_vditype:
fullpath = os.path.join(self.path, name)
uuid = fields[0]
try:
self.vdis[uuid] = self.vdi(uuid)
self.vdis[uuid].marked_deleted = self.vdis[uuid].hidden
except SR.SRException as inst:
pass
# Mark parent VDIs as Read-only and generate virtual allocation
self.virtual_allocation = 0
for uuid, vdi in self.vdis.iteritems():
if vdi.parent:
if vdi.parent in self.vdis:
self.vdis[vdi.parent].read_only = True
if vdi.parent in geneology:
geneology[vdi.parent].append(uuid)
else:
geneology[vdi.parent] = [uuid]
self.virtual_allocation += (vdi.size + \
VDI.VDIMetadataSize(SR.DEFAULT_TAP, vdi.size))
def print_LUNs(self):
self.LUNs = {}
if os.path.exists(self.path):
for file in util.listdir(self.path):
if file.find("LUN") != -1 and file.find("_") == -1:
vdi_path = os.path.join(self.path,file)
LUNid = file.replace("LUN","")
obj = self.vdi(self.uuid)
obj._query(vdi_path, LUNid)
self.LUNs[obj.uuid] = obj
dom = xml.dom.minidom.Document()
element = dom.createElement("iscsi-target")
dom.appendChild(element)
for uuid in self.LUNs:
val = self.LUNs[uuid]
entry = dom.createElement('LUN')
element.appendChild(entry)
for attr in ('vendor', 'serial', 'LUNid', \
'size', 'SCSIid'):
try:
aval = getattr(val, attr)
except AttributeError:
continue
if aval:
subentry = dom.createElement(attr)
entry.appendChild(subentry)
textnode = dom.createTextNode(str(aval))
subentry.appendChild(textnode)
print >>sys.stderr,dom.toprettyxml()
def get_lun_scsiid_devicename_mapping(targetIQN, portal):
iscsilib.refresh_luns(targetIQN, portal)
lunToScsiId={}
path = os.path.join("/dev/iscsi",targetIQN,portal)
try:
deviceToLun = {}
deviceToScsiId = {}
for file in util.listdir(path):
realPath = os.path.realpath(os.path.join(path, file))
if file.find("LUN") == 0 and file.find("_") == -1:
lun=file.replace("LUN","")
if deviceToScsiId.has_key(realPath):
lunToScsiId[lun] = (deviceToScsiId[realPath], realPath)
else:
deviceToLun[realPath] = lun
continue
if file.find("SERIAL-") == 0:
scsiid = file.replace("SERIAL-", "")
# found the SCSI ID, check if LUN has already been seen for this SCSI ID.
if deviceToLun.has_key(realPath):
# add this to the map
lunId = deviceToLun[realPath]
lunToScsiId[lunId] = (scsiid, realPath)
else:
# The SCSI ID was seen first so add this to the map
deviceToScsiId[realPath] = scsiid
return lunToScsiId
except util.CommandException, inst:
XenCertPrint("Failed to find any LUNs for IQN: %s and portal: %s" % targetIQN, portal)
return {}
def create(self, sr_uuid, size):
self.__check_license()
if self.checkmount():
raise xs_errors.XenError('SMBAttached')
try:
self.mount()
except SMBException as exc:
try:
os.rmdir(self.mountpoint)
except:
pass
raise xs_errors.XenError('SMBMount', opterr=exc.errstr)
if util.ioretry(lambda: util.pathexists(self.linkpath)):
if len(util.ioretry(lambda: util.listdir(self.linkpath))) != 0:
self.detach(sr_uuid)
raise xs_errors.XenError('SRExists')
else:
try:
util.ioretry(lambda: util.makedirs(self.linkpath))
os.symlink(self.linkpath, self.path)
except util.CommandException as inst:
if inst.code != errno.EEXIST:
try:
self.unmount(self.mountpoint, True)
except SMBException:
util.logException('SMBSR.unmount()')
raise xs_errors.XenError(
'SMBCreate',
opterr="remote directory creation error: {}"
.format(os.strerror(inst.code))
)
self.detach(sr_uuid)
def scan_srlist(path, dconf):
"""Scan and report SR, UUID."""
dom = xml.dom.minidom.Document()
element = dom.createElement("SRlist")
dom.appendChild(element)
for val in filter(util.match_uuid, util.ioretry(
lambda: util.listdir(path))):
fullpath = os.path.join(path, val)
if not util.ioretry(lambda: util.isdir(fullpath)):
continue
entry = dom.createElement('SR')
element.appendChild(entry)
subentry = dom.createElement("UUID")
entry.appendChild(subentry)
textnode = dom.createTextNode(val)
subentry.appendChild(textnode)
from NFSSR import PROBEVERSION
if dconf.has_key(PROBEVERSION):
util.SMlog("Add supported nfs versions to sr-probe")
supported_versions = get_supported_nfs_versions(dconf.get('server'))
supp_ver = dom.createElement("SupportedVersions")
element.appendChild(supp_ver)
for ver in supported_versions:
version = dom.createElement('Version')
supp_ver.appendChild(version)
textnode = dom.createTextNode(ver)
version.appendChild(textnode)
return dom.toprettyxml()
def listDir(self, filesyspath, urlpath, outputStream):
f = outputStream
filelist, dirlist = util.listdir(filesyspath)
filelist.sort(lambda a, b: cmp(a.lower(), b.lower()))
dirlist.sort(lambda a, b: cmp(a.lower(), b.lower()))
f.write(
'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">\n'
)
f.write('<html>\n')
f.write(("<head><title>Directory listing for %s</title></head>\n" %
urlpath) +
("<body>\n<h2>Directory listing for %s</h2>\n<hr>\n" %
urlpath))
if urlpath:
f.write("<p><a href=\"..\">↑ parent directory</a>")
if dirlist:
f.write("<h4>Directories</h4>\n<ul>\n")
f.write('\n'.join([
'<li><a href="%s/">%s</a>' % (urllib.quote(name), name)
for name in dirlist
]))
f.write("</ul>\n")
if filelist:
f.write("<h4>Files</h4>\n<ul>\n")
f.write('\n'.join([
'<li><a href="%s">%s</a>' % (urllib.quote(name), name)
for name in filelist
]))
f.write("</ul>\n")
else:
f.write("<p>There are no files in this location.\n")
f.write("<hr>\n\n<address>Served by Snakelets</address>")
f.write("\n</body></html>")
def _loadvdis(self):
"""Scan the directory and get uuids either from the VDI filename,
or by creating a new one."""
if self.vdis:
return
for name in filter(is_image_utf8_compatible,
util.listdir(self.path, quiet=True)):
self.vdis[name] = LocalISOVDI(self, name)
# Set the VDI UUID if the filename is of the correct form.
# Otherwise, one will be generated later in VDI._db_introduce.
m = uuid_file_regex.match(name)
if m:
self.vdis[name].uuid = m.group(1)
# Synchronise the read-only status with existing VDI records
__xenapi_records = util.list_VDI_records_in_sr(self)
__xenapi_locations = {}
for vdi in __xenapi_records.keys():
__xenapi_locations[__xenapi_records[vdi]['location']] = vdi
for vdi in self.vdis.values():
if vdi.location in __xenapi_locations:
v = __xenapi_records[__xenapi_locations[vdi.location]]
sm_config = v['sm_config']
if 'created' in sm_config:
vdi.sm_config['created'] = sm_config['created']
vdi.read_only = False
def scan_srlist(path, dconf):
"""Scan and report SR, UUID."""
dom = xml.dom.minidom.Document()
element = dom.createElement("SRlist")
dom.appendChild(element)
for val in filter(util.match_uuid,
util.ioretry(lambda: util.listdir(path))):
fullpath = os.path.join(path, val)
if not util.ioretry(lambda: util.isdir(fullpath)):
continue
entry = dom.createElement('SR')
element.appendChild(entry)
subentry = dom.createElement("UUID")
entry.appendChild(subentry)
textnode = dom.createTextNode(val)
subentry.appendChild(textnode)
from NFSSR import PROBEVERSION
if dconf.has_key(PROBEVERSION):
util.SMlog("Add supported nfs versions to sr-probe")
supported_versions = get_supported_nfs_versions(dconf.get('server'))
supp_ver = dom.createElement("SupportedVersions")
element.appendChild(supp_ver)
for ver in supported_versions:
version = dom.createElement('Version')
supp_ver.appendChild(version)
textnode = dom.createTextNode(ver)
version.appendChild(textnode)
return dom.toprettyxml()
def create(self, sr_uuid, size):
util._testHost(self.dconf['server'], NFSPORT, 'NFSTarget')
self.validate_remotepath(True)
if self._checkmount():
raise xs_errors.XenError('NFSAttached')
# Set the target path temporarily to the base dir
# so that we can create the target SR directory
self.remotepath = self.dconf['serverpath'].encode('utf-8')
try:
self.mount_remotepath(sr_uuid)
except Exception as exn:
try:
os.rmdir(self.path)
except:
pass
raise exn
if not self.nosubdir:
newpath = os.path.join(self.path, sr_uuid)
if util.ioretry(lambda: util.pathexists(newpath)):
if len(util.ioretry(lambda: util.listdir(newpath))) != 0:
self.detach(sr_uuid)
raise xs_errors.XenError('SRExists')
else:
try:
util.ioretry(lambda: util.makedirs(newpath))
except util.CommandException as inst:
if inst.code != errno.EEXIST:
self.detach(sr_uuid)
raise xs_errors.XenError(
'NFSCreate',
opterr='remote directory creation error is %d' %
inst.code)
self.detach(sr_uuid)
def _probe_hba(self):
try:
# use sysfs tree to gather FC data
dom = xml.dom.minidom.Document()
hbalist = dom.createElement("HBAInfoList")
dom.appendChild(hbalist)
hostlist = util.listdir("/sys/class/fc_host")
for host in hostlist:
hbainfo = dom.createElement("HBAInfo")
hbalist.appendChild(hbainfo)
cmd = ["cat", "/sys/class/fc_host/%s/symbolic_name" % host]
sname = util.pread(cmd)[:-1]
entry = dom.createElement("model")
hbainfo.appendChild(entry)
textnode = dom.createTextNode(sname)
entry.appendChild(textnode)
cmd = ["cat", "/sys/class/fc_host/%s/node_name" % host]
nname = util.pread(cmd)[:-1]
nname = util.make_WWN(nname)
entry = dom.createElement("nodeWWN")
hbainfo.appendChild(entry)
# adjust nodename to look like expected string
textnode = dom.createTextNode(nname)
entry.appendChild(textnode)
port = dom.createElement("Port")
hbainfo.appendChild(port)
cmd = ["cat", "/sys/class/fc_host/%s/port_name" % host]
pname = util.pread(cmd)[:-1]
pname = util.make_WWN(pname)
entry = dom.createElement("portWWN")
port.appendChild(entry)
# adjust nodename to look like expected string
textnode = dom.createTextNode(pname)
entry.appendChild(textnode)
cmd = ["cat", "/sys/class/fc_host/%s/port_state" % host]
state = util.pread(cmd)[:-1]
entry = dom.createElement("state")
port.appendChild(entry)
# adjust nodename to look like expected string
textnode = dom.createTextNode(state)
entry.appendChild(textnode)
entry = dom.createElement("deviceName")
port.appendChild(entry)
# adjust nodename to look like expected string
textnode = dom.createTextNode("/sys/class/scsi_host/%s" % host)
entry.appendChild(textnode)
return dom.toxml()
except:
raise xs_errors.XenError('CSLGXMLParse', \
opterr='HBA probe failed')
def create(self, sr_uuid, size):
if util.ioretry(lambda: self._checkmount()):
raise xs_errors.XenError('NFSAttached')
# Set the target path temporarily to the base dir
# so that we can create the target SR directory
self.remotepath = self.dconf['serverpath']
try:
self.attach(sr_uuid)
except:
try:
os.rmdir(self.path)
except:
pass
raise xs_errors.XenError('NFSMount')
newpath = os.path.join(self.path, sr_uuid)
if util.ioretry(lambda: util.pathexists(newpath)):
if len(util.ioretry(lambda: util.listdir(newpath))) != 0:
self.detach(sr_uuid)
raise xs_errors.XenError('SRExists')
else:
try:
util.ioretry(lambda: util.makedirs(newpath))
except util.CommandException, inst:
if inst.code != errno.EEXIST:
self.detach(sr_uuid)
raise xs_errors.XenError('NFSCreate', \
opterr='remote directory creation error is %d' \
% inst.code)
def disk_info():
logdir = dsz.lp.GetLogsDirectory()
projectdir = os.path.split(logdir)[0]
infofile = os.path.join(projectdir, 'disk-version.txt')
if os.path.exists(infofile):
dsz.ui.Echo(('Disk version already logged; if you switched disks for some reason, rename %s and restart the LP please.' % infofile), dsz.GOOD)
return True
opsdisk_root = os.path.normpath((dsz.lp.GetResourcesDirectory() + '/..'))
dszfiles = util.listdir(opsdisk_root, '^DSZOpsDisk-.+\\.zip$')
disk = None
if (len(dszfiles) == 1):
disk = dszfiles[0]
elif (len(dszfiles) > 1):
menu = util.menu.Menu('Found mulitple DSZOpsDisk zips:', dszfiles, None, 'Which one are you executing? ')
index = (menu.show()[0] - 1)
if ((index > 0) and (index < len(dszfiles))):
disk = dszfiles[index]
else:
dsz.ui.Echo('Could not determine which opsdisk is running. Version NOT recorded.', dsz.ERROR)
return False
else:
dsz.ui.Echo('Could not find DSZOpsDisk zip. Disk version NOT recorded.', dsz.ERROR)
return False
with open(infofile, 'w') as output:
output.write(('%s\n' % disk))
dsz.ui.Echo(('Disk version %s recorded to %s.' % (disk, infofile)), dsz.GOOD)
return True
def debug_train():
os.environ["CUDA_VISIBLE_DEVICES"] = "2"
traindir = "/data4T1/samhu/shapenet_split_complete/train"
settings = {}
settings['batch_size'] = 32
settings['height'] = 192
settings['width'] = 256
net_name = 'KPARAM_33_DUAL'
net_dict = net.build_model(net_name, settings)
train_fetcher = DataFetcher()
train_fetcher.BATCH_SIZE = settings['batch_size']
train_fetcher.PTS_DIM = 3
train_fetcher.HEIGHT = settings['height']
train_fetcher.WIDTH = settings['width']
train_fetcher.Dir = util.listdir(traindir, ".h5")
train_fetcher.shuffleDir()
if 'rand' in net_dict.keys():
train_fetcher.randfunc = net_dict['rand']
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
lrate = 3e-5
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
try:
train_fetcher.start()
lastEpoch = 0
data_dict = train_fetcher.fetch()
x2D = data_dict['x2D']
x3D = data_dict['x3D']
yGT = 0.8 * data_dict['x3D_final']
GT_PTS_NUM = int(yGT.shape[1])
print x3D.shape
print yGT.shape
yGT = yGT.reshape((-1, 3))
x3D = x3D.reshape((-1, 3))
feed = {
net_dict['yGT']: yGT,
net_dict['ix3D']: x3D,
net_dict['ix2D']: x2D,
net_dict['lr']: lrate
}
if 'eidx' in data_dict.keys():
i = 0
while 'eidx_%d' % i in net_dict.keys():
feed[net_dict['eidx_%d' % i]] = data_dict['eidx'][i]
i += 1
if ('fidx' in data_dict.keys()) and ('fidx' in net_dict.keys()):
feed[net_dict['fidx']] = data_dict['fidx'][-1]
yout = sess.run(net_dict['ox3D'], feed_dict=feed)
print yout.shape
train_fetcher.shutdown()
finally:
train_fetcher.shutdown()
return
def probe(self):
try:
err = "CIFSMount"
self.mount(PROBE_MOUNTPOINT)
sr_list = filter(util.match_uuid, util.listdir(PROBE_MOUNTPOINT))
err = "CIFSUnMount"
self.unmount(PROBE_MOUNTPOINT, True)
except CifsException, inst:
raise xs_errors.XenError(err, opterr=inst.errstr)
def print_LUNs(self):
self.LUNs = {}
if os.path.exists(self.path):
for file in util.listdir(self.path):
if file.find("LUN") != -1 and file.find("_") == -1:
vdi_path = os.path.join(self.path,file)
LUNid = file.replace("LUN","")
obj = self.vdi(self.uuid)
obj._query(vdi_path, LUNid)
self.LUNs[obj.uuid] = obj
def probe(self):
try:
self.mount(PROBE_MOUNTPOINT)
sr_list = filter(util.match_uuid, util.listdir(PROBE_MOUNTPOINT))
self.unmount(PROBE_MOUNTPOINT, True)
except (util.CommandException, xs_errors.XenError):
raise
# Create a dictionary from the SR uuids to feed SRtoXML()
sr_dict = {sr_uuid: {} for sr_uuid in sr_list}
return util.SRtoXML(sr_dict)
def print_LUNs(self):
self.LUNs = {}
if os.path.exists(self.path):
for file in util.listdir(self.path):
if file.find("LUN") != -1 and file.find("_") == -1:
vdi_path = os.path.join(self.path, file)
LUNid = file.replace("LUN", "")
obj = self.vdi(self.uuid)
obj._query(vdi_path, LUNid)
self.LUNs[obj.uuid] = obj
def _loadvdis(self):
"""Scan the location directory."""
if self.vdis:
return
try:
for name in util.listdir(self.dconf['location']):
if name != "":
self.vdis[name] = SHMVDI(self, util.gen_uuid(), name)
except:
pass
def _loadvdis(self):
"""Scan the location directory."""
if self.vdis:
return
try:
for name in util.listdir(self.dconf['location']):
if name != "":
self.vdis[name] = SHMVDI(self, util.gen_uuid(), name)
except:
pass
def get_luns(targetIQN, portal):
refresh_luns(targetIQN, portal)
luns=[]
path = os.path.join("/dev/iscsi",targetIQN,portal)
try:
for file in util.listdir(path):
if file.find("LUN") == 0 and file.find("_") == -1:
lun=file.replace("LUN","")
luns.append(lun)
return luns
except util.CommandException, inst:
raise xs_errors.XenError('ISCSIDevice', opterr='Failed to find any LUNs')
def get_luns(targetIQN, portal):
refresh_luns(targetIQN, portal)
luns=[]
path = os.path.join("/dev/iscsi",targetIQN,portal)
try:
for file in util.listdir(path):
if file.find("LUN") == 0 and file.find("_") == -1:
lun=file.replace("LUN","")
luns.append(lun)
return luns
except util.CommandException, inst:
raise xs_errors.XenError('ISCSIDevice', opterr='Failed to find any LUNs')
def locate_files(file, subdir=None):
if (subdir is None):
subdir = '.'
resdirs = util.listdir(ops.RESDIR, includeFiles=False)
files = []
for resdir in resdirs:
if ((resdir.lower() == 'ops') or (resdir.lower() == 'dsz')):
continue
fullpath = os.path.normpath(os.path.join(ops.RESDIR, resdir, subdir, file))
if os.path.exists(fullpath):
files.append((resdir, fullpath))
files.sort(cmp=(lambda x, y: cmp(x[0].lower(), y[0].lower())))
return files
def get_journals():
files = util.listdir(JOURNAL_FOLDER)
util.mkdir(JOURNAL_CRALWED_FOLDER)
cnt = 0
for file_name in files:
save_path = os.path.join(JOURNAL_CRALWED_FOLDER, file_name)
data = util.load_json(os.path.join(JOURNAL_FOLDER, file_name))
html = util.get_page(data['url'])
full_name = get_full_name(html)
data['name'] = full_name
cnt += 1
print cnt, len(files), data['short']
data['links'] = get_links(data['short'], html)
def get_journals(): files = util.listdir(JOURNAL_FOLDER) util.mkdir(JOURNAL_CRALWED_FOLDER) cnt = 0 for file_name in files: save_path = os.path.join(JOURNAL_CRALWED_FOLDER, file_name) data = util.load_json(os.path.join(JOURNAL_FOLDER, file_name)) html = util.get_page(data['url']) full_name = get_full_name(html) data['name'] = full_name cnt += 1 print cnt, len(files), data['short'] data['links'] = get_links(data['short'], html)
def create(self, sr_uuid, size):
# Check SCSIid not already in use by other PBDs
if util.test_SCSIid(self.session, sr_uuid, self.SCSIid):
raise xs_errors.XenError('SRInUse')
self.iscsi.attach(sr_uuid)
try:
if not self.iscsi._attach_LUN_bySCSIid(self.SCSIid):
# UPGRADE FROM GEORGE: take care of ill-formed SCSIid
upgraded = False
matchSCSIid = False
for file in filter(self.iscsi.match_lun, util.listdir(self.iscsi.path)):
path = os.path.join(self.iscsi.path,file)
if not util.wait_for_path(path, ISCSISR.MAX_TIMEOUT):
util.SMlog("Unable to detect LUN attached to host [%s]" % path)
continue
try:
SCSIid = scsiutil.getSCSIid(path)
except:
continue
try:
matchSCSIid = scsiutil.compareSCSIid_2_6_18(self.SCSIid, path)
except:
continue
if (matchSCSIid):
util.SMlog("Performing upgrade from George")
try:
pbd = util.find_my_pbd(self.session, self.host_ref, self.sr_ref)
device_config = self.session.xenapi.PBD.get_device_config(pbd)
device_config['SCSIid'] = SCSIid
self.session.xenapi.PBD.set_device_config(pbd, device_config)
self.dconf['SCSIid'] = SCSIid
self.SCSIid = self.dconf['SCSIid']
except:
continue
if not self.iscsi._attach_LUN_bySCSIid(self.SCSIid):
raise xs_errors.XenError('InvalidDev')
else:
upgraded = True
break
else:
util.SMlog("Not a matching LUN, skip ... scsi_id is: %s" % SCSIid)
continue
if not upgraded:
raise xs_errors.XenError('InvalidDev')
self._pathrefresh(LVHDoISCSISR)
LVHDSR.LVHDSR.create(self, sr_uuid, size)
except Exception, inst:
self.iscsi.detach(sr_uuid)
raise xs_errors.XenError("SRUnavailable", opterr=inst)
def get_authors(): files = util.listdir(AUTHOR_FOLDER) util.mkdir(AUTHOR_CRALWED_FOLDER) for file_name in files: save_path = os.path.join(AUTHOR_CRALWED_FOLDER, file_name) if util.exists(save_path): continue data = util.load_json(os.path.join(AUTHOR_FOLDER, file_name)) html = util.get_page(data['url']) full_name = get_full_name(html) data['name'] = full_name print data['short'], full_name data['links'] = get_links(data['short'], html) util.save_json(save_path, data)
def get_authors():
files = util.listdir(AUTHOR_FOLDER)
util.mkdir(AUTHOR_CRALWED_FOLDER)
for file_name in files:
save_path = os.path.join(AUTHOR_CRALWED_FOLDER, file_name)
if util.exists(save_path):
continue
data = util.load_json(os.path.join(AUTHOR_FOLDER, file_name))
html = util.get_page(data['url'])
full_name = get_full_name(html)
data['name'] = full_name
print data['short'], full_name
data['links'] = get_links(data['short'], html)
util.save_json(save_path, data)
def _genHostList(procname):
# loop through and check all adapters
ids = []
try:
for dir in util.listdir('/sys/class/scsi_host'):
filename = os.path.join('/sys/class/scsi_host',dir,'proc_name')
if os.path.exists(filename):
f = open(filename, 'r')
if f.readline().find(procname) != -1:
ids.append(dir.replace("host",""))
f.close()
except:
pass
return ids
def _loadvdis(self):
count = 0
if not os.path.exists(self.path):
return 0
for file in filter(self.match_lun, util.listdir(self.path)):
vdi_path = os.path.join(self.path,file)
LUNid = file.replace("LUN","")
uuid = scsiutil.gen_uuid_from_string(scsiutil.getuniqueserial(vdi_path))
obj = self.vdi(uuid)
obj._query(vdi_path, LUNid)
self.vdis[uuid] = obj
self.physical_size += obj.size
count += 1
return count
def _loadvdis(self):
count = 0
if not os.path.exists(self.path):
return 0
for file in filter(self.match_lun, util.listdir(self.path)):
vdi_path = os.path.join(self.path,file)
LUNid = file.replace("LUN","")
uuid = scsiutil.gen_uuid_from_string(scsiutil.getuniqueserial(vdi_path))
obj = self.vdi(uuid)
obj._query(vdi_path, LUNid)
self.vdis[uuid] = obj
self.physical_size += obj.size
count += 1
return count
def print_LUNs(self):
self.LUNs = {}
if os.path.exists(self.path):
dom0_disks = util.dom0_disks()
for file in util.listdir(self.path):
if file.find("LUN") != -1 and file.find("_") == -1:
vdi_path = os.path.join(self.path, file)
if os.path.realpath(vdi_path) in dom0_disks:
util.SMlog("Hide dom0 boot disk LUN")
else:
LUNid = file.replace("LUN", "")
obj = self.vdi(self.uuid)
obj._query(vdi_path, LUNid)
self.LUNs[obj.uuid] = obj
def _genHostList(procname):
# loop through and check all adapters
ids = []
try:
for dir in util.listdir('/sys/class/scsi_host'):
filename = os.path.join('/sys/class/scsi_host', dir, 'proc_name')
if os.path.exists(filename):
f = open(filename, 'r')
if f.readline().find(procname) != -1:
ids.append(dir.replace("host", ""))
f.close()
except:
pass
return ids
def print_LUNs(self):
self.LUNs = {}
if os.path.exists(self.path):
dom0_disks = util.dom0_disks()
for file in util.listdir(self.path):
if file.find("LUN") != -1 and file.find("_") == -1:
vdi_path = os.path.join(self.path,file)
if os.path.realpath(vdi_path) in dom0_disks:
util.SMlog("Hide dom0 boot disk LUN")
else:
LUNid = file.replace("LUN","")
obj = self.vdi(self.uuid)
obj._query(vdi_path, LUNid)
self.LUNs[obj.uuid] = obj
def locate_files(file, subdir=None):
if (subdir is None):
subdir = '.'
resdirs = util.listdir(ops.RESDIR, includeFiles=False)
files = []
for resdir in resdirs:
if ((resdir.lower() == 'ops') or (resdir.lower() == 'dsz')):
continue
fullpath = os.path.normpath(
os.path.join(ops.RESDIR, resdir, subdir, file))
if os.path.exists(fullpath):
files.append((resdir, fullpath))
files.sort(cmp=(lambda x, y: cmp(x[0].lower(), y[0].lower())))
return files
def _loadvdis(self):
if self.vdis:
return
for name in filter(util.match_uuid, util.ioretry( \
lambda: util.listdir(self.path))):
fields = name.split('.')
if len(fields) == 2 and fields[1] == self.sr_vditype:
fullpath = os.path.join(self.path, name)
uuid = fields[0]
try:
self.vdis[uuid] = self.vdi(uuid)
self.vdis[uuid].marked_deleted = self.vdis[uuid].hidden
except SR.SRException, inst:
pass
def _loadvdis(self):
if self.vdis:
return
for name in filter(util.match_uuid, util.ioretry( \
lambda: util.listdir(self.path))):
fields = name.split('.')
if len(fields) == 2 and fields[1] == self.sr_vditype:
fullpath = os.path.join(self.path,name)
uuid = fields[0]
try:
self.vdis[uuid] = self.vdi(uuid)
self.vdis[uuid].marked_deleted = self.vdis[uuid].hidden
except SR.SRException, inst:
pass
def get_lun_scsiid_devicename_mapping(targetIQN, portal):
iscsilib.refresh_luns(targetIQN, portal)
lunToScsiId={}
path = os.path.join("/dev/iscsi",targetIQN,portal)
try:
for file in util.listdir(path):
realPath = os.path.realpath(os.path.join(path, file))
if file.find("LUN") == 0 and file.find("_") == -1:
lun=file.replace("LUN","")
scsi_id = scsiutil.getSCSIid(os.path.join(path, file))
lunToScsiId[lun] = (scsi_id, realPath)
return lunToScsiId
except util.CommandException, inst:
XenCertPrint("Failed to find any LUNs for IQN: %s and portal: %s" % (targetIQN, portal))
return {}
def _filelist(self,root,path):
try:
(filez,dirz) = util.listdir(path)
except OSError:
raise errors.NamingError('group not found')
files=[]
for f in filez:
if path==root:
files.append(':'+f)
else:
p=string.replace(path[len(root):], os.sep, '.')
files.append(':'+p+'.'+f)
for d in dirz:
files.extend(self._filelist(root,os.path.join(path,d)))
return files
def __init__(self, l_dir=None, l_file=None):
# (Iter, Iter) -> None
# Create directories
from itertools import ifilterfalse
i_folder = ifilterfalse(os.path.exists, (irpf90_t.irpdir, irpf90_t.mandir))
map(os.mkdir, i_folder)
# List file
l_dir = l_dir if l_dir else (command_line.include_dir + ['.'])
l_not_dir = [d for d in l_dir if not (os.path.exists(d) and os.path.isdir(d))]
if l_not_dir:
logger.error('Try to include no existing directory: [%s]' % ','.join(l_not_dir))
sys.exit(1)
# Create folder in IRPDIR
i_folder = ifilterfalse(os.path.exists, (os.path.join(irpf90_t.irpdir, d) for d in l_dir))
map(os.mkdir, i_folder)
s_folder_abs = set(os.path.abspath(path) for path in l_dir)
s_file_folder_all = set(flatten(listdir(path, abspath=True) for path in s_folder_abs))
# Take everything!
s_file_folder = filter(lambda f: os.path.isfile(f) and not f.startswith("."),
s_file_folder_all)
s_file_tot = set(l_file) if l_file else set()
s_file_tot.update(s_file_folder)
s_file_rel = set(os.path.relpath(f, self.cwd) for f in s_file_tot)
# Lazy Copy file
for f in s_file_rel:
src = os.path.join(self.cwd, f)
text_ref = open(src, 'rb').read()
dest = os.path.join(self.cwd, irpf90_t.irpdir, f)
lazy_write_file(dest, text_ref)
if command_line.do_codelet:
s_file_tot.update(command_line.codelet[3])
# No filter the irpf90 file
self.irpf90_files_ordered = sorted(filter(lambda f: f.endswith(".irp.f"), s_file_rel))
def get_conferences(): files = util.listdir(CONFERENCE_FOLDER) util.mkdir(CONFERENCE_CRALWED_FOLDER) cnt = 0 for file_name in files: cnt += 1 if cnt < 1970: continue save_path = os.path.join(CONFERENCE_CRALWED_FOLDER, file_name) data = util.load_json(os.path.join(CONFERENCE_FOLDER, file_name)) html = util.get_page(data['url']) full_name = get_full_name(html) data['name'] = full_name try: print cnt, len(files), data['short'] except: pass data['links'] = get_links(data['short'], html)
def print_LUNs(self):
self.LUNs = {}
rbd_size = ''
rbd_identity = ''
pool_path = os.path.join('/var/lib/rbd', self.dconf['targetIQN'])
for file in util.listdir(pool_path):
lun_path = os.path.join(pool_path, file)
lun_file = open(lun_path, 'rt')
lun_info = lun_file.read()
for params in lun_info.split(','):
if '"size' in params:
rbd_size = params.split(':')[1]
if 'block_name_prefix' in params:
rbd_identity = params.split(':')[1]
if rbd_size and rbd_identity:
obj = self.vdi(self.uuid)
self._divert_query(obj, lun_path,rbd_identity,rbd_size,file)
self.LUNs[obj.uuid] = obj
def create(self, sr_uuid, size):
""" Create the SR. The path must not already exist, or if it does,
it must be empty. (This accounts for the case where the user has
mounted a device onto a directory manually and want to use this as the
root of a file-based SR.) """
try:
if util.ioretry(lambda: util.pathexists(self.remotepath)):
if len(util.ioretry(lambda: util.listdir(self.remotepath))) != 0:
raise xs_errors.XenError("SRExists")
else:
try:
util.ioretry(lambda: os.mkdir(self.remotepath))
except util.CommandException, inst:
if inst.code == errno.EEXIST:
raise xs_errors.XenError("SRExists")
else:
raise xs_errors.XenError("FileSRCreate", opterr="directory creation failure %d" % inst.code)
except:
raise xs_errors.XenError("FileSRCreate")
def scan_srlist(path):
dom = xml.dom.minidom.Document()
element = dom.createElement("SRlist")
dom.appendChild(element)
for val in filter(util.match_uuid, util.ioretry( \
lambda: util.listdir(path))):
fullpath = os.path.join(path, val)
if not util.ioretry(lambda: util.isdir(fullpath)):
continue
entry = dom.createElement('SR')
element.appendChild(entry)
subentry = dom.createElement("UUID")
entry.appendChild(subentry)
textnode = dom.createTextNode(val)
subentry.appendChild(textnode)
return dom.toprettyxml()
def get_conferences():
files = util.listdir(CONFERENCE_FOLDER)
util.mkdir(CONFERENCE_CRALWED_FOLDER)
cnt = 0
conf = util.load_json('conf_name.json')
for file_name in files:
save_path = os.path.join(CONFERENCE_CRALWED_FOLDER, file_name)
if util.exists(save_path):
continue
data = util.load_json(os.path.join(CONFERENCE_FOLDER, file_name))
if data['short'] not in conf.keys():
continue
html = util.get_page(data['url'])
subs = get_subs(data['short'], html)
data['name'] = conf[data['short']]
data['sub'] = {}
if len(subs) == 0:
data['sub']['#'] = get_publications(html)
util.save_json(save_path, data)
cnt += 1
def get_journals():
files = util.listdir(JOURNAL_FOLDER)
util.mkdir(JOURNAL_CRALWED_FOLDER)
cnt = 0
jour = util.load_json('jour_name.json')
for file_name in files:
save_path = os.path.join(JOURNAL_CRALWED_FOLDER, file_name)
if util.exists(save_path):
continue
data = util.load_json(os.path.join(JOURNAL_FOLDER, file_name))
if data['short'] not in jour.keys():
continue
html = util.get_page(data['url'])
subs = get_subs(data['short'], html)
data['name'] = jour[data['short']]
data['sub'] = {}
if len(subs) == 0:
data['sub']['#'] = get_publications(html)
util.save_json(save_path, data)
cnt += 1
print cnt, len(files), data['short']