def catch_error(env, start_response): try: return app(env, start_response) except Exception, err: logger.exception("Unhandled error in WSGI app: %s", err) start_response("500 Internal Server Error", [("content-type", "text/html")]) return ["An error occurred handling your request."]
def copy_file_out(self, path, callback=None): """ copy file to the iscsi device """ try: self.copy_volume(path, self.device, callback=callback) except IOError, e: logger.exception("copy_file_out failed with '%s'" % e) raise ISCSICopyFailed()
def catch_error(env, start_response): try: return app(env, start_response) except Exception, err: logger.exception('Unhandled error in WSGI app: %s', err) start_response('500 Internal Server Error', [('content-type', 'text/html')]) return ['An error occurred handling your request.']
def disconnect(self): """ logout or close the iscsi connection """ try: execute('iscsiadm', mode='node', targetname=self.iqn, portal=self.ip, logout=None) except ProcessError, e: logger.exception("iscsi logout failed with '%s'" % e) raise ISCSILogoutFailed()
def ietadm(self, *args, **kwargs): try: return ietadm(*args, **kwargs) except ConnectionRefused, e: if not os.path.exists(self.proc_iet_volume): raise ServiceUnavailable("'%s' does not exist, ietd is not " "running." % self.proc_iet_volume) msg = "Unexpected ConnectionRefused: %s" % e logger.exception(msg) raise ServiceUnavailable(msg)
def ietadm(self, *args, **kwargs): try: return ietadm(*args, **kwargs) except ConnectionRefused, e: if not os.path.exists(self.proc_iet_volume): raise ServiceUnavailable("'%s' does not exist, ietd is not " "running." % self.proc_iet_volume) msg = 'Unexpected ConnectionRefused: %s' % e logger.exception(msg) raise ServiceUnavailable(msg)
def connect(self): """ login or open an iscsi connection """ try: portal = "%s:%s" % (self.ip, self.port) execute('iscsiadm', mode='discovery', type='sendtargets', portal=self.ip) execute('iscsiadm', mode='node', targetname=self.iqn, portal=self.ip, login=None) except ProcessError, e: logger.exception("iscsi login failed with '%s'" % e) raise ISCSILoginFailed()
def format_config_line(export): config_template = Template( "# ${volume}\n" "Target ${name}\n" " Lun ${lun} Path=${path},Type=${iotype},IOMode=${iomode}\n" ) try: return config_template.substitute(export) except KeyError: # this seems to happen if the volume of behind an export was deleted logger.exception("invalid export %s" % repr(export)) return ""
def status(self): options = ('vg_size', 'vg_free', 'lv_count') try: out = execute('vgs', self.volume_group, noheadings=None, unit='b', options=','.join(options), separator=':') except ProcessError, e: if e.errcode == 5 and 'not found' in e.err: raise ServiceUnavailable("Volume group '%s' not found." % self.volume_group) logger.exception("Unknown error trying to query status of " "volume group '%s'" % self.volume_group) raise ServiceUnavailable("[Errno %d] %s" % (e.errcode, e.err))
def format_config_line(export): config_template = Template( "# ${volume}\n" "Target ${name}\n" " Lun ${lun} Path=${path},Type=${iotype},IOMode=${iomode}\n") try: return config_template.substitute(export) except KeyError: # this seems to happen if the volume of behind an export was deleted logger.exception('invalid export %s' % repr(export)) return ''
class LunrWsgiApp(object): def __init__(self, conf, urlmap, helper=None): self.conf = conf self.urlmap = urlmap if helper: self.helper = helper else: self.helper = self._get_helper(conf) def _get_helper(conf): pass def match(self, request): route = self.urlmap.match(environ=request.environ) if route is None: raise HTTPNotFound("No route matched for path '%s'" % request.path) logger.debug("Request path: %s matched: %s" % (request.path, repr(route))) try: # Try to call the controller and action controller = route['controller'](route, self) return controller.__getattribute__(route['action']) except AttributeError, e: # Couldn't find the action on the controller logger.debug( 'No action (%s) for controller (%s) Error: %s' % (route.get('action', ''), route.get('controller', ''), e)) raise HTTPNotImplemented("No action for path '%s'" % request.path) except Exception, e: logger.exception('controller/action FAIL: %s %s' % (e, route)) raise HTTPInternalServerError('Internal route error')
def remove_lvm_snapshot(self, snapshot): try: op_start = time() volume = self.get(snapshot['origin']) logger.rename('lunr.storage.helper.volume.remove_lvm_snapshot') self.scrub.scrub_snapshot(snapshot, volume) self.remove(snapshot['path']) # TODO: Failure to scrub a snapshot is un-acceptable # If we catch an exception, we should mark the snapshot # Or make this as recoverable as possible duration = time() - op_start logger.info("STAT: remove_lvm_snapshot(%r) Time: %r" % (volume['path'], duration)) except Scrub, e: logger.exception( "scrub snapshot failed with '%r' after %r seconds" % (e, time() - op_start))
def call(self, request): # Match the Request URL to an action action = self.match(request) for attempts in range(0, 3): try: result = action(request) self.helper.commit() return result except OperationalError, e: if hasattr(e, 'orig') and e.orig.args[0] == 2006: # MySQL server has gone away logger.warning("DB connection error attempt #%d" % attempts, exc_info=True) sleep(2 ** attempts) continue logger.exception("Database Error: %s" % e) raise HTTPInternalServerError("Internal database error") finally:
def remove_lvm_volume(self, volume): try: op_start = time() size = volume['size'] / 1024 / 1024 / 1024 logger.rename('lunr.storage.helper.volume.remove_lvm_volume') setproctitle("lunr-remove: " + volume['id']) # Scrub the volume self.scrub.scrub_volume(volume['path']) # Remove the device self.remove(volume['path']) duration = time() - op_start logger.info( 'STAT: remove_lvm_volume(%r) ' 'Size: %r GB Time: %r s Speed: %r MB/s' % (volume['path'], size, duration, size * 1024 / duration)) except ProcessError, e: logger.exception( "delete volume failed with '%r' after %r seconds" % (e, time() - op_start))
def remove_lvm_volume(self, volume): try: op_start = time() size = volume['size'] / 1024 / 1024 / 1024 logger.rename('lunr.storage.helper.volume.remove_lvm_volume') setproctitle("lunr-remove: " + volume['id']) # Scrub the volume self.scrub.scrub_volume(volume['path']) # Remove the device self.remove(volume['path']) duration = time() - op_start logger.info('STAT: remove_lvm_volume(%r) ' 'Size: %r GB Time: %r s Speed: %r MB/s' % (volume['path'], size, duration, size * 1024 / duration)) except ProcessError, e: logger.exception( "delete volume failed with '%r' after %r seconds" % (e, time() - op_start))
def delete(self, id, force=False, initiator=None): exports = self._get_exports(id) if not exports: raise NotFound("No export for volume '%s'" % id) # we hope there will only ever be one export, and try to ensure with # syncronization in create for export in exports: tid = export['tid'] try: out = self.ietadm(op='delete', tid=tid) except DeviceBusy, e: if force: for i in range(3): try: return self.force_delete(id) except IscsitargetError: logger.exception( 'force delete attempt %s failed' % (i + 1)) sleep(1 + i ** 2) # try one more time and let it die return self.force_delete(id) sessions = self._sessions(id) if initiator: for session in sessions: if (initiator == session['initiator'] and session['connected']): raise ResourceBusy( "Volume '%s' is currently attached " "to '%s' for initiator: %s" % (id, session['ip'], initiator)) # Fall through, our initiator didn't match, someone else # is attached. Delay "failure" until delete. return else: for session in sessions: if session['connected']: raise ResourceBusy( "Volume '%s' is currently attached " "to '%s'" % (id, session['ip'])) logger.exception("Unable to remove target for '%s' " "because it was busy" % id) raise ResourceBusy("Volume '%s' is currently attached" % id) self.rewrite_config()
def delete(self, id, force=False, initiator=None): exports = self._get_exports(id) if not exports: raise NotFound("No export for volume '%s'" % id) # we hope there will only ever be one export, and try to ensure with # syncronization in create for export in exports: tid = export['tid'] try: out = self.ietadm(op='delete', tid=tid) except DeviceBusy, e: if force: for i in range(3): try: return self.force_delete(id) except IscsitargetError: logger.exception('force delete attempt %s failed' % (i + 1)) sleep(1 + i**2) # try one more time and let it die return self.force_delete(id) sessions = self._sessions(id) if initiator: for session in sessions: if (initiator == session['initiator'] and session['connected']): raise ResourceBusy( "Volume '%s' is currently attached " "to '%s' for initiator: %s" % (id, session['ip'], initiator)) # Fall through, our initiator didn't match, someone else # is attached. Delay "failure" until delete. return else: for session in sessions: if session['connected']: raise ResourceBusy( "Volume '%s' is currently attached " "to '%s'" % (id, session['ip'])) logger.exception("Unable to remove target for '%s' " "because it was busy" % id) raise ResourceBusy("Volume '%s' is currently attached" % id) self.rewrite_config()
def copy_image(self, volume, image, glance, tmp_vol, scrub_callback): logger.rename('lunr.storage.helper.volume.copy_image') setproctitle("lunr-copy-image: " + volume['id']) copy_image_start = time() convert_dir = None try: if image.disk_format == 'raw': self.write_raw_image(glance, image, volume['path']) return convert_dir = self.prepare_tmp_vol(tmp_vol) if not os.path.exists(convert_dir): raise ValueError("Convert dir doesn't exist!") try: path = mkdtemp(dir=convert_dir) logger.info("Image convert tmp dir: %s" % path) image_file = os.path.join(path, 'image') self.write_raw_image(glance, image, image_file) if (image.disk_format == 'vhd' and image.container_format == 'ovf'): self.untar_image(path, image) image_file = self.get_coalesced_vhd(path) op_start = time() out = execute('qemu-img', 'convert', '-O', 'raw', image_file, volume['path']) duration = time() - op_start mbytes = os.path.getsize(image_file) / 1024 / 1024 logger.info('STAT: image convert %r. Image Size: %r MB ' 'Time: %r Speed: %r' % (image.id, mbytes, duration, mbytes / duration)) except Exception, e: logger.exception("Exception in image conversion") raise except Exception, e: # We have to clean this up no matter what happened. # Delete volume syncronously. Clean up db in callback. logger.exception('Unhandled exception in copy_image') self.remove_lvm_volume(volume)
def _scan_file(self, file): """ Scans IET files in the form tid:1 name:iqn.2010-11.com.rackspace:volume-00000001.uuid \tsid:562950527844864 initiator:iqn.1993-08.org.debian:01:47441681ba44 \t\tcid:0 ip:127.0.0.1 state:active hd:none dd:none \tsid:281474997486080 initiator:iqn.1993-08.org.debian:01:669c9c15f124 \t\tcid:0 ip:192.168.56.1 state:active hd:none dd:none and returns an array of key:value dicts for each entry it finds in the file """ # TODO(clayg): this needs a refactor, it's really hard to follow, sry records = [] try: with open(file) as f: record = {} for line in f: if not line.strip(): continue # new record, get it in the list and update via ref if not line.startswith('\t'): record = {} records.append(record) subrecord = self.parse_export_line(line) if not record: base_record = dict(subrecord) if any(k in record for k in subrecord): # this is a second entry record = dict(base_record) records.append(record) record.update(subrecord) except IOError, e: if e.errno == errno.ENOENT: raise ServiceUnavailable("'%s' does not exist, iscsitarget" "is not running." % file) else: msg = "Unexpected error trying to read '%s'", file logger.exception(msg) raise ServiceUnavailable(msg)
def create(self, id, ip=None): with lock.ResourceFile(self._build_lock_path(id)): try: # see if an export was created while we were locking self.get(id) except NotFound: pass else: raise AlreadyExists("An export already exists for " "volume '%s'" % id) # create target params = {'Name': self._generate_target_name(id)} try: out = self.ietadm(op='new', tid='auto', params=params) except InvalidArgument: logger.exception("Unable to create target for '%s'" % id) raise ServiceUnavailable("Invalid argument while trying to " "create export for '%s'" % id) # lookup tid tid = self._get_tid(id) # add lun path = self._lun_path(id) params = {'Path': path, 'Type': 'blockio'} try: out += self.ietadm(op='new', tid=tid, lun=0, params=params) except (NoSuchFile, NotPermitted): # clean up the target self.delete(id) if not os.path.exists(path): raise NotFound("No volume named '%s'" % id) logger.exception('Unable to create export for %s' % id) raise ServiceUnavailable("Invalid param trying to create " "export for '%s'" % id) # Write the new exports to the iet config exports = self.rewrite_config() if ip: self.add_initiator_allow(id, ip) return self.get(id, exports=exports)
def __call__(self, request): """Handle WSGI request.""" try: try: # Call the implementation to handle the action return self.encode_response(self.call(request)) except HTTPException: raise except Exception, e: logger.exception("Caught unknown exception, traceback dumped") raise HTTPInternalServerError("Internal controller error") except HTTPException, e: # Avoid logging HTTPOk Exceptions if isinstance(e, HTTPError): logger.error("%s" % e) # (thrawn01) returning a HTTPError exception to WSGI # container would result in a text/html content type # provided by webob, this is not desired body = { 'reason': e.detail, 'request-id': request.headers.get('x-request-id', '-') } return self.encode_response(Response(body=body, status=e.status))
def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI API server""" # Reload the paster config, since paster only passes us our config conf = LunrConfig.from_conf(global_conf['__file__']) # ensure global logger is named logger.rename(__name__) app = StorageWsgiApp(conf, urlmap) # Check for a valid volume config app.helper.volumes.check_config() try: app.helper.check_registration() except Exception: logger.exception('Registration failed') volumes = app.helper.volumes.list() app.helper.cgroups.load_initial_cgroups(volumes) app.helper.exports.init_initiator_allows() return app
def read_local_manifest(local_cache_filename): try: with open(local_cache_filename) as f: raw_json_string = f.read() except IOError, e: if e.errno != errno.ENOENT: raise raw_json_string = None if not raw_json_string: raise ManifestEmptyError('Manifest file at %s contained no content' % local_cache_filename) try: return Manifest.loads(raw_json_string) except ValueError: msg = 'Unable to parse manifest %s' % local_cache_filename logger.exception(msg + ':\n"""%s"""' % raw_json_string) raise ManifestParseError(msg + '.') def load_manifest(conn, volume_id, lock_file): fd = aquire_lock(lock_file) op_start = time() _headers, raw_json_string = conn.get_object(volume_id, 'manifest', newest=True) duration = time() - op_start logger.info("STAT: load_manifest for %r Duration: %r" % (volume_id, duration)) manifest = Manifest.loads(raw_json_string) os.write(fd, raw_json_string) return manifest
logger.rename('lunr.storage.helper.volume.remove_lvm_snapshot') self.scrub.scrub_snapshot(snapshot, volume) self.remove(snapshot['path']) # TODO: Failure to scrub a snapshot is un-acceptable # If we catch an exception, we should mark the snapshot # Or make this as recoverable as possible duration = time() - op_start logger.info("STAT: remove_lvm_snapshot(%r) Time: %r" % (volume['path'], duration)) except Scrub, e: logger.exception( "scrub snapshot failed with '%r' after %r seconds" % (e, time() - op_start)) except ProcessError, e: logger.exception( "delete snapshot failed with '%r' after %r seconds" % (e, time() - op_start)) except Exception, e: logger.exception("unknown exception caught '%r' after %s seconds" % (e, time() - op_start)) def remove_lvm_volume(self, volume): try: op_start = time() size = volume['size'] / 1024 / 1024 / 1024 logger.rename('lunr.storage.helper.volume.remove_lvm_volume') setproctitle("lunr-remove: " + volume['id']) # Scrub the volume self.scrub.scrub_volume(volume['path']) # Remove the device self.remove(volume['path'])
self.scrub.scrub_snapshot(snapshot, volume) self.remove(snapshot['path']) # TODO: Failure to scrub a snapshot is un-acceptable # If we catch an exception, we should mark the snapshot # Or make this as recoverable as possible duration = time() - op_start logger.info("STAT: remove_lvm_snapshot(%r) Time: %r" % (volume['path'], duration)) except Scrub, e: logger.exception( "scrub snapshot failed with '%r' after %r seconds" % (e, time() - op_start)) except ProcessError, e: logger.exception( "delete snapshot failed with '%r' after %r seconds" % (e, time() - op_start)) except Exception, e: logger.exception( "unknown exception caught '%r' after %s seconds" % (e, time() - op_start)) def remove_lvm_volume(self, volume): try: op_start = time() size = volume['size'] / 1024 / 1024 / 1024 logger.rename('lunr.storage.helper.volume.remove_lvm_volume') setproctitle("lunr-remove: " + volume['id']) # Scrub the volume self.scrub.scrub_volume(volume['path']) # Remove the device