def _db_introduce(self):
uuid = util.default(self, "uuid", lambda: util.gen_uuid())
sm_config = util.default(self, "sm_config", lambda: {})
vdi = self.sr.session.xenapi.VDI.db_introduce(uuid, self.label, self.description, self.sr.sr_ref, "user", self.shareable, self.read_only, {}, self.location, {}, {})
self.sr.session.xenapi.VDI.set_sm_config(vdi, sm_config)
self.sr.session.xenapi.VDI.set_managed(vdi, self.managed)
self.sr.session.xenapi.VDI.set_virtual_size(vdi, str(self.size))
self.sr.session.xenapi.VDI.set_physical_utilisation(vdi, str(self.utilisation))
return vdi
def _db_update(self):
vdi = self.sr.session.xenapi.VDI.get_by_uuid(self.uuid)
self.sr.session.xenapi.VDI.set_virtual_size(vdi, str(self.size))
self.sr.session.xenapi.VDI.set_physical_utilisation(vdi, str(self.utilisation))
self.sr.session.xenapi.VDI.set_read_only(vdi, self.read_only)
sm_config = util.default(self, "sm_config", lambda: {})
self.sr.session.xenapi.VDI.set_sm_config(vdi, sm_config)
def initialize(self, device_type="auto", width=320, height=240):
Camera.device_type = device_type
if device_type == "auto":
Camera.device_type = thermal() if lepton_in(
modules) else raspberrypi() if is_linux() else default()
self.width = width
self.height = height
if Camera.thread is None:
Camera.thread = threading.Thread(target=self._thread)
Camera.thread.stop_event = threading.Event()
Camera.thread.start()
# Return control to calling class when frame becomes available or when thread terminates
print("Camera class initialised with device '{}'".format(
Camera.device_type))
while self.frame is None and self.thread is not None:
sleep(0)
print("Returning control to calling class")
return self.thread is None
else:
print("Camera thread already initialised")
return False
def __init__(self, host, port=None, username="******", password="******",
mechanism="PLAIN", heartbeat=None, **options):
"""
Creates a connection. A newly created connection must be connected
with the Connection.connect() method before it can be used.
@type host: str
@param host: the name or ip address of the remote host
@type port: int
@param port: the port number of the remote host
@rtype: Connection
@return: a disconnected Connection
"""
self.host = host
self.port = default(port, AMQP_PORT)
self.username = username
self.password = password
self.mechanism = mechanism
self.heartbeat = heartbeat
self.id = str(uuid4())
self.session_counter = 0
self.sessions = {}
self.reconnect = options.get("reconnect", False)
self._connected = False
self._lock = RLock()
self._condition = Condition(self._lock)
self._waiter = Waiter(self._condition)
self._modcount = Serial(0)
self.error = None
from driver import Driver
self._driver = Driver(self)
self._driver.start()
def in_sync_with_xenapi_record(self, x):
"""Returns true if this VDI is in sync with the supplied XenAPI record"""
if self.location <> util.to_plain_string(x['location']):
util.SMlog("location %s <> %s" % (self.location, x['location']))
return False
if self.read_only <> x['read_only']:
util.SMlog("read_only %s <> %s" % (self.read_only, x['read_only']))
return False
if str(self.size) <> x['virtual_size']:
util.SMlog("virtual_size %s <> %s" % (self.size, x['virtual_size']))
return False
if str(self.utilisation) <> x['physical_utilisation']:
util.SMlog("utilisation %s <> %s" % (self.utilisation, x['physical_utilisation']))
return False
sm_config = util.default(self, "sm_config", lambda: {})
if set(sm_config.keys()) <> set(x['sm_config'].keys()):
util.SMlog("sm_config %s <> %s" % (repr(sm_config), repr(x['sm_config'])))
return False
for k in sm_config.keys():
if sm_config[k] <> x['sm_config'][k]:
util.SMlog("sm_config %s <> %s" % (repr(sm_config), repr(x['sm_config'])))
return False
if self.cbt_enabled != x['cbt_enabled']:
util.SMlog("cbt_enabled %s <> %s" % (
self.cbt_enabled, x['cbt_enabled']))
return False
return True
def in_sync_with_xenapi_record(self, x):
"""Returns true if this VDI is in sync with the supplied XenAPI record"""
if self.location <> util.to_plain_string(x['location']):
util.SMlog("location %s <> %s" % (self.location, x['location']))
return False
if self.read_only <> x['read_only']:
util.SMlog("read_only %s <> %s" % (self.read_only, x['read_only']))
return False
if str(self.size) <> x['virtual_size']:
util.SMlog("virtual_size %s <> %s" %
(self.size, x['virtual_size']))
return False
if str(self.utilisation) <> x['physical_utilisation']:
util.SMlog("utilisation %s <> %s" %
(self.utilisation, x['physical_utilisation']))
return False
sm_config = util.default(self, "sm_config", lambda: {})
if set(sm_config.keys()) <> set(x['sm_config'].keys()):
util.SMlog("sm_config %s <> %s" %
(repr(sm_config), repr(x['sm_config'])))
return False
for k in sm_config.keys():
if sm_config[k] <> x['sm_config'][k]:
util.SMlog("sm_config %s <> %s" %
(repr(sm_config), repr(x['sm_config'])))
return False
if self.cbt_enabled != x['cbt_enabled']:
util.SMlog("cbt_enabled %s <> %s" %
(self.cbt_enabled, x['cbt_enabled']))
return False
return True
def __init__(self, sr):
self.sr = sr
self.__xenapi_locations = {}
self.__xenapi_records = util.list_VDI_records_in_sr(sr)
for vdi in self.__xenapi_records.keys():
self.__xenapi_locations[self.__xenapi_records[vdi]['location']] = vdi
self.__sm_records = {}
for vdi in sr.vdis.values():
# We initialise the sm_config field with the values from the database
# The sm_config_overrides contains any new fields we want to add to
# sm_config, and also any field to delete (by virtue of having
# sm_config_overrides[key]=None)
try:
if not hasattr(vdi, "sm_config"):
vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()
except:
util.SMlog("missing config for vdi: %s" % vdi.location)
vdi.sm_config = {}
if vdi.sm_config_override:
if len(vdi.sm_config_override.keys())>0:
util.SMlog("original sm_config: %s" % repr(vdi.sm_config))
for key in vdi.sm_config_override.keys():
if vdi.sm_config_override[key]:
vdi.sm_config[key]=vdi.sm_config_override[key]
else:
if vdi.sm_config.has_key(key):
del vdi.sm_config[key]
if len(vdi.sm_config_override.keys())>0:
util.SMlog("overridden sm_config: %s" % repr(vdi.sm_config))
self.__sm_records[vdi.location] = vdi
xenapi_locations = set(self.__xenapi_locations.keys())
sm_locations = set(self.__sm_records.keys())
# These ones are new on disk
self.new = sm_locations.difference(xenapi_locations)
# These have disappeared from the disk
self.gone = xenapi_locations.difference(sm_locations)
# These are the ones which are still present but might have changed...
existing = sm_locations.intersection(xenapi_locations)
# Synchronise the uuid fields using the location as the primary key
# This ensures we know what the UUIDs are even though they aren't stored
# in the storage backend.
for location in existing:
sm_vdi = self.get_sm_vdi(location)
xenapi_vdi = self.get_xenapi_vdi(location)
sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])
# Only consider those whose configuration looks different
self.existing = filter(lambda x:not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x))), existing)
if len(self.new) <> 0:
util.SMlog("new VDIs on disk: " + repr(self.new))
if len(self.gone) <> 0:
util.SMlog("VDIs missing from disk: " + repr(self.gone))
if len(self.existing) <> 0:
util.SMlog("VDIs changed on disk: " + repr(self.existing))
def _db_introduce(self):
uuid = util.default(self, "uuid", lambda: util.gen_uuid())
sm_config = util.default(self, "sm_config", lambda: {})
ty = util.default(self, "ty", lambda: "user")
is_a_snapshot = util.default(self, "is_a_snapshot", lambda: False)
metadata_of_pool = util.default(self, "metadata_of_pool", lambda: "OpaqueRef:NULL")
snapshot_time = util.default(self, "snapshot_time", lambda: "19700101T00:00:00Z")
snapshot_of = util.default(self, "snapshot_of", lambda: "OpaqueRef:NULL")
vdi = self.sr.session.xenapi.VDI.db_introduce(
uuid,
self.label,
self.description,
self.sr.sr_ref,
ty,
self.shareable,
self.read_only,
{},
self.location,
{},
sm_config,
self.managed,
str(self.size),
str(self.utilisation),
metadata_of_pool,
is_a_snapshot,
xmlrpclib.DateTime(snapshot_time),
snapshot_of,
)
return vdi
def _db_update(self):
vdi = self.sr.session.xenapi.VDI.get_by_uuid(self.uuid)
self.sr.session.xenapi.VDI.set_virtual_size(vdi, str(self.size))
self.sr.session.xenapi.VDI.set_physical_utilisation(
vdi, str(self.utilisation))
self.sr.session.xenapi.VDI.set_read_only(vdi, self.read_only)
sm_config = util.default(self, "sm_config", lambda: {})
self._override_sm_config(sm_config)
self._db_update_sm_config(vdi, sm_config)
def __init__(self, dim, dim_out=None, mult=4, glu=True, dropout=0.):
super().__init__()
inner_dim = int(dim * mult)
dim_out = default(dim_out, dim)
project_in = nn.Sequential(nn.Linear(
dim, inner_dim), nn.GELU()) if not glu else GEGLU(dim, inner_dim)
self.net = nn.Sequential(project_in, nn.Dropout(dropout),
nn.Linear(inner_dim, dim_out))
def _db_update(self):
vdi = self.sr.session.xenapi.VDI.get_by_uuid(self.uuid)
self.sr.session.xenapi.VDI.set_virtual_size(vdi, str(self.size))
self.sr.session.xenapi.VDI.set_physical_utilisation(vdi, str(self.utilisation))
self.sr.session.xenapi.VDI.set_read_only(vdi, self.read_only)
sm_config = util.default(self, "sm_config", lambda: {})
self._override_sm_config(sm_config)
self._db_update_sm_config(vdi, sm_config)
self.sr.session.xenapi.VDI.set_cbt_enabled(vdi,
self._get_blocktracking_status())
def __init__(self, sr):
self.sr = sr
self.__xenapi_locations = {}
self.__xenapi_records = util.list_VDI_records_in_sr(sr)
for vdi in self.__xenapi_records.keys():
self.__xenapi_locations[util.to_plain_string(
self.__xenapi_records[vdi]['location'])] = vdi
self.__sm_records = {}
for vdi in sr.vdis.values():
# We initialise the sm_config field with the values from the database
# The sm_config_overrides contains any new fields we want to add to
# sm_config, and also any field to delete (by virtue of having
# sm_config_overrides[key]=None)
try:
if not hasattr(vdi, "sm_config"):
vdi.sm_config = self.__xenapi_records[
self.__xenapi_locations[
vdi.location]]['sm_config'].copy()
except:
util.SMlog("missing config for vdi: %s" % vdi.location)
vdi.sm_config = {}
vdi._override_sm_config(vdi.sm_config)
self.__sm_records[vdi.location] = vdi
xenapi_locations = set(self.__xenapi_locations.keys())
sm_locations = set(self.__sm_records.keys())
# These ones are new on disk
self.new = sm_locations.difference(xenapi_locations)
# These have disappeared from the disk
self.gone = xenapi_locations.difference(sm_locations)
# These are the ones which are still present but might have changed...
existing = sm_locations.intersection(xenapi_locations)
# Synchronise the uuid fields using the location as the primary key
# This ensures we know what the UUIDs are even though they aren't stored
# in the storage backend.
for location in existing:
sm_vdi = self.get_sm_vdi(location)
xenapi_vdi = self.get_xenapi_vdi(location)
sm_vdi.uuid = util.default(sm_vdi, "uuid",
lambda: xenapi_vdi['uuid'])
# Only consider those whose configuration looks different
self.existing = filter(
lambda x: not (self.get_sm_vdi(x).in_sync_with_xenapi_record(
self.get_xenapi_vdi(x))), existing)
if len(self.new) <> 0:
util.SMlog("new VDIs on disk: " + repr(self.new))
if len(self.gone) <> 0:
util.SMlog("VDIs missing from disk: " + repr(self.gone))
if len(self.existing) <> 0:
util.SMlog("VDIs changed on disk: " + repr(self.existing))
def _divert_query(self, vdi, path, rbd_id, rbd_size, lun_name):
vdi.uuid = scsiutil.gen_uuid_from_string(rbd_id)
vdi.location = self.uuid
vdi.vendor = 'RADOS'
vdi.serial = lun_name
vdi.LUNid = rbd_id.split('.')[1]
vdi.size = rbd_size
vdi.SCSIid = lun_name
vdi.path = path
sm_config = util.default(vdi, "sm_config", lambda: {})
sm_config['LUNid'] = str(vdi.LUNid)
sm_config['SCSIid'] = vdi.SCSIid
vdi.sm_config = sm_config
def _divert_query(self, vdi, path, rbd_id, rbd_size, lun_name):
vdi.uuid = scsiutil.gen_uuid_from_string(rbd_id)
vdi.location = self.uuid
vdi.vendor = 'RADOS'
vdi.serial = lun_name
vdi.LUNid = rbd_id.split('.')[1]
vdi.size = rbd_size
vdi.SCSIid = lun_name
vdi.path = path
sm_config = util.default(vdi, "sm_config", lambda: {})
sm_config['LUNid'] = str(vdi.LUNid)
sm_config['SCSIid'] = vdi.SCSIid
vdi.sm_config = sm_config
def _query(self, path, id):
self.uuid = scsiutil.gen_uuid_from_string(scsiutil.getuniqueserial(path))
self.location = self.uuid
self.vendor = scsiutil.getmanufacturer(path)
self.serial = scsiutil.getserial(path)
self.LUNid = id
self.size = scsiutil.getsize(path)
self.SCSIid = scsiutil.getSCSIid(path)
self.path = path
sm_config = util.default(self, "sm_config", lambda: {})
sm_config['LUNid'] = str(self.LUNid)
sm_config['SCSIid'] = self.SCSIid
self.sm_config = sm_config
def _db_introduce(self):
uuid = util.default(self, "uuid", lambda: util.gen_uuid())
sm_config = util.default(self, "sm_config", lambda: {})
if self.sr.srcmd.params.has_key("vdi_sm_config"):
for key in SM_CONFIG_PASS_THROUGH_FIELDS:
val = self.sr.srcmd.params["vdi_sm_config"].get(key)
if val:
sm_config[key] = val
ty = util.default(self, "ty", lambda: "user")
is_a_snapshot = util.default(self, "is_a_snapshot", lambda: False)
metadata_of_pool = util.default(self, "metadata_of_pool", lambda: "OpaqueRef:NULL")
snapshot_time = util.default(self, "snapshot_time", lambda: "19700101T00:00:00Z")
snapshot_of = util.default(self, "snapshot_of", lambda: "OpaqueRef:NULL")
cbt_enabled = util.default(self, "cbt_enabled", lambda: False)
vdi = self.sr.session.xenapi.VDI.db_introduce(uuid, self.label, self.description, self.sr.sr_ref, ty, self.shareable, self.read_only, {}, self.location, {}, sm_config, self.managed, str(self.size), str(self.utilisation), metadata_of_pool, is_a_snapshot, xmlrpclib.DateTime(snapshot_time), snapshot_of, cbt_enabled)
return vdi
def _query(self, path, id):
self.uuid = scsiutil.gen_uuid_from_string(
scsiutil.getuniqueserial(path))
self.location = self.uuid
self.vendor = scsiutil.getmanufacturer(path)
self.serial = scsiutil.getserial(path)
self.LUNid = id
self.size = scsiutil.getsize(path)
self.SCSIid = scsiutil.getSCSIid(path)
self.path = path
sm_config = util.default(self, "sm_config", lambda: {})
sm_config['LUNid'] = str(self.LUNid)
sm_config['SCSIid'] = self.SCSIid
self.sm_config = sm_config
def _divert_query(self, vdi, path, rbd_id, rbd_size, lun_name, count):
dbg_prt("[rbdsr] _divert_query, rbd_id:%s, rbd_size:%s, lun_name:%s", rbd_id, rbd_size, lun_name )
vdi.uuid = scsiutil.gen_uuid_from_string(rbd_id)
vdi.location = self.uuid
vdi.vendor = 'RADOS'
vdi.serial = self._format_uuid_from_string(lun_name)
vdi.LUNid = str(count)
vdi.size = rbd_size
vdi.SCSIid = lun_name
vdi.path = path
sm_config = util.default(vdi, "sm_config", lambda: {})
sm_config['LUNid'] = vdi.LUNid
sm_config['SCSIid'] = vdi.SCSIid
vdi.sm_config = sm_config
dbg_prt("[rbdsr] vdi[uuid:%s,location:%s,vendor:%s,serial:%s,LUNid:%s,size:%s,SCSIid:%s,path:%s]", vdi.uuid, vdi.location,vdi.vendor,vdi.serial,vdi.LUNid,vdi.size,vdi.SCSIid,vdi.path)
dbg_prt("[rbdsr] sm_config[%s]", sm_config)
def _query(self, path, id, uuid=None, scsi_id=None):
"""Overloaded function with mostly duplicated code"""
if uuid:
self.uuid = uuid
else:
util.SMlog("RawHBA: uuid should not be generated..")
self.uuid = scsiutil.gen_uuid_from_string(
scsiutil.getuniqueserial(path)
)
if scsi_id:
self.SCSIid = scsi_id
else:
# It is usually unnecessary to calculate it again but scsi_id
# is used as a flag in this function and we cannot guarantee
# this info is already available at call time
self.SCSIid = scsiutil.getSCSIid(path)
self.location = self.uuid
self.vendor = scsiutil.getmanufacturer(path)
self.serial = scsiutil.getserial(path)
self.LUNid = id
# Handle resize done at the array size. The resize gets reflected
# only when the vdi is not in detached state. Do this if we the vdi
# is known to xapi
try:
vdi_ref = self.sr.session.xenapi.VDI.get_by_uuid(self.uuid)
# Check if the vbd is not in attached state, do a LUN rescan
# to reflect the array LUN
dev = [path]
if scsi_id:
# We want all the devices with this scsi_id
dev = scsiutil._genReverseSCSIidmap(scsi_id)
if self.sr.srcmd.cmd == "vdi_attach":
scsiutil.refreshdev(dev)
elif not B_util.is_vdi_attached(self.sr.session, vdi_ref):
scsiutil.refreshdev(dev)
except:
pass
self.size = scsiutil.getsize(path)
self.path = path
sm_config = util.default(self, "sm_config", lambda: {})
sm_config['LUNid'] = str(self.LUNid)
sm_config['SCSIid'] = self.SCSIid
# Make sure to use kernel blkback (not blktap3) for raw LUNs
sm_config['backend-kind'] = 'vbd'
self.sm_config = sm_config
def _query(self, path, id, uuid=None, scsi_id=None):
"""Overloaded function with mostly duplicated code"""
if uuid:
self.uuid = uuid
else:
util.SMlog("RawHBA: uuid should not be generated..")
self.uuid = scsiutil.gen_uuid_from_string(
scsiutil.getuniqueserial(path)
)
if scsi_id:
self.SCSIid = scsi_id
else:
# It is usually unnecessary to calculate it again but scsi_id
# is used as a flag in this function and we cannot guarantee
# this info is already available at call time
self.SCSIid = scsiutil.getSCSIid(path)
self.location = self.uuid
self.vendor = scsiutil.getmanufacturer(path)
self.serial = scsiutil.getserial(path)
self.LUNid = id
# Handle resize done at the array size. The resize gets reflected
# only when the vdi is not in detached state. Do this if we the vdi
# is known to xapi
try:
vdi_ref = self.sr.session.xenapi.VDI.get_by_uuid(self.uuid)
# Check if the vbd is not in attached state, do a LUN rescan
# to reflect the array LUN
dev = [path]
if scsi_id:
# We want all the devices with this scsi_id
dev = scsiutil._genReverseSCSIidmap(scsi_id)
if self.sr.srcmd.cmd == "vdi_attach":
scsiutil.refreshdev(dev)
elif not B_util.is_vdi_attached(self.sr.session, vdi_ref):
scsiutil.refreshdev(dev)
except:
pass
self.size = scsiutil.getsize(path)
self.path = path
sm_config = util.default(self, "sm_config", lambda: {})
sm_config['LUNid'] = str(self.LUNid)
sm_config['SCSIid'] = self.SCSIid
# Make sure to use kernel blkback (not blktap3) for raw LUNs
sm_config['backend-kind'] = 'vbd'
self.sm_config = sm_config
def _db_introduce(self):
uuid = util.default(self, "uuid", lambda: util.gen_uuid())
sm_config = util.default(self, "sm_config", lambda: {})
ty = util.default(self, "ty", lambda: "user")
is_a_snapshot = util.default(self, "is_a_snapshot", lambda: False)
metadata_of_pool = util.default(self, "metadata_of_pool",
lambda: "OpaqueRef:NULL")
snapshot_time = util.default(self, "snapshot_time",
lambda: "19700101T00:00:00Z")
snapshot_of = util.default(self, "snapshot_of",
lambda: "OpaqueRef:NULL")
vdi = self.sr.session.xenapi.VDI.db_introduce(
uuid, self.label, self.description, self.sr.sr_ref, ty,
self.shareable, self.read_only, {},
self.location, {}, sm_config, self.managed, str(self.size),
str(self.utilisation), metadata_of_pool, is_a_snapshot,
xmlrpclib.DateTime(snapshot_time), snapshot_of)
return vdi
def _divert_query(self, vdi, path, rbd_id, rbd_size, lun_name, count):
dbg_prt("[rbdsr] _divert_query, rbd_id:%s, rbd_size:%s, lun_name:%s",
rbd_id, rbd_size, lun_name)
vdi.uuid = scsiutil.gen_uuid_from_string(rbd_id)
vdi.location = self.uuid
vdi.vendor = 'RADOS'
vdi.serial = self._format_uuid_from_string(lun_name)
vdi.LUNid = str(count)
vdi.size = rbd_size
vdi.SCSIid = lun_name
vdi.path = path
sm_config = util.default(vdi, "sm_config", lambda: {})
sm_config['LUNid'] = vdi.LUNid
sm_config['SCSIid'] = vdi.SCSIid
vdi.sm_config = sm_config
dbg_prt(
"[rbdsr] vdi[uuid:%s,location:%s,vendor:%s,serial:%s,LUNid:%s,size:%s,SCSIid:%s,path:%s]",
vdi.uuid, vdi.location, vdi.vendor, vdi.serial, vdi.LUNid,
vdi.size, vdi.SCSIid, vdi.path)
dbg_prt("[rbdsr] sm_config[%s]", sm_config)
def _db_introduce(self):
uuid = util.default(self, "uuid", lambda: util.gen_uuid())
sm_config = util.default(self, "sm_config", lambda: {})
if self.sr.srcmd.params.has_key("vdi_sm_config"):
for key in SM_CONFIG_PASS_THROUGH_FIELDS:
val = self.sr.srcmd.params["vdi_sm_config"].get(key)
if val:
sm_config[key] = val
ty = util.default(self, "ty", lambda: "user")
is_a_snapshot = util.default(self, "is_a_snapshot", lambda: False)
metadata_of_pool = util.default(self, "metadata_of_pool",
lambda: "OpaqueRef:NULL")
snapshot_time = util.default(self, "snapshot_time",
lambda: "19700101T00:00:00Z")
snapshot_of = util.default(self, "snapshot_of",
lambda: "OpaqueRef:NULL")
cbt_enabled = util.default(self, "cbt_enabled", lambda: False)
vdi = self.sr.session.xenapi.VDI.db_introduce(
uuid, self.label, self.description, self.sr.sr_ref, ty,
self.shareable, self.read_only, {},
self.location, {}, sm_config, self.managed, str(self.size),
str(self.utilisation), metadata_of_pool, is_a_snapshot,
xmlrpclib.DateTime(snapshot_time), snapshot_of, cbt_enabled)
return vdi
def forward(self,
x,
context=None,
mask=None,
context_mask=None,
rel_pos=None,
sinusoidal_emb=None,
prev_attn=None,
mem=None):
b, n, _, h, device = *x.shape, self.heads, x.device
kv_input = default(context, x) # For Cross-Attention
q_input = x
k_input = kv_input
v_input = kv_input
if exists(mem):
k_input = torch.cat((mem, k_input), dim=-2)
v_input = torch.cat((mem, v_input), dim=-2)
if exists(sinusoidal_emb):
offset = k_input.shape[-2] - q_input.shape[
-2] # for cross-attention,
# difference in seq length
q_input = q_input + sinusoidal_emb(q_input, offset=offset)
k_input = k_input + sinusoidal_emb(k_input)
q = self.to_q(q_input)
k = self.to_k(k_input)
v = self.to_v(v_input)
# compute each head embedding from number_of_heads * y = embed_size
q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h),
(q, k, v))
input_mask = None
if any(map(exists, (mask, context_mask))):
q_mask = default(mask, lambda: torch.ones(
(b, n), device=device).bool())
k_mask = q_mask if not exists(context) else context_mask
k_mask = default(
k_mask, lambda: torch.ones(
(b, k.shape[-2]), device=device).bool())
# Create like context mask * context mask matrix with each value in the
# matrix representing whether to compute attention for that or not.
q_mask = rearrange(q_mask,
'b i -> b () i ()') # (B, N) -> (B, 1, N, 1)
k_mask = rearrange(k_mask,
'b j -> b () () j') # (B, M) -> (B, 1, 1, M)
input_mask = q_mask * k_mask # (B, 1, N, 1) * (B, 1, 1, M) -> (B, 1, N, M)
dots = einsum('b h i d, b h j d -> b h i j', q, k) * self.scale
mask_value = max_neg_value(dots)
if exists(prev_attn):
dots += prev_attn
pre_softmax_attn = dots
if exists(rel_pos):
dots = rel_pos(dots)
if exists(input_mask):
dots.masked_fill_(~input_mask, mask_value)
del input_mask
attn = self.attn_fn(dots, dim=-1)
post_softmax_attn = attn
attn = self.dropout(attn)
out = einsum('b h i j, b h j d -> b h i d', attn, v)
out = rearrange(out, 'b h n d -> b n (h d)')
intermediates = Intermediates(pre_softmax_attn=pre_softmax_attn,
post_softmax_attn=post_softmax_attn)
return self.to_out(out), intermediates
