Пример #1
0
def LambdaHandler(event, context):
    print('event:', event, 'context:', context)
    config = configparser.ConfigParser()
    print('Reading the config.ini')
    config.read('config.ini')

    print('Fetching the sources')
    mf_info, mf_links = [], []
    for s in config['SOURCES']:
        print('config[SOURCES]', s)
    for source in config['SOURCES'][event['resources'][0].split('/')
                                    [-1]].split(','):
        html = FetchPage(source)
        _, info, links = ExtractTable(html,
                                      attrs={'id': 'dataTableId'},
                                      extract_links=True)
        mf_info.extend(info)
        mf_links.extend(links)

    print('Tranforming the links to portfolio-holdings')
    for i, x in enumerate(mf_links):
        if (len(x) > 0):
            s = x.split('/')
            mf_links[i] = '/'.join(s[:-1] + ['portfolio-holdings', s[-1]])

    print('Creating the dynamodb resource')
    dynamodb = boto3.resource('dynamodb', region_name='ap-south-1')

    # client = boto3.client('dynamodb',
    #   aws_access_key_id='AKIA2VLYX2H6IG7HOBBR',
    #   aws_secret_access_key='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',
    #   region_name='ap-south-1')
    # print(client.scan(TableName='baskets'))
    # sys.exit()

    print('Getting the funds table')
    funds_table = dynamodb.Table('funds')

    params = []

    for ind, (i, l) in enumerate(zip(mf_info, mf_links)):
        if (i[1] == 'Regular' or len(l) == 0):
            continue
        params.append((ind, i, l, funds_table))

    print('Fetching the sources\' portfolio-holdings')
    # Execute(params[0][0], params[0][1], params[0][2], funds_table)
    Pool(Execute, params, max_workers=5)
Пример #2
0
	def __init__(self, mdsip, mdsport):
		self.guid = Guid.generate()
		self.chkpool = Pool(ChunkServerClient, ChunkServerClient.__del__)
		self.mds = MDSClient(self.guid, mdsip, mdsport)
		self.dmclient = DMClient()
		self.CheckVolume()
Пример #3
0
class Client:
	def __init__(self, mdsip, mdsport):
		self.guid = Guid.generate()
		self.chkpool = Pool(ChunkServerClient, ChunkServerClient.__del__)
		self.mds = MDSClient(self.guid, mdsip, mdsport)
		self.dmclient = DMClient()
		self.CheckVolume()
	
	def __del__(self):
		self.chkpool.dispose()
		del self.guid
		del self.mds
		del self.chkpool
		del self.dmclient

	# give a list of chunk sizes, return a list of volumes
	# volume : path node msg.volume
	def NewChunkList(self, chksizes):
		volumelist = []
		
		serlist = self.mds.GetChunkServers(5)
		if len(serlist) == 0:
			logging.error('Currently not ChunkServer is available')
			return None

		for size in chksizes:
			server = serlist[0]
			addr = server.ServiceAddress
			port = server.ServicePort
			chkclient = self.chkpool.get(self.guid, addr, port)
			chunks = chkclient.NewChunk(size, 1)
			if chunks == []:
				pass
			
			volume = Object()
			volume.size = size
			volume.assembler = 'chunk'
			volume.guid = Guid.toStr(chunks.guids[0])
			volume.parameters = [volume.guid, '', '', addr, str(port)]
			path, nodename = self.MountChunk(volume)
			volume.parameters[1] = path
			volume.parameters[2] = nodename
			
			if path == None:
				for volume in volumelist:
					self.UnmountChunk(volume)
				logging.error('Mount chunk failed')
				return None

			volumelist.append(volume)
		return volumelist

	def GetChunkNode(self, name, addr, port=3260):
		try:
			nodelist = libiscsi.discover_sendtargets(addr, port)
		except IOError as ex:
			#This IOError is because there's no iscsi node is found now
			if str(ex).find('connection login retries (reopen_max) 5 exceeded')>-1:
				return None
			else:
				raise ex
		for node in nodelist:
			if name == node.name:
				return node
		return None

	def MountChunk(self, volume):
		addr = volume.parameters[3]
		port = int(volume.parameters[4])
		chkclient = self.chkpool.get(self.guid, addr, port)
		target = chkclient.AssembleChunk(volume)
		node = self.GetChunkNode(target.access_point, addr)
		if node != None:
			try:
				node.login()
			except Exception as ex:
				#If this node is already mounted
				if str(ex).find('session exists') > -1:
					logging.warn('chunk '+node.name+' is already mounted')
				else:
					raise ex
			dev = scandev.get_blockdev_by_targetname(node.name)
			if dev != None:
				return dev, node.name
		return None, None

	def UnmountChunk(self, volume):
		addr = volume.parameters[3]
		port = int(volume.parameters[4])
		nodename = volume.parameters[2]
		node = self.GetChunkNode(nodename, addr)
		if node == None:
			logging.error('Could not find the chunk to unmount')
			return False
		node.logout()
		chkclient = self.chkpool.get(self.guid, addr, port)
		if chkclient.DisassembleChunk(nodename) == False:
			return False
		return True
	
	def ChangeVolumeStatus(self, volume, status):
		if volume.assembler == 'chunk':
			return
		volume.parameters[3] = status
		for subvol in volume.subvolumes:
			self.ChangeVolumeStatus(subvol, status)
		self.mds.WriteVolumeInfo(volume)

	def MountVolume(self, volume_name):
		try:
			volume = self.mds.ReadVolumeInfo(volume_name)
		except IOError as ex:
			if str(ex).find('No such file or directory') > -1:
				logging.error('Volume '+volume_name+' does not exist')
				return False
			else:
				raise ex
		status = volume.parameters[3]
		if status == 'active':
			logging.warn('Volume '+volume_name+' is already mounted')
			return False
		isSubvolume = volume.parameters[2]
		if isSubvolume == 'subvolume':
			logging.warn('Volume '+volume_name+' is a subvolume')
			return False
		if self.MountVolumeTree(volume) == False:
			return False
		if self.dmclient.MountVolume(volume) == False:
			return False
		self.ChangeVolumeStatus(volume, 'active')
		return True

	def MountVolumeTree(self, volume):
		if volume.assembler == 'chunk':
			ret,_ = self.MountChunk(volume)
			if ret == None:
				return False
			return True
		for subvol in volume.subvolumes:
			if self.MountVolumeTree(subvol) == False:
				return False
		return True

	def UnmountVolume(self, volume_name):
		try:
			volume = self.mds.ReadVolumeInfo(volume_name)
		except IOError as ex:
			if str(ex).find('No such file or directory') > -1:
				logging.error('Volume '+volume_name+' does not exist')
				return False
			else:
				raise ex
		status = volume.parameters[3]
		if status == 'inactive':
			logging.warn('Volume '+volume_name+' is already unmounted')
			return False
		isSubvolume = volume.parameters[2]
		if isSubvolume == 'subvolume':
			logging.warn('Volume '+volume_name+' is a subvolume')
			return False
		if self.UnmountVolumeTree(volume) == False:
			return False
		if self.dmclient.DeleteVolume(volume) == False:
			return False
		self.ChangeVolumeStatus(volume, 'inactive')
		return True

	def UnmountVolumeTree(self, volume):
		if volume.assembler == 'chunk':
			return self.UnmountChunk(volume)
		for subvol in volume.subvolumes:
			if self.UnmountVolumeTree(subvol) == False:
				return False
		return True

	def SplitVolume(self, volume_name):
		try:
			volume = self.mds.ReadVolumeInfo(volume_name)
		except IOError as ex:
			if str(ex).find('No such file or directory') > -1:
				logging.error('Volume '+volume_name+' does not exist')
				return False
			else:
				raise ex
		if volume.subvolumes[0].assembler == 'chunk':
			logging.error('This volume is not divisible!')
			return False

		if self.dmclient.SplitVolume(volume_name) == False:
			return False
		self.mds.DeleteVolumeInfo(volume)
		for subvol in volume.subvolumes:
			subvol.parameters[2] = 'volume'
			self.mds.WriteVolumeInfo(subvol)
		return True

	def CreateVolume(self, volname, volsize, voltype, chksizes, volnames, params):
		vollist = []
		if len(volnames) > 0:
			volsize = 0
			for name in volnames:
				try:
					vol = self.mds.ReadVolumeInfo(name)
				except IOError as ex:
					if str(ex).find('No such file or directory') > -1:
						logging.error('Volume '+name+' does not exist')
						return False
					else:
						raise ex
				if vol.parameters[2] == 'subvolume':
					logging.error('volume '+name+' is a subvolume of another volume')
					return False
				if vol.parameters[3] == 'inactive':
					logging.warn('Volume '+name+' is inactive')
					logging.warn('Mounting volume '+name)
					if not self.MountVolume(name):
						logging.error('Mount volume '+name+' failed')
						return False
				volsize += vol.size
				vollist.append(vol)
		
		if len(chksizes) == 0 and len(volnames) == 0:
			totsize = volsize
			while totsize > CHUNKSIZE:
				chksizes.append(CHUNKSIZE)
				totsize -= CHUNKSIZE
			chksizes.append(totsize)

		if voltype == 'gfs':
			temlist = []
			for chksize in chksizes:
				temlist.append(chksize)
				temlist.append(chksize)
				temlist.append(chksize)
			chksizes = temlist

		if len(chksizes) > 0:
			vollist = self.NewChunkList(chksizes)
			if vollist == None:
				logging.error('New Chunk(s) failed')
				return False
			
		volume = Object()
		volume.size = volsize
		volume.assembler = voltype
		volume.subvolumes = vollist
		volume.guid = Guid.toStr(Guid.generate())
		volume.parameters = [volname, '/dev/mapper/'+volname, 'volume', 'active']
		volume.parameters.extend(params)

		if self.dmclient.MapVolume(volume) == False:
			logging.error('device mapper: create volume '+volname+' failed')
			if volume.subvolumes[0].assembler == 'chunk':
				for subvol in volume.subvolumes:
					self.DeleteVolumeTree(subvol)
			return False

		self.mds.WriteVolumeInfo(volume)
		for vol in vollist:
			if vol.assembler != 'chunk':
				vol.parameters[2] = 'subvolume'
				vol.parameters[3] = 'active'
			self.mds.WriteVolumeInfo(vol)

		return True			
 
	def DeleteVolume(self, volume_name):
		try:
			volume = self.mds.ReadVolumeInfo(volume_name)
		except IOError as ex:
			if str(ex).find('No such file or directory') > -1:
				logging.error(sys.stderr,'Volume '+volume_name+' does not exist')
				return False
			else:
				raise ex
		if self.DeleteVolumeTree(volume) == False:
			return False
		if self.dmclient.DeleteVolume(volume) == False:
			return False
		return True

	def DeleteVolumeTree(self, volume):
		if volume.assembler == 'chunk':
			addr = volume.parameters[3]
			port = int(volume.parameters[4])
			chkclient = self.chkpool.get(self.guid, addr, port)
			if self.UnmountChunk(volume) == False:
				return False
			chkclient.DeleteChunk(volume)
			self.mds.DeleteVolumeInfo(volume)
			return True
		for subvolume in volume.subvolumes:
			if self.DeleteVolumeTree(subvolume) == False:
				return False
		self.mds.DeleteVolumeInfo(volume)
		return True

	def CheckVolume(self):
		volumes = self.mds.ListDirectory('/')
		for vol in volumes:
			if self.dmclient.GetVolumeMap(vol.name) == None:
				status = 'inactive'
			else:
				status = 'active'
			volume = self.mds.ReadVolumeInfo(vol.name)
			volume_status = volume.parameters[3]
			if volume_status != status:
				volume.parameters[3] = status
				self.mds.WriteVolumeInfo(volume)


# def test():
# 	global PARAM
# 	client = Client('0.0.0.0', 22136)
# 	items = client.mds.ListDirectory('/')
# 	for item in items:
# 		print item
# 	client.ListVolume('hello_softsan_striped')

# 	create a stiped type volume
# 	arg = Object()
# 	arg.type = 'striped'
# 	arg.chunksizes = []
# 	arg.subvolumes = []
# 	arg.parameters = []
# 	arg.name = 'hello_softsan_striped'
# 	arg.size = 128
# 	client.CreateVolume(arg)

# 	create 2 linear type volumes and then 
# 	build a striped volume with these two linear volumes

# 	arg = Object()
# 	arg.type = 'linear'
# 	arg.chunksizes = []
# 	arg.subvolumes = []
# 	arg.parameters = []
# 	arg.name = 'hello_softsan_1'
# 	arg.size = 100
# 	client.CreateVolume(arg)

# 	arg = Object()
# 	arg.type = 'linear'
# 	arg.chunksizes = []
# 	arg.subvolumes = []
# 	arg.parameters = []
# 	arg.name = 'hello_softsan_2'
# 	arg.size = 100
# 	client.CreateVolume(arg)

# 	arg = Object()
# 	arg.type = 'striped'
# 	arg.chunksizes = []
# 	arg.subvolumes = ['hello_softsan_1', 'hello_softsan_2']
# 	arg.parameters = []
# 	arg.name = 'hello_softsan_3'
# 	arg.size = 200
# 	client.CreateVolume(arg)
# 	# client.DeleteVolume('hello_softsan_1')
# 	# client.DeleteVolume('hello_softsan_2')
# 	client.DeleteVolume('hello_softsan_3')

# 	arg = Object()
# 	arg.type = 'gfs'
# 	arg.chunksizes = []
# 	arg.subvolumes = []
# 	arg.parameters = []
# 	arg.name = 'gfs'
# 	arg.size = 80
# 	client.CreateVolume(arg)

# 	client.DeleteVolume('gfs')
# 	client.Clear()


# if __name__=='__main__':
# 	test()