def LambdaHandler(event, context):
print('event:', event, 'context:', context)
config = configparser.ConfigParser()
print('Reading the config.ini')
config.read('config.ini')
print('Fetching the sources')
mf_info, mf_links = [], []
for s in config['SOURCES']:
print('config[SOURCES]', s)
for source in config['SOURCES'][event['resources'][0].split('/')
[-1]].split(','):
html = FetchPage(source)
_, info, links = ExtractTable(html,
attrs={'id': 'dataTableId'},
extract_links=True)
mf_info.extend(info)
mf_links.extend(links)
print('Tranforming the links to portfolio-holdings')
for i, x in enumerate(mf_links):
if (len(x) > 0):
s = x.split('/')
mf_links[i] = '/'.join(s[:-1] + ['portfolio-holdings', s[-1]])
print('Creating the dynamodb resource')
dynamodb = boto3.resource('dynamodb', region_name='ap-south-1')
# client = boto3.client('dynamodb',
# aws_access_key_id='AKIA2VLYX2H6IG7HOBBR',
# aws_secret_access_key='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',
# region_name='ap-south-1')
# print(client.scan(TableName='baskets'))
# sys.exit()
print('Getting the funds table')
funds_table = dynamodb.Table('funds')
params = []
for ind, (i, l) in enumerate(zip(mf_info, mf_links)):
if (i[1] == 'Regular' or len(l) == 0):
continue
params.append((ind, i, l, funds_table))
print('Fetching the sources\' portfolio-holdings')
# Execute(params[0][0], params[0][1], params[0][2], funds_table)
Pool(Execute, params, max_workers=5)
def __init__(self, mdsip, mdsport): self.guid = Guid.generate() self.chkpool = Pool(ChunkServerClient, ChunkServerClient.__del__) self.mds = MDSClient(self.guid, mdsip, mdsport) self.dmclient = DMClient() self.CheckVolume()
