def handler(event, context):
# set the log function
libtaylor.setLogFunction(info)
libtaylor.setParseRanges(parseRanges)
info("Event: " + str(event))
for record in event['Records']:
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
download_path = '/tmp/{}{}'.format(uuid.uuid4(), key.replace('/', '-'))
upload_path_indexes = '/tmp/{}{}'.format(key.replace('/', '-'),
'.indexes')
upload_path_force = '/tmp/{}{}'.format(key.replace('/', '-'), '.force')
upload_path_disp = '/tmp/{}{}'.format(key.replace('/', '-'), '.disp')
s3_client.download_file(bucket, key, download_path)
f = open(download_path)
e = f.readlines()
e = ' '.join(e)
f.close()
processRecord(e, upload_path_indexes, upload_path_force,
upload_path_disp)
s3_client.upload_file(upload_path_indexes, '{}'.format(bucket),
'{}{}'.format(key, '.indexes'))
s3_client.upload_file(upload_path_force, '{}'.format(bucket),
'{}{}'.format(key, '.force'))
s3_client.upload_file(upload_path_disp, '{}'.format(bucket),
'{}{}'.format(key, '.disp'))
def handler(event, context):
# set the log function
libtaylor.setLogFunction(info)
libtaylor.setParseRanges(parseRanges)
info("Event: " + str(event))
for record in event['Records']:
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
download_path = '/tmp/{}{}'.format(uuid.uuid4(), key.replace('/','-'))
upload_path_indexes = '/tmp/{}{}'.format(key.replace('/','-'), '.indexes')
upload_path_force = '/tmp/{}{}'.format(key.replace('/','-'), '.force')
upload_path_disp = '/tmp/{}{}'.format(key.replace('/','-'), '.disp')
s3_client.download_file(bucket, key, download_path)
f = open(download_path)
e = f.readlines()
e = ' '.join(e)
f.close()
processRecord(e, upload_path_indexes, upload_path_force, upload_path_disp)
s3_client.upload_file(upload_path_indexes, '{}'.format(bucket), '{}{}'.format(key, '.indexes'))
s3_client.upload_file(upload_path_force, '{}'.format(bucket), '{}{}'.format(key, '.force'))
s3_client.upload_file(upload_path_disp, '{}'.format(bucket), '{}{}'.format(key, '.disp'))
info("Error: Parameter not formatted properly: " + rstr)
info(
"Do you have both a start and end value separated with a dash?"
)
exit(1)
start = int(startend[0])
end = int(startend[1])
results[num].append([start, end])
info("# Parsed range " + str(rangestr) + " as " + str(results))
return results
# ******* Begin main program **********
# set the log function
libtaylor.setLogFunction(info)
libtaylor.setParseRanges(parseRanges)
# digits are zero based so add 1 to user input
args.digits += 1
# Set up indexes
# If parallelization is enabled, calculate the start and end based on the node number
# and number of nodes
if args.parallel:
if args.verbose and not args.silent:
print 'Calculating parallelization values'
p = args.parallel.split(":")
if len(p) != 2:
print "Error: Parameter not formatted properly: " + args.parallel
print "Is your start/end node list properly formatted as '(start):(end)'?"
for range in ranges[1:-1].split(","):
startend=range.split("-")
if len(startend) != 2:
info("Error: Parameter not formatted properly: " + rstr)
info("Do you have both a start and end value separated with a dash?")
exit(1)
start=int(startend[0])
end=int(startend[1])
results[num].append([start,end])
info("# Parsed range " + str(rangestr) + " as " + str(results))
return results
# ******* Begin main program **********
# set the log function
libtaylor.setLogFunction(info)
libtaylor.setParseRanges(parseRanges)
# Set up indexes
# If parallelization is enabled, calculate the start and end based on the node number
# and number of nodes
if args.parallel:
if args.verbose and not args.silent:
print 'Calculating parallelization values'
p = args.parallel.split(":");
if len(p) != 2:
print "Error: Parameter not formatted properly: " + args.parallel
print "Is your start/end node list properly formatted as '(start):(end)'?"
exit(1)
if int(p[0]) == 1:
# If it's the first node in the node set, then the start index is 0