def download_files(s3, bucket, track_keys, existing_crap=None):
print("Downloading {} files ...".format(len(track_keys)))
result = list()
try:
count = 0
for i, track_key in enumerate(track_keys):
if existing_crap is not None and track_key in existing_crap:
key = track_key
target_path = os.path.join(AWS_LAMBDA_WORKING_DIR, os.path.basename(track_key))
else:
key = '{0}{1}'.format(track_key, AWS_MULTRACK_FILE_EXT.format(i))
target_path = os.path.join(AWS_LAMBDA_WORKING_DIR,
os.path.basename(track_key)) + \
AWS_MULTRACK_FILE_EXT.format(i)
print("About to fetch key '{0}' into file '{1}' ...".format(key, target_path))
with open(target_path, "wb") as file_object:
s3.download_fileobj(bucket, key, file_object)
count += 1
result.append(target_path)
execute_command(['ls', '-alh', AWS_LAMBDA_WORKING_DIR]) # ??
except botocore.exceptions.ClientError as ex:
if ex.response['Error']['Code'] == "404":
print("The object does not exist.")
raise
print("Downloaded {} files.".format(count))
return result
def download_file(s3, event, metadata):
print("Downloading file ...")
try:
bucket, key = get_bucket_key(event)
if 'url' in metadata:
url = metadata['url']
print("URL: {}".format(url))
timestamp = build_timestamp()
file_name = 'avatar_{}.jpg'.format(timestamp)
print("File name: {}".format(file_name))
target_path = os.path.join(AWS_LAMBDA_WORKING_DIR, file_name)
print("About to fetch file '{0}' ...".format(target_path))
urllib.request.urlretrieve(url, target_path)
else:
target_path = os.path.join(AWS_LAMBDA_WORKING_DIR, build_src_file_name(key))
print("About to fetch file '{0}' ...".format(target_path))
with open(target_path, "wb") as file_object:
s3.download_fileobj(bucket, key, file_object)
result = target_path
execute_command(['ls', '-alh', AWS_LAMBDA_WORKING_DIR]) # ??
except botocore.exceptions.ClientError as ex:
if ex.response['Error']['Code'] == "404":
print("The object does not exist.")
raise
print("Downloaded file '{0}'.".format(result))
return result
def convert_audio_file(avconv_file, ffmpeg_file, input_file, metadata):
print("Converting audio file '{0}' with {1} ...".format(input_file, ffmpeg_file))
file_name, _ = os.path.splitext(input_file)
result = file_name + '-zopa'
start = metadata[AWS_METADATA_AUDIO_START]
end = metadata[AWS_METADATA_AUDIO_END]
duration = float(end) - float(start)
if os.path.exists(result):
os.remove(result)
returncode, _, _ = execute_command([avconv_file,
'-y',
'-ss',
start,
'-t',
'{}'.format(duration),
'-i',
input_file,
result + '.wav'])
if returncode != 0:
raise RuntimeError("Unable to complete avconv call.")
# returncode, _, _ = execute_command([ffmpeg_file,
# '-y',
# # '-ss',
# # start,
# # '-t',
# # '{}'.format(duration),
# '-i',
# result + '.wav',
# '-filter:a',
# 'loudnorm',
# result + '-norm.wav'])
# if returncode != 0:
# raise RuntimeError("Unable to complete ffmpeg call.")
returncode, _, _ = execute_command([avconv_file,
'-y',
# '-ss',
# start,
# '-t',
# '{}'.format(duration),
'-i',
# result + '-norm.wav',
result + '.wav',
result + '.mp3'])
if returncode != 0:
raise RuntimeError("Unable to complete avconv call after normalization.")
result = result + '.mp3'
print("Converted audio file '{0}' into '{1}'.".format(input_file, result))
return result
def cleanup():
print("Cleaning up '/tmp' ...")
if os .path.exists(AWS_LAMBDA_WORKING_DIR):
execute_command(['ls', '-alh', '/tmp']) # ??
execute_command(['rm', '-rf', AWS_LAMBDA_WORKING_DIR])
execute_command(['ls', '-alh', '/tmp']) # ??
print("Cleaned up '/tmp'.")
def task(context):
Logger().info("Running task {0} ...".format(context.id))
k = len(context.scripts)
Logger().info("Task {0}, executing {1} commands ...".format(
context.id, len(context.scripts)))
for script, return_code in context.scripts:
result, stdout, stderr = execute_command(script_to_command(script))
if result != return_code:
break
k -= 1
Logger().info("Task {0}, executed {1} commands ...".format(
context.id,
len(context.scripts) - k))
if k == 0:
connection = None
cursor = None
try:
Logger().info("Task {0}, updating AppCommands status ...".format(
context.id))
connection = pymysql.connect(user=context.db_user,
passwd=context.db_password,
db=context.db_name,
host=context.db_host,
port=context.db_port,
charset='utf8',
cursorclass=pymysql.cursors.Cursor)
connection.autocommit = False
cursor = connection.cursor()
query = "UPDATE {0}.{1} SET Active = false WHERE UserId = {2} AND GroupId = '{3}' "\
.format(context.db_name, context.db_table, context.user_id, context.group_id)
cursor.execute(query)
connection.commit()
Logger().info("Task {0}, updated AppCommands status.".format(
context.id))
except pymysql.Error as ex:
Logger().exception("DBMS error. {}".format(str(ex)))
if connection is not None:
connection.rollback()
except Exception as ex:
Logger().exception("DBMS error. {}".format(str(ex)))
if connection is not None:
connection.rollback()
finally:
if cursor is not None:
cursor.close()
if connection is not None:
connection.close()
Logger().info("Completed task '{0}'.".format(context.id))
return True
def convert_image_file(input_file, size):
print("Converting image file '{0}' ...".format(input_file))
file_name, _ = os.path.splitext(input_file)
result = file_name + ('-thumbnail' if size < 256 else '') + '-zopa.jpg'
returncode, _, _ = execute_command(['convert',
input_file,
'-resize',
'{}'.format(size),
'-background',
'white',
'-flatten',
result])
if returncode != 0:
raise RuntimeError("Unable to complete avconv call.")
print("Converted image file '{0}' into '{1}'.".format(input_file, result))
return result
def convert_audio_files(s3,
metadata,
avconv_file,
bucket,
input_files,
track_keys,
existing_crap=None):
# avconv -i 1.mp3 -i 2.mp3 -filter_complex
# "[0:a]atrim=0:10[a0]; [1:a]atrim=0:29[a1]; [a0][a1]amix=inputs=2"
# -c:a libmp3lame -q:a 4 output.mp3
print("Converting '{0}' audio files ...".format(len(input_files)))
command = list()
command.append(avconv_file)
for input_file in input_files:
command.append('-i')
command.append('{}'.format(input_file))
command.append('-filter_complex')
trims = ''
mixes = ''
volumes = ''
pans = ''
new_metadata = None
durations = list()
for i, track_key in enumerate(track_keys):
if existing_crap is not None and track_key in existing_crap:
key = track_key
else:
key = '{0}{1}'.format(track_key, AWS_MULTRACK_FILE_EXT.format(i))
if existing_crap is not None and track_key in existing_crap:
start = metadata['existing-audio-start']
end = metadata['existing-audio-end']
volume = metadata['existing-audio-volume']
pan = metadata['existing-audio-pan']
else:
print('Fetching head object of {0} ...'.format(key))
ho_response = s3.head_object(Bucket=bucket, Key=key)
print(ho_response)
start = ho_response['Metadata']['audio-start']
end = ho_response['Metadata']['audio-end']
volume = ho_response['Metadata']['audio-volume']
pan = ho_response['Metadata']['audio-pan']
durations.append(int((float(end) - float(start)) * 1000.0))
trims += '[{0}:a]atrim={1}:{2}[a{0}]; '.format(i, start, end)
volumes += '[a{0}]volume=volume={1}[a{0}]; '.format(i, float(volume))
gain_l, gain_r = calculate_pan(float(pan))
pans += '[a{0}]pan=stereo| c0={1}*c0| c1={2}*c1[a{0}]; '.format(i, gain_l, gain_r)
mixes += '[a{0}]'.format(i)
# if new_metadata is None:
# new_metadata = ho_response['Metadata']
command.append('{0}{1}{2}{3}amix=inputs={4}'.format(trims,
volumes,
pans,
mixes,
len(input_files)))
command.append('-c:a')
command.append('libmp3lame')
command.append('-q:a')
command.append('4')
file_name, _ = os.path.splitext(input_files[0])
result = file_name + '-zopa'
command.append(result + '.mp3')
if os.path.exists(result):
os.remove(result)
returncode, _, _ = execute_command(command)
if returncode != 0:
raise RuntimeError("Unable to complete avconv call.")
result = result + '.mp3'
# duration = extract_duration(stdout, stderr)
# metadata['duration'] = convert_time_to_msecs(duration)
if durations:
metadata['duration'] = str(max(durations))
print("Converted audio files into '{0}'.".format(result))
return new_metadata, result
def patch_executables(context):
print("Patching executable files ...")
avconv = os.path.join(AWS_LAMBDA_WORKING_DIR, context.aws_request_id + '-avconv')
execute_command(['cp', '/var/task/avconv', avconv])
execute_command(['chmod', '755', avconv])
ffmpeg = os.path.join(AWS_LAMBDA_WORKING_DIR, context.aws_request_id + '-ffmpeg')
execute_command(['cp', '/var/task/ffmpeg', ffmpeg])
execute_command(['chmod', '755', ffmpeg])
os.environ['LD_LIBRARY_PATH'] = AWS_LAMBDA_WORKING_DIR
libmp3lame = os.path.join(AWS_LAMBDA_WORKING_DIR, 'libmp3lame.so.0.0.0')
if not os.path.exists(libmp3lame):
execute_command(['cp', '/var/task/libmp3lame.so.0.0.0', AWS_LAMBDA_WORKING_DIR])
for link_libmp3lame in [os.path.join(AWS_LAMBDA_WORKING_DIR, 'libmp3lame.so'),
os.path.join(AWS_LAMBDA_WORKING_DIR, 'libmp3lame.so.0')]:
if os.path.exists(link_libmp3lame):
os.remove(link_libmp3lame)
execute_command(['ln', '-s', libmp3lame, link_libmp3lame])
result = ffmpeg, ffmpeg # always use ffmpeg for now ...
# result = avconv, ffmpeg
print("Patched executable files '{0}'".format(', '.join(str(exe) for exe in result)))
return result