def get_freshair(outputdir, date_s, order_totnum=None, file_data=None, debug=False, do_mp4=False, exec_dict=None, check_if_exist=False, mp3_exist=False, to_file_debug=True): # check if outputdir is a directory if not os.path.isdir(outputdir): raise ValueError("Error, %s is not a directory." % outputdir) # check if actually a weekday assert (npr_utils.is_weekday(date_s)) if exec_dict is None: exec_dict = npr_utils.find_necessary_executables() assert (exec_dict is not None) avconv_exec = exec_dict['avconv'] if order_totnum is None: order_totnum = npr_utils.get_order_number_weekday_in_year(date_s) order_in_year, tot_in_year = order_totnum if file_data is None: file_data = get_freshair_image() decdate = date_s.strftime('%d.%m.%Y') m4afile = os.path.join(outputdir, 'NPR.FreshAir.%s.m4a' % decdate) if check_if_exist and os.path.isfile(m4afile): return nprURL = npr_utils.get_NPR_URL(date_s, _npr_FreshAir_progid, npr_utils.get_api_key()) year = date_s.year # download this data into an lxml elementtree tree = lxml.etree.fromstring(requests.get(nprURL).content) if debug: # print 'URL = %s' % nprURL if to_file_debug: with open( os.path.join(outputdir, 'NPR.FreshAir.tree.%s.xml' % decdate), 'w') as openfile: openfile.write(lxml.etree.tostring(tree)) return tree # check for unavailable tag if len( list( filter( lambda elem: 'value' in elem.keys() and elem.get('value') == 'true', tree.iter('unavailable')))) != 0: unavailable_elem = max( filter( lambda elem: 'value' in elem.keys() and elem.get('value') == 'true', tree.iter('unavailable'))) if unavailable_elem.text is None: print( 'Could not create Fresh Air episode for date %s for some reason' % npr_utils.get_datestring(date_s)) else: print( 'Could not create Fresh Air episode for date %s for this reason: %s' % (npr_utils.get_datestring(date_s), unavailable_elem.text.strip())) return # now get tuple of title to mp3 file if not do_mp4: try: title_mp3_urls = _process_freshair_titlemp3_tuples_one(tree) except ValueError: title_mp3_urls = _process_freshair_titlemp3_tuples_two(tree) if len(title_mp3_urls) == 0: print('Error, could not find any Fresh Air episodes for date %s.' % npr_utils.get_datestring(date_s)) return titles, songurls = zip(*title_mp3_urls) outfiles = [ os.path.join(outputdir, 'freshair.%s.%d.mp3' % (decdate, num + 1)) for (num, mp3url) in enumerate(songurls) ] if mp3_exist: assert (all([os.path.isfile(outfile) for outfile in outfiles])) else: title_mp4_urls = _process_freshair_titlemp4_tuples(tree) if len(title_mp4_urls) == 0: print('Error, could not find any Fresh Air episodes for date %s.' % npr_utils.get_datestring(date_s)) return titles, songurls = zip(*title_mp4_urls) outfiles = [ os.path.join(outputdir, 'freshair.%s.%d.mp4' % (decdate, num + 1)) for (num, mp4url) in enumerate(songurls) ] title = date_s.strftime('%A, %B %d, %Y') title = '%s: %s.' % (title, '; '.join( ['%d) %s' % (num + 1, titl) for (num, titl) in enumerate(titles)])) # download those files time0 = time.time() pool = multiprocessing.Pool(processes=len(songurls)) if not mp3_exist: outfiles = list( filter(None, pool.map(_download_file, zip(songurls, outfiles)))) if do_mp4: # replace mp4 with ac3 newouts = [] for outfile in outfiles: newfile = re.sub('\.mp4$', '.ac3', outfile) split_cmd = [avconv_exec, '-y', '-i', outfile, newfile] proc = subprocess.Popen(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_val, stderr_val = proc.communicate() os.remove(outfile) newouts.append(newfile) outfiles = newouts # sox magic command #wgdate = date_s.strftime('%d-%b-%Y') #wavfile = os.path.join(outputdir, 'freshair%s.wav' % wgdate ).replace(' ', '\ ') #split_cmd = [ '(for', 'file', 'in', ] + fnames + [ # ';', sox_exec, '$file', '-t', 'cdr', '-', ';', 'done)' ] + [ # '|', sox_exec, 't-', 'cdr', '-', wavfile ] #split_cmd = [ sox_exec, ] + fnames + [ wavfile, ] #print split_cmd #return #proc = subprocess.Popen(split_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) #stdout_val, stderr_val = proc.communicate() #for filename in outfiles: # os.remove(filename) # now convert to m4a file # /usr/bin/avconv -y -i freshair$wgdate.wav -ar 44100 -ac 2 -aq 400 -acodec libfaac NPR.FreshAir."$decdate".m4a ; time0 = time.time() fnames = [filename.replace(' ', '\ ') for filename in outfiles] avconv_concat_cmd = 'concat:%s' % '|'.join(fnames) split_cmd = [ avconv_exec, '-y', '-i', avconv_concat_cmd, '-ar', '44100', '-ac', '2', '-threads', '%d' % multiprocessing.cpu_count(), '-strict', 'experimental', '-acodec', 'aac', m4afile ] proc = subprocess.Popen(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_val, stderr_val = proc.communicate() # remove wav file for filename in outfiles: os.remove(filename) # now put in metadata mp4tags = mutagen.mp4.MP4(m4afile) mp4tags.tags['\xa9nam'] = [ title, ] mp4tags.tags['\xa9alb'] = [ 'Fresh Air From WHYY: %d' % year, ] mp4tags.tags['\xa9ART'] = [ 'Terry Gross', ] mp4tags.tags['\xa9day'] = [ '%d' % year, ] mp4tags.tags['\xa9cmt'] = [ "more info at : Fresh Air from WHYY and NPR Web site", ] mp4tags.tags['trkn'] = [ (order_in_year, tot_in_year), ] mp4tags.tags['covr'] = [ mutagen.mp4.MP4Cover(file_data, mutagen.mp4.MP4Cover.FORMAT_PNG), ] mp4tags.tags['\xa9gen'] = [ 'Podcast', ] mp4tags.tags['aART'] = [ 'Terry Gross', ] mp4tags.save() return m4afile
def get_waitwait(outputdir, date_s, order_totnum=None, file_data=None, debugonly=False, exec_dict=None): # check if outputdir is a directory if not os.path.isdir(outputdir): raise ValueError("Error, %s is not a directory." % outputdir) # check if actually saturday if not npr_utils.is_saturday(date_s): raise ValueError("Error, date = %s not a Saturday." % npr_utils.get_datestring(date_s)) if exec_dict is None: exec_dict = npr_utils.find_necessary_executables() assert exec_dict is not None avconv_exec = exec_dict["avconv"] if order_totnum is None: order_totnum = npr_utils.get_order_number_saturday_in_year(date_s) order_in_year, tot_in_year = order_totnum if file_data is None: file_data = get_waitwait_image() year = date_s.year decdate = npr_utils.get_decdate(date_s) m4afile = os.path.join(outputdir, "NPR.WaitWait.%s.m4a" % decdate) if year >= 2006: tup = get_title_wavfile_standard(date_s, outputdir, avconv_exec, debugonly=debugonly) if tup is None: return title, outfiles = tup fnames = [filename.replace(" ", "\ ") for filename in outfiles] sox_string_cmd = "concat:%s" % "|".join(fnames) split_cmd = [ avconv_exec, "-y", "-i", sox_string_cmd, "-ar", "44100", "-ac", "2", "-threads", "%d" % multiprocessing.cpu_count(), "-strict", "experimental", "-acodec", "aac", m4afile, ] proc = subprocess.Popen(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_val, stderr_val = proc.communicate() for filename in outfiles: os.remove(filename) else: title = waitwait_realmedia.rm_get_title_from_url(date_s) rmfile = waitwait_realmedia.rm_download_file(date_s, outdir=outputdir) wavfile = waitwait_realmedia.rm_create_wav_file(date_s, rmfile, outdir=outputdir) os.remove(rmfile) # now convert to m4a file m4afile = os.path.join(outputdir, "NPR.WaitWait.%s.m4a" % decdate) split_cmd = [ avconv_exec, "-y", "-i", wavfile, "-ar", "44100", "-ac", "2", "-threads", "%d" % multiprocessing.cpu_count(), "-strict", "experimental", "-acodec", "aac", m4afile, ] proc = subprocess.Popen(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_val, stderr_val = proc.communicate() # remove wav file os.remove(wavfile) # now put in metadata mp4tags = mutagen.mp4.MP4(m4afile) mp4tags.tags["\xa9nam"] = [title] mp4tags.tags["\xa9alb"] = ["Wait Wait...Don't Tell Me: %d" % year] mp4tags.tags["\xa9ART"] = ["Peter Sagal"] mp4tags.tags["\xa9day"] = ["%d" % year] mp4tags.tags["\xa9cmt"] = ["more info at : NPR Web site"] mp4tags.tags["trkn"] = [(order_in_year, tot_in_year)] mp4tags.tags["covr"] = [mutagen.mp4.MP4Cover(file_data, mutagen.mp4.MP4Cover.FORMAT_PNG)] mp4tags.tags["\xa9gen"] = ["Podcast"] mp4tags.save() return m4afile
def get_freshair(outputdir, date_s, order_totnum = None, file_data = None, debug = False, do_mp4 = False, exec_dict = None, check_if_exist = False, mp3_exist = False, to_file_debug = True ): # check if outputdir is a directory if not os.path.isdir(outputdir): raise ValueError("Error, %s is not a directory." % outputdir) # check if actually a weekday assert( npr_utils.is_weekday(date_s) ) if exec_dict is None: exec_dict = npr_utils.find_necessary_executables() assert( exec_dict is not None ) avconv_exec = exec_dict['avconv'] if order_totnum is None: order_totnum = npr_utils.get_order_number_weekday_in_year(date_s) order_in_year, tot_in_year = order_totnum if file_data is None: file_data = get_freshair_image() decdate = date_s.strftime('%d.%m.%Y') m4afile = os.path.join(outputdir, 'NPR.FreshAir.%s.m4a' % decdate ) if check_if_exist and os.path.isfile(m4afile): return nprURL = npr_utils.get_NPR_URL(date_s, _npr_FreshAir_progid, npr_utils.get_api_key() ) year = date_s.year # download this data into an lxml elementtree tree = lxml.etree.fromstring( requests.get(nprURL).content ) if debug: # print 'URL = %s' % nprURL if to_file_debug: with open(os.path.join(outputdir, 'NPR.FreshAir.tree.%s.xml' % decdate), 'w') as openfile: openfile.write( lxml.etree.tostring( tree ) ) return tree # check for unavailable tag if len(filter(lambda elem: 'value' in elem.keys() and elem.get('value') == 'true', tree.iter('unavailable'))) != 0: unavailable_elem = max(filter(lambda elem: 'value' in elem.keys() and elem.get('value') == 'true', tree.iter('unavailable'))) if unavailable_elem.text is None: print 'Could not create Fresh Air episode for date %s for some reason' % npr_utils.get_datestring( date_s ) else: print 'Could not create Fresh Air episode for date %s for this reason: %s' % \ ( npr_utils.get_datestring( date_s ), unavailable_elem.text.strip() ) return # now get tuple of title to mp3 file if not do_mp4: try: title_mp3_urls = _process_freshair_titlemp3_tuples_one(tree) except ValueError: title_mp3_urls = _process_freshair_titlemp3_tuples_two(tree) if len(title_mp3_urls) == 0: print 'Error, could not find any Fresh Air episodes for date %s.' % \ npr_utils.get_datestring( date_s ) return titles, songurls = zip(*title_mp3_urls) outfiles = [ os.path.join(outputdir, 'freshair.%s.%d.mp3' % ( decdate, num + 1) ) for (num, mp3url) in enumerate( songurls ) ] if mp3_exist: assert(all([ os.path.isfile( outfile ) for outfile in outfiles ]) ) else: title_mp4_urls = _process_freshair_titlemp4_tuples( tree ) if len(title_mp4_urls) == 0: print 'Error, could not find any Fresh Air episodes for date %s.' % \ npr_utils.get_datestring( date_s ) return titles, songurls = zip(*title_mp4_urls) outfiles = [ os.path.join(outputdir, 'freshair.%s.%d.mp4' % ( decdate, num + 1) ) for (num, mp4url) in enumerate( songurls ) ] title = date_s.strftime('%A, %B %d, %Y') title = '%s: %s.' % ( title, '; '.join([ '%d) %s' % ( num + 1, titl ) for (num, titl) in enumerate(titles) ]) ) # download those files time0 = time.time() pool = multiprocessing.Pool(processes = len(songurls) ) if not mp3_exist: outfiles = filter(None, pool.map(_download_file, zip( songurls, outfiles ) ) ) if do_mp4: # replace mp4 with ac3 newouts = [] for outfile in outfiles: newfile = re.sub('\.mp4$', '.ac3', outfile ) split_cmd = [ avconv_exec, '-y', '-i', outfile, newfile ] proc = subprocess.Popen( split_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout_val, stderr_val = proc.communicate( ) os.remove( outfile ) newouts.append( newfile ) outfiles = newouts # sox magic command #wgdate = date_s.strftime('%d-%b-%Y') #wavfile = os.path.join(outputdir, 'freshair%s.wav' % wgdate ).replace(' ', '\ ') #split_cmd = [ '(for', 'file', 'in', ] + fnames + [ # ';', sox_exec, '$file', '-t', 'cdr', '-', ';', 'done)' ] + [ # '|', sox_exec, 't-', 'cdr', '-', wavfile ] #split_cmd = [ sox_exec, ] + fnames + [ wavfile, ] #print split_cmd #return #proc = subprocess.Popen(split_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) #stdout_val, stderr_val = proc.communicate() #for filename in outfiles: # os.remove(filename) # now convert to m4a file # /usr/bin/avconv -y -i freshair$wgdate.wav -ar 44100 -ac 2 -aq 400 -acodec libfaac NPR.FreshAir."$decdate".m4a ; time0 = time.time() fnames = [ filename.replace(' ', '\ ') for filename in outfiles ] avconv_concat_cmd = 'concat:%s' % '|'.join(fnames) split_cmd = [ avconv_exec, '-y', '-i', avconv_concat_cmd, '-ar', '44100', '-ac', '2', '-threads', '%d' % multiprocessing.cpu_count(), '-strict', 'experimental', '-acodec', 'aac', m4afile ] proc = subprocess.Popen(split_cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE) stdout_val, stderr_val = proc.communicate() # remove wav file for filename in outfiles: os.remove(filename) # now put in metadata mp4tags = mutagen.mp4.MP4(m4afile) mp4tags.tags['\xa9nam'] = [ title, ] mp4tags.tags['\xa9alb'] = [ 'Fresh Air From WHYY: %d' % year, ] mp4tags.tags['\xa9ART'] = [ 'Terry Gross', ] mp4tags.tags['\xa9day'] = [ '%d' % year, ] mp4tags.tags['\xa9cmt'] = [ "more info at : Fresh Air from WHYY and NPR Web site", ] mp4tags.tags['trkn'] = [ ( order_in_year, tot_in_year ), ] mp4tags.tags['covr'] = [ mutagen.mp4.MP4Cover(file_data, mutagen.mp4.MP4Cover.FORMAT_PNG ), ] mp4tags.tags['\xa9gen'] = [ 'Podcast', ] mp4tags.tags['aART'] = [ 'Terry Gross', ] mp4tags.save() return m4afile
def get_waitwait(outputdir, date_s, order_totnum=None, file_data=None, debugonly=False, exec_dict=None, verify=True, justFix=False): # check if outputdir is a directory if not os.path.isdir(outputdir): raise ValueError("Error, %s is not a directory." % outputdir) # check if actually saturday if not npr_utils.is_saturday(date_s): raise ValueError("Error, date = %s not a Saturday." % npr_utils.get_datestring(date_s)) if exec_dict is None: exec_dict = npr_utils.find_necessary_executables() assert (exec_dict is not None) avconv_exec = exec_dict['avconv'] if order_totnum is None: order_totnum = npr_utils.get_order_number_saturday_in_year(date_s) order_in_year, tot_in_year = order_totnum if file_data is None: file_data = get_waitwait_image(verify=verify) year = date_s.year decdate = npr_utils.get_decdate(date_s) m4afile = os.path.join(outputdir, 'NPR.WaitWait.%s.m4a' % decdate) if year >= 2006: tup = get_title_wavfile_standard(date_s, outputdir, avconv_exec, debugonly=debugonly, verify=verify, justFix=justFix) if tup is None: return title, outfiles = tup if justFix: # works only for year >= 2006 if not os.path.isfile(m4afile): print "Error, %s does not exist." % os.path.basename(m4afile) return mp4tags = mutagen.mp4.MP4(m4afile) mp4tags.tags['\xa9nam'] = [ title, ] mp4tags.save() logging.debug('fixed title for %s.' % m4afile) return m4afile fnames = map(lambda filename: filename.replace(' ', '\ '), outfiles) sox_string_cmd = 'concat:%s' % '|'.join(fnames) split_cmd = [ avconv_exec, '-y', '-i', sox_string_cmd, '-ar', '44100', '-ac', '2', '-threads', '%d' % multiprocessing.cpu_count(), '-strict', 'experimental', '-acodec', 'aac', m4afile ] proc = subprocess.Popen(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_val, stderr_val = proc.communicate() if 'Protocol not found' in stderr_val.strip(): for filename in outfiles: os.remove(filename) raise ValueError( "Error, AVCONV does not have the concatenation protocol.") for filename in outfiles: os.remove(filename) else: title = waitwait_realmedia.rm_get_title_from_url(date_s) rmfile = waitwait_realmedia.rm_download_file(date_s, outdir=outputdir) wavfile = waitwait_realmedia.rm_create_wav_file(date_s, rmfile, outdir=outputdir) os.remove(rmfile) # now convert to m4a file m4afile = os.path.join(outputdir, 'NPR.WaitWait.%s.m4a' % decdate) split_cmd = [ avconv_exec, '-y', '-i', wavfile, '-ar', '44100', '-ac', '2', '-threads', '%d' % multiprocessing.cpu_count(), '-strict', 'experimental', '-acodec', 'aac', m4afile ] proc = subprocess.Popen(split_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_val, stderr_val = proc.communicate() # remove wav file os.remove(wavfile) # now put in metadata mp4tags = mutagen.mp4.MP4(m4afile) mp4tags.tags['\xa9nam'] = [ title, ] mp4tags.tags['\xa9alb'] = [ "Wait Wait...Don't Tell Me: %d" % year, ] mp4tags.tags['\xa9ART'] = [ 'Peter Sagal', ] mp4tags.tags['\xa9day'] = [ '%d' % year, ] mp4tags.tags['\xa9cmt'] = [ "more info at : NPR Web site", ] mp4tags.tags['trkn'] = [ (order_in_year, tot_in_year), ] mp4tags.tags['covr'] = [ mutagen.mp4.MP4Cover(file_data, mutagen.mp4.MP4Cover.FORMAT_PNG), ] mp4tags.tags['\xa9gen'] = [ 'Podcast', ] mp4tags.save() return m4afile