def main(): ''' do the thing ''' global conf conf = rawconfig.config() parser = argparse.ArgumentParser(description="processes magnetic tape transfers") parser.add_argument('-m',dest='m',choices=['batch','single'],default=False,help='mode, for processing a single transfer or a batch in new_ingest') parser.add_argument('-i','--input',dest='i',help="the rawcapture file.wav to process, single mode only") args = parser.parse_args() ###SINGLE MODE### if args.m == 'single': ###INIT### file = args.i rawfname,ext = os.path.splitext(file) ###END INIT### ###GET ANUMBER FACE AND CHANNELCONFIG FROM FILEMAKER### #output = subprocess.check_output(["python","fm-stuff.py","-pi","-t","-p","nameFormat","-i",rawfname]) #get aNumber, channelconfig, face from FileMaker kwargs = ut.dotdict({"rawcapNumber":args.i}) processWorked = process(kwargs) if processWorked is not True: print "there was an error processing that file" ###END SINGLE MODE### ###BATCH MODE### elif args.m == 'batch': for dirs,subdirs,files in os.walk(conf.magneticTape.new_ingest): for file in files: ###GET RID OF BS### if file.endswith(".gpk") or file.endswith(".mrk") or file.endswith(".bak") or file.endswith(".pkf"): try: os.remove(file) except: pass ###END BS### ###PROCESS CAPTURE### elif os.path.exists(os.path.join(conf.magneticTape.new_ingest,file)) and file.endswith(".wav"): try: #control for files currently in use subprocess.call("ffprobe " + os.path.join(dirs,file)) except: continue ###INIT### print file rawfname,ext = os.path.splitext(file) ###END INIT### ###GET ANUMBER FACE AND CHANNELCONFIG FROM FILEMAKER### kwargs = ut.dotdict({"rawcapNumber":rawfname}) processWorked = process(kwargs) if not processWorked: continue
def config(): ''' do the thing ''' scriptRepo, fn = os.path.split(os.path.abspath(__file__)) config = ConfigParser.RawConfigParser(allow_no_value=True) config.read(os.path.join(scriptRepo, "microservices-config.ini")) #init dictionary of the different material types/ processes conf = {'log':{}, 'NationalJukebox':{}, 'cylinders':{}, 'discs':{}, 'video':{}, 'magneticTape':{}} #init list of filepath names tags = ['location', 'AudioArchDir', 'AudioBroadDir', 'PreIngestQCDir', 'VisualArchRawDir', 'BatchDir', 'scratch', 'new_ingest', 'repo', 'avlab', 'lto_stage', 'vid_leads', 'master_format_policy', 'access_format_policy', 'ff_master_format_policy', 'ff_access_format_policy', 'cnxn'] for c in conf: for t in tags: try: if not t == 'cnxn': #replace _ necessary for config file with . which xml attributes use, assign the value in config #drivematch puts the right prefix to the path per OS conf[c][t] = ut.drivematch(config.get(c, t)) else: conf[c][t]=config.get(c, t) except: #if no config tag exists, do nothing pass conf[c] = ut.dotdict(conf[c]) ff = {} tags = ['filter_silence', 'filter_halfspeed', 'filter_doublespeed', 'filter_loudnorm', 'filter_afade', 'filter_deinterlace', 'acodec_master', 'acodec_master_format', 'acodec_master_arate', 'acodec_master_writebext', 'acodec_broadcast_format', 'acodec_broadcast', 'acodec_broadcast_rate', 'acodec_access_format', 'acodec_access_arate', 'acodec_access_bitrate', 'acodec_writeid3', 'vcodec_master', 'vcodec_master_pixel_format', 'vcodec_master_dimensions', 'vcodec_master_vrate', 'vcodec_master_acodec', 'vcodec_master_arate', 'vcodec_broadcast_format', 'vcodec_access', 'vcodec_master_format', 'vcodec_access_pixel_format', 'vcodec_access_acodec', 'vcodec_access_arate', 'vcodec_access_format', 'vcodec_access_dimensions', 'vcodec_access_vrate', 'vcodec_access_bitrate'] for t in tags: try: ff[t] = config.get("ffmpeg", t) except: pass conf['ffmpeg'] = ut.dotdict(ff) conf['scriptRepo'] = scriptRepo conf['python'] = ut.pythonpath() conf = ut.dotdict(conf) return conf
def parse_input(args): ''' makes a dictionary of file attributes ''' thefile = ut.dotdict({}) startObj = subprocess.check_output([conf.python,os.path.join(conf.scriptRepo,"makestartobject.py"),"-i",args.i]) thefile.infullpath = startObj.replace("\\","/").strip() thefile.fname, thefile.ext = os.path.splitext(os.path.basename(os.path.abspath(thefile.infullpath))) #splits filename and extension thefile.startUseChar = thefile.fname[-1:] #grabs the last char of file name which is ~sometimes~ the use character thefile.dir = os.path.dirname(thefile.infullpath) #grabs the directory that this object is in (we'll cd into it later) if args.nj is False: thefile = make_endUseChar(thefile) #grip the right filename endings, canonical name of the asset else: thefile.endUseChar = "x" thefile.assetName = thefile.fname thefile.outfname = thefile.assetName + thefile.endUseChar + "." + conf.ffmpeg.acodec_broadcast_format thefile.outfullpath = os.path.join(thefile.dir, thefile.outfname) if args.t: match = '' match = re.search("a\d{4,5}",thefile.assetName) #grip just the a1234 part of the filename if match: thefile.aNumber = match.group() elif args.c: match = '' match = re.search(r"\d{4,5}",thefile.assetName) #grip just the number of the cylinder if match: thefile.cylNumber = match.group() return thefile
def process(thefile): print thefile.rawcapNumber kwargs = thefile acf = mtd.get_aNumber_channelConfig_face(conf.magneticTape.cnxn,**kwargs) print acf if acf is not None: for k,v in acf.iteritems(): kwargs[k] = v kwargs = ut.dotdict(kwargs) kwargs.aNumber = "a" + kwargs.aNumber ###END GET ANUMBER FACE CHANNELCONFIG FROM FILEMAKER### ###DO THE FFMPEG### #init folder to do the work in kwargs.processDir = os.path.join(conf.magneticTape.new_ingest,kwargs.aNumber) if not os.path.exists(kwargs.processDir): os.makedirs(kwargs.processDir) time.sleep(1) #make the full ffstr using the paths we have ffproc = ff.audio_init_ffproc(conf,kwargs) full_ffstr = ff.prefix(os.path.join(conf.magneticTape.new_ingest,kwargs.rawcapNumber) + "." + conf.ffmpeg.acodec_master_format) + ffproc.ff_suffix print full_ffstr #run ffmpeg on the file and make sure it completes successfully with ut.cd(kwargs.processDir): ffWorked = ff.go(full_ffstr) makebext(kwargs.aNumber,kwargs.processDir)
def run_simulations(game, simulations, nodes, model=None): """ Given a TSP problem, run MCTS with `n_simulations` simulations before choosing an action. Returns value of MCTS path """ args = dotdict({'numMCTSSims': simulations, 'n_nodes': nodes, 'cpuct': 1}) mcts = MCTS(game, model, args) current = [0] pay = 0 decisions = [] end = game.getGameEnded(current) while not end: action = np.argmax(mcts.getActionProb(current)) current, payoff = game.getNextState(current, action) decisions.append(action) pay += payoff end = game.getGameEnded(current) return game.path_pay(tuple(decisions))
def move_package_toRepo(**kwargs): ''' use hashmove to move the dir to the batch dir ''' args = ut.dotdict(kwargs) output = subprocess.Popen([ 'python', os.path.join(args.scriptRepo, 'hashmove.py'), args.fullpath, os.path.join(args.repo, args.assetName) ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) foo, err = output.communicate() if err: print err return False else: print foo sourcehash = re.search('srce\s\S+\s\w{32}', foo) desthash = re.search('dest\s\S+\s\w{32}', foo) dh = desthash.group() sh = sourcehash.group() if sh[-32:].lower() == dh[-32:].lower(): return True else: return False
def parse_input(args): ''' returns dictionary of file attributes ''' thefile = ut.dotdict({}) startObj = subprocess.check_output([ 'python', os.path.join(conf.scriptRepo, 'makestartobject.py'), '-i', args.i ]) thefile.infullpath = startObj.replace("\\", '/').strip() #for the windows peeps thefile.fname, thefile.ext = os.path.splitext( os.path.basename(os.path.abspath(thefile.infullpath))) thefile.startUseChar = thefile.fname[ -1:] #grabs the last char of file name which is ~sometimes~ the use character thefile.dir = os.path.dirname( thefile.infullpath ) #grabs the directory that this object is in (we'll cd into it later) if thefile.ext == '.mxf' or thefile.ext == '.mp4' or thefile.ext == '.mpeg': thefile.outfname = thefile.fname.replace("-pres", "").replace( "-broadcast", "") + "-acc." + conf.ffmpeg.vcodec_access_format elif thefile.ext == '.wav': thefile = make_endUseChar( thefile ) #grip the right filename endings, canonical name of the asset thefile.outfname = thefile.assetName + thefile.endUseChar + "." + conf.ffmpeg.acodec_access_format thefile.outfullpath = os.path.join(thefile.dir, thefile.outfname) return thefile
def get_tape_ID3(cnxn,**kwargs): args = ut.dotdict(kwargs) cnxn = pyodbc.connect(cnxn) sqlstr = """select Tape_Title, Collection_Name, Original_Recording_Date from Audio_Originals where Original_Tape_Number like '""" + args.aNumber.upper() + "%'" row = queryFM_single(sqlstr,cnxn) return row
def main(): ''' do the thing ''' kwargs = ut.dotdict({}) kwargs.charsInLine = 90 filepath = '/Volumes/2017_0215/LourdesPortillo/Transcript_LourdesPortillo_Eng_Public_CD_081618.docx' fulltext = get_text_lines_docx(filepath, kwargs) print(fulltext)
def init(): ''' initialize VARS ''' global conf conf = rawconfig.config() parser = argparse.ArgumentParser( description="processes disc transfer files during digitization") parser.add_argument("input", help="the barcode of the disc you'd like to process") args = parser.parse_args() kwargs = ut.dotdict({}) kwargs.qcDir = conf.NationalJukebox.PreIngestQCDir kwargs.batchDir = conf.NationalJukebox.BatchDir kwargs.archDir = conf.NationalJukebox.AudioArchDir kwargs.broadDir = conf.NationalJukebox.AudioBroadDir kwargs.barcode = args.input #grab the lone argument that FM provides kwargs._fname = kwargs.barcode + ".wav" log(**{"message": "processing " + kwargs.barcode, "print": True}) if not os.path.exists(os.path.join( kwargs.archDir, kwargs._fname)) or not os.path.exists( os.path.join(kwargs.broadDir, kwargs._fname)): log( **{ "message": "file " + kwargs._fname + " missing from arch or broad dir, not processed", "level": "error", "print": True }) #print "Please check that you saved the file to the right directory in Wavelab before indicating that it was transferred" foo = raw_input( "Please check that the file was named correctly and saved to the correct directory" ) sys.exit() else: if 'ucsb_' in kwargs._fname: os.rename( os.path.join(kwargs.archDir, kwargs._fname), os.path.join(kwargs.archDir, kwargs._fname).replace("ucsb_", "cusb_")) os.rename( os.path.join(kwargs.broadDir, kwargs._fname), os.path.join(kwargs.broadDir, kwargs._fname).replace("ucsb_", "cusb_")) kwargs.barcode = kwargs.barcode.replace( "ucsb", "cusb" ) #stupid, stupid bug. The printed barcodes start with 'ucsb_' rather than using the sigla 'cusb_' kwargs.fname = kwargs.barcode + ".wav" kwargs.archive_fname = kwargs.barcode + "m.wav" #make the new filename kwargs.broadcast_fname = kwargs.barcode + ".wav" kwargs.archiveFP_pre = os.path.join(kwargs.archDir, kwargs.fname) kwargs.archiveFP_post = os.path.join(kwargs.archDir, kwargs.archive_fname) kwargs.broadcastFP = os.path.join(kwargs.broadDir, kwargs.fname) return args, kwargs
def main(): for dirs, subdirs, files, in os.walk(sys.argv[1]): for f in files: thefile = ut.dotdict({}) thefile.fnamext = f thefile.rawcapNumber, thefile.ext = os.path.splitext(f) thefile.fullpath = os.path.join(dirs, f) if thefile.ext == '.md5': continue elif thefile.ext == '.wav': process(thefile)
def get_cylinder_ID3(cnxn,**kwargs): ''' takes a db connection and argument for the cylinder number (no prefix) and returns a raw list of Title, Performer, Label, and date ''' args = ut.dotdict(kwargs) cnxn = pyodbc.connect(cnxn) sqlstr = """select Title, Performer, Composer, Label_Cat, yr from Cylinders where Call_Number_asText='""" + args.cylNumber + "'" print sqlstr row = queryFM_single(sqlstr,cnxn) return row
def parse_input(args): file = ut.dotdict({}) file.name = 'cusb-' + args.i file.ext = '.iso' file.dir = os.path.join(conf.video.new_ingest, args.i) file.isoFullPath = os.path.join(file.dir, file.name + file.ext) file.logFullPath = os.path.join(file.dir, file.name + '.log') file.broadcastFullPath = os.path.join(file.dir, file.name + "-broadcast." + "mov") file.accessFullPath = os.path.join( file.dir, file.name + "-acc." + conf.ffmpeg.vcodec_access_format) return file
def get_raw_captures(cnxn,**kwargs): ''' returns the rawcapturenumber uuid for suppled aNumber ''' args = ut.dotdict(kwargs) cnxn = pyodbc.connect(cnxn) sqlstr = """select Audio_Masters.rawCaptureName_fAB, Audio_Masters.rawCaptureName_fCD from Audio_Masters join Audio_Originals on Audio_Masters.Original_Key=Audio_Originals.Original_Key where Audio_Originals.Tape_Number='""" + args.aNumber.replace("A","").replace("a","") + "'" row = queryFM_single(sqlstr,cnxn) if row: return row else: return False
def MDSPlotTest(): import json import experiment resPath = "../experiments/ebook_color_pca_3" experiment.experimentCase("../params/ebook_color_pca_28x28_3.json",resPath) info = json.loads(util.fileString("../params/ebook_color_pca_28x28_3.json")) info = util.dotdict(info) x = util.load(resPath+"/x.pkl") print x.dtype compressed = util.load(resPath+"/compressed.pkl") MDSPlots(x,compressed,info.dataSet.shape) import matplotlib.pyplot as plt fig.savefig() print("show figure") plt.show()
def make_video(args): ''' handles making broadcast masters from VIDEO_TS DVD/ VOB files ''' file = ut.dotdict({}) file.vobDir = args.i file.vNum = os.path.basename(os.path.dirname(file.vobDir)) file.name = 'cusb-' + file.vNum file.vobOutFullPath = os.path.join(os.path.dirname(file.vobDir), file.name + '-concat.VOB') file.outputFullPath = os.path.join(os.path.dirname(file.vobDir), file.name + '-acc.' + conf.ffmpeg.vcodec_access_format) file.concatxt = os.path.join(file.vobDir, 'concat.txt') worked = concat_vobs(file) #worked = True if worked is not True: print 'makebroadcast encountered an error' sys.exit()
def make_id3str(kwargs): ''' take existing id3 info and turn it into something ffmpeg understands ''' args = ut.dotdict(kwargs) id3str = '' for index, tag in enumerate(args.id3fields): #loop thru the raw list of id3 values, grip the index if tag is not None: if args.id3rawlist[index] is not None: id3str = id3str + " -metadata " + tag + '"' + args.id3rawlist[index].replace('"','') + '"' if not "album=" in args.id3fields: id3str = id3str + ' -metadata album="' + args.assetName + '" -metadata publisher="UCSB Special Research Collections"' else: id3str = id3str + ' -metadata publisher="UCSB Special Research Collections"' id3str = UnicodeDammit.detwingle(id3str) return id3str
def main(): ''' do the thing ''' p('starting...') kwargs = ut.dotdict({}) kwargs.auPath = '/home/brnco/devs/media_sources/AlfonsoArau_spa_full.wav' kwargs.auFile = load_audio(kwargs.auPath) p('parsing audio...') kwargs.duration = 8 kwargs.offset = 88 kwargs.language = 'pt-BR' kwargs.audio = r_audio(kwargs) p('audio parsed') transcript = transcribe(kwargs) p(transcript)
def verify_package_contents(**kwargs): ''' checks that every file type is in the SIPable dir ''' args = ut.dotdict(kwargs) packageIsComplete = {} for filetype in args.filetypes: packageIsComplete[filetype] = False for file in os.listdir(args.fullpath): if file.endswith(filetype): packageIsComplete[filetype] = True continue for pic, v in packageIsComplete.iteritems(): if v is False: return False return True
def get_aNumber_channelConfig_face(cnxn,**kwargs): ''' formats a query for FM to grip a tape's aNumber, channel configuration, and face, from that's tape's rawCaptureName in new_ingest returns list of [aNumber,channelConfig,face] e.g., ["a1234","1/4-inch Half Track Mono","fAB"] ''' args = ut.dotdict(kwargs) facelist = ["fAB","fCD"] nameFormat = {} row = '' count = 0 cnxn = pyodbc.connect(cnxn) while not row: #try to find the Tape Number and Format based on the rawCaptureName if count > 1: return None else: sqlstr="""select Audio_Originals.Tape_Number, Audio_Originals.Original_Recording_Format from Audio_Originals join Audio_Masters on Audio_Originals.Original_Key=Audio_Masters.Original_Key where Audio_Masters.rawCaptureName_""" + facelist[count] + "='" + args.rawcapNumber + "' or Audio_Masters.rawCaptureName_" + facelist[count] + "='" + args.rawcapNumber + ".wav'" #OR above necessary because sometimes the rawCaptureName has a ".wav" at the end :( row = queryFM_single(sqlstr,cnxn) face = facelist[count] #assign this now, if we assign at bottom of loop, count = count + 1 and it'll be the wrong index count = count+1 if row: #print row nameFormat["aNumber"] = row[0] nameFormat["face"] = face #rowstr = str(row) #convert to string if any("Cassette" in s for s in row): #if the rawCaptureName is of a cassette tape nameFormat["channelConfig"] = row[1] elif any("Open Reel" in s for s in row) or any("Disc" in s for s in row): #if the rawCaptureName is of an open reel #having the format isn't enough, we need the channel configuration for open reels sqlstr = '''select Audio_Originals.Tape_Number, Audio_Originals.Tape_Format from Audio_Originals inner join Audio_Masters on Audio_Originals.Original_Key=Audio_Masters.Original_Key where Audio_Masters.rawCaptureName_''' + face + "='" + args.rawcapNumber + "' or Audio_Masters.rawCaptureName_" + face + "='" + args.rawcapNumber + ".wav'" row = queryFM_single(sqlstr,cnxn) if row: nameFormat["channelConfig"] = row[1] return nameFormat else: return None
def main(): ''' thefile = ut.dotdict({}) thefile.fnamext = sys.argv[1] thefile.rawcapNumber, thefile.ext = os.path.splitext(thefile.fnamext) thefile.fullpath = os.path.join(conf.magneticTape.new_ingest, thefile.fnamext) ''' for dirs, subdris, files in os.walk(conf.magneticTape.new_ingest): for f in files: thefile = ut.dotdict({}) thefile.fnamext = f thefile.rawcapNumber, thefile.ext = os.path.splitext(f) thefile.fullpath = os.path.join(dirs, f) print thefile.fnamext if thefile.ext == '.gpk': try: os.remove(thefile.fullpath) except: continue else: process(thefile) print ""
def makebext_complete(cnxn,**kwargs): ''' handler to generate a complete bext string that can be read by BWFMetaEdit ''' args = ut.dotdict(kwargs) bext_description = makebext_description(cnxn,args) if args.aNumber: bext_originatorReference = "cusb-" + args.aNumber elif args.cylNumber: bext_originatorReference = "cusb-cyl" + args.cylNumber elif args.discID: bext_originatorReference = args.discBarcode.replace("ucsb","cusb") if args.bextVersion == '1': umid = makebext_umid() else: umid = '' bextstr = "--Originator=US,CUSB,SRC --originatorReference="+ bext_originatorReference + " " + bext_description + " " + umid try: bext = bextstr.encode('utf-8') except: bext = bextstr return bext
def init(): ''' initialize variable container object ''' parser = argparse.ArgumentParser( description="Generate a transcript using automated tools") parser.add_argument("-i", "--input", dest="i", help="the input audio file") parser.add_argument("-a", "--algorithm", dest="a", choices=["Google", "Watson"], help="the speech-to-text algorithm to use") parser.add_argument("-m", "--model", dest="m", choices=["en-US", "en-UK"], help="the speech model to use") parser.add_argument( "-n", "--number", dest="n", default=False, help="request number for completed Google Cloud Speech call") args = parser.parse_args() kwargs = ut.dotdict({}) kwargs.i = args.i kwargs.a = args.a kwargs.m = args.m kwargs.n = args.n kwargs.google_json = "request.json" '''kwargs.transcode = False if not kwargs.i.endswith(".ogg"): kwargs.transcode = True if kwargs.a == "Watson": kwargs.transcodeDestination = kwargs.i + ".ogg"''' return kwargs
def init(): ''' initialize variable container object ''' parser = argparse.ArgumentParser( description="Sync a transcript for OHMS using Google Speech Recognition" ) parser.add_argument( '--transcript', dest='t', help="path to the transcript which you would like to sync") parser.add_argument('--audio', dest='a', help="path to audio file which matches transcript") parser.add_argument( '--language', dest='l', help= "Google language code: cloud.google.com/speech-to-text/docs/languages") parser.add_argument( '--charsInLine', dest='charsInLine', default=77, help="the numbers of characters in a line of the transcript") parser.add_argument('--export', dest='export', action='store_true', default=False, help="export txt file with sync data") parser.add_argument( '--syncdata', dest='syncdata', help="input sync data for transformation, surround with double-quotes") parser.add_argument( '--convert', dest='convert', help="number of lines to add (+) or subtract (-) from sync data") parser.add_argument( '--syncdata-print', dest='syncdataprint', action='store_true', default=False, help="print sync data to console on separate lines (for debugging)") parser.add_argument( '--translate', dest='translate', help="translation language, 2-letter code. Use to sync a translation") parser.add_argument('--verbose', dest='loglevel', action='store_const', const=log.INFO, help="display more info during operation") parser.add_argument('--debug', dest='loglevel', action='store_const', const=log.DEBUG, default=log.WARNING, help="print everything to console") args = parser.parse_args() kwargs = ut.dotdict({}) #keyword arguments wrapper kwargs.auPath = args.a kwargs.txPath = args.t kwargs.duration = 8 #default duration for each transcription request, in seconds kwargs.auOffset = 56 #default offset for first transcription request, in seconds kwargs.txOffset = 0 # defaultoffset for transcription text file, in lines kwargs.charsInLine = int(args.charsInLine) kwargs.language = args.l #default language for each transcription request kwargs.fakeSyncWordIndex = 2 #default word index for sync points which have to be faked if args.syncdata: kwargs.syncData = args.syncdata else: kwargs.syncData = [] kwargs.convert = args.convert kwargs.export = args.export kwargs.syncdataprint = args.syncdataprint kwargs.translate = args.translate log.basicConfig(level=args.loglevel) log.debug('args %s', args) log.debug('kwargs %s', kwargs) return kwargs
def process(kwargs): ''' directs the processing of a file ''' #check if it's bigger than WAVE spec tooBig = size_check(kwargs) if tooBig is True: print "this file is too large to process with ffmpeg" return False #get the aNumber, channelConfig, face of the file acf = mtd.get_aNumber_channelConfig_face(conf.magneticTape.cnxn,**kwargs) print acf #verify that we got an object back from mtd.py if acf is None: return False else: #verify that FM record filled out compeltely processNone = 0 for p in acf: if acf[p] is None: processNone = 1 return False if processNone > 0: print "" print "ERROR - FileMaker record incomplete" print "Please check FileMaker record for rawcapture:" print args.i return False #add aNumber, channelConfig, face info to kwargs for k,v in acf.iteritems(): kwargs[k] = v kwargs = ut.dotdict(kwargs) #verify that there's not two channelConfigs set in FM if "\r" in kwargs.channelConfig: return False kwargs.aNumber = "a" + kwargs.aNumber #init folder to do the work in kwargs.processDir = os.path.join(conf.magneticTape.new_ingest,kwargs.aNumber) if not os.path.exists(kwargs.processDir): os.makedirs(kwargs.processDir) time.sleep(1) #make the full ffstr using the paths we have ffproc = ff.audio_init_ffproc(conf,kwargs) full_ffstr = ff.prefix(os.path.join(conf.magneticTape.new_ingest,kwargs.rawcapNumber) + "." + conf.ffmpeg.acodec_master_format) + ffproc.ff_suffix print full_ffstr #run ffmpeg on the file and make sure it completes successfully with ut.cd(kwargs.processDir): ffWorked = ff.go(full_ffstr) if not ffWorked: return False #special add for mono files if not "Stereo" in kwargs.channelConfig and not "Cassette" in kwargs.channelConfig: ffWorked = mono_silence(kwargs) #if we need to reverse do it if ffproc.revface: revWorked = reverse(ffproc,kwargs) if revWorked is not True: print "there was a problem reversing the file" print revWorked return False #if we need to normalize our sample rate to 96kHz, because we sped up or slowed down a recording, do it here if ffproc.hlvface or ffproc.dblface: print "sample rate normalize" srnWorked = sampleratenormalize(ffproc,kwargs) if not srnWorked: print "there was a problem normalizing the sampel rate of the file" return False #embed bext info makebext(kwargs.aNumber,kwargs.processDir) #hashmove them to the repo dir move(kwargs) return True
import config as rawconfig import util as ut import logger as log import mtd import makestartobject as makeso import makesip import ff ###INIT VARS### global conf conf = rawconfig.config() obj = "R:/Visual/avlab/new_ingest/v0061/cusb-v0061-pres.mxf" streams = ff.probe_streams(obj) streams = ut.dotdict(streams) log.log(streams['0.pix_fmt'], **{"print": True}) '''args = {} args = ut.dotdict(args) args.so = "0b70bd7b-eaee-4819-beb8-995a0ea996ad3" args.face = "fCD" args.channelConfig = '1/4-inch Full Track Mono' #args.channelConfig = '1/4-inch Half Track Stereo' #args.channelConfig = '1/4-inch Quarter Track Stereo' #args.channelConfig = '1/4-inch 4 Track' args.aNumber = 'a19260' ffproc = ff.audio_init_ffproc(conf.magneticTape.cnxn,**args) full_ffstr = ff.prefix(args.so + "." + conf.ffmpeg.acodec_master_format) + ffproc.ff_suffix print full_ffstr print "" print ffproc
initial_args = dotdict({ 'numIters': 10, 'numEps': 50, # Number of complete self-play games to simulate during a new iteration. 'tempThreshold': 15, # 'updateThreshold': 0.55, # During arena playoff, new neural net will be accepted if threshold or more of games are won. 'maxlenOfQueue': 200000, # Number of game examples to train the neural networks. 'numMCTSSims': 50, # Number of games moves for MCTS to simulate. 'arenaCompare': 40, # Number of games to play during arena play to determine if new net will be accepted. 'cpuct': 1, 'cuda': False, 'checkpoint': './temp2/', 'load_model': False, 'load_folder_file': ('./temp2/', 'best.pth.tar'), 'numItersForTrainExamplesHistory': 20, # 'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu'), 'device': 'cpu', 'n_nodes': 8, 'n_node_features': 5, 'n_executions': 100, 'lr': 0.001, 'dropout': 0.3, 'epochs': 10, 'batch_size': 64, 'use_gdc': True, 'invert_probs': True })
from util import dotdict # train the model: check training examples # training example: (current graph, pi, v) # pi is the MCTS informed policy vector, v is +1 if # the player eventually won the game, else -1. N = 8 args = dotdict({ 'numMCTSSims': 200, 'n_nodes': N, 'cpuct': 1, 'use_gdc': False, 'n_node_features': 3 }) training_examples = [] n_games_to_train = 1000 for i in range(n_games_to_train): R = 0 game_examples = [] game = TspGame(args) mcts = MCTS(game, None, args) state = game.getInitState() while not game.getGameEnded(state): pi = mcts.getActionProb(state) game_examples.append([game.build_graph_from_state(state), pi])
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') args = dotdict({ 'lr': 0.001, 'dropout': 0.3, 'epochs': 10, 'batch_size': 64, 'cuda': torch.cuda.is_available(), 'use_gdc': True, 'n_node_features': 5, 'n_nodes': n_nodes, 'n_nodes_nnet': 10, 'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu'), 'numMCTSSims': n_simulations, 'max_dist': 100, 'cpuct': 1, }) nnet = NNetWrapper(args)
from util import dotdict o = dict(model="FDN", device="cuda", batch_size=128, num_workers=4, epoch=5, lr=0.03) # lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False o = dotdict(o)
def audio_init_ffproc(conf, kwargs): ''' generates ffmpeg process data for magnetic tape transfers ''' args = ut.dotdict(kwargs) cnxn = pyodbc.connect(conf.magneticTape.cnxn) ffproc = mtd.get_ff_processes(args, cnxn) #get faces/ processes from filemaker" ffproc = ut.dotdict(ffproc) print args.channelConfig if not "Stereo" in args.channelConfig and not "Cassette" in args.channelConfig: print "mono" ###FOR MONO TAPES### channel0 = ut.dotdict({'map': "-map_channel 0.0.0"}) channel1 = ut.dotdict({'map': "-map_channel 0.0.1"}) for k, v in ffproc.iteritems(): #convert fm outputs to ffmpeg strings if v is not None: if k == 'dblface' and (v == 'fA' or v == 'fC'): channel0.af = 'asetrate=192000' elif k == 'dblface' and (v == 'fB' or v == 'fD'): channel1.af = 'asetrate=192000' elif k == 'hlvface' and (v == 'fA' or v == 'fC'): channel0.af = 'asetrate=48000' elif k == 'hlvface' and (v == 'fB' or v == 'fD'): channel1.af = 'asetrate=48000' if k == 'delface' and (v == 'fA' or v == 'fC'): channel0 = {} elif k == 'delface' and (v == 'fB' or v == 'fD'): channel1 = {} ff_suffix0 = ff_suffix1 = '' #for the first face, if it exists if channel0: ff_suffix0 = channel0.map if channel0.af: ff_suffix0 = ff_suffix0 + ' -af ' + channel0.af ###GENERATE FILENAME FOR FACE0### ffproc.filename0 = filename0 = "cusb-" + args.aNumber + args.face[ 1] + "a." + conf.ffmpeg.acodec_master_format ff_suffix0 = ff_suffix0 + ' -c:a ' + conf.ffmpeg.acodec_master + ' ' + filename0 #for the second face, if it exists if channel1: ff_suffix1 = channel1.map if channel1.af: ff_suffix1 = ff_suffix1 + ' -af ' + channel1.af ###GENERATE FILENAME FOR FACE1""" ffproc.filename1 = filename1 = "cusb-" + args.aNumber + args.face[ 2] + "a." + conf.ffmpeg.acodec_master_format ff_suffix1 = ff_suffix1 + ' -c:a ' + conf.ffmpeg.acodec_master + ' ' + filename1 ###PUT IT TOGETHER### if ff_suffix0 and ff_suffix1: ff_suffix = ff_suffix0 + ' ' + ff_suffix1 elif ff_suffix0: ff_suffix = ff_suffix0 elif ff_suffix1: ff_suffix = ff_suffix1 else: ff_suffix = None ###END MONO### else: ###FOR STEREO TAPES### channel0 = ut.dotdict({ "silence": conf.ffmpeg.filter_silence, "af": '', "faceChar": '' }) if "Quarter" in args.channelConfig: channel0.faceChar = args.face[1] elif "Half" in args.channelConfig: channel0.faceChar = '' for k, v in ffproc.iteritems(): if v is not None: if k == 'dblface': channel0.af = 'asetrate=192000' elif k == 'hlvface': channel0.af = 'asetrate=48000' ff_suffix0 = '-af ' + channel0.silence if channel0.af: ff_suffix0 = ff_suffix0 + ',' + channel0.af ff_suffix0 = ff_suffix0 + ' -c:a ' + conf.ffmpeg.acodec_master ###GENERATE FILENAME FOR THE OBJECT### ffproc.filename0 = filename0 = 'cusb-' + args.aNumber + channel0.faceChar + 'a.' + conf.ffmpeg.acodec_master_format ###PUT IT TOGETHER### ff_suffix = ff_suffix0 + ' ' + filename0 ###END STEREO### ffproc.ff_suffix = ff_suffix return ffproc