def main():
'''
do the thing
'''
global conf
conf = rawconfig.config()
parser = argparse.ArgumentParser(description="processes magnetic tape transfers")
parser.add_argument('-m',dest='m',choices=['batch','single'],default=False,help='mode, for processing a single transfer or a batch in new_ingest')
parser.add_argument('-i','--input',dest='i',help="the rawcapture file.wav to process, single mode only")
args = parser.parse_args()
###SINGLE MODE###
if args.m == 'single':
###INIT###
file = args.i
rawfname,ext = os.path.splitext(file)
###END INIT###
###GET ANUMBER FACE AND CHANNELCONFIG FROM FILEMAKER###
#output = subprocess.check_output(["python","fm-stuff.py","-pi","-t","-p","nameFormat","-i",rawfname]) #get aNumber, channelconfig, face from FileMaker
kwargs = ut.dotdict({"rawcapNumber":args.i})
processWorked = process(kwargs)
if processWorked is not True:
print "there was an error processing that file"
###END SINGLE MODE###
###BATCH MODE###
elif args.m == 'batch':
for dirs,subdirs,files in os.walk(conf.magneticTape.new_ingest):
for file in files:
###GET RID OF BS###
if file.endswith(".gpk") or file.endswith(".mrk") or file.endswith(".bak") or file.endswith(".pkf"):
try:
os.remove(file)
except:
pass
###END BS###
###PROCESS CAPTURE###
elif os.path.exists(os.path.join(conf.magneticTape.new_ingest,file)) and file.endswith(".wav"):
try: #control for files currently in use
subprocess.call("ffprobe " + os.path.join(dirs,file))
except:
continue
###INIT###
print file
rawfname,ext = os.path.splitext(file)
###END INIT###
###GET ANUMBER FACE AND CHANNELCONFIG FROM FILEMAKER###
kwargs = ut.dotdict({"rawcapNumber":rawfname})
processWorked = process(kwargs)
if not processWorked:
continue
def config():
'''
do the thing
'''
scriptRepo, fn = os.path.split(os.path.abspath(__file__))
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.read(os.path.join(scriptRepo, "microservices-config.ini"))
#init dictionary of the different material types/ processes
conf = {'log':{}, 'NationalJukebox':{}, 'cylinders':{}, 'discs':{}, 'video':{}, 'magneticTape':{}}
#init list of filepath names
tags = ['location', 'AudioArchDir', 'AudioBroadDir', 'PreIngestQCDir',
'VisualArchRawDir', 'BatchDir', 'scratch', 'new_ingest', 'repo',
'avlab', 'lto_stage', 'vid_leads', 'master_format_policy', 'access_format_policy', 'ff_master_format_policy', 'ff_access_format_policy',
'cnxn']
for c in conf:
for t in tags:
try:
if not t == 'cnxn':
#replace _ necessary for config file with . which xml attributes use, assign the value in config
#drivematch puts the right prefix to the path per OS
conf[c][t] = ut.drivematch(config.get(c, t))
else:
conf[c][t]=config.get(c, t)
except: #if no config tag exists, do nothing
pass
conf[c] = ut.dotdict(conf[c])
ff = {}
tags = ['filter_silence', 'filter_halfspeed', 'filter_doublespeed',
'filter_loudnorm', 'filter_afade', 'filter_deinterlace',
'acodec_master', 'acodec_master_format',
'acodec_master_arate', 'acodec_master_writebext', 'acodec_broadcast_format',
'acodec_broadcast', 'acodec_broadcast_rate', 'acodec_access_format',
'acodec_access_arate', 'acodec_access_bitrate', 'acodec_writeid3',
'vcodec_master', 'vcodec_master_pixel_format', 'vcodec_master_dimensions',
'vcodec_master_vrate', 'vcodec_master_acodec', 'vcodec_master_arate',
'vcodec_broadcast_format', 'vcodec_access', 'vcodec_master_format', 'vcodec_access_pixel_format',
'vcodec_access_acodec', 'vcodec_access_arate', 'vcodec_access_format',
'vcodec_access_dimensions', 'vcodec_access_vrate', 'vcodec_access_bitrate']
for t in tags:
try:
ff[t] = config.get("ffmpeg", t)
except:
pass
conf['ffmpeg'] = ut.dotdict(ff)
conf['scriptRepo'] = scriptRepo
conf['python'] = ut.pythonpath()
conf = ut.dotdict(conf)
return conf
def parse_input(args):
'''
makes a dictionary of file attributes
'''
thefile = ut.dotdict({})
startObj = subprocess.check_output([conf.python,os.path.join(conf.scriptRepo,"makestartobject.py"),"-i",args.i])
thefile.infullpath = startObj.replace("\\","/").strip()
thefile.fname, thefile.ext = os.path.splitext(os.path.basename(os.path.abspath(thefile.infullpath))) #splits filename and extension
thefile.startUseChar = thefile.fname[-1:] #grabs the last char of file name which is ~sometimes~ the use character
thefile.dir = os.path.dirname(thefile.infullpath) #grabs the directory that this object is in (we'll cd into it later)
if args.nj is False:
thefile = make_endUseChar(thefile) #grip the right filename endings, canonical name of the asset
else:
thefile.endUseChar = "x"
thefile.assetName = thefile.fname
thefile.outfname = thefile.assetName + thefile.endUseChar + "." + conf.ffmpeg.acodec_broadcast_format
thefile.outfullpath = os.path.join(thefile.dir, thefile.outfname)
if args.t:
match = ''
match = re.search("a\d{4,5}",thefile.assetName) #grip just the a1234 part of the filename
if match:
thefile.aNumber = match.group()
elif args.c:
match = ''
match = re.search(r"\d{4,5}",thefile.assetName) #grip just the number of the cylinder
if match:
thefile.cylNumber = match.group()
return thefile
def process(thefile): print thefile.rawcapNumber kwargs = thefile acf = mtd.get_aNumber_channelConfig_face(conf.magneticTape.cnxn,**kwargs) print acf if acf is not None: for k,v in acf.iteritems(): kwargs[k] = v kwargs = ut.dotdict(kwargs) kwargs.aNumber = "a" + kwargs.aNumber ###END GET ANUMBER FACE CHANNELCONFIG FROM FILEMAKER### ###DO THE FFMPEG### #init folder to do the work in kwargs.processDir = os.path.join(conf.magneticTape.new_ingest,kwargs.aNumber) if not os.path.exists(kwargs.processDir): os.makedirs(kwargs.processDir) time.sleep(1) #make the full ffstr using the paths we have ffproc = ff.audio_init_ffproc(conf,kwargs) full_ffstr = ff.prefix(os.path.join(conf.magneticTape.new_ingest,kwargs.rawcapNumber) + "." + conf.ffmpeg.acodec_master_format) + ffproc.ff_suffix print full_ffstr #run ffmpeg on the file and make sure it completes successfully with ut.cd(kwargs.processDir): ffWorked = ff.go(full_ffstr) makebext(kwargs.aNumber,kwargs.processDir)
def run_simulations(game, simulations, nodes, model=None):
"""
Given a TSP problem, run MCTS with `n_simulations`
simulations before choosing an action.
Returns value of MCTS path
"""
args = dotdict({'numMCTSSims': simulations, 'n_nodes': nodes, 'cpuct': 1})
mcts = MCTS(game, model, args)
current = [0]
pay = 0
decisions = []
end = game.getGameEnded(current)
while not end:
action = np.argmax(mcts.getActionProb(current))
current, payoff = game.getNextState(current, action)
decisions.append(action)
pay += payoff
end = game.getGameEnded(current)
return game.path_pay(tuple(decisions))
def move_package_toRepo(**kwargs):
'''
use hashmove to move the dir to the batch dir
'''
args = ut.dotdict(kwargs)
output = subprocess.Popen([
'python',
os.path.join(args.scriptRepo, 'hashmove.py'), args.fullpath,
os.path.join(args.repo, args.assetName)
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
foo, err = output.communicate()
if err:
print err
return False
else:
print foo
sourcehash = re.search('srce\s\S+\s\w{32}', foo)
desthash = re.search('dest\s\S+\s\w{32}', foo)
dh = desthash.group()
sh = sourcehash.group()
if sh[-32:].lower() == dh[-32:].lower():
return True
else:
return False
def parse_input(args):
'''
returns dictionary of file attributes
'''
thefile = ut.dotdict({})
startObj = subprocess.check_output([
'python',
os.path.join(conf.scriptRepo, 'makestartobject.py'), '-i', args.i
])
thefile.infullpath = startObj.replace("\\",
'/').strip() #for the windows peeps
thefile.fname, thefile.ext = os.path.splitext(
os.path.basename(os.path.abspath(thefile.infullpath)))
thefile.startUseChar = thefile.fname[
-1:] #grabs the last char of file name which is ~sometimes~ the use character
thefile.dir = os.path.dirname(
thefile.infullpath
) #grabs the directory that this object is in (we'll cd into it later)
if thefile.ext == '.mxf' or thefile.ext == '.mp4' or thefile.ext == '.mpeg':
thefile.outfname = thefile.fname.replace("-pres", "").replace(
"-broadcast", "") + "-acc." + conf.ffmpeg.vcodec_access_format
elif thefile.ext == '.wav':
thefile = make_endUseChar(
thefile
) #grip the right filename endings, canonical name of the asset
thefile.outfname = thefile.assetName + thefile.endUseChar + "." + conf.ffmpeg.acodec_access_format
thefile.outfullpath = os.path.join(thefile.dir, thefile.outfname)
return thefile
def get_tape_ID3(cnxn,**kwargs): args = ut.dotdict(kwargs) cnxn = pyodbc.connect(cnxn) sqlstr = """select Tape_Title, Collection_Name, Original_Recording_Date from Audio_Originals where Original_Tape_Number like '""" + args.aNumber.upper() + "%'" row = queryFM_single(sqlstr,cnxn) return row
def main():
'''
do the thing
'''
kwargs = ut.dotdict({})
kwargs.charsInLine = 90
filepath = '/Volumes/2017_0215/LourdesPortillo/Transcript_LourdesPortillo_Eng_Public_CD_081618.docx'
fulltext = get_text_lines_docx(filepath, kwargs)
print(fulltext)
def init():
'''
initialize VARS
'''
global conf
conf = rawconfig.config()
parser = argparse.ArgumentParser(
description="processes disc transfer files during digitization")
parser.add_argument("input",
help="the barcode of the disc you'd like to process")
args = parser.parse_args()
kwargs = ut.dotdict({})
kwargs.qcDir = conf.NationalJukebox.PreIngestQCDir
kwargs.batchDir = conf.NationalJukebox.BatchDir
kwargs.archDir = conf.NationalJukebox.AudioArchDir
kwargs.broadDir = conf.NationalJukebox.AudioBroadDir
kwargs.barcode = args.input #grab the lone argument that FM provides
kwargs._fname = kwargs.barcode + ".wav"
log(**{"message": "processing " + kwargs.barcode, "print": True})
if not os.path.exists(os.path.join(
kwargs.archDir, kwargs._fname)) or not os.path.exists(
os.path.join(kwargs.broadDir, kwargs._fname)):
log(
**{
"message": "file " + kwargs._fname +
" missing from arch or broad dir, not processed",
"level": "error",
"print": True
})
#print "Please check that you saved the file to the right directory in Wavelab before indicating that it was transferred"
foo = raw_input(
"Please check that the file was named correctly and saved to the correct directory"
)
sys.exit()
else:
if 'ucsb_' in kwargs._fname:
os.rename(
os.path.join(kwargs.archDir, kwargs._fname),
os.path.join(kwargs.archDir,
kwargs._fname).replace("ucsb_", "cusb_"))
os.rename(
os.path.join(kwargs.broadDir, kwargs._fname),
os.path.join(kwargs.broadDir,
kwargs._fname).replace("ucsb_", "cusb_"))
kwargs.barcode = kwargs.barcode.replace(
"ucsb", "cusb"
) #stupid, stupid bug. The printed barcodes start with 'ucsb_' rather than using the sigla 'cusb_'
kwargs.fname = kwargs.barcode + ".wav"
kwargs.archive_fname = kwargs.barcode + "m.wav" #make the new filename
kwargs.broadcast_fname = kwargs.barcode + ".wav"
kwargs.archiveFP_pre = os.path.join(kwargs.archDir, kwargs.fname)
kwargs.archiveFP_post = os.path.join(kwargs.archDir,
kwargs.archive_fname)
kwargs.broadcastFP = os.path.join(kwargs.broadDir, kwargs.fname)
return args, kwargs
def main():
for dirs, subdirs, files, in os.walk(sys.argv[1]):
for f in files:
thefile = ut.dotdict({})
thefile.fnamext = f
thefile.rawcapNumber, thefile.ext = os.path.splitext(f)
thefile.fullpath = os.path.join(dirs, f)
if thefile.ext == '.md5':
continue
elif thefile.ext == '.wav':
process(thefile)
def get_cylinder_ID3(cnxn,**kwargs): ''' takes a db connection and argument for the cylinder number (no prefix) and returns a raw list of Title, Performer, Label, and date ''' args = ut.dotdict(kwargs) cnxn = pyodbc.connect(cnxn) sqlstr = """select Title, Performer, Composer, Label_Cat, yr from Cylinders where Call_Number_asText='""" + args.cylNumber + "'" print sqlstr row = queryFM_single(sqlstr,cnxn) return row
def parse_input(args):
file = ut.dotdict({})
file.name = 'cusb-' + args.i
file.ext = '.iso'
file.dir = os.path.join(conf.video.new_ingest, args.i)
file.isoFullPath = os.path.join(file.dir, file.name + file.ext)
file.logFullPath = os.path.join(file.dir, file.name + '.log')
file.broadcastFullPath = os.path.join(file.dir,
file.name + "-broadcast." + "mov")
file.accessFullPath = os.path.join(
file.dir, file.name + "-acc." + conf.ffmpeg.vcodec_access_format)
return file
def get_raw_captures(cnxn,**kwargs):
'''
returns the rawcapturenumber uuid for suppled aNumber
'''
args = ut.dotdict(kwargs)
cnxn = pyodbc.connect(cnxn)
sqlstr = """select Audio_Masters.rawCaptureName_fAB, Audio_Masters.rawCaptureName_fCD
from Audio_Masters join Audio_Originals on Audio_Masters.Original_Key=Audio_Originals.Original_Key
where Audio_Originals.Tape_Number='""" + args.aNumber.replace("A","").replace("a","") + "'"
row = queryFM_single(sqlstr,cnxn)
if row:
return row
else:
return False
def MDSPlotTest():
import json
import experiment
resPath = "../experiments/ebook_color_pca_3"
experiment.experimentCase("../params/ebook_color_pca_28x28_3.json",resPath)
info = json.loads(util.fileString("../params/ebook_color_pca_28x28_3.json"))
info = util.dotdict(info)
x = util.load(resPath+"/x.pkl")
print x.dtype
compressed = util.load(resPath+"/compressed.pkl")
MDSPlots(x,compressed,info.dataSet.shape)
import matplotlib.pyplot as plt
fig.savefig()
print("show figure")
plt.show()
def make_video(args):
'''
handles making broadcast masters from VIDEO_TS DVD/ VOB files
'''
file = ut.dotdict({})
file.vobDir = args.i
file.vNum = os.path.basename(os.path.dirname(file.vobDir))
file.name = 'cusb-' + file.vNum
file.vobOutFullPath = os.path.join(os.path.dirname(file.vobDir), file.name + '-concat.VOB')
file.outputFullPath = os.path.join(os.path.dirname(file.vobDir), file.name + '-acc.' + conf.ffmpeg.vcodec_access_format)
file.concatxt = os.path.join(file.vobDir, 'concat.txt')
worked = concat_vobs(file)
#worked = True
if worked is not True:
print 'makebroadcast encountered an error'
sys.exit()
def make_id3str(kwargs):
'''
take existing id3 info and turn it into something ffmpeg understands
'''
args = ut.dotdict(kwargs)
id3str = ''
for index, tag in enumerate(args.id3fields): #loop thru the raw list of id3 values, grip the index
if tag is not None:
if args.id3rawlist[index] is not None:
id3str = id3str + " -metadata " + tag + '"' + args.id3rawlist[index].replace('"','') + '"'
if not "album=" in args.id3fields:
id3str = id3str + ' -metadata album="' + args.assetName + '" -metadata publisher="UCSB Special Research Collections"'
else:
id3str = id3str + ' -metadata publisher="UCSB Special Research Collections"'
id3str = UnicodeDammit.detwingle(id3str)
return id3str
def main():
'''
do the thing
'''
p('starting...')
kwargs = ut.dotdict({})
kwargs.auPath = '/home/brnco/devs/media_sources/AlfonsoArau_spa_full.wav'
kwargs.auFile = load_audio(kwargs.auPath)
p('parsing audio...')
kwargs.duration = 8
kwargs.offset = 88
kwargs.language = 'pt-BR'
kwargs.audio = r_audio(kwargs)
p('audio parsed')
transcript = transcribe(kwargs)
p(transcript)
def verify_package_contents(**kwargs):
'''
checks that every file type is in the SIPable dir
'''
args = ut.dotdict(kwargs)
packageIsComplete = {}
for filetype in args.filetypes:
packageIsComplete[filetype] = False
for file in os.listdir(args.fullpath):
if file.endswith(filetype):
packageIsComplete[filetype] = True
continue
for pic, v in packageIsComplete.iteritems():
if v is False:
return False
return True
def get_aNumber_channelConfig_face(cnxn,**kwargs):
'''
formats a query for FM to grip a tape's aNumber, channel configuration, and face, from that's tape's rawCaptureName in new_ingest
returns list of [aNumber,channelConfig,face] e.g., ["a1234","1/4-inch Half Track Mono","fAB"]
'''
args = ut.dotdict(kwargs)
facelist = ["fAB","fCD"]
nameFormat = {}
row = ''
count = 0
cnxn = pyodbc.connect(cnxn)
while not row: #try to find the Tape Number and Format based on the rawCaptureName
if count > 1:
return None
else:
sqlstr="""select Audio_Originals.Tape_Number, Audio_Originals.Original_Recording_Format
from Audio_Originals join Audio_Masters on Audio_Originals.Original_Key=Audio_Masters.Original_Key
where Audio_Masters.rawCaptureName_""" + facelist[count] + "='" + args.rawcapNumber + "' or Audio_Masters.rawCaptureName_" + facelist[count] + "='" + args.rawcapNumber + ".wav'"
#OR above necessary because sometimes the rawCaptureName has a ".wav" at the end :(
row = queryFM_single(sqlstr,cnxn)
face = facelist[count] #assign this now, if we assign at bottom of loop, count = count + 1 and it'll be the wrong index
count = count+1
if row:
#print row
nameFormat["aNumber"] = row[0]
nameFormat["face"] = face
#rowstr = str(row) #convert to string
if any("Cassette" in s for s in row): #if the rawCaptureName is of a cassette tape
nameFormat["channelConfig"] = row[1]
elif any("Open Reel" in s for s in row) or any("Disc" in s for s in row): #if the rawCaptureName is of an open reel
#having the format isn't enough, we need the channel configuration for open reels
sqlstr = '''select Audio_Originals.Tape_Number, Audio_Originals.Tape_Format
from Audio_Originals inner join Audio_Masters on Audio_Originals.Original_Key=Audio_Masters.Original_Key
where Audio_Masters.rawCaptureName_''' + face + "='" + args.rawcapNumber + "' or Audio_Masters.rawCaptureName_" + face + "='" + args.rawcapNumber + ".wav'"
row = queryFM_single(sqlstr,cnxn)
if row:
nameFormat["channelConfig"] = row[1]
return nameFormat
else:
return None
def main():
'''
thefile = ut.dotdict({})
thefile.fnamext = sys.argv[1]
thefile.rawcapNumber, thefile.ext = os.path.splitext(thefile.fnamext)
thefile.fullpath = os.path.join(conf.magneticTape.new_ingest, thefile.fnamext)
'''
for dirs, subdris, files in os.walk(conf.magneticTape.new_ingest):
for f in files:
thefile = ut.dotdict({})
thefile.fnamext = f
thefile.rawcapNumber, thefile.ext = os.path.splitext(f)
thefile.fullpath = os.path.join(dirs, f)
print thefile.fnamext
if thefile.ext == '.gpk':
try:
os.remove(thefile.fullpath)
except:
continue
else:
process(thefile)
print ""
def makebext_complete(cnxn,**kwargs):
'''
handler to generate a complete bext string that can be read by BWFMetaEdit
'''
args = ut.dotdict(kwargs)
bext_description = makebext_description(cnxn,args)
if args.aNumber:
bext_originatorReference = "cusb-" + args.aNumber
elif args.cylNumber:
bext_originatorReference = "cusb-cyl" + args.cylNumber
elif args.discID:
bext_originatorReference = args.discBarcode.replace("ucsb","cusb")
if args.bextVersion == '1':
umid = makebext_umid()
else:
umid = ''
bextstr = "--Originator=US,CUSB,SRC --originatorReference="+ bext_originatorReference + " " + bext_description + " " + umid
try:
bext = bextstr.encode('utf-8')
except:
bext = bextstr
return bext
def init():
'''
initialize variable container object
'''
parser = argparse.ArgumentParser(
description="Generate a transcript using automated tools")
parser.add_argument("-i", "--input", dest="i", help="the input audio file")
parser.add_argument("-a",
"--algorithm",
dest="a",
choices=["Google", "Watson"],
help="the speech-to-text algorithm to use")
parser.add_argument("-m",
"--model",
dest="m",
choices=["en-US", "en-UK"],
help="the speech model to use")
parser.add_argument(
"-n",
"--number",
dest="n",
default=False,
help="request number for completed Google Cloud Speech call")
args = parser.parse_args()
kwargs = ut.dotdict({})
kwargs.i = args.i
kwargs.a = args.a
kwargs.m = args.m
kwargs.n = args.n
kwargs.google_json = "request.json"
'''kwargs.transcode = False
if not kwargs.i.endswith(".ogg"):
kwargs.transcode = True
if kwargs.a == "Watson":
kwargs.transcodeDestination = kwargs.i + ".ogg"'''
return kwargs
def init():
'''
initialize variable container object
'''
parser = argparse.ArgumentParser(
description="Sync a transcript for OHMS using Google Speech Recognition"
)
parser.add_argument(
'--transcript',
dest='t',
help="path to the transcript which you would like to sync")
parser.add_argument('--audio',
dest='a',
help="path to audio file which matches transcript")
parser.add_argument(
'--language',
dest='l',
help=
"Google language code: cloud.google.com/speech-to-text/docs/languages")
parser.add_argument(
'--charsInLine',
dest='charsInLine',
default=77,
help="the numbers of characters in a line of the transcript")
parser.add_argument('--export',
dest='export',
action='store_true',
default=False,
help="export txt file with sync data")
parser.add_argument(
'--syncdata',
dest='syncdata',
help="input sync data for transformation, surround with double-quotes")
parser.add_argument(
'--convert',
dest='convert',
help="number of lines to add (+) or subtract (-) from sync data")
parser.add_argument(
'--syncdata-print',
dest='syncdataprint',
action='store_true',
default=False,
help="print sync data to console on separate lines (for debugging)")
parser.add_argument(
'--translate',
dest='translate',
help="translation language, 2-letter code. Use to sync a translation")
parser.add_argument('--verbose',
dest='loglevel',
action='store_const',
const=log.INFO,
help="display more info during operation")
parser.add_argument('--debug',
dest='loglevel',
action='store_const',
const=log.DEBUG,
default=log.WARNING,
help="print everything to console")
args = parser.parse_args()
kwargs = ut.dotdict({}) #keyword arguments wrapper
kwargs.auPath = args.a
kwargs.txPath = args.t
kwargs.duration = 8 #default duration for each transcription request, in seconds
kwargs.auOffset = 56 #default offset for first transcription request, in seconds
kwargs.txOffset = 0 # defaultoffset for transcription text file, in lines
kwargs.charsInLine = int(args.charsInLine)
kwargs.language = args.l #default language for each transcription request
kwargs.fakeSyncWordIndex = 2 #default word index for sync points which have to be faked
if args.syncdata:
kwargs.syncData = args.syncdata
else:
kwargs.syncData = []
kwargs.convert = args.convert
kwargs.export = args.export
kwargs.syncdataprint = args.syncdataprint
kwargs.translate = args.translate
log.basicConfig(level=args.loglevel)
log.debug('args %s', args)
log.debug('kwargs %s', kwargs)
return kwargs
def process(kwargs): ''' directs the processing of a file ''' #check if it's bigger than WAVE spec tooBig = size_check(kwargs) if tooBig is True: print "this file is too large to process with ffmpeg" return False #get the aNumber, channelConfig, face of the file acf = mtd.get_aNumber_channelConfig_face(conf.magneticTape.cnxn,**kwargs) print acf #verify that we got an object back from mtd.py if acf is None: return False else: #verify that FM record filled out compeltely processNone = 0 for p in acf: if acf[p] is None: processNone = 1 return False if processNone > 0: print "" print "ERROR - FileMaker record incomplete" print "Please check FileMaker record for rawcapture:" print args.i return False #add aNumber, channelConfig, face info to kwargs for k,v in acf.iteritems(): kwargs[k] = v kwargs = ut.dotdict(kwargs) #verify that there's not two channelConfigs set in FM if "\r" in kwargs.channelConfig: return False kwargs.aNumber = "a" + kwargs.aNumber #init folder to do the work in kwargs.processDir = os.path.join(conf.magneticTape.new_ingest,kwargs.aNumber) if not os.path.exists(kwargs.processDir): os.makedirs(kwargs.processDir) time.sleep(1) #make the full ffstr using the paths we have ffproc = ff.audio_init_ffproc(conf,kwargs) full_ffstr = ff.prefix(os.path.join(conf.magneticTape.new_ingest,kwargs.rawcapNumber) + "." + conf.ffmpeg.acodec_master_format) + ffproc.ff_suffix print full_ffstr #run ffmpeg on the file and make sure it completes successfully with ut.cd(kwargs.processDir): ffWorked = ff.go(full_ffstr) if not ffWorked: return False #special add for mono files if not "Stereo" in kwargs.channelConfig and not "Cassette" in kwargs.channelConfig: ffWorked = mono_silence(kwargs) #if we need to reverse do it if ffproc.revface: revWorked = reverse(ffproc,kwargs) if revWorked is not True: print "there was a problem reversing the file" print revWorked return False #if we need to normalize our sample rate to 96kHz, because we sped up or slowed down a recording, do it here if ffproc.hlvface or ffproc.dblface: print "sample rate normalize" srnWorked = sampleratenormalize(ffproc,kwargs) if not srnWorked: print "there was a problem normalizing the sampel rate of the file" return False #embed bext info makebext(kwargs.aNumber,kwargs.processDir) #hashmove them to the repo dir move(kwargs) return True
import config as rawconfig
import util as ut
import logger as log
import mtd
import makestartobject as makeso
import makesip
import ff
###INIT VARS###
global conf
conf = rawconfig.config()
obj = "R:/Visual/avlab/new_ingest/v0061/cusb-v0061-pres.mxf"
streams = ff.probe_streams(obj)
streams = ut.dotdict(streams)
log.log(streams['0.pix_fmt'], **{"print": True})
'''args = {}
args = ut.dotdict(args)
args.so = "0b70bd7b-eaee-4819-beb8-995a0ea996ad3"
args.face = "fCD"
args.channelConfig = '1/4-inch Full Track Mono'
#args.channelConfig = '1/4-inch Half Track Stereo'
#args.channelConfig = '1/4-inch Quarter Track Stereo'
#args.channelConfig = '1/4-inch 4 Track'
args.aNumber = 'a19260'
ffproc = ff.audio_init_ffproc(conf.magneticTape.cnxn,**args)
full_ffstr = ff.prefix(args.so + "." + conf.ffmpeg.acodec_master_format) + ffproc.ff_suffix
print full_ffstr
print ""
print ffproc
initial_args = dotdict({
'numIters': 10,
'numEps': 50, # Number of complete self-play games to simulate during a new iteration.
'tempThreshold': 15, #
'updateThreshold': 0.55,
# During arena playoff, new neural net will be accepted if threshold or more of games are won.
'maxlenOfQueue': 200000, # Number of game examples to train the neural networks.
'numMCTSSims': 50, # Number of games moves for MCTS to simulate.
'arenaCompare': 40, # Number of games to play during arena play to determine if new net will be accepted.
'cpuct': 1,
'cuda': False,
'checkpoint': './temp2/',
'load_model': False,
'load_folder_file': ('./temp2/', 'best.pth.tar'),
'numItersForTrainExamplesHistory': 20,
# 'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
'device': 'cpu',
'n_nodes': 8,
'n_node_features': 5,
'n_executions': 100,
'lr': 0.001,
'dropout': 0.3,
'epochs': 10,
'batch_size': 64,
'use_gdc': True,
'invert_probs': True
})
from util import dotdict
# train the model: check training examples
# training example: (current graph, pi, v)
# pi is the MCTS informed policy vector, v is +1 if
# the player eventually won the game, else -1.
N = 8
args = dotdict({
'numMCTSSims': 200,
'n_nodes': N,
'cpuct': 1,
'use_gdc': False,
'n_node_features': 3
})
training_examples = []
n_games_to_train = 1000
for i in range(n_games_to_train):
R = 0
game_examples = []
game = TspGame(args)
mcts = MCTS(game, None, args)
state = game.getInitState()
while not game.getGameEnded(state):
pi = mcts.getActionProb(state)
game_examples.append([game.build_graph_from_state(state), pi])
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
args = dotdict({
'lr':
0.001,
'dropout':
0.3,
'epochs':
10,
'batch_size':
64,
'cuda':
torch.cuda.is_available(),
'use_gdc':
True,
'n_node_features':
5,
'n_nodes':
n_nodes,
'n_nodes_nnet':
10,
'device':
torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
'numMCTSSims':
n_simulations,
'max_dist':
100,
'cpuct':
1,
})
nnet = NNetWrapper(args)
from util import dotdict o = dict(model="FDN", device="cuda", batch_size=128, num_workers=4, epoch=5, lr=0.03) # lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False o = dotdict(o)
def audio_init_ffproc(conf, kwargs):
'''
generates ffmpeg process data for magnetic tape transfers
'''
args = ut.dotdict(kwargs)
cnxn = pyodbc.connect(conf.magneticTape.cnxn)
ffproc = mtd.get_ff_processes(args,
cnxn) #get faces/ processes from filemaker"
ffproc = ut.dotdict(ffproc)
print args.channelConfig
if not "Stereo" in args.channelConfig and not "Cassette" in args.channelConfig:
print "mono"
###FOR MONO TAPES###
channel0 = ut.dotdict({'map': "-map_channel 0.0.0"})
channel1 = ut.dotdict({'map': "-map_channel 0.0.1"})
for k, v in ffproc.iteritems():
#convert fm outputs to ffmpeg strings
if v is not None:
if k == 'dblface' and (v == 'fA' or v == 'fC'):
channel0.af = 'asetrate=192000'
elif k == 'dblface' and (v == 'fB' or v == 'fD'):
channel1.af = 'asetrate=192000'
elif k == 'hlvface' and (v == 'fA' or v == 'fC'):
channel0.af = 'asetrate=48000'
elif k == 'hlvface' and (v == 'fB' or v == 'fD'):
channel1.af = 'asetrate=48000'
if k == 'delface' and (v == 'fA' or v == 'fC'):
channel0 = {}
elif k == 'delface' and (v == 'fB' or v == 'fD'):
channel1 = {}
ff_suffix0 = ff_suffix1 = ''
#for the first face, if it exists
if channel0:
ff_suffix0 = channel0.map
if channel0.af:
ff_suffix0 = ff_suffix0 + ' -af ' + channel0.af
###GENERATE FILENAME FOR FACE0###
ffproc.filename0 = filename0 = "cusb-" + args.aNumber + args.face[
1] + "a." + conf.ffmpeg.acodec_master_format
ff_suffix0 = ff_suffix0 + ' -c:a ' + conf.ffmpeg.acodec_master + ' ' + filename0
#for the second face, if it exists
if channel1:
ff_suffix1 = channel1.map
if channel1.af:
ff_suffix1 = ff_suffix1 + ' -af ' + channel1.af
###GENERATE FILENAME FOR FACE1"""
ffproc.filename1 = filename1 = "cusb-" + args.aNumber + args.face[
2] + "a." + conf.ffmpeg.acodec_master_format
ff_suffix1 = ff_suffix1 + ' -c:a ' + conf.ffmpeg.acodec_master + ' ' + filename1
###PUT IT TOGETHER###
if ff_suffix0 and ff_suffix1:
ff_suffix = ff_suffix0 + ' ' + ff_suffix1
elif ff_suffix0:
ff_suffix = ff_suffix0
elif ff_suffix1:
ff_suffix = ff_suffix1
else:
ff_suffix = None
###END MONO###
else:
###FOR STEREO TAPES###
channel0 = ut.dotdict({
"silence": conf.ffmpeg.filter_silence,
"af": '',
"faceChar": ''
})
if "Quarter" in args.channelConfig:
channel0.faceChar = args.face[1]
elif "Half" in args.channelConfig:
channel0.faceChar = ''
for k, v in ffproc.iteritems():
if v is not None:
if k == 'dblface':
channel0.af = 'asetrate=192000'
elif k == 'hlvface':
channel0.af = 'asetrate=48000'
ff_suffix0 = '-af ' + channel0.silence
if channel0.af:
ff_suffix0 = ff_suffix0 + ',' + channel0.af
ff_suffix0 = ff_suffix0 + ' -c:a ' + conf.ffmpeg.acodec_master
###GENERATE FILENAME FOR THE OBJECT###
ffproc.filename0 = filename0 = 'cusb-' + args.aNumber + channel0.faceChar + 'a.' + conf.ffmpeg.acodec_master_format
###PUT IT TOGETHER###
ff_suffix = ff_suffix0 + ' ' + filename0
###END STEREO###
ffproc.ff_suffix = ff_suffix
return ffproc
