def main():
args = parse_args()
_input = args.inputPath
ingestID = args.ingestID
canonicalName = args.canonical_name
wrapper = args.wrapper
success = False
problems = ""
if os.path.isdir(_input):
# get rid of any hidden files
pymmFunctions.remove_hidden_system_files(_input)
# get the abs path of each input file
sourceList = pymmFunctions.list_files(_input)
elif isinstance(_input, list):
sourceList = _input
else:
problems += "\ninput is not a dir or list of files. exiting!"
print(problems)
# sys.exit()
# safe_to_concat returns either True or a list of problems
safeToConcat = safe_to_concat(sourceList)
concattedFile = False
if safeToConcat == True:
try:
concattedFile = concat(sourceList, canonicalName, wrapper)
except:
problems += "\nsome problem with the concat process."
print(problems)
else:
problems += safeToConcat
# rename the file so it sorts to the top of the output directory
# maybe this is a stupid idea? but it will be handy
if not concattedFile == False:
concatBase = os.path.basename(concattedFile)
concatDir = os.path.dirname(concattedFile)
newBase = "00_concatenated_{}".format(concatBase)
newPath = os.path.join(concatDir, newBase)
# actually rename the file
os.rename(concattedFile, newPath)
# reset the var to the new path name
concattedFile = newPath
success = True
else:
success = False
concattedFile = problems
return concattedFile, success
def remove_system_files(CurrentIngest):
'''
### HUNT FOR HIDDEN SYSTEM FILES ###
### DESTROY!! DESTROY!! DESTROY! ###
'''
inputPath = CurrentIngest.InputObject.inputPath
removedFiles = pymmFunctions.remove_hidden_system_files(inputPath)
removeFailures = []
status = "OK"
event = "deletion"
# check again for system files... just in case.
for item in os.scandir(inputPath):
if item.name.startswith('.'):
try:
removedFiles.append(item.name)
os.remove(item.path)
except:
removeFailures.append(item.name)
print("tried to remove a pesky system file and failed.")
if not removeFailures == []:
if not removedFiles == []:
outcome = ("System files deleted at \n{}\n"
"Some additional system files were NOT removed "
"at \n{}\n".format("\n".join(removedFiles),
"\n".join(removeFailures)))
status == "INCONCLUSIVE"
else:
# if both passes failed, log it as such
outcome = "Tried and failed to remove system files. Sorry."
status = "FAIL"
loggers.short_log(CurrentIngest, event, outcome, status)
else:
if not removedFiles == []:
outcome = "System files deleted at \n{}\n".format(
"\n".join(removedFiles))
loggers.short_log(CurrentIngest, event, outcome, status)
else:
pass
def __init__(self, inputPath):
##################
# CORE ATTRIBUTES
##################
self.inputPath = inputPath
self.inputParent = os.path.dirname(inputPath)
self.basename = os.path.basename(inputPath)
self.filename = None
# this is the "canonical name" of an object, either the
# filename of a single file or the dirname, which should
# encompass the whole work/package being ingested.
self.canonicalName = os.path.basename(self.inputPath)
self.outlierComponents = []
self.inputType = self.sniff_input(inputPath)
self.inputTypeDetail = None
self.structureCompliance = None
# Initialize a list to be filled with component objects.
# There should be at least one in the case of a single file input.
# Objects should either be AV or a single documentation folder called
# 'documentation'
self.ComponentObjects = []
# self.inputTypeDetail possible values are:
# - 'file'
# - 'single discrete file with documentation'
# - 'multiple discrete files'
# - 'multiple discrete files with documentation'
# - 'single-reel dpx'
# - 'single-reel dpx with documentation'
# - 'multi-reel dpx'
# - 'multi-reel dpx with documentation'
if self.inputType == 'dir':
pymmFunctions.remove_hidden_system_files(inputPath)
self.structureCompliance, self.inputTypeDetail = directoryScanner.main(
inputPath)
if not self.structureCompliance:
self.inputTypeDetail = (
"Directory structure and/or file format problems!"
" See: {}".format(self.inputTypeDetail))
if not 'dpx' in self.inputTypeDetail:
# Note: film scanner component parts get added
# later during ingestSip
for item in os.scandir(self.inputPath):
self.ComponentObjects.append(
ComponentObject(item.path, topLevelObject=True))
else:
if 'multi' in self.inputTypeDetail:
for item in os.scandir(self.inputPath):
self.ComponentObjects.append(
ComponentObject(item.path, topLevelObject=True))
else:
documentation = [
item for item in os.scandir(self.inputPath) \
if item.name == 'documentation'
]
if documentation != []:
documentation = documentation[0]
self.ComponentObjects.append(
ComponentObject(documentation.path,
topLevelObject=True))
self.ComponentObjects.append(
ComponentObject(self.inputPath, topLevelObject=True))
elif self.inputType == 'file':
self.inputTypeDetail = 'file'
self.filename = self.basename
self.ComponentObjects.insert(
0, ComponentObject(inputPath, topLevelObject=True))
#######################################
# ASSIGNED / MUTABLE DURING PROCESSING
#######################################
self.pbcoreXML = pbcore.PBCoreDocument()
self.pbcoreFile = None
def main():
args = parse_args()
_input = args.inputPath
canonicalName = args.canonical_name
wrapper = args.wrapper
success = False
problems = ""
if os.path.isdir(_input):
# get rid of any hidden files
pymmFunctions.remove_hidden_system_files(_input)
# get the abs path of each input file
sourceList = pymmFunctions.list_files(_input)
elif isinstance(_input, list):
sourceList = _input
else:
problems += "\ninput is not a dir or list of files. exiting!"
print(problems)
# sys.exit()
if not canonicalName:
canonicalName = os.path.basename(sourceList[0])
print("You didn't specify a canonical_name "
"so we will treat the first item in sourceList as "
"the canonical name for your object.")
#######################
# START TESTING FILES
stream_compatability, streams = count_streams(sourceList)
if not stream_compatability:
problems += ("\nCan't concatenate. There are stream count differences "
"in your input files. See this: \n{}".format(streams))
success = False
print(problems)
return problems, success
# safe_to_concat returns either True or a list of problems
safeToConcat = None
simpleConcat = safe_to_concat(sourceList)
complexConcat = None # placeholder
if not simpleConcat == True:
complexConcat = False
# placeholder:
# complexConcat = safe_to_concat(sourceList,complex=True)
if True in (simpleConcat, complexConcat):
safeToConcat = True
concattedFile = False
if safeToConcat == True:
try:
concattedFile = concat(sourceList,
canonicalName,
wrapper,
simple=True)
except:
problems += "\nsome problem with the concat process."
# print(problems)
else:
problems += simpleConcat
# rename the file so it sorts to the top of the output directory
# maybe this is a stupid idea? but it will be handy
if not concattedFile == False:
concatBase = os.path.basename(concattedFile)
concatDir = os.path.dirname(concattedFile)
newBase = "00_concatenated_{}".format(concatBase)
newPath = os.path.join(concatDir, newBase)
# actually rename the file
os.rename(concattedFile, newPath)
# reset the var to the new path name
concattedFile = newPath
if pymmFunctions.is_av(concattedFile) in ('VIDEO', 'AUDIO'):
success = True
else:
success = False
try:
os.remove(concattedFile)
except:
pass
problems += ("\nOutput concat file is not recognized as AV!! "
"Something must have gone wrong. Sorry.")
concattedFile = problems
else:
success = False
concattedFile = problems
return concattedFile, success
def scan_dir(inputPath):
'''
expected image sequence dir structure:
- title_accesssion#_barcode_r01of01/
dpx/
title_accesssion#_barcode_r01of01_0008600.dpx
title_accesssion#_barcode_r01of01_0008601.dpx
<etc>
title_accesssion#_barcode_r01of01.wav
~OR~
- title_accesssion#/
title_accesssion#_barcode_r01of02/
dpx/
title_accesssion#_barcode_r01of02_0008600.dpx
title_accesssion#_barcode_r01of02_0008601.dpx
<etc>
title_accesssion#_barcode_r01of02.wav
title_accesssion#_barcode_r01of02/
dpx/
title_accesssion#_barcode_r02of02_0008600.dpx
title_accesssion#_barcode_r02of02_0008601.dpx
<etc>
title_accesssion#_barcode_r02of02.wav
return a tuple: (True/False, [list of illegal subdirs])
'''
# system files should already be removed by the time it's called but...
pymmFunctions.remove_hidden_system_files(inputPath)
# remove any empty folders
for root, dirs, files in os.walk(inputPath):
for _dir in dirs:
path = os.path.join(root, _dir)
if os.path.isdir(path):
contents = os.listdir(path)
if len(contents) == 0:
print("removing empty dir at " + path)
os.rmdir(path)
# start looking for inappropriate/unrecorgnized folders
problems = []
dirs = []
outcome = None
for entry in os.scandir(inputPath):
# cast the fiery circle,
# summon all the subdirectories for judgement before my wrath
if entry.is_dir():
dirs.append(entry.path)
if len(dirs) > 0:
if (len(dirs) == 1) and (os.path.basename(dirs[0]).lower() != 'dpx'):
# if there is only one subdir and it isn't DPX, shit it out
problems = dirs
return False, problems
elif len(dirs) > 1:
baddies = []
for _dir in dirs:
for root, subs, _ in os.walk(_dir):
# grab any non-dpx dirs to return
things = [
os.path.join(root, sub) for sub in subs
if sub.lower() != 'dpx'
]
for thing in things:
baddies.append(thing)
if baddies != []:
# there shouldn't be anything other than dpx folders at this level
outcome = False
for baddie in baddies:
problems.append(baddie)
else:
print("DPX dirs are ok")
outcome = True
else:
outcome = True
return outcome, problems