def check_pymm_log_exists():
# open a local instance of config here in case
# it has changed since importing this file
pymmConfig = pymmFunctions.read_config()
pymmLogDir = pymmConfig['logging']['pymm_log_dir']
pymmLogPath = os.path.join(pymmLogDir, 'pymm_log.txt')
# sys.exit()
opener = (((("#" * 75) + '\n') * 2) + ((' ') * 4) +
'THIS IS THE LOG FOR PYMEDIAMICROSERVICES'
'\n\n' + ((' ') * 4) + 'THIS VERSION WAS STARTED ON ' +
pymmFunctions.today + '\n' + ((("#" * 75) + '\n') * 2) + '\n')
if not os.path.isfile(pymmLogPath):
print('wait i need to make a logfile')
if pymmLogDir == '':
print("!~" * 75)
print("CONFIGURATION PROBLEM:\n"
"THERE IS NO DIRECTORY SET FOR THE SYSTEM-WIDE LOG.\n"
"PLEASE RUN pymmconfig.py OR EDIT config.ini DIRECTLY\n"
"TO ADD A VALID DIRECTORY FOR pymm_log.txt TO LIVE IN.\n"
"WE'RE JUST GOING TO PUT IT ON YOUR DESKTOP FOR NOW...")
desktop = pymmFunctions.get_desktop()
pymmLogPath = os.path.join(desktop, 'pymm_log.txt')
from pymmconfig import pymmconfig
pymmconfig.set_value('logging', 'pymm_log_dir', desktop)
with open(pymmLogPath, 'w+') as pymmLog:
pymmLog.write(opener)
else:
open(pymmLogPath, 'x')
with open(pymmLogPath, 'w+') as pymmLog:
pymmLog.write(opener)
else:
pass
def main():
config = pymmFunctions.read_config() # THIS IS PROBABLY NOT GOING TO BE NEEDED
parser = argparse.ArgumentParser()
parser.add_argument('-i','--inputPath',help='path of input file')
parser.add_argument('-m','--mediainfo',action='store_true',help='generate a mediainfo sidecar file')
parser.add_argument('-f','--frame_md5',action='store_true',help='make frame md5 report')
parser.add_argument('-j','--getJSON',action='store_true',help='get JSON output as applicable')
parser.add_argument('-d','--destination',help='set destination for output metadata files')
args = parser.parse_args()
inputPath = args.inputPath
destination = args.destination
frame_md5 = args.frame_md5
mediainfo_report = args.mediainfo
getJSON = args.getJSON
if not inputPath:
print("\n\nHEY THERE, YOU NEED TO SET AN INPUT FILE TO RUN THIS SCRIPT ON.\rNOW EXITING")
sys.exit()
if not destination:
print('''
YOU DIDN'T TELL ME WHERE TO PUT THE OUTPUT OF THIS SCRIPT,
SO WE'LL PUT ANY SIDECAR FILES IN THE SAME DIRECTORY AS YOUR INPUT FILE.
''')
destination = os.path.dirname(os.path.abspath(inputPath))
if mediainfo_report:
get_mediainfo_report(inputPath,destination,getJSON)
if frame_md5:
frameMd5Filepath = make_frame_md5(inputPath,destination)
print(frameMd5Filepath)
def main():
config = pymmFunctions.read_config()
args = set_args()
requiredArgs = ['inputPath', 'destination']
inputPath = args.inputPath
algorithm = args.algorithm
removeOriginals = args.removeOriginals
destination = args.destination
loglevel = args.loglevel
logDir = args.logDir
now = pymmFunctions.timestamp('now')
# Quit if there are required variables missing
missingArgs = 0
for _arg in requiredArgs:
if getattr(args, _arg) == None:
print("CONFIGURATION PROBLEM:\n"
"You forgot to set " + _arg + ". It is required.\n"
"Try again, but set " + _arg + " with the flag --" + _arg +
"\n")
missingArgs += 1
if missingArgs > 0:
sys.exit()
# set up rsync log
if loglevel == 'all':
pymmLogpath = os.path.join(config['logging']['pymm_log_dir'],
'pymm_log.txt')
# AT WHAT POINT WILL WE ACTUALLY WANT TO PYMMLOG A COPY? FINAL AIP XFER?
try:
rsyncLogpath = os.path.join(
logDir, 'rsync_log_' + pymmFunctions.get_base(inputPath) +
'_' + pymmFunctions.timestamp('now') + '.txt')
except:
print("there was a problem getting the rsync log path ....")
rsyncLogpath = ''
else:
rsyncLogpath = ''
# sniff what the input is
dir_or_file = pymmFunctions.dir_or_file(inputPath)
if dir_or_file == False:
print("oy you've got big problems. " + inputPath +
" is not a directory or a file. what is it? is it a ghost?")
sys.exit()
# copy the input according to its type
elif dir_or_file == 'dir':
# add trailing slash for rsync destination directory
if not destination[-1] == '/':
destination = destination + '/'
copy_dir(inputPath, rsyncLogpath, destination)
elif dir_or_file == 'file':
copy_file(inputPath, rsyncLogpath, destination)
else:
print("o_O what is going on here? you up to something?")
sys.exit()
def __init__(self,
user=None,
objectJSON=None,
databaseReporting=None,
ingestType=None,
makeProres=None,
concatChoice=None,
cleanupStrategy=None,
overrideOutdir=None,
overrideAIPdir=None,
overrideRS=None,
mono=None,
combineAudio=None):
#######
# GLOBAL CONFIG (this is ConfigParser object, callable as a dict)
self.config = pymmFunctions.read_config()
#######
# INPUT ARGUMENTS (FROM CLI)
self.user = user
# path to optional JSON file of descriptive mentadata
self.objectJSON = objectJSON
self.databaseReporting = databaseReporting
self.ingestType = ingestType
self.makeProres = makeProres
self.concatChoice = concatChoice
self.cleanupStrategy = cleanupStrategy
self.mono = mono
self.combineAudio = combineAudio
if overrideOutdir == None:
self.outdir_ingestsip = self.config['paths']['outdir_ingestsip']
else:
self.outdir_ingestsip = overrideOutdir
if overrideAIPdir == None:
self.aip_staging = self.config['paths']['aip_staging']
else:
self.aip_staging = overrideAIPdir
if overrideRS == None:
self.resourcespace_deliver = self.config['paths'][
'resourcespace_deliver']
else:
self.resourcespace_deliver = overrideRS
self.databaseReporting = self.test_db_access(self.user)
#######
# ENVIRONMENT VARIABLES
self.computer = pymmFunctions.get_node_name()
self.ffmpegVersion = 'ffmpeg ver.: '+\
pymmFunctions.get_ffmpeg_version()
def pymm_log(CurrentIngest, event, outcome, status):
check_pymm_log_exists()
pymmConfig = pymmFunctions.read_config()
pymmLogDir = pymmConfig['logging']['pymm_log_dir']
pymmLogPath = os.path.join(pymmLogDir, 'pymm_log.txt')
stamp = pymmFunctions.timestamp('iso8601')
systemInfo = CurrentIngest.systemInfo
objectRootPath = CurrentIngest.InputObject.inputPath
canonicalName = CurrentIngest.InputObject.canonicalName
inputTypeDetail = CurrentIngest.InputObject.inputTypeDetail
user = CurrentIngest.ProcessArguments.user
ingestUUID = CurrentIngest.ingestUUID
tempID = CurrentIngest.tempID
workingDir = CurrentIngest.ProcessArguments.outdir_ingestsip
prefix = ''
suffix = '\n'
# I think basename gets updated depending on what is getting logged... ? @fixme
basename = os.path.basename(objectRootPath)
if status == 'STARTING':
prefix = ('&' * 50) + '\n\n'
stuffToLog = [
prefix, stamp, "\nEvent type: Ingestion start\n",
"Object canonical name: {}\n".format(canonicalName),
"Object filepath: {}\n".format(objectRootPath),
"Object type: {}\n".format(inputTypeDetail),
"Ingest UUID: {}\n".format(ingestUUID),
"Operator: {}\n".format(user),
"Ingest working directory: {}\n".format(workingDir),
"\n### SYSTEM INFO: ### \n{}".format(systemInfo), suffix
]
elif status in ("ENDING", "ABORTING"):
suffix = '\n\n' + ('#' * 50) + "\n\n"
stuffToLog = [
prefix, stamp, " | Status: {} |".format(status),
" | Event type: Ingestion end | ", "Outcome: {}".format(outcome),
suffix
]
else:
stuffToLog = [
prefix, stamp, " | Status: {}".format(status),
" | Object name: {}".format(basename),
" | Event type: {}".format(event),
" | Event outcome: {}".format(outcome), suffix
]
with open(pymmLogPath, 'a') as log:
for item in stuffToLog:
log.write(item)
import loggers
import makeDerivs
import moveNcopy
import makeMetadata
import pymmFunctions
except:
from .bampfa_pbcore import pbcore, makePbcore
from . import concatFiles
from . import ingestClasses
from . import loggers
from . import makeDerivs
from . import moveNcopy
from . import makeMetadata
from . import pymmFunctions
# read in from the config file
config = pymmFunctions.read_config()
def set_args():
parser = argparse.ArgumentParser()
parser.add_argument('-i',
'--inputPath',
help='path of input file',
required=True)
parser.add_argument('-u',
'--user',
help='name of the person doing the ingest',
required=True)
parser.add_argument(
'-j',
'--metadataJSON',
def main():
config = pymmFunctions.read_config()
args = set_args()
requiredArgs = ['inputPath', 'destination']
inputPath = args.inputPath
movingSIP = args.movingSIP
algorithm = args.algorithm
removeOriginals = args.removeOriginals
destination = args.destination
loglevel = args.loglevel
logDir = args.logDir
useMV = args.useMV
now = pymmFunctions.timestamp('now')
# Quit if there are required variables missing
missingArgs = 0
try:
# see if the input/destination are on the same filesystem
# if so, we will use mv rather than rsync for efficiency
inputFS = pymmFunctions.get_filesystem_id(inputPath)
destFS = pymmFunctions.get_filesystem_id(destination)
print(inputFS, destFS)
if inputFS == destFS:
print("HEYYYY")
sameFilesystem = True
else:
sameFilesystem = False
except:
sameFilesystem = False
for _arg in requiredArgs:
if getattr(args, _arg) == None:
print("CONFIGURATION PROBLEM:\n"
"You forgot to set {0}. It is required.\n"
"Try again, but set {0} with the flag --{0}\n".format(_arg))
missingArgs += 1
if missingArgs > 0:
sys.exit()
if not movingSIP:
# set up rsync log
if loglevel == 'all':
pymmLogpath = os.path.join(config['logging']['pymm_log_dir'],
'pymm_log.txt')
try:
rsyncLogPath = os.path.join(
logDir, 'rsync_log_{}_{}.txt'.format(
pymmFunctions.get_base(inputPath),
pymmFunctions.timestamp('now')))
except:
print("there was a problem getting the rsync log path ....")
rsyncLogPath = ''
else:
rsyncLogPath = '.'
# sniff what the input is
dir_or_file = pymmFunctions.dir_or_file(inputPath)
if dir_or_file == False:
print(
"oy you've got big problems. {} is not a directory or a file."
" what is it? is it a ghost?".format(inputPath))
return False, False
# sys.exit(1)
# copy the input according to its type
elif dir_or_file == 'dir':
# add trailing slash for rsync destination directory
if not destination[-1] == '/':
destination = destination + '/'
if not sameFilesystem == True:
rsync_object(inputPath, rsyncLogPath, destination)
else:
mv_object(inputPath, destination)
elif dir_or_file == 'file':
if not sameFilesystem == True or useMV == False:
rsync_object(inputPath, rsyncLogPath, destination)
else:
mv_object(inputPath, destination)
else:
print("o_O what is going on here? you up to something?")
# sys.exit()
else:
stagedSIPpath, safe = move_n_verify_sip(inputPath, destination)
print(stagedSIPpath)
return stagedSIPpath, safe
import mysql.connector
except ImportError:
print("Try installing mysql connector/python again.")
sys.exit()
# local modules:
try:
from pymmFunctions import read_config
except:
from .pymmFunctions import read_config
# check this out for mysql.connector install: https://gist.github.com/stefanfoulis/902296/f466a8dba3a75c172ac88627298f18eaaf0aa4c3
# brew install mysql-connector-c
# pip3 install mysql-connector
# pip3 error ``Unable to find Protobuf include directory.`` --> `brew install protobuf`
# and if that still doesn't work try pip3 install mysql-connector==2.1.6
config = read_config()
pymmDB = config['database settings']['pymm_db']
# set db name to None in case it doesn't exist yet
if pymmDB in ('', None):
pymmDB = None
class DB:
connection = None
cursor = None
def __init__(self, user):
self.user = user
def connect(self):
# create a connection using root ...
