def check_pymm_log_exists(): # open a local instance of config here in case # it has changed since importing this file pymmConfig = pymmFunctions.read_config() pymmLogDir = pymmConfig['logging']['pymm_log_dir'] pymmLogPath = os.path.join(pymmLogDir, 'pymm_log.txt') # sys.exit() opener = (((("#" * 75) + '\n') * 2) + ((' ') * 4) + 'THIS IS THE LOG FOR PYMEDIAMICROSERVICES' '\n\n' + ((' ') * 4) + 'THIS VERSION WAS STARTED ON ' + pymmFunctions.today + '\n' + ((("#" * 75) + '\n') * 2) + '\n') if not os.path.isfile(pymmLogPath): print('wait i need to make a logfile') if pymmLogDir == '': print("!~" * 75) print("CONFIGURATION PROBLEM:\n" "THERE IS NO DIRECTORY SET FOR THE SYSTEM-WIDE LOG.\n" "PLEASE RUN pymmconfig.py OR EDIT config.ini DIRECTLY\n" "TO ADD A VALID DIRECTORY FOR pymm_log.txt TO LIVE IN.\n" "WE'RE JUST GOING TO PUT IT ON YOUR DESKTOP FOR NOW...") desktop = pymmFunctions.get_desktop() pymmLogPath = os.path.join(desktop, 'pymm_log.txt') from pymmconfig import pymmconfig pymmconfig.set_value('logging', 'pymm_log_dir', desktop) with open(pymmLogPath, 'w+') as pymmLog: pymmLog.write(opener) else: open(pymmLogPath, 'x') with open(pymmLogPath, 'w+') as pymmLog: pymmLog.write(opener) else: pass
def main(): config = pymmFunctions.read_config() # THIS IS PROBABLY NOT GOING TO BE NEEDED parser = argparse.ArgumentParser() parser.add_argument('-i','--inputPath',help='path of input file') parser.add_argument('-m','--mediainfo',action='store_true',help='generate a mediainfo sidecar file') parser.add_argument('-f','--frame_md5',action='store_true',help='make frame md5 report') parser.add_argument('-j','--getJSON',action='store_true',help='get JSON output as applicable') parser.add_argument('-d','--destination',help='set destination for output metadata files') args = parser.parse_args() inputPath = args.inputPath destination = args.destination frame_md5 = args.frame_md5 mediainfo_report = args.mediainfo getJSON = args.getJSON if not inputPath: print("\n\nHEY THERE, YOU NEED TO SET AN INPUT FILE TO RUN THIS SCRIPT ON.\rNOW EXITING") sys.exit() if not destination: print(''' YOU DIDN'T TELL ME WHERE TO PUT THE OUTPUT OF THIS SCRIPT, SO WE'LL PUT ANY SIDECAR FILES IN THE SAME DIRECTORY AS YOUR INPUT FILE. ''') destination = os.path.dirname(os.path.abspath(inputPath)) if mediainfo_report: get_mediainfo_report(inputPath,destination,getJSON) if frame_md5: frameMd5Filepath = make_frame_md5(inputPath,destination) print(frameMd5Filepath)
def main(): config = pymmFunctions.read_config() args = set_args() requiredArgs = ['inputPath', 'destination'] inputPath = args.inputPath algorithm = args.algorithm removeOriginals = args.removeOriginals destination = args.destination loglevel = args.loglevel logDir = args.logDir now = pymmFunctions.timestamp('now') # Quit if there are required variables missing missingArgs = 0 for _arg in requiredArgs: if getattr(args, _arg) == None: print("CONFIGURATION PROBLEM:\n" "You forgot to set " + _arg + ". It is required.\n" "Try again, but set " + _arg + " with the flag --" + _arg + "\n") missingArgs += 1 if missingArgs > 0: sys.exit() # set up rsync log if loglevel == 'all': pymmLogpath = os.path.join(config['logging']['pymm_log_dir'], 'pymm_log.txt') # AT WHAT POINT WILL WE ACTUALLY WANT TO PYMMLOG A COPY? FINAL AIP XFER? try: rsyncLogpath = os.path.join( logDir, 'rsync_log_' + pymmFunctions.get_base(inputPath) + '_' + pymmFunctions.timestamp('now') + '.txt') except: print("there was a problem getting the rsync log path ....") rsyncLogpath = '' else: rsyncLogpath = '' # sniff what the input is dir_or_file = pymmFunctions.dir_or_file(inputPath) if dir_or_file == False: print("oy you've got big problems. " + inputPath + " is not a directory or a file. what is it? is it a ghost?") sys.exit() # copy the input according to its type elif dir_or_file == 'dir': # add trailing slash for rsync destination directory if not destination[-1] == '/': destination = destination + '/' copy_dir(inputPath, rsyncLogpath, destination) elif dir_or_file == 'file': copy_file(inputPath, rsyncLogpath, destination) else: print("o_O what is going on here? you up to something?") sys.exit()
def __init__(self, user=None, objectJSON=None, databaseReporting=None, ingestType=None, makeProres=None, concatChoice=None, cleanupStrategy=None, overrideOutdir=None, overrideAIPdir=None, overrideRS=None, mono=None, combineAudio=None): ####### # GLOBAL CONFIG (this is ConfigParser object, callable as a dict) self.config = pymmFunctions.read_config() ####### # INPUT ARGUMENTS (FROM CLI) self.user = user # path to optional JSON file of descriptive mentadata self.objectJSON = objectJSON self.databaseReporting = databaseReporting self.ingestType = ingestType self.makeProres = makeProres self.concatChoice = concatChoice self.cleanupStrategy = cleanupStrategy self.mono = mono self.combineAudio = combineAudio if overrideOutdir == None: self.outdir_ingestsip = self.config['paths']['outdir_ingestsip'] else: self.outdir_ingestsip = overrideOutdir if overrideAIPdir == None: self.aip_staging = self.config['paths']['aip_staging'] else: self.aip_staging = overrideAIPdir if overrideRS == None: self.resourcespace_deliver = self.config['paths'][ 'resourcespace_deliver'] else: self.resourcespace_deliver = overrideRS self.databaseReporting = self.test_db_access(self.user) ####### # ENVIRONMENT VARIABLES self.computer = pymmFunctions.get_node_name() self.ffmpegVersion = 'ffmpeg ver.: '+\ pymmFunctions.get_ffmpeg_version()
def pymm_log(CurrentIngest, event, outcome, status): check_pymm_log_exists() pymmConfig = pymmFunctions.read_config() pymmLogDir = pymmConfig['logging']['pymm_log_dir'] pymmLogPath = os.path.join(pymmLogDir, 'pymm_log.txt') stamp = pymmFunctions.timestamp('iso8601') systemInfo = CurrentIngest.systemInfo objectRootPath = CurrentIngest.InputObject.inputPath canonicalName = CurrentIngest.InputObject.canonicalName inputTypeDetail = CurrentIngest.InputObject.inputTypeDetail user = CurrentIngest.ProcessArguments.user ingestUUID = CurrentIngest.ingestUUID tempID = CurrentIngest.tempID workingDir = CurrentIngest.ProcessArguments.outdir_ingestsip prefix = '' suffix = '\n' # I think basename gets updated depending on what is getting logged... ? @fixme basename = os.path.basename(objectRootPath) if status == 'STARTING': prefix = ('&' * 50) + '\n\n' stuffToLog = [ prefix, stamp, "\nEvent type: Ingestion start\n", "Object canonical name: {}\n".format(canonicalName), "Object filepath: {}\n".format(objectRootPath), "Object type: {}\n".format(inputTypeDetail), "Ingest UUID: {}\n".format(ingestUUID), "Operator: {}\n".format(user), "Ingest working directory: {}\n".format(workingDir), "\n### SYSTEM INFO: ### \n{}".format(systemInfo), suffix ] elif status in ("ENDING", "ABORTING"): suffix = '\n\n' + ('#' * 50) + "\n\n" stuffToLog = [ prefix, stamp, " | Status: {} |".format(status), " | Event type: Ingestion end | ", "Outcome: {}".format(outcome), suffix ] else: stuffToLog = [ prefix, stamp, " | Status: {}".format(status), " | Object name: {}".format(basename), " | Event type: {}".format(event), " | Event outcome: {}".format(outcome), suffix ] with open(pymmLogPath, 'a') as log: for item in stuffToLog: log.write(item)
import loggers import makeDerivs import moveNcopy import makeMetadata import pymmFunctions except: from .bampfa_pbcore import pbcore, makePbcore from . import concatFiles from . import ingestClasses from . import loggers from . import makeDerivs from . import moveNcopy from . import makeMetadata from . import pymmFunctions # read in from the config file config = pymmFunctions.read_config() def set_args(): parser = argparse.ArgumentParser() parser.add_argument('-i', '--inputPath', help='path of input file', required=True) parser.add_argument('-u', '--user', help='name of the person doing the ingest', required=True) parser.add_argument( '-j', '--metadataJSON',
def main(): config = pymmFunctions.read_config() args = set_args() requiredArgs = ['inputPath', 'destination'] inputPath = args.inputPath movingSIP = args.movingSIP algorithm = args.algorithm removeOriginals = args.removeOriginals destination = args.destination loglevel = args.loglevel logDir = args.logDir useMV = args.useMV now = pymmFunctions.timestamp('now') # Quit if there are required variables missing missingArgs = 0 try: # see if the input/destination are on the same filesystem # if so, we will use mv rather than rsync for efficiency inputFS = pymmFunctions.get_filesystem_id(inputPath) destFS = pymmFunctions.get_filesystem_id(destination) print(inputFS, destFS) if inputFS == destFS: print("HEYYYY") sameFilesystem = True else: sameFilesystem = False except: sameFilesystem = False for _arg in requiredArgs: if getattr(args, _arg) == None: print("CONFIGURATION PROBLEM:\n" "You forgot to set {0}. It is required.\n" "Try again, but set {0} with the flag --{0}\n".format(_arg)) missingArgs += 1 if missingArgs > 0: sys.exit() if not movingSIP: # set up rsync log if loglevel == 'all': pymmLogpath = os.path.join(config['logging']['pymm_log_dir'], 'pymm_log.txt') try: rsyncLogPath = os.path.join( logDir, 'rsync_log_{}_{}.txt'.format( pymmFunctions.get_base(inputPath), pymmFunctions.timestamp('now'))) except: print("there was a problem getting the rsync log path ....") rsyncLogPath = '' else: rsyncLogPath = '.' # sniff what the input is dir_or_file = pymmFunctions.dir_or_file(inputPath) if dir_or_file == False: print( "oy you've got big problems. {} is not a directory or a file." " what is it? is it a ghost?".format(inputPath)) return False, False # sys.exit(1) # copy the input according to its type elif dir_or_file == 'dir': # add trailing slash for rsync destination directory if not destination[-1] == '/': destination = destination + '/' if not sameFilesystem == True: rsync_object(inputPath, rsyncLogPath, destination) else: mv_object(inputPath, destination) elif dir_or_file == 'file': if not sameFilesystem == True or useMV == False: rsync_object(inputPath, rsyncLogPath, destination) else: mv_object(inputPath, destination) else: print("o_O what is going on here? you up to something?") # sys.exit() else: stagedSIPpath, safe = move_n_verify_sip(inputPath, destination) print(stagedSIPpath) return stagedSIPpath, safe
import mysql.connector except ImportError: print("Try installing mysql connector/python again.") sys.exit() # local modules: try: from pymmFunctions import read_config except: from .pymmFunctions import read_config # check this out for mysql.connector install: https://gist.github.com/stefanfoulis/902296/f466a8dba3a75c172ac88627298f18eaaf0aa4c3 # brew install mysql-connector-c # pip3 install mysql-connector # pip3 error ``Unable to find Protobuf include directory.`` --> `brew install protobuf` # and if that still doesn't work try pip3 install mysql-connector==2.1.6 config = read_config() pymmDB = config['database settings']['pymm_db'] # set db name to None in case it doesn't exist yet if pymmDB in ('', None): pymmDB = None class DB: connection = None cursor = None def __init__(self, user): self.user = user def connect(self): # create a connection using root ...