示例#1
0
def main(user_args):
    key = user_args.key
    namespace = user_args.namespace

    if namespace is None:
        namespace = default_namespace

    log_level = user_args.log_level

    #***************** Setup Logging **************
    logger = logging.getLogger('service_final' + key)
    level = logging.getLevelName(log_level)
    logger.setLevel(level)
    logFileHandle = logging.FileHandler(pds_log + 'Service.log')

    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s, %(message)s')
    logFileHandle.setFormatter(formatter)
    logger.addHandler(logFileHandle)

    logger.info('Starting Final Process')
    #************Set up REDIS Queues ****************
    zipQueue = RedisQueue(key + '_ZIP', namespace)
    loggyQueue = RedisQueue(key + '_loggy', namespace)
    infoHash = RedisHash(key + '_info')
    recipeQueue = RedisQueue(key + '_recipe', namespace)
    errorHash = RedisHash(key + '_error')

    DBQO = PDS_DBquery('JOBS')

    if errorHash.HashCount() > 0:
        root = ET.Element('errors')

        test = errorHash.getKeys()
        for item in test:
            sub = ET.Element('error')
            root.append(sub)

            field1 = ET.SubElement(sub, 'file')
            if isinstance(item, bytes):
                item = item.decode('utf-8')
            field1.text = item

            Eval = errorHash.getError(item)
            if isinstance(Eval, bytes):
                Eval = Eval.decode('utf-8')

            field2 = ET.SubElement(sub, 'message')
            field2.text = Eval

        tree = ET.ElementTree(root)

        #        testfile = 'test.xml'
        #        with open(testfile, "w") as fh:

        fh = BytesIO()
        tree.write(fh, encoding='utf-8', xml_declaration=True)
        errorxml = str(fh.getvalue(), 'utf-8').replace("\n", "")
        testval = DBQO.addErrors(key, errorxml)
        if testval == 'Success':
            logger.info('Error XML added to JOBS DB')
        elif testval == 'Error':
            logger.error('Adding Error XML to JOBS DB: Error')
        print(errorxml)

    Fdir = os.path.join(pow_map2_base, infoHash.Service(), key)
    Wpath = os.path.join(workarea, key)

    # Make final directory
    if not os.path.exists(Fdir):
        try:
            os.makedirs(Fdir)
            logger.info('Final Location Success: %s', Fdir)
        except:
            logger.error('Error Making Final Directory')

    # Block to build job log file

    outputLOG = Wpath + "/" + key + '.log'
    logOBJ = open(outputLOG, "w")

    logOBJ.write("       U.S. Geological Survey Cloud Processing Services\n")
    logOBJ.write("                http://astrocloud.wr.usgs.gov\n\n")

    if infoHash.Service() == 'POW':
        logOBJ.write("                 Processing On the Web(POW)\n\n")

    logOBJ.write("       Processing Provided by ASTROGEOLOGY USGS Flagstaff\n")
    logOBJ.write("              Contact Information: [email protected]\n\n")
    logOBJ.write(
        "____________________________________________________________________\n\n"
    )

    logOBJ.write("JOB INFORMATION\n\n")
    logOBJ.write("     SERVICE:         " + infoHash.Service() + "\n")
    logOBJ.write("     JOB KEY:         " + key + "\n")
    logOBJ.write("     PROCESSING DATE: " +
                 datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + "\n")

    logOBJ.write("     ISIS VERSION:   " + isis_version)

    if infoHash.getStatus() == 'ERROR':
        logOBJ.write("     JOB STATUS:      " + infoHash.getStatus() +
                     " See Details Below\n")
    else:
        logOBJ.write("     JOB STATUS:      " + infoHash.getStatus() + "\n")
    logOBJ.write("     FILE COUNT:      " + infoHash.getFileCount() + "\n\n")
    logOBJ.write(
        "_____________________________________________________________________\n\n"
    )

    logOBJ.write("PROCESSING INFORMATION\n\n")
    for element in loggyQueue.ListGet():
        procDICT = json.loads(element, object_pairs_hook=OrderedDict)
        for infile in procDICT:
            logOBJ.write("     IMAGE: " + infile + "\n")
            for proc, _ in procDICT[infile].items():
                logOBJ.write("          PROCESS:  " + str(proc) + "\n")
                for k, val in procDICT[infile][proc].items():
                    if k == 'status':
                        logOBJ.write("               STATUS:     " + val +
                                     "\n")
                    elif k == 'parameters':
                        logOBJ.write("               PARAMETERS: " + val +
                                     "\n")
                    elif k == 'helplink':
                        logOBJ.write("               HELP LINK:  " + val +
                                     "\n\n")
                    elif k == 'error':
                        logOBJ.write("               ERROR:      " + val +
                                     "\n\n")

    logOBJ.write("END-PROCESSING\n")
    logOBJ.close()

    #******** Block for to copy and zip files to final directory ******
    map_file = os.path.join(Wpath, key + '.map')
    Zfile = os.path.join(Wpath, key + '.zip')

    final_file_list = zipQueue.ListGet()
    final_file_list.append(outputLOG)
    final_file_list.append(map_file)

    logger.info('Making Zip File %s', Zfile)
    with zipfile.ZipFile(Zfile, 'w') as out_zip:
        for item in final_file_list:
            try:
                logger.info('File %s added to zip file: success', item)
                out_zip.write(item, os.path.basename(item))
            except:
                logger.error('File %s NOT added to zip file', item)

    logger.info('Copying output files to %s', Fdir)
    for item in final_file_list:
        try:
            shutil.copyfile(item, os.path.join(Fdir, os.path.basename(item)))
            logger.info('Copied file %s to download directory: Success', item)
        except:
            logger.error('File %s NOT COPIED to download directory', item)
            logger.error(e)

    try:
        shutil.copy(Zfile, os.path.join(Fdir, os.path.basename(Zfile)))
        logger.info('Copied zip file %s to download directory', Zfile)
    except IOError as e:
        logger.error(
            'Error while attempting to copy zip file %s to download area',
            Zfile)
        logger.error(e)


#************** Clean up *******************
    try:
        shutil.rmtree(Wpath)
        logger.info('Working Directory Removed: Success')
    except:
        logger.error('Working Directory NOT Removed')

    DBQO2 = PDS_DBquery('JOBS')
    DBQO2.setJobsFinished(key)

    errorHash.RemoveAll()
    infoHash.RemoveAll()
    loggyQueue.RemoveAll()
    zipQueue.RemoveAll()
    recipeQueue.RemoveAll()

    logger.info('Job %s is Complete', key)
示例#2
0
def main(user_args):
    key = user_args.key
    namespace = user_args.namespace
    log_level = user_args.log_level

#***************** Setup Logging **************
    logger = logging.getLogger('service_final' + key)
    level = logging.getLevelName(log_level)
    logger.setLevel(level)
    logFileHandle = logging.FileHandler(pds_log+'Service.log')

    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s, %(message)s')
    logFileHandle.setFormatter(formatter)
    logger.addHandler(logFileHandle)

    logger.info('Starting Final Process')
#************Set up REDIS Queues ****************
    zipQueue = RedisQueue(key + '_ZIP', namespace)
    loggyQueue = RedisQueue(key + '_loggy', namespace)
    infoHash = RedisHash(key + '_info')
    recipeQueue = RedisQueue(key + '_recipe', namespace)
    errorHash = RedisHash(key + '_error')

    DBQO = PDS_DBquery('JOBS')

    if errorHash.HashCount() > 0:
        root = ET.Element('errors')

        test = errorHash.getKeys()
        for key in test:
            sub = ET.Element('error')
            root.append(sub)

            field1 = ET.SubElement(sub, 'file')
            field1.text = key

            Eval = errorHash.getError(key)

            field2 = ET.SubElement(sub, 'message')
            field2.text = Eval

        tree = ET.ElementTree(root)

#        testfile = 'test.xml'
#        with open(testfile, "w") as fh:

        fh = BytesIO()
        tree.write(fh, encoding='utf-8', xml_declaration=True)
        testval = DBQO.addErrors(key , fh.getvalue())
        if testval == 'Success':
            logger.info('Error XML add to JOBS DB')
        elif testval == 'Error':
            logger.error('Addin Error XML to JOBS DB: Error')
        print(fh.getvalue())

    Fdir = pow_map2_base + infoHash.Service() + '/' + key
    Wpath = scratch + key

    # Make final directory
    if not os.path.exists(Fdir):
        try:
            os.makedirs(Fdir)
            logger.info('Final Location Success: %s', Fdir)
        except:
            logger.error('Error Making Final Directory')

    # Block to build job log file

    outputLOG = Wpath + "/" + key + '.log'
    logOBJ = open(outputLOG, "w")

    logOBJ.write("       U.S. Geological Survey Cloud Processing Services\n")
    logOBJ.write("                http://astrocloud.wr.usgs.gov\n\n")

    if infoHash.Service() == 'POW':
        logOBJ.write("                 Processing On the Web(POW)\n\n")

    logOBJ.write("       Processing Provided by ASTROGEOLOGY USGS Flagstaff\n")
    logOBJ.write("              Contact Information: [email protected]\n\n")
    logOBJ.write(
        "____________________________________________________________________\n\n")

    logOBJ.write("JOB INFORMATION\n\n")
    logOBJ.write("     SERVICE:         " + infoHash.Service() + "\n")
    logOBJ.write("     JOB KEY:         " + key + "\n")
    logOBJ.write("     PROCESSING DATE: " +
                 datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + "\n")

    logOBJ.write("     ISIS VERSION:   " + isis_version)

    if infoHash.getStatus() == 'ERROR':
        logOBJ.write("     JOB STATUS:      " +
                     infoHash.getStatus() + " See Details Below\n")
    else:
        logOBJ.write("     JOB STATUS:      " + infoHash.getStatus() + "\n")
    logOBJ.write("     FILE COUNT:      " + infoHash.getFileCount() + "\n\n")
    logOBJ.write(
        "_____________________________________________________________________\n\n")

    logOBJ.write("PROCESSING INFORMATION\n\n")
    for element in loggyQueue.ListGet():
        procDICT = json.loads(element, object_pairs_hook=OrderedDict)
        for infile in procDICT:
            logOBJ.write("     IMAGE: " + infile + "\n")
            for proc, _ in procDICT[infile].items():
                logOBJ.write("          PROCESS:  " + str(proc) + "\n")
                for k, val in procDICT[infile][proc].items():
                    if k == 'status':
                        logOBJ.write(
                            "               STATUS:     " + val + "\n")
                    elif k == 'command':
                        logOBJ.write(
                            "               COMMAND:    " + val + "\n")
                    elif k == 'helplink':
                        logOBJ.write(
                            "               HELP LINK:  " + val + "\n\n")
                    elif k == 'error':
                        logOBJ.write(
                            "               ERROR:      " + val + "\n\n")

    logOBJ.write("END-PROCESSING\n")
    logOBJ.close()

#******** Block for to copy and zip files to final directory ******
    Zfile = Wpath + '/' + key + '.zip'
    logger.info('Making Zip File %s', Zfile)

# log file stuff
    try:
        Lfile = key + '.log'
        Zcmd = 'zip -j ' + Zfile + " -q " + outputLOG
        process = subprocess.Popen(
            Zcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        (stdout, stderr) = process.communicate()
#        zOBJ.write(outputLOG, arcname=Lfile)
        logger.info('Log file %s Added to Zip File: Success', Lfile)
        logger.info('zip stdout: ' + stdout)
        logger.info('zip stderr: ' + stderr)
    except:
        logger.error('Log File %s NOT Added to Zip File', Lfile)

    try:
        shutil.copyfile(outputLOG, Fdir + "/" + Lfile)
        logger.info('Copied Log File %s to Final Area: Success', Lfile)
        os.remove(outputLOG)
    except IOError as e:
        logger.error('Log File %s NOT COPIED to Final Area', Lfile)
        logger.error(e)

    # Add map file to zip
    try:
        map_file = Wpath + "/" + key + '.map'
        Zcmd = 'zip -j ' + Zfile + " -q " + map_file
        process = subprocess.Popen(
            Zcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        (stdout, stderr) = process.communicate()

        logger.info('Map file %s added to zip file: Success', map_file)
        logger.info('zip stdout: ' + stdout)
        logger.info('zip stderr: ' + stderr)
    except:
        logger.error('Map File %s NOT added to Zip File', map_file)

    try:
        shutil.copyfile(map_file, Fdir + "/" + key + '.map')
        logger.info('Copied map file %s to final area: success', key + '.map')
    except IOError as e:
        logger.error('Map file %s NOT COPIED to final area', key + '.map')
        logger.error(e)

# file stuff
    for Lelement in zipQueue.ListGet():
        Pfile = os.path.basename(Lelement)
#        auxfile = os.path.basename(Lelement) + '.aux.xml'
        auxfile = Wpath + '/' + os.path.basename(Lelement) + '.aux.xml'

        try:
            Zcmd = 'zip -j ' + Zfile + " -q " + Lelement
            process = subprocess.Popen(
                Zcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
            (stdout, stderr) = process.communicate()
            logger.info('File %s Added to Zip File: Success', Pfile)
            if os.path.isfile(auxfile):
                Zcmd = 'zip -j ' + Zfile + " -q " + auxfile
                process = subprocess.Popen(
                    Zcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                (stdout, stderr) = process.communicate()
                logger.info('File %s Added to Zip File: Success',
                            os.path.basename(Lelement) + '.aux.xml')
        except:
            logger.error('Error During Zip Operation')

        try:
            shutil.copyfile(Wpath + '/' + Pfile, Fdir + '/' + Pfile)
            logger.info('Copy File %s : Success', Pfile)
            os.remove(Wpath + "/" + Pfile)
            if os.path.isfile(auxfile):
                shutil.copyfile(auxfile, Fdir + '/' +
                                os.path.basename(Lelement) + '.aux.xml')
                logger.info('Copy File %s : Success',
                            os.path.basename(Lelement) + '.aux.xml')
                os.remove(auxfile)
        except IOError as e:
            logger.error('Error During File Copy Operation')
            logger.error(e)

#    zOBJ.close()

    try:
        shutil.copy(Zfile, Fdir + '/' + key + '.zip')
        os.remove(Zfile)
        logger.info('Zip File Copied to Final Directory')
    except IOError as e:
        logger.error('Error During Zip File Copy Operation')
        logger.error(e)

#************** Clean up *******************
    os.remove(Wpath + '/' + key + '.map')
    os.remove(Wpath + '/' + key + '.sbatch')
    try:
        #        os.rmdir(Wpath)
        shutil.rmtree(Wpath)
        logger.info('Working Directory Removed: Success')
    except:
        logger.error('Working Directory NOT Removed')

    DBQO2 = PDS_DBquery('JOBS')
    DBQO2.setJobsFinished(key )

    infoHash.RemoveAll()
    loggyQueue.RemoveAll()
    zipQueue.RemoveAll()
    recipeQueue.RemoveAll()

    logger.info('Job %s is Complete', key)