Esempio n. 1
0
def package_submission(basepath, recid, hep_submission_obj):
    """
    Zips up a submission directory. This is in advance of its download
    for example by users
    :param basepath: path of directory containing all submission files
    :param recid: the publication record ID
    :param hep_submission_obj: the HEPSubmission object representing
           the overall position
    """
    if not os.path.exists(
            os.path.join(current_app.config['CFG_DATADIR'], str(recid))):
        os.makedirs(os.path.join(current_app.config['CFG_DATADIR'],
                                 str(recid)))

    version = hep_submission_obj.version
    if version == 0:
        version = 1

    zip_location = os.path.join(
        current_app.config['CFG_DATADIR'], str(recid),
        current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(
            recid, version))
    if os.path.exists(zip_location):
        os.remove(zip_location)

    zipf = zipfile.ZipFile(zip_location, 'w')
    os.chdir(basepath)
    zipdir(".", zipf)
    zipf.close()
Esempio n. 2
0
def package_submission(basepath, recid, hep_submission_obj):
    """
    Zips up a submission directory. This is in advance of its download
    for example by users
    :param basepath: path of directory containing all submission files
    :param recid: the publication record ID
    :param hep_submission_obj: the HEPSubmission object representing
           the overall position
    """
    if not os.path.exists(os.path.join(current_app.config['CFG_DATADIR'], str(recid))):
        os.makedirs(os.path.join(current_app.config['CFG_DATADIR'], str(recid)))

    version = hep_submission_obj.version
    if version == 0:
        version = 1

    zip_location = os.path.join(
        current_app.config['CFG_DATADIR'], str(recid),
        current_app.config['SUBMISSION_FILE_NAME_PATTERN']
            .format(recid, version))
    if os.path.exists(zip_location):
        os.remove(zip_location)

    zipf = zipfile.ZipFile(zip_location, 'w')
    os.chdir(basepath)
    zipdir(".", zipf)
    zipf.close()
Esempio n. 3
0
def split_files(file_location, output_location,
                archive_location=None):
    """
    :param file_location: input yaml file location
    :param output_location: output directory path
    :param archive_location: if present will create a zipped
    representation of the split files
    """
    try:
        try:
            file_documents = yaml.load_all(open(file_location, 'r'), Loader=yaml.CSafeLoader)
        except:
            file_documents = yaml.safe_load_all(open(file_location, 'r'))

        # make a submission directory where all the files will be stored.
        # delete a directory in the event that it exists.
        if os.path.exists(output_location):
            shutil.rmtree(output_location)

        os.makedirs(output_location)

        with open(os.path.join(output_location, "submission.yaml"),
                  'w') as submission_yaml:
            for document in file_documents:
                if "record_ids" in document:
                    write_submission_yaml_block(
                        document, submission_yaml)
                else:
                    file_name = document["name"].replace(' ', '') + ".yaml"
                    document["data_file"] = file_name

                    with open(os.path.join(output_location, file_name),
                              'w') as data_file:
                        yaml.add_representer(str, str_presenter)
                        yaml.dump(
                            {"independent_variables":
                                cleanup_data_yaml(
                                    document["independent_variables"]),
                                "dependent_variables":
                                    cleanup_data_yaml(
                                        document["dependent_variables"])},
                            data_file, allow_unicode=True)

                    write_submission_yaml_block(document,
                                                submission_yaml,
                                                type="record")

        if archive_location:
            if os.path.exists(archive_location):
                os.remove(archive_location)

            zipf = zipfile.ZipFile(archive_location, 'w')
            os.chdir(output_location)
            zipdir(".", zipf)
            zipf.close()
    except Exception as e:
        current_app.logger.exception(e)
        current_app.logger.error(
            'Error parsing {0}, {1}'.format(file_location, e.message))
Esempio n. 4
0
def split_files(file_location, output_location, archive_location=None):
    """
    :param file_location: input yaml file location
    :param output_location: output directory path
    :param archive_location: if present will create a zipped
    representation of the split files
    """
    last_updated = datetime.now()
    try:
        file_documents = yaml.load_all(open(file_location, 'r'), Loader=Loader)

        # make a submission directory where all the files will be stored.
        # delete a directory in the event that it exists.
        if os.path.exists(output_location):
            shutil.rmtree(output_location)

        os.makedirs(output_location)

        with open(os.path.join(output_location, "submission.yaml"),
                  'w') as submission_yaml:
            for document in file_documents:
                if not document:
                    continue
                elif "name" not in document:
                    if "dateupdated" in document:
                        try:
                            last_updated = parse(document['dateupdated'],
                                                 dayfirst=True)
                        except ValueError as ve:
                            last_updated = datetime.now()
                    write_submission_yaml_block(document, submission_yaml)
                else:
                    file_name = document["name"].replace(' ', '') + ".yaml"
                    document["data_file"] = file_name

                    with open(os.path.join(output_location, file_name),
                              'w') as data_file:
                        Dumper.add_representer(str, str_presenter)
                        yaml.dump(
                            {
                                "independent_variables":
                                cleanup_data_yaml(
                                    document["independent_variables"]),
                                "dependent_variables":
                                cleanup_data_yaml(
                                    document["dependent_variables"])
                            },
                            data_file,
                            allow_unicode=True,
                            Dumper=Dumper)

                    write_submission_yaml_block(document,
                                                submission_yaml,
                                                type="record")

        if archive_location:
            if os.path.exists(archive_location):
                os.remove(archive_location)

            zipf = zipfile.ZipFile(archive_location, 'w')
            os.chdir(output_location)
            zipdir(".", zipf)
            zipf.close()
    except yaml.scanner.ScannerError as se:
        return se, last_updated
    except yaml.parser.ParserError as pe:
        return pe, last_updated
    except Exception as e:
        log.error('Error parsing %s, %s', file_location, e.message)
        return e, last_updated
    return None, last_updated